atris 3.74.28 → 3.75.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/fleet.js CHANGED
@@ -33,7 +33,7 @@ const {
33
33
  buildOneLapValidatorPrompt,
34
34
  parseOneLapValidatorVerdict,
35
35
  } = require('./one-lap-validator');
36
- const { listWorktrees, defaultMainlineBase, gitFailureDetail, gitNetworkEnvironment } = require('../commands/worktree');
36
+ const { listWorktrees, defaultMainlineBase, gitFailureDetail, gitNetworkEnvironment, lowDiskRefusal } = require('../commands/worktree');
37
37
  const { isConductorStatusLine } = require('./conductor-artifacts');
38
38
  const { matchLessons } = require('./lesson-preflight');
39
39
  const { sleepSync } = require('./sleep-sync');
@@ -44,7 +44,7 @@ const { buildVerifiedScoutPack, appendVerifiedScoutPack } = require('./dispatch-
44
44
  const { runPrepPass, withPrepBrief, prepRecordFields } = require('./roster-prep');
45
45
  const { mapNotesPath } = require('./map-refs');
46
46
  const { SHARED_BRIEF_RULES } = require('./brief-rules');
47
- const { PROOF_NOT_SAVED } = require('./release-kind');
47
+ const { PROOF_NOT_SAVED, COULD_NOT_START } = require('./release-kind');
48
48
  const {
49
49
  applySecretGrantEnvironment,
50
50
  buildGatewaySupervisorScript,
@@ -53,18 +53,27 @@ const {
53
53
  } = require('./secret-gateway');
54
54
  const { appendEngineLiveLogChunk, createEngineLiveLog, engineTerminalReason } = require('./engine-job-lifecycle');
55
55
  const { recordEngineResource } = require('./engine-resource');
56
+ const {
57
+ taskTagTokens,
58
+ isDecisionHoldTag,
59
+ isDecisionTask,
60
+ DECISION_REFUSE_REASON,
61
+ } = require('./task-decision');
56
62
 
57
63
  // Lanes a fleet may never staff on its own: the human keeps irreversible
58
64
  // calls. Mirrors the autoland denied lanes.
59
65
  const DENIED_TAGS = ['billing', 'money', 'payments', 'deploy', 'security', 'customer', 'external', 'feedback'];
60
66
 
61
- // The protected lane a task sits in (a denied tag on the task or its todo
62
- // tags), or null. The ship gate and the dispatch flight both ask this, so they
63
- // cannot disagree about which work a person must land.
67
+ // The protected lane a task sits in, or null. This is the one place that
68
+ // decides protection: staffing (isSafeLane), the ship gate, the dispatch flight
69
+ // and the builder prompt all ask it, so a task one of them holds back as
70
+ // protected is never one that another would merge. A task's tags are its tag,
71
+ // its tags list, the tags `atris task tag` adds later and hashtags in its
72
+ // title (all through taskTagTokens), plus the todo tags it was imported with.
64
73
  function protectedLaneOfTask(task) {
65
74
  if (!task || typeof task !== 'object') return null;
66
75
  const todoTags = task.metadata && Array.isArray(task.metadata.todo_tags) ? task.metadata.todo_tags : [];
67
- const tags = [task.tag, ...todoTags].filter(Boolean).map((tag) => String(tag).toLowerCase());
76
+ const tags = [...taskTagTokens(task), ...todoTags.filter(Boolean).map((tag) => String(tag).trim().toLowerCase())];
68
77
  return tags.find((tag) => DENIED_TAGS.includes(tag)) || null;
69
78
  }
70
79
 
@@ -126,7 +135,8 @@ function shellQuote(value) {
126
135
 
127
136
  // The bounded prompt every engine gets: isolated worktree, MAP first, the
128
137
  // task's own check, one landing order per mode. Yolo: the engine lands it
129
- // itself with worktree ship. Otherwise: commit only, the conductor lands it.
138
+ // itself with worktree ship, unless the task is in a protected lane. Otherwise:
139
+ // commit only, the conductor lands it.
130
140
  function buildFleetPrompt(task, { worktreePath, yolo = false } = {}) {
131
141
  const ref = task.display_id || task.id || 'TASK';
132
142
  const title = String(task.title || '').trim();
@@ -134,11 +144,27 @@ function buildFleetPrompt(task, { worktreePath, yolo = false } = {}) {
134
144
  const done = titleDone || String((task.metadata && task.metadata.done_looks_like) || '').trim();
135
145
  const check = taskDispatchCheck(task, worktreePath || process.cwd());
136
146
  const shipMessage = title.split(/[.:](?:\s|$)/)[0].replace(/"/g, "'").slice(0, 90) || ref;
137
- const commitRule = yolo
138
- ? `Land it yourself. Commit on the current branch with a plain-English message, then run: atris worktree ship --message "${shipMessage} (${ref})" --verify ${shellQuote(check)} --merge, and report the PR URL. That ship command is the only push allowed. If it reports a conflict or a failed check, stop and report; never resolve conflicts yourself.`
139
- : 'Commit on the current branch with a clear message. Do not push and do not create branches; the conductor lands your commit.';
147
+ // A protected-lane task is never shipped by its builder, yolo or not. The
148
+ // builder commits and stops, and the dispatch flight ships that commit once,
149
+ // without --merge, so a pull request opens and a person lands it. A builder
150
+ // that shipped too would open the pull request before the flight's own
151
+ // check, and if the flight's check or ship then failed, the claim would be
152
+ // released with that pull request open and unrecorded. Yolo still gives the
153
+ // builder its network for running checks.
154
+ const lane = yolo ? protectedLaneOfTask(task) : null;
155
+ let commitRule;
156
+ if (lane) {
157
+ commitRule = `This task is in the protected ${lane} lane, so a person lands it, not you. Commit on the current branch with a plain-English message. Do not push, do not run \`atris worktree ship\`, and do not create branches; the conductor ships your commit for a person to review.`;
158
+ } else if (yolo) {
159
+ commitRule = `Land it yourself. Commit on the current branch with a plain-English message, then run: atris worktree ship --message "${shipMessage} (${ref})" --verify ${shellQuote(check)} --merge, and report the PR URL. That ship command is the only push allowed. If it reports a conflict or a failed check, stop and report; never resolve conflicts yourself.`;
160
+ } else {
161
+ commitRule = 'Commit on the current branch with a clear message. Do not push and do not create branches; the conductor lands your commit.';
162
+ }
163
+ // The guard refuses a checkout that has edits in it, so it can only pass at
164
+ // the start. A builder that re-ran it in its review step, after its edits,
165
+ // took the failure as a stop order and left finished work uncommitted.
140
166
  const lines = [
141
- 'First, run `atris worktree guard`; if it fails, stop immediately, report back, and do not edit anything. Do this before any file edit.',
167
+ 'Before your first edit, run `atris worktree guard` once; if it fails then, stop immediately, report back, and do not edit anything. It checks for a clean start, so it fails once edits exist: never run it again after editing.',
142
168
  '',
143
169
  ATRIS_BUILD_PROCESS_PREAMBLE,
144
170
  '',
@@ -162,7 +188,7 @@ function buildFleetPrompt(task, { worktreePath, yolo = false } = {}) {
162
188
  `- ${commitRule}`,
163
189
  ...METHOD_KERNEL.map((rule) => `- ${rule}`),
164
190
  '',
165
- `Final report (plain text): files changed, check command + result, commit sha${yolo ? ', PR URL' : ''} (or say what failed and why).`
191
+ `Final report (plain text): files changed, check command + result, commit sha${yolo && !lane ? ', PR URL' : ''} (or say what failed and why).`
166
192
  );
167
193
  const lessonFiles = [
168
194
  ...(Array.isArray(task.files) ? task.files : []),
@@ -926,10 +952,18 @@ async function dispatchEntryWithRestaff({
926
952
  defaultPin = null,
927
953
  job = 'build',
928
954
  prepAsk = undefined,
955
+ beforeEngine = null,
929
956
  }) {
930
957
  const attempts = [];
931
- const result = await dispatchEntryAttempts({ entry, engine, root, dispatch, installedEngines, restaffState, scoutAsk, clock, defaultPin, attempts, job, prepAsk });
932
- return { ...result, attempts };
958
+ try {
959
+ const result = await dispatchEntryAttempts({ entry, engine, root, dispatch, installedEngines, restaffState, scoutAsk, clock, defaultPin, attempts, job, prepAsk, beforeEngine });
960
+ return { ...result, attempts };
961
+ } catch (err) {
962
+ // One job's attempt blowing up (a retry brief that cannot be written, a store that throws) is that job's
963
+ // trouble. It comes back as a job whose attempt failed, for the flight to let its claim go where it failed,
964
+ // and the run goes on to its cleanup instead of ending with the other jobs' claims still held.
965
+ return { not_started: 'dispatch', detail: String(err && err.message || err), attempts };
966
+ }
933
967
  }
934
968
 
935
969
  // One prep pass per prep job per task: a backup whose line asks for the same
@@ -968,6 +1002,10 @@ async function dispatchEntryAttempts({
968
1002
  attempts = [],
969
1003
  job = 'build',
970
1004
  prepAsk = undefined,
1005
+ // Called right before an engine is asked, every time one is: the assigned engine, a retry on another engine
1006
+ // after a dead one, a handover to the next worker (all after any scout, prep, and brief), so the flight can
1007
+ // check it still holds the claim and mark it started. If it throws, that engine is not asked.
1008
+ beforeEngine = null,
971
1009
  }) {
972
1010
  const basePrompt = entry.prompt || buildFleetPrompt(entry.task, { worktreePath: entry.worktreePath, yolo: entry.yolo });
973
1011
  const prepFor = prepPassCache({ entry, root, clock, prepAsk, job });
@@ -989,16 +1027,39 @@ async function dispatchEntryAttempts({
989
1027
  // A worker whose roster line asks for prep gets its brief first.
990
1028
  const prep = ranWith.prep ? await prepFor(ranWith.prep, scouted) : null;
991
1029
  const prompt = withPrepBrief(scouted, prep);
992
- const brief = captureDispatchBrief({
993
- root,
994
- task: entry.task,
995
- engine: engineName,
996
- worktreePath: entry.worktreePath,
997
- prompt,
998
- missionId: entry.mission_id || entry.missionId || '',
999
- author: entry.author || '',
1000
- yolo: entry.yolo,
1001
- });
1030
+ let brief;
1031
+ try {
1032
+ brief = captureDispatchBrief({
1033
+ root,
1034
+ task: entry.task,
1035
+ engine: engineName,
1036
+ worktreePath: entry.worktreePath,
1037
+ prompt,
1038
+ missionId: entry.mission_id || entry.missionId || '',
1039
+ author: entry.author || '',
1040
+ yolo: entry.yolo,
1041
+ });
1042
+ } catch (err) {
1043
+ // Before any engine has run, a brief that cannot be written means the job never started: hand that
1044
+ // back, so the flight pauses it and lets the claim go as one that could not start. After an engine has
1045
+ // run, it fails the flight as it always did.
1046
+ if (attempts.length) throw err;
1047
+ return { not_started: 'brief_capture', detail: String(err && err.message || err) };
1048
+ }
1049
+ // An engine is about to be asked, the assigned one or a later one (a retry, a handover), and each start gets
1050
+ // the same check on the claim as it stands now: a claim can be lost between one attempt and the next. A scout
1051
+ // or prep pass before this point never counts as the build starting. A mark that cannot be saved means the
1052
+ // start did not happen, so the job is handed back as not started and the engine is not asked. A mark refused
1053
+ // because the flight no longer holds the claim (claimLost, see markClaimStarted) is handed back as such: the
1054
+ // job is someone else's now. The mark of an attempt already made stands (marking twice adds nothing).
1055
+ if (beforeEngine) {
1056
+ try {
1057
+ beforeEngine(entry);
1058
+ } catch (err) {
1059
+ if (err && err.claimLost) return { not_started: 'claim_lost', detail: String(err.message || err), ...(err.unnamed ? { unnamed: true } : {}) };
1060
+ return { not_started: 'prepare', detail: String(err && err.message || err) };
1061
+ }
1062
+ }
1002
1063
  const startedMs = clock();
1003
1064
  let runResult;
1004
1065
  try {
@@ -1019,6 +1080,8 @@ async function dispatchEntryAttempts({
1019
1080
  };
1020
1081
 
1021
1082
  const first = await runOnce(engine);
1083
+ // No engine was asked, so there is no engine health to record and no one to hand the job to.
1084
+ if (first.not_started) return first;
1022
1085
  if (!first.max_seconds && entry.maxSeconds) first.max_seconds = entry.maxSeconds;
1023
1086
  const deadEngine = detectDeadEngineDispatch(first);
1024
1087
  recordDispatchEngineHealth(first, deadEngine, root);
@@ -1043,6 +1106,9 @@ async function dispatchEntryAttempts({
1043
1106
 
1044
1107
  restaffState.used = true;
1045
1108
  const fallbackResult = await runOnce(fallback);
1109
+ // The start check refused the retry (the claim is not this run's any more): no engine was asked, so the job is
1110
+ // handed back as not started, and there is no engine health to record.
1111
+ if (fallbackResult.not_started) return fallbackResult;
1046
1112
  recordDispatchEngineHealth(fallbackResult, detectDeadEngineDispatch(fallbackResult), root);
1047
1113
  return {
1048
1114
  ...fallbackResult,
@@ -1103,6 +1169,9 @@ async function handOverStalledDispatch({ entry, engine, first, root, runOnce, jo
1103
1169
  prep: next.roster_prep || '',
1104
1170
  };
1105
1171
  const result = await runOnce(next.id, pin);
1172
+ // The start check refused this handover (see dispatchEntryAttempts): no engine was asked, so the job is
1173
+ // handed back as not started and no one else is tried.
1174
+ if (result.not_started) return result;
1106
1175
  const dead = detectDeadEngineDispatch(result);
1107
1176
  recordDispatchEngineHealth(result, dead, root);
1108
1177
  last = result;
@@ -1358,20 +1427,6 @@ function dispatchToEngine({ task, engine, worktreePath, root = process.cwd(), ti
1358
1427
  // ---------------------------------------------------------------------------
1359
1428
  // T2: staffing
1360
1429
 
1361
- const {
1362
- taskTagTokens,
1363
- isDecisionHoldTag,
1364
- isDecisionTask,
1365
- DECISION_REFUSE_REASON,
1366
- } = require('./task-decision');
1367
-
1368
- function taskTags(task) {
1369
- // Tags added after creation live in metadata.tags (`atris task tag`); a
1370
- // fleet that only read task.tags/title hashtags would ignore an owner-hold
1371
- // flag stamped on a live task and keep restaffing it (CLI-879).
1372
- return taskTagTokens(task);
1373
- }
1374
-
1375
1430
  // A task flagged for a human decision is never fleet-staffable, whatever its
1376
1431
  // lane. Mirrors the sweep's needs-human hold so both loops agree. Also honors
1377
1432
  // the clearer `decision` tag so policy questions stay off autonomous lanes.
@@ -1379,10 +1434,10 @@ function isHumanHoldTag(tag) {
1379
1434
  return isDecisionHoldTag(tag);
1380
1435
  }
1381
1436
 
1437
+ // The fleet may staff a task only when no person has to decide it and it sits
1438
+ // in no protected lane. Protection is protectedLaneOfTask's call alone.
1382
1439
  function isSafeLane(task) {
1383
- if (isDecisionTask(task)) return false;
1384
- const tags = taskTags(task);
1385
- return !tags.some((t) => DENIED_TAGS.includes(t));
1440
+ return !isDecisionTask(task) && !protectedLaneOfTask(task);
1386
1441
  }
1387
1442
 
1388
1443
  // Rough file-surface guess from a task's text: paths and bare `commands/x.js`
@@ -1689,6 +1744,10 @@ module.exports = {
1689
1744
  shipPullRequest,
1690
1745
  DEAD_ENGINE_OUTPUT_PATTERNS,
1691
1746
  releasePausedClaimsWithoutWork,
1747
+ dispatchEntryWithRestaff,
1748
+ flightExitCode,
1749
+ markClaimStarted,
1750
+ releaseOwnClaim,
1692
1751
  reviewOnlyEngineEnvironment,
1693
1752
  runInReapedProcessGroup,
1694
1753
  shipWithRetry,
@@ -1713,12 +1772,10 @@ module.exports = {
1713
1772
  assertIsolatedWorktree,
1714
1773
  buildEngineCommand,
1715
1774
  isSafeLane,
1716
- taskTags,
1717
1775
  nextInstalledFleetEngine,
1718
1776
  rankFleetEnginesDetailed,
1719
1777
  dispatchToEngine,
1720
1778
  recordDispatchOutcome,
1721
- taskTags,
1722
1779
  isHumanHoldTag,
1723
1780
  isDecisionTask,
1724
1781
  DECISION_REFUSE_REASON,
@@ -3208,6 +3265,9 @@ async function runFleetFlight({
3208
3265
  scoutAsk = null,
3209
3266
  clock = Date.now,
3210
3267
  prepAsk = undefined,
3268
+ // The disk check before anything is claimed (see lowDiskRefusal). Tests inject both.
3269
+ freeSpace = undefined,
3270
+ spaceGuard = undefined,
3211
3271
  } = {}) {
3212
3272
  const cli = ownCli || defaultOwnCli(root);
3213
3273
  const landTarget = resolveDispatchLandTarget(root, checkoutBase);
@@ -3257,6 +3317,12 @@ async function runFleetFlight({
3257
3317
  return flight;
3258
3318
  }
3259
3319
 
3320
+ // One look at the disk before anything is claimed or written to any job, with worktree start's own
3321
+ // check and reaping. Under its floor every start would refuse anyway, and each refused start used to
3322
+ // let a claim go that the rating then scored as a drop.
3323
+ const lowDisk = lowDiskRefusal({ root, agent: 'fleet', task: 'flight', freeSpace, spaceGuard });
3324
+ if (lowDisk) return stopForLowDisk(flight, lowDisk, staffed.map((s) => ({ task: s.task.display_id, engine: s.engine })), log);
3325
+
3260
3326
  const liveLogPath = startLiveFlight(root, receiptPath, flight);
3261
3327
 
3262
3328
  // Claim + cut a worktree per assignment, then dispatch all in parallel.
@@ -3291,8 +3357,25 @@ async function runFleetFlight({
3291
3357
  // jobs this flight claimed are in them, so a listed job it refused cannot be mistaken for one.
3292
3358
  const claimedRefs = new Map();
3293
3359
  const claimedActors = new Map();
3360
+ // The claim this flight made on each job, remembered from the claim write itself (see claimJob).
3361
+ const claims = claimLedger();
3362
+ // A claim let go before the engine ran is recorded in this process, on the claim this flight made, as one
3363
+ // that could not start. Any other claim the flight lets go (the engine was asked) is an ordinary release.
3364
+ const letGoUnstarted = ({ ref, actor, reason }) => releaseOwnClaim({ taskDb: leaseDb, db: leaseConnection, root, ref, actor, reason, claimEventId: claims.eventId(ref), neverStarted: true });
3365
+ const letGoAsked = ({ ref, actor, reason }) => releaseOwnClaim({ taskDb: leaseDb, db: leaseConnection, root, ref, actor, reason, claimEventId: claims.eventId(ref) });
3366
+ // The start-failure releases the store failed, for the closing pass to try again (see releaseUnstartedDispatchClaim).
3367
+ const releaseFailures = new Map();
3368
+ // Right before the assigned engine is asked, the claim is marked started.
3369
+ const markStarted = (entry) => markClaimStarted({
3370
+ taskDb: leaseDb,
3371
+ db: leaseConnection,
3372
+ claims,
3373
+ ref: claimedRefs.get(String(entry.task.display_id)) || entry.task.display_id,
3374
+ actor: claimedActors.get(String(entry.task.display_id)) || `fleet-${entry.engine}`,
3375
+ task: entry.task.display_id,
3376
+ });
3294
3377
  const leaseTimer = setInterval(() => {
3295
- for (const [id, holder] of leased) leaseDb.renewLease(leaseConnection, { id, holder, pid: process.pid });
3378
+ for (const [id, holder] of leased) leaseDb.renewLease(leaseConnection, { id, holder, pid: process.pid, claimEventId: claims.eventId(id) });
3296
3379
  }, 5 * 60000);
3297
3380
  leaseTimer.unref();
3298
3381
  try {
@@ -3308,7 +3391,7 @@ async function runFleetFlight({
3308
3391
  // The row the list named is the row this claims, leases, proves and lets go of: named once,
3309
3392
  // by its row id, at every step (see jobRef).
3310
3393
  const rowRef = jobRef(entry.task, entry.task.display_id);
3311
- const claimed = cli(['task', 'claim', rowRef, '--as', `fleet-${entry.engine}`, '--lease-minutes', '60', '--lease-pid', String(process.pid)]);
3394
+ const claimed = claimJob({ cli, taskDb: leaseDb, db: leaseConnection, root, ref: rowRef, actor: `fleet-${entry.engine}` });
3312
3395
  if (!claimed || claimed.status !== 0) {
3313
3396
  const detail = String(claimed && (claimed.stderr || claimed.stdout) || 'claim failed').trim().slice(0, 300);
3314
3397
  flight.paused.push({ task: entry.task.display_id, engine: entry.engine, stage: 'claim', detail });
@@ -3319,20 +3402,67 @@ async function runFleetFlight({
3319
3402
  const holder = `fleet-${entry.engine}`;
3320
3403
  claimedRefs.set(String(entry.task.display_id), rowRef);
3321
3404
  claimedActors.set(String(entry.task.display_id), holder);
3322
- const leaseSet = leaseDb.setLease(leaseConnection, { id: leaseId, holder });
3323
- if (!leaseSet && (!ownCli || leaseDb.getTask(leaseConnection, leaseId))) throw new Error('could not set claim lease');
3324
- if (leaseSet) leased.set(leaseId, holder);
3325
- const started = cli(['worktree', 'start', '--agent', entry.engine, '--task', `fleet-${String(entry.task.display_id).toLowerCase()}`, ...startBaseArgs]);
3326
- const wt = (started.stdout.match(/next: cd (.+)/) || [])[1];
3327
- if (!wt) {
3328
- flight.paused.push({ task: entry.task.display_id, stage: 'worktree_start', detail: started.stderr.slice(0, 200) });
3329
- continue;
3405
+ claims.remember(rowRef, claimed.eventId);
3406
+ // From here until the engine is asked, a job that cannot start is let go where it fails, as one that could
3407
+ // not start, so a failure later in the flight cannot leave its claim to run out as a miss.
3408
+ const cannotStart = (stage, detail) => {
3409
+ const released = releaseUnstartedDispatchClaim(cli, { taskId: entry.task.display_id, ref: rowRef, actor: holder, detail, stage, release: letGoUnstarted, failures: releaseFailures, log });
3410
+ flight.paused.push({ task: entry.task.display_id, engine: entry.engine, stage, detail, claim_released: released.released });
3411
+ };
3412
+ try {
3413
+ const leaseSet = leaseDb.setLease(leaseConnection, { id: leaseId, holder, claimEventId: claims.eventId(leaseId) });
3414
+ if (!leaseSet && (!ownCli || leaseDb.getTask(leaseConnection, leaseId))) throw new Error('could not set claim lease');
3415
+ if (leaseSet) leased.set(leaseId, holder);
3416
+ const started = cli(['worktree', 'start', '--agent', entry.engine, '--task', `fleet-${String(entry.task.display_id).toLowerCase()}`, ...startBaseArgs]);
3417
+ const wt = (started.stdout.match(/next: cd (.+)/) || [])[1];
3418
+ if (!wt) {
3419
+ cannotStart('worktree_start', started.stderr.slice(0, 200));
3420
+ continue;
3421
+ }
3422
+ prepared.push({ ...entry, worktreePath: wt.trim() });
3423
+ log(` ${entry.engine.padEnd(8)} building ${entry.task.display_id} in ${path.basename(wt.trim())}`);
3424
+ } catch (err) {
3425
+ cannotStart('prepare', String(err && err.message || err).slice(0, 200));
3426
+ log(` ${entry.engine.padEnd(8)} ✗ ${entry.task.display_id} prepare failed`);
3330
3427
  }
3331
- prepared.push({ ...entry, worktreePath: wt.trim() });
3332
- log(` ${entry.engine.padEnd(8)} building ${entry.task.display_id} in ${path.basename(wt.trim())}`);
3333
3428
  }
3334
3429
 
3335
- const results = await Promise.all(prepared.map((entry) =>
3430
+ // A job that never reached its engine (its brief could not be written, or its start could not be marked)
3431
+ // pauses and its claim goes back as one that could not start, here, where it failed, while the flight's
3432
+ // other jobs go on. The jobs that did start carry on.
3433
+ const letGoNotStarted = (entry, result) => {
3434
+ const taskLabel = entry.task.display_id;
3435
+ const stage = result.not_started;
3436
+ const ref = claimedRefs.get(String(taskLabel)) || taskLabel;
3437
+ if (stage === 'claim_lost') {
3438
+ // The job is another run's now (see markClaimStarted). The flight lets go of nothing, writes nothing to it, and
3439
+ // stops renewing a lease on it.
3440
+ leased.delete(ref);
3441
+ flight.paused.push({ task: taskLabel, engine: entry.engine, stage, detail: result.unnamed ? UNNAMED_CLAIM_DETAIL : LOST_CLAIM_DETAIL, worktree: entry.worktreePath, claim_released: false });
3442
+ log(result.unnamed ? unnamedClaimLine(taskLabel) : claimLostLine(taskLabel));
3443
+ return;
3444
+ }
3445
+ // Nothing in here may end the run: the other jobs still have their cleanup to come.
3446
+ let detail = result.detail;
3447
+ let released = false;
3448
+ try {
3449
+ ({ released, detail } = releaseUnstartedDispatchClaim(cli, {
3450
+ taskId: taskLabel,
3451
+ ref,
3452
+ actor: claimedActors.get(String(taskLabel)) || `fleet-${entry.engine}`,
3453
+ detail: result.detail,
3454
+ stage,
3455
+ release: letGoUnstarted,
3456
+ failures: releaseFailures,
3457
+ log,
3458
+ }));
3459
+ } catch (err) {
3460
+ detail = `${String(result.detail || '').slice(0, 120)}. claim release failed: ${String(err && err.message || err).slice(0, 80)}`;
3461
+ }
3462
+ flight.paused.push({ task: taskLabel, engine: entry.engine, stage, detail, worktree: entry.worktreePath, claim_released: released });
3463
+ log(` ${entry.engine.padEnd(8)} ✗ ${taskLabel} ${NOT_STARTED_WORDS[stage] || 'prepare'} failed`);
3464
+ };
3465
+ const settled = await Promise.all(prepared.map((entry) =>
3336
3466
  dispatchEntryWithRestaff({
3337
3467
  entry,
3338
3468
  engine: entry.engine,
@@ -3344,8 +3474,13 @@ async function runFleetFlight({
3344
3474
  clock,
3345
3475
  defaultPin: fleetPrepPin(entry.engine, root),
3346
3476
  prepAsk,
3347
- }).then((result) => ({ entry, result }))
3477
+ beforeEngine: markStarted,
3478
+ }).then((result) => {
3479
+ if (result.not_started) letGoNotStarted(entry, result);
3480
+ return { entry, result };
3481
+ })
3348
3482
  ));
3483
+ const results = settled.filter(({ result }) => !result.not_started);
3349
3484
  flight.results = results.map(({ entry, result }) => {
3350
3485
  const row = { task: entry.task.display_id, engine: result.engine || entry.engine, exitCode: result.exitCode };
3351
3486
  if (result.brief_id) row.brief_id = result.brief_id;
@@ -3438,6 +3573,8 @@ async function runFleetFlight({
3438
3573
  refByTask: claimedRefs,
3439
3574
  baseRef: landTarget,
3440
3575
  log,
3576
+ release: letGoAsked,
3577
+ failures: releaseFailures,
3441
3578
  });
3442
3579
 
3443
3580
  recordFlightRosterRuns(root, flight, results, { job: 'build', source: 'fleet' });
@@ -3452,7 +3589,7 @@ async function runFleetFlight({
3452
3589
  clearInterval(leaseTimer);
3453
3590
  for (const [id, holder] of leased) {
3454
3591
  const row = leaseDb.getTask(leaseConnection, id);
3455
- if (row && row.status !== 'claimed') leaseDb.clearLease(leaseConnection, { id, holder });
3592
+ if (row && row.status !== 'claimed') leaseDb.clearLease(leaseConnection, { id, holder, claimEventId: claims.eventId(id) });
3456
3593
  }
3457
3594
  }
3458
3595
  }
@@ -3472,7 +3609,7 @@ function releaseReason(lead, detail) {
3472
3609
  }
3473
3610
 
3474
3611
  // Pauses where the engine never got to finish its turn.
3475
- const ENGINE_DID_NOT_FINISH = new Set(['worktree_start', 'remote_quarantine', 'prepare', 'build']);
3612
+ const ENGINE_DID_NOT_FINISH = new Set(['worktree_start', 'remote_quarantine', 'prepare', 'brief_capture', 'build']);
3476
3613
 
3477
3614
  // The word a flight uses to name a job to the task command at every step: its row id when the
3478
3615
  // task command (or the task list) gave one, else the display id the flight has. The display id a
@@ -3496,17 +3633,225 @@ function jobOutsideWorkspace(taskDb, task, workspace) {
3496
3633
 
3497
3634
  const namesAnotherWorkspace = (displayId) => `${displayId} names a job in another workspace`;
3498
3635
 
3636
+ // What a flight does when the disk is under worktree start's floor. It has claimed nothing and written
3637
+ // nothing to any job, so it says why in one plain line (the free space and the floor, in worktree start's
3638
+ // own words) and hands back a failed flight that names the refusal. flightExitCode turns that into the
3639
+ // exit code worktree start itself gives. jobs are the rows the flight would have worked on.
3640
+ function stopForLowDisk(flight, refusal, jobs, log) {
3641
+ log(` refusing: ${refusal.message}`);
3642
+ flight.refusal = { reason: 'low_disk', detail: refusal.message, free_bytes: refusal.free, needs_bytes: refusal.needs };
3643
+ flight.paused = jobs.map((job) => ({ ...job, stage: 'low_disk', detail: refusal.message }));
3644
+ flight.status = 'failed';
3645
+ flight.finished_at = new Date().toISOString();
3646
+ return flight;
3647
+ }
3648
+
3649
+ // The exit code for a finished flight: 2 when it stopped on a low disk, as worktree start does, else 1
3650
+ // when any job paused, else 0.
3651
+ function flightExitCode(flight) {
3652
+ if (flight && flight.refusal && flight.refusal.reason === 'low_disk') return 2;
3653
+ return flight && flight.paused && flight.paused.length > 0 ? 1 : 0;
3654
+ }
3655
+
3656
+ // The two files people read, drawn again from the saved state the way the task command does after
3657
+ // every release and every claim: the task list file, and atris/TODO.md of the flight's own workspace.
3658
+ // The write is already saved, so a file that cannot be written does not undo it, and one failing does
3659
+ // not stop the other. The task command is loaded only here: this file is read on every start, and only
3660
+ // the paths that write to a job need it.
3661
+ function redrawTaskFiles({ taskDb, db, root }) {
3662
+ for (const [what, refresh] of [
3663
+ ['task list file', () => require('../commands/task').writeDefaultProjection(taskDb, db, { root })],
3664
+ ['TODO.md', () => require('../commands/task').autoRenderTodoFromDb(taskDb.workspaceRoot(root))],
3665
+ ]) {
3666
+ try { refresh(); } catch (err) {
3667
+ if (process.env.ATRIS_DEBUG) process.stderr.write(`${what} refresh failed: ${String(err?.message ?? err).replace(/[\r\n]+/g, ' ')}\n`);
3668
+ }
3669
+ }
3670
+ }
3671
+
3672
+ // Which claim this flight made on each job, by the row id it claimed. A mark or a release checks the job is
3673
+ // still held under that claim before it writes anything: another run can take a job over under the same
3674
+ // builder name (a lease that ran out, then a second flight), and a late release from this flight must not
3675
+ // land on, or stamp, a claim that belongs to someone else. The name of the claim comes from the claim write
3676
+ // itself (see claimJob) and is remembered the moment it is made; it is never looked up afterwards, because
3677
+ // a look-up cannot tell this flight's claim from one made just after it. A job claimed through the task
3678
+ // command (a stand-in task plane, a job shown with no row id) has no name to remember.
3679
+ function claimLedger() {
3680
+ const made = new Map();
3681
+ return {
3682
+ remember(ref, eventId) {
3683
+ if (ref && eventId) made.set(String(ref), String(eventId));
3684
+ },
3685
+ // The event that is this flight's claim on the job, or '' when none was remembered.
3686
+ eventId(ref) {
3687
+ return made.get(String(ref)) || '';
3688
+ },
3689
+ };
3690
+ }
3691
+
3692
+ // True when the task store holds a row under this name. It decides whether a flight may act on a job without a
3693
+ // claim to name, so a store that cannot be read counts as holding it: doubt means it may not.
3694
+ function storeHoldsJob(taskDb, db, ref) {
3695
+ if (!ref) return false;
3696
+ try {
3697
+ return Boolean(taskDb.getTask(db, ref));
3698
+ } catch {
3699
+ return true;
3700
+ }
3701
+ }
3702
+
3703
+ // How a flight claims a job. A job the task store holds a row for is claimed in the flight's own process,
3704
+ // through the store it already holds for leases, and the claim's name (its claimed event) comes from the claim
3705
+ // write itself: claimTask saves the row and the event in one transaction and answers with the event, so there is
3706
+ // no moment in which the flight has claimed a job but cannot say which claim is its own. What the task command
3707
+ // did besides the write stays: the check on the actor's name, the plain failure text, the two files people
3708
+ // read drawn again (see redrawTaskFiles), and the clearing of the claims that have run out, which every task
3709
+ // command begins with, so a job whose old claim is dead is cleared and claimed in one go. A job the store holds
3710
+ // no row for (a stand-in task plane, a job shown with no row id) has no claim to name and is claimed through the
3711
+ // task command, as it always was.
3712
+ //
3713
+ // It answers like the task command (status 0, or status 1 and why), with eventId when it claimed in process. A
3714
+ // claim that landed but came back with no name is refused: the flight cannot tell later whether the job is
3715
+ // still its own, so it builds nothing on it and leaves it alone.
3716
+ function claimJob({ cli, taskDb, db, root, ref, actor, minutes = 60 }) {
3717
+ if (!storeHoldsJob(taskDb, db, ref)) {
3718
+ return cli(['task', 'claim', ref, '--as', actor, '--lease-minutes', String(minutes), '--lease-pid', String(process.pid)]);
3719
+ }
3720
+ const check = require('./review-integrity').validateActor(actor, { root });
3721
+ if (!check.ok) {
3722
+ return {
3723
+ status: 1,
3724
+ stdout: '',
3725
+ stderr: check.reason === 'reserved_actor'
3726
+ ? `atris task claim: reserved_actor: '${actor}' is a system actor and cannot be used with --as`
3727
+ : `atris task claim: actor_not_on_roster: '${actor}' is not a workspace member or engine (actor validation is enforced)`,
3728
+ };
3729
+ }
3730
+ // The step the task command takes first, with its reach (every claim in the store that has run out) and the
3731
+ // history it writes (a claim_expired entry for each), because it is the command's own function. It never throws.
3732
+ taskDb.sweepExpiredLeases();
3733
+ let result;
3734
+ try {
3735
+ result = taskDb.claimTask(db, { id: ref, claimedBy: actor, leaseMinutes: minutes, leasePid: process.pid });
3736
+ } catch (err) {
3737
+ return { status: 1, stdout: '', stderr: `claim failed: ${String(err?.message ?? err).trim() || 'unknown'}` };
3738
+ }
3739
+ if (!result.claimed) {
3740
+ return { status: 1, stdout: '', stderr: `claim failed: ${result.reason}${result.claimed_by ? ` (held by ${result.claimed_by})` : ''}` };
3741
+ }
3742
+ redrawTaskFiles({ taskDb, db, root });
3743
+ const eventId = result.event && result.event.event_id;
3744
+ if (!eventId) return { status: 1, stdout: '', stderr: UNNAMED_CLAIM_DETAIL, unnamed: true };
3745
+ return { status: 0, stdout: `claimed ${ref} as ${actor}`, stderr: '', eventId };
3746
+ }
3747
+
3748
+ // The one plain line for a job another run has claimed since this flight made its own claim.
3749
+ const claimTakenOverLine = (task) => ` left ${task} alone: another run claimed it after this one did`;
3750
+
3751
+ // The one plain line for a job whose claim this flight no longer holds when it is about to ask the engine.
3752
+ const LOST_CLAIM_DETAIL = 'this run no longer holds the claim, so the engine was not started and the job was left alone';
3753
+ const claimLostLine = (task) => ` ${task}: ${LOST_CLAIM_DETAIL}`;
3754
+
3755
+ // The one plain line for a job the flight has claimed but cannot name the claim of: it cannot tell whether the job
3756
+ // is still its own, so it writes nothing to it, releases nothing, and starts no engine on it.
3757
+ const UNNAMED_CLAIM_DETAIL = 'this run has no record of which claim is its own, so the job was left alone';
3758
+ const unnamedClaimLine = (task) => ` ${task}: ${UNNAMED_CLAIM_DETAIL}`;
3759
+
3760
+ // What a flight calls the step a job failed at, in the one line it logs when it lets that job's claim go.
3761
+ const NOT_STARTED_WORDS = { brief_capture: 'brief capture', dispatch: 'attempt', prepare: 'prepare' };
3762
+
3763
+ // Right before an engine is asked, the first or a later one (a retry, a handover), the flight marks the claim it
3764
+ // made as started (see markBuildStarted in task-db.js). That is what keeps the could-not-start kind honest: the
3765
+ // store and the rating take it only for a claim with no mark. A job the store does not hold under this holder (a
3766
+ // stand-in task plane) has no claim to mark and goes on. A mark refused for a claim the flight remembered means
3767
+ // the claim is not the flight's any more (it was let go, or taken over, even by a run of the same name): the
3768
+ // engine is not asked, and the error says so (claimLost) for the flight to stop on, writing nothing more. A job
3769
+ // the store holds that the flight has no name for the claim of stops the same way (claimLost and unnamed).
3770
+ // A mark that cannot be saved throws like any store trouble, and the engine is not asked either.
3771
+ function markClaimStarted({ taskDb, db, claims, ref, actor, task }) {
3772
+ const claimEventId = claims.eventId(ref);
3773
+ if (!claimEventId && storeHoldsJob(taskDb, db, ref)) {
3774
+ // A job the store holds is claimed in process, so a flight with no name for its claim cannot say the job is
3775
+ // still its own: the engine is not asked, and nothing is written.
3776
+ const unnamed = new Error(`${task}: ${UNNAMED_CLAIM_DETAIL}`);
3777
+ unnamed.claimLost = true;
3778
+ unnamed.unnamed = true;
3779
+ throw unnamed;
3780
+ }
3781
+ const result = taskDb.markBuildStarted(db, { id: ref, actor, claimEventId });
3782
+ if (!result.marked && claimEventId) {
3783
+ const lost = new Error(`the claim on ${task} is no longer this run's (${result.reason})`);
3784
+ lost.claimLost = true;
3785
+ throw lost;
3786
+ }
3787
+ return result;
3788
+ }
3789
+
3790
+ // A flight lets go of a claim in its own process, on the exact claim it made, through the task store it already
3791
+ // holds for leases. ref is the row id the claim named and claimEventId the claim event the flight remembered
3792
+ // (see claimLedger): the store lets go only if the job is still held under that claim, in the same write, so a
3793
+ // job another run took over since, even under the same builder name and in the same millisecond, is never
3794
+ // released by a late call of this flight's. `atris task release` takes no kind and no claim, because any worker
3795
+ // holding a failed job could pass one to excuse its own miss; only the flights say these things, here.
3796
+ //
3797
+ // neverStarted asks for the could-not-start kind (lib/release-kind.js): the build machine could not start the
3798
+ // job, so it is nobody's miss. The store keeps that honest: it drops the kind from a claim marked started (see
3799
+ // markClaimStarted), and the release then goes through as an ordinary one, so a claim is never left held.
3800
+ //
3801
+ // It answers like the task command (status 0, or status 1 and why), with a word more: not_ours when the job is
3802
+ // not held under the flight's claim any more (nothing was written, nothing is retried, and the caller says so),
3803
+ // final when it is already let go or gone, retryable when the store failed (the closing pass tries once more),
3804
+ // unnamed when the store holds the job but the flight has no name for its claim (nothing is written and nothing
3805
+ // is retried: a release that names no claim could land on anyone's, so the job is left alone and the caller says
3806
+ // so). Null only when the store holds no row for the job (a job shown with no row id, a stand-in task plane): the
3807
+ // caller then lets it go through the task command, as a plain release, the way it always did.
3808
+ function releaseOwnClaim({ taskDb, db, root, ref, actor, reason, claimEventId = '', neverStarted = false }) {
3809
+ if (!claimEventId) {
3810
+ return storeHoldsJob(taskDb, db, ref) ? { status: 1, stdout: '', stderr: UNNAMED_CLAIM_DETAIL, unnamed: true } : null;
3811
+ }
3812
+ let result;
3813
+ try {
3814
+ result = taskDb.releaseTask(db, { id: ref, actor, reason, claimEventId, ...(neverStarted ? { kind: COULD_NOT_START } : {}) });
3815
+ } catch (err) {
3816
+ return { status: 1, stdout: '', stderr: String(err?.message ?? err).trim() || 'unknown', retryable: true };
3817
+ }
3818
+ if (!result.released) {
3819
+ const stderr = `release failed: ${result.reason}${result.claimed_by ? ` (held by ${result.claimed_by})` : ''}`;
3820
+ if (result.reason === 'claim_changed' || result.reason === 'held_by_other') return { status: 1, stdout: '', stderr, not_ours: true };
3821
+ if (result.reason === 'not_found' || String(result.reason).startsWith('already_')) return { status: 1, stdout: '', stderr, final: true };
3822
+ return { status: 1, stdout: '', stderr, retryable: true };
3823
+ }
3824
+ redrawTaskFiles({ taskDb, db, root });
3825
+ return { status: 0, stdout: 'released', stderr: '' };
3826
+ }
3827
+
3499
3828
  // A claim taken before the engine starts must not stay held if the flight
3500
3829
  // refuses or errors first. Release through the same task plane the claim used,
3501
- // naming the job by ref: the row id the claim named.
3502
- function releaseUnstartedDispatchClaim(cli, { taskId, ref = taskId, actor, detail, stage }) {
3503
- const released = cli(['task', 'release', ref, '--as', actor, '--reason', releaseReason(`paused at ${pausedStageLabel(stage)}`, detail)]);
3830
+ // naming the job by ref: the row id the claim named. release, when a flight gives
3831
+ // one, lets it go in process as a job that could not start (see releaseOwnClaim); it
3832
+ // answers null for a claim it cannot record, and the task plane lets that go. A
3833
+ // claim another run has taken over since is left alone, with one plain line. A
3834
+ // release the store failed (retryable) is noted in failures, by the job's name, for the
3835
+ // closing pass to try again.
3836
+ function releaseUnstartedDispatchClaim(cli, { taskId, ref = taskId, actor, detail, stage, release = null, failures = null, log = () => {} }) {
3837
+ const reason = releaseReason(`paused at ${pausedStageLabel(stage)}`, detail);
3838
+ const released = (release && release({ ref, actor, reason })) || cli(['task', 'release', ref, '--as', actor, '--reason', reason]);
3504
3839
  const releasedOk = Boolean(released && released.status === 0);
3840
+ const notOurs = Boolean(released && released.not_ours);
3841
+ const unnamed = Boolean(released && released.unnamed);
3842
+ if (notOurs) log(claimTakenOverLine(taskId));
3843
+ if (unnamed) log(unnamedClaimLine(taskId));
3505
3844
  const suffix = releasedOk
3506
3845
  ? 'claim released, safe to retry'
3507
- : `claim release failed: ${String(released && (released.stderr || released.stdout) || 'unknown').trim().slice(0, 80)}`;
3846
+ : notOurs
3847
+ ? 'claim left alone, another run claimed it after this one did'
3848
+ : unnamed
3849
+ ? 'claim left alone, this run has no record of which claim is its own'
3850
+ : `claim release failed: ${String(released && (released.stderr || released.stdout) || 'unknown').trim().slice(0, 80)}`;
3508
3851
  const base = String(detail || '').trim().slice(0, 200);
3509
- return { released: releasedOk, detail: base ? `${base}. ${suffix}` : suffix };
3852
+ const text = base ? `${base}. ${suffix}` : suffix;
3853
+ if (failures && released && released.retryable) failures.set(String(taskId), { ref, actor, reason, base, detail: text, release });
3854
+ return { released: releasedOk, detail: text };
3510
3855
  }
3511
3856
 
3512
3857
  // A pause with no work must not keep the task. On 2026-08-21, 49 bug tasks sat
@@ -3524,14 +3869,44 @@ function releaseUnstartedDispatchClaim(cli, { taskId, ref = taskId, actor, detai
3524
3869
  // refByTask names each claimed job by the row id the flight claimed it with, so the release
3525
3870
  // lands on that row even if the display id a person reads now means another job. A job with
3526
3871
  // no entry there is named by the display id in its pause row.
3527
- function releasePausedClaimsWithoutWork(cli, { paused, worktreeByTask, actorByTask, refByTask = new Map(), baseRef = '', log = () => {}, keepClaimsWithWork = true }) {
3872
+ //
3873
+ // release, when a flight gives one, lets a claim go in the flight's own process, on the exact claim it
3874
+ // made (see releaseOwnClaim); a job it answers null for is let go through the task command. A job another
3875
+ // run has claimed since is left alone, with one plain line, and its row says the claim was not released.
3876
+ // A release the store failed is tried once more, and a claim that still cannot be let go says so in one
3877
+ // plain line.
3878
+ //
3879
+ // failures holds the start-failure releases that failed at the point of failure (see
3880
+ // releaseUnstartedDispatchClaim): a row that says its claim was not released, with an entry there, gets its
3881
+ // release tried again here, the way it was first tried.
3882
+ function releasePausedClaimsWithoutWork(cli, { paused, worktreeByTask, actorByTask, refByTask = new Map(), baseRef = '', log = () => {}, keepClaimsWithWork = true, release = null, failures = null }) {
3883
+ const letGo = (how, { ref, actor, reason }) => (how && how({ ref, actor, reason })) || cli(['task', 'release', ref, '--as', actor, '--reason', reason]);
3884
+ const saysNo = (taskId, released) => log(` could not release the claim on ${taskId}: ${String(released && (released.stderr || released.stdout) || 'unknown').replace(/\s+/g, ' ').trim().slice(0, 80)}. it stays until its lease runs out, unless the task record is repaired first.`);
3528
3885
  for (const row of paused || []) {
3529
3886
  // No claim (the job was never found, was refused as another workspace's, or the claim
3530
- // failed), work already landed, or work waiting on a pull request for a person (the flight
3531
- // let that claim go itself when saving its proof failed).
3532
- if (row.stage === 'claim' || row.stage === 'task_lookup' || row.stage === 'other_workspace' || row.landed === true || row.held_for_review === true) continue;
3533
- if (row.claim_released !== undefined) continue;
3887
+ // failed), a claim the flight lost before its engine ran (it is the other run's now), work
3888
+ // already landed, or work waiting on a pull request for a person (the flight let that claim go
3889
+ // itself when saving its proof failed).
3890
+ if (row.stage === 'claim' || row.stage === 'task_lookup' || row.stage === 'other_workspace' || row.stage === 'claim_lost' || row.landed === true || row.held_for_review === true) continue;
3534
3891
  const taskId = String(row.task || '');
3892
+ const failed = row.claim_released === false && failures ? failures.get(taskId) : null;
3893
+ if (failed) {
3894
+ failures.delete(taskId);
3895
+ const released = letGo(failed.release, failed);
3896
+ if (released && released.status === 0) {
3897
+ row.claim_released = true;
3898
+ if (row.detail === failed.detail) row.detail = failed.base ? `${failed.base}. claim released, safe to retry` : 'claim released, safe to retry';
3899
+ log(` released claim on ${taskId} on a second try`);
3900
+ } else if (released && released.not_ours) {
3901
+ log(claimTakenOverLine(taskId));
3902
+ } else if (released && released.unnamed) {
3903
+ log(unnamedClaimLine(taskId));
3904
+ } else if (!(released && released.final)) {
3905
+ saysNo(taskId, released);
3906
+ }
3907
+ continue;
3908
+ }
3909
+ if (row.claim_released !== undefined) continue;
3535
3910
  const actor = actorByTask.get(taskId);
3536
3911
  if (!taskId || !actor) continue;
3537
3912
  const worktreePath = row.worktree || worktreeByTask.get(taskId) || '';
@@ -3543,9 +3918,19 @@ function releasePausedClaimsWithoutWork(cli, { paused, worktreeByTask, actorByTa
3543
3918
  const reason = noWork && !ENGINE_DID_NOT_FINISH.has(row.stage)
3544
3919
  ? 'engine produced no work'
3545
3920
  : releaseReason(`paused at ${pausedStageLabel(row.stage)}`, row.stage === 'build' ? row.reason || row.detail : row.detail || row.reason);
3546
- const released = cli(['task', 'release', refByTask.get(taskId) || taskId, '--as', actor, '--reason', reason]);
3921
+ const ref = refByTask.get(taskId) || taskId;
3922
+ let released = letGo(release, { ref, actor, reason });
3923
+ if (released && released.retryable) released = letGo(release, { ref, actor, reason });
3924
+ if (released && (released.not_ours || released.unnamed)) {
3925
+ row.claim_released = false;
3926
+ log(released.unnamed ? unnamedClaimLine(taskId) : claimTakenOverLine(taskId));
3927
+ continue;
3928
+ }
3547
3929
  row.claim_released = Boolean(released && released.status === 0);
3548
- if (!row.claim_released) continue;
3930
+ if (!row.claim_released) {
3931
+ if (released && released.retryable) saysNo(taskId, released);
3932
+ continue;
3933
+ }
3549
3934
  log(keepClaimsWithWork
3550
3935
  ? ` released claim on ${taskId} (engine produced no work)`
3551
3936
  : ` released claim on ${taskId} so a retry can pick it up${worktreePath ? `; any work stays in ${path.basename(worktreePath)}` : ''}`);
@@ -3572,14 +3957,26 @@ function releasePausedClaimsWithoutWork(cli, { paused, worktreeByTask, actorByTa
3572
3957
  // same ids, so it is never looked up again. With no row id there is nothing exact to
3573
3958
  // release, and nothing is.
3574
3959
  //
3960
+ // It also names the claim it made (claimEventId, see claimLedger). A job another run has claimed since is
3961
+ // left alone, with one plain line: the pull request is still open for a person, and the claim is not this
3962
+ // flight's to put the kind on. With no name for its claim, the flight cannot say the claim is its own, so it
3963
+ // writes nothing and says so in one plain line.
3964
+ //
3575
3965
  // Tried once: if the release fails too, the row says so.
3576
- function releaseClaimOfUnsavedPullRequest({ taskDb, db, root, row, rowId, actor, log = () => {} }) {
3966
+ function releaseClaimOfUnsavedPullRequest({ taskDb, db, root, row, rowId, actor, log = () => {}, claimEventId = '' }) {
3577
3967
  const reason = releaseReason(`PR ${row.pull_request.url} is open but its proof could not be saved`, row.detail);
3578
3968
  let released = false;
3969
+ let notOurs = false;
3970
+ let unnamed = false;
3579
3971
  let failure = 'unknown';
3580
3972
  try {
3581
3973
  if (!rowId) {
3582
3974
  failure = 'the flight has no row id for it';
3975
+ } else if (!claimEventId && storeHoldsJob(taskDb, db, rowId)) {
3976
+ // The store holds the job and the flight has no name for its claim: a release that names no claim
3977
+ // could land on anyone's, so the job is left alone. A row the store does not know goes on to the
3978
+ // release below, which answers not_found, as it always did.
3979
+ unnamed = true;
3583
3980
  } else {
3584
3981
  const result = taskDb.releaseTask(db, {
3585
3982
  id: rowId,
@@ -3587,9 +3984,11 @@ function releaseClaimOfUnsavedPullRequest({ taskDb, db, root, row, rowId, actor,
3587
3984
  reason,
3588
3985
  kind: PROOF_NOT_SAVED,
3589
3986
  pullRequest: row.pull_request,
3987
+ claimEventId,
3590
3988
  });
3591
3989
  if (!result.released) {
3592
3990
  failure = `release failed: ${result.reason}${result.claimed_by ? ` (held by ${result.claimed_by})` : ''}`;
3991
+ notOurs = result.reason === 'claim_changed' || result.reason === 'held_by_other';
3593
3992
  } else {
3594
3993
  released = true;
3595
3994
  // The two files people read are drawn again from the saved state, the way the
@@ -3614,7 +4013,11 @@ function releaseClaimOfUnsavedPullRequest({ taskDb, db, root, row, rowId, actor,
3614
4013
  row.claim_released = released;
3615
4014
  log(row.claim_released
3616
4015
  ? ` released the claim on ${row.task} with the pull request link as the reason`
3617
- : ` could not release the claim on ${row.task}: ${failure.replace(/\s+/g, ' ').trim().slice(0, 80)}. it stays until its lease runs out, unless the task record is repaired first.`);
4016
+ : notOurs
4017
+ ? claimTakenOverLine(row.task)
4018
+ : unnamed
4019
+ ? unnamedClaimLine(row.task)
4020
+ : ` could not release the claim on ${row.task}: ${failure.replace(/\s+/g, ' ').trim().slice(0, 80)}. it stays until its lease runs out, unless the task record is repaired first.`);
3618
4021
  return row.claim_released;
3619
4022
  }
3620
4023
 
@@ -3688,6 +4091,9 @@ async function runDispatchFlight({
3688
4091
  // Extra house rules a caller appends to the builder's brief (atris idea
3689
4092
  // tick sends the four values and the no-force-push, no-hook-skip rules).
3690
4093
  brief = '',
4094
+ // The disk check before anything is claimed (see lowDiskRefusal). Tests inject both.
4095
+ freeSpace = undefined,
4096
+ spaceGuard = undefined,
3691
4097
  } = {}) {
3692
4098
  if (!engine) throw new Error('runDispatchFlight: engine is required');
3693
4099
  if (!DISPATCH_CAPABLE.includes(engine)) {
@@ -3710,8 +4116,44 @@ async function runDispatchFlight({
3710
4116
  }
3711
4117
 
3712
4118
  const cli = ownCli || defaultOwnCli(root);
4119
+ // One look at the disk before anything is claimed or written to any job, not even the sweep of expired
4120
+ // claims below: worktree start's own check and reaping. Under its floor every start would refuse anyway,
4121
+ // and each refused start used to let a claim go that the rating then scored as a drop. A sealed review-only
4122
+ // flight builds in the temp folder, which that check leaves alone by default, so it is asked about that
4123
+ // folder with the check on.
4124
+ const sealedReview = reviewOnly && !dispatcher;
4125
+ const lowDisk = lowDiskRefusal({
4126
+ root,
4127
+ agent: engine,
4128
+ task: 'dispatch',
4129
+ freeSpace,
4130
+ spaceGuard: sealedReview ? (spaceGuard ?? true) : spaceGuard,
4131
+ ...(sealedReview ? { target: os.tmpdir() } : {}),
4132
+ });
4133
+ if (lowDisk) {
4134
+ return stopForLowDisk({
4135
+ schema: 'atris.dispatch_receipt.v1',
4136
+ at: new Date().toISOString(),
4137
+ root,
4138
+ engine,
4139
+ tasks: ids,
4140
+ results: [],
4141
+ landed: [],
4142
+ ready: [],
4143
+ held_for_review: [],
4144
+ paused: [],
4145
+ }, lowDisk, ids.map((task) => ({ task })), log);
4146
+ }
3713
4147
  const leaseDb = require('./task-db');
3714
4148
  const leaseConnection = leaseDb.open();
4149
+ // The claim this flight made on each job, remembered from the claim write itself (see claimJob).
4150
+ const claims = claimLedger();
4151
+ // A claim let go before the engine ran is recorded in this process, on the claim this flight made, as one
4152
+ // that could not start. Any other claim the flight lets go (the engine was asked) is an ordinary release.
4153
+ const letGoUnstarted = ({ ref, actor, reason }) => releaseOwnClaim({ taskDb: leaseDb, db: leaseConnection, root, ref, actor, reason, claimEventId: claims.eventId(ref), neverStarted: true });
4154
+ const letGoAsked = ({ ref, actor, reason }) => releaseOwnClaim({ taskDb: leaseDb, db: leaseConnection, root, ref, actor, reason, claimEventId: claims.eventId(ref) });
4155
+ // The start-failure releases the store failed, for the closing pass to try again (see releaseUnstartedDispatchClaim).
4156
+ const releaseFailures = new Map();
3715
4157
  // The workspace this flight works in, spelled the way the store spells it. It is read once, before
3716
4158
  // the flight writes anything into its folder.
3717
4159
  const workspace = leaseDb.workspaceRoot(root);
@@ -3764,8 +4206,17 @@ async function runDispatchFlight({
3764
4206
  // What each claimed job is called to the task command, by the id the caller gave. Only jobs this
3765
4207
  // flight claimed are in it.
3766
4208
  const claimedRefs = new Map();
4209
+ // Right before the assigned engine is asked, the claim is marked started.
4210
+ const markStarted = (entry) => markClaimStarted({
4211
+ taskDb: leaseDb,
4212
+ db: leaseConnection,
4213
+ claims,
4214
+ ref: claimedRefs.get(String(entry.taskId)) || entry.taskId,
4215
+ actor: taskActor,
4216
+ task: entry.taskId,
4217
+ });
3767
4218
  const leaseTimer = setInterval(() => {
3768
- for (const id of leasedIds) leaseDb.renewLease(leaseConnection, { id, holder: taskActor, pid: process.pid });
4219
+ for (const id of leasedIds) leaseDb.renewLease(leaseConnection, { id, holder: taskActor, pid: process.pid, claimEventId: claims.eventId(id) });
3769
4220
  }, 5 * 60000);
3770
4221
  leaseTimer.unref();
3771
4222
  try {
@@ -3789,17 +4240,18 @@ async function runDispatchFlight({
3789
4240
  // named once, by its row id, at every step (see jobRef). Log lines and the receipt keep saying
3790
4241
  // the display id.
3791
4242
  const rowRef = jobRef(task, taskId);
3792
- const claimed = cli(['task', 'claim', rowRef, '--as', taskActor, '--lease-minutes', '60', '--lease-pid', String(process.pid)]);
4243
+ const claimed = claimJob({ cli, taskDb: leaseDb, db: leaseConnection, root, ref: rowRef, actor: taskActor });
3793
4244
  if (!claimed || claimed.status !== 0) {
3794
4245
  const detail = String(claimed && (claimed.stderr || claimed.stdout) || 'claim failed').trim().slice(0, 300);
3795
4246
  flight.paused.push({ task: taskId, stage: 'claim', detail });
3796
- log(` x ${taskId} claim failed`);
4247
+ log(claimed && claimed.unnamed ? unnamedClaimLine(taskId) : ` x ${taskId} claim failed`);
3797
4248
  continue;
3798
4249
  }
3799
4250
  claimedRefs.set(String(taskId), rowRef);
4251
+ claims.remember(rowRef, claimed.eventId);
3800
4252
  try {
3801
4253
  const leaseId = rowRef;
3802
- const leased = leaseDb.setLease(leaseConnection, { id: leaseId, holder: taskActor });
4254
+ const leased = leaseDb.setLease(leaseConnection, { id: leaseId, holder: taskActor, claimEventId: claims.eventId(leaseId) });
3803
4255
  if (!leased && (!ownCli || leaseDb.getTask(leaseConnection, leaseId))) throw new Error('could not set claim lease');
3804
4256
  if (leased) leasedIds.add(leaseId);
3805
4257
  let landingWorktreePath = '';
@@ -3820,6 +4272,9 @@ async function runDispatchFlight({
3820
4272
  actor: taskActor,
3821
4273
  stage: 'worktree_start',
3822
4274
  detail: String(started.stderr || '').slice(0, 200),
4275
+ release: letGoUnstarted,
4276
+ failures: releaseFailures,
4277
+ log,
3823
4278
  });
3824
4279
  flight.paused.push({ task: taskId, stage: 'worktree_start', detail: released.detail, claim_released: released.released });
3825
4280
  log(` ✗ ${taskId} worktree start failed`);
@@ -3834,6 +4289,9 @@ async function runDispatchFlight({
3834
4289
  actor: taskActor,
3835
4290
  stage: 'remote_quarantine',
3836
4291
  detail: headTail(String(remoteBoundary && remoteBoundary.detail || 'could not prepare a sealed review sandbox')),
4292
+ release: letGoUnstarted,
4293
+ failures: releaseFailures,
4294
+ log,
3837
4295
  });
3838
4296
  flight.paused.push({
3839
4297
  task: taskId,
@@ -3879,6 +4337,9 @@ async function runDispatchFlight({
3879
4337
  actor: taskActor,
3880
4338
  stage: 'prepare',
3881
4339
  detail: String(err && err.message || err).slice(0, 200),
4340
+ release: letGoUnstarted,
4341
+ failures: releaseFailures,
4342
+ log,
3882
4343
  });
3883
4344
  flight.paused.push({ task: taskId, stage: 'prepare', detail: released.detail, claim_released: released.released });
3884
4345
  log(` ✗ ${taskId} prepare failed`);
@@ -3913,7 +4374,41 @@ async function runDispatchFlight({
3913
4374
  log(` fable handoff started: receipt ${path.relative(root, receiptPath)}`);
3914
4375
  }
3915
4376
 
3916
- const results = await Promise.all(prepared.map((entry) => {
4377
+ // A job that never reached its engine (its brief could not be written, or its start could not be marked)
4378
+ // pauses and its claim goes back as one that could not start, here, where it failed, while the flight's
4379
+ // other jobs go on. The jobs that did start carry on.
4380
+ const letGoNotStarted = (entry, result) => {
4381
+ const stage = result.not_started;
4382
+ const ref = claimedRefs.get(String(entry.taskId)) || entry.taskId;
4383
+ if (stage === 'claim_lost') {
4384
+ // The job is another run's now (see markClaimStarted). The flight lets go of nothing, writes nothing to it, and
4385
+ // stops renewing a lease on it.
4386
+ leasedIds.delete(ref);
4387
+ flight.paused.push({ task: entry.taskId, stage, detail: result.unnamed ? UNNAMED_CLAIM_DETAIL : LOST_CLAIM_DETAIL, worktree: entry.worktreePath, claim_released: false });
4388
+ log(result.unnamed ? unnamedClaimLine(entry.taskId) : claimLostLine(entry.taskId));
4389
+ return;
4390
+ }
4391
+ // Nothing in here may end the run: the other jobs still have their cleanup to come.
4392
+ let detail = result.detail;
4393
+ let released = false;
4394
+ try {
4395
+ ({ released, detail } = releaseUnstartedDispatchClaim(cli, {
4396
+ taskId: entry.taskId,
4397
+ ref,
4398
+ actor: taskActor,
4399
+ stage,
4400
+ detail: result.detail,
4401
+ release: letGoUnstarted,
4402
+ failures: releaseFailures,
4403
+ log,
4404
+ }));
4405
+ } catch (err) {
4406
+ detail = `${String(result.detail || '').slice(0, 120)}. claim release failed: ${String(err && err.message || err).slice(0, 80)}`;
4407
+ }
4408
+ flight.paused.push({ task: entry.taskId, stage, detail, worktree: entry.worktreePath, claim_released: released });
4409
+ log(` ✗ ${entry.taskId} ${NOT_STARTED_WORDS[stage] || 'prepare'} failed`);
4410
+ };
4411
+ const settled = await Promise.all(prepared.map((entry) => {
3917
4412
  const startedAtMs = Date.now();
3918
4413
  return dispatchEntryWithRestaff({
3919
4414
  entry,
@@ -3927,19 +4422,22 @@ async function runDispatchFlight({
3927
4422
  defaultPin: { model, effort, maxSeconds: capSeconds(maxSeconds), ...(prep ? { prep } : {}) },
3928
4423
  job: rosterJob,
3929
4424
  prepAsk,
4425
+ beforeEngine: markStarted,
3930
4426
  }).then((result) => {
4427
+ if (result.not_started) letGoNotStarted(entry, result);
3931
4428
  const completedAtMs = Date.now();
3932
4429
  return {
3933
4430
  entry,
3934
4431
  result,
3935
4432
  startedAtMs,
3936
4433
  completedAtMs,
3937
- candidate: enforceRemoteBoundary && !detectDeadEngineDispatch(result)
4434
+ candidate: enforceRemoteBoundary && !result.not_started && !detectDeadEngineDispatch(result)
3938
4435
  ? reviewCandidateSnapshot(entry.remoteBoundary)
3939
4436
  : null,
3940
4437
  };
3941
4438
  });
3942
4439
  }));
4440
+ const results = settled.filter(({ result }) => !result.not_started);
3943
4441
  flight.results = results.map(({ entry, result, candidate, startedAtMs, completedAtMs }) => {
3944
4442
  const row = {
3945
4443
  task: entry.taskId,
@@ -4482,7 +4980,10 @@ async function runDispatchFlight({
4482
4980
  stampDispatchBrief(root, result.brief_id, 'partial', `opened a pull request for ${entry.taskId}, but task ready failed`);
4483
4981
  log(' the pull request is open, but saving its proof failed. repair the task record before starting more work.');
4484
4982
  const unsaved = flight.paused.find((row) => row.task === entry.taskId && row.stage === 'task_ready' && row.held_for_review === true);
4485
- if (unsaved) releaseClaimOfUnsavedPullRequest({ taskDb: leaseDb, db: leaseConnection, root, row: unsaved, rowId: entry.task && entry.task.id, actor: taskActor, log });
4983
+ if (unsaved) {
4984
+ const rowId = entry.task && entry.task.id;
4985
+ releaseClaimOfUnsavedPullRequest({ taskDb: leaseDb, db: leaseConnection, root, row: unsaved, rowId, actor: taskActor, log, claimEventId: claims.eventId(rowId) });
4986
+ }
4486
4987
  continue;
4487
4988
  }
4488
4989
  stampDispatchBrief(root, result.brief_id, 'pass', `held ${entry.taskId} for a person to land`);
@@ -4531,6 +5032,8 @@ async function runDispatchFlight({
4531
5032
  baseRef: landTarget,
4532
5033
  log,
4533
5034
  keepClaimsWithWork: proofOnly,
5035
+ release: letGoAsked,
5036
+ failures: releaseFailures,
4534
5037
  });
4535
5038
 
4536
5039
  recordFlightRosterRuns(root, flight, results, {
@@ -4557,7 +5060,7 @@ async function runDispatchFlight({
4557
5060
  clearInterval(leaseTimer);
4558
5061
  for (const id of leasedIds) {
4559
5062
  const row = leaseDb.getTask(leaseConnection, id);
4560
- if (row && row.status !== 'claimed') leaseDb.clearLease(leaseConnection, { id, holder: taskActor });
5063
+ if (row && row.status !== 'claimed') leaseDb.clearLease(leaseConnection, { id, holder: taskActor, claimEventId: claims.eventId(id) });
4561
5064
  }
4562
5065
  }
4563
5066
  }