atris 3.74.29 → 3.76.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/fleet.js CHANGED
@@ -33,7 +33,7 @@ const {
33
33
  buildOneLapValidatorPrompt,
34
34
  parseOneLapValidatorVerdict,
35
35
  } = require('./one-lap-validator');
36
- const { listWorktrees, defaultMainlineBase, gitFailureDetail, gitNetworkEnvironment } = require('../commands/worktree');
36
+ const { listWorktrees, defaultMainlineBase, gitFailureDetail, gitNetworkEnvironment, lowDiskRefusal } = require('../commands/worktree');
37
37
  const { isConductorStatusLine } = require('./conductor-artifacts');
38
38
  const { matchLessons } = require('./lesson-preflight');
39
39
  const { sleepSync } = require('./sleep-sync');
@@ -44,7 +44,7 @@ const { buildVerifiedScoutPack, appendVerifiedScoutPack } = require('./dispatch-
44
44
  const { runPrepPass, withPrepBrief, prepRecordFields } = require('./roster-prep');
45
45
  const { mapNotesPath } = require('./map-refs');
46
46
  const { SHARED_BRIEF_RULES } = require('./brief-rules');
47
- const { PROOF_NOT_SAVED } = require('./release-kind');
47
+ const { PROOF_NOT_SAVED, COULD_NOT_START } = require('./release-kind');
48
48
  const {
49
49
  applySecretGrantEnvironment,
50
50
  buildGatewaySupervisorScript,
@@ -952,10 +952,18 @@ async function dispatchEntryWithRestaff({
952
952
  defaultPin = null,
953
953
  job = 'build',
954
954
  prepAsk = undefined,
955
+ beforeEngine = null,
955
956
  }) {
956
957
  const attempts = [];
957
- const result = await dispatchEntryAttempts({ entry, engine, root, dispatch, installedEngines, restaffState, scoutAsk, clock, defaultPin, attempts, job, prepAsk });
958
- return { ...result, attempts };
958
+ try {
959
+ const result = await dispatchEntryAttempts({ entry, engine, root, dispatch, installedEngines, restaffState, scoutAsk, clock, defaultPin, attempts, job, prepAsk, beforeEngine });
960
+ return { ...result, attempts };
961
+ } catch (err) {
962
+ // One job's attempt blowing up (a retry brief that cannot be written, a store that throws) is that job's
963
+ // trouble. It comes back as a job whose attempt failed, for the flight to let its claim go where it failed,
964
+ // and the run goes on to its cleanup instead of ending with the other jobs' claims still held.
965
+ return { not_started: 'dispatch', detail: String(err && err.message || err), attempts };
966
+ }
959
967
  }
960
968
 
961
969
  // One prep pass per prep job per task: a backup whose line asks for the same
@@ -994,6 +1002,10 @@ async function dispatchEntryAttempts({
994
1002
  attempts = [],
995
1003
  job = 'build',
996
1004
  prepAsk = undefined,
1005
+ // Called right before an engine is asked, every time one is: the assigned engine, a retry on another engine
1006
+ // after a dead one, a handover to the next worker (all after any scout, prep, and brief), so the flight can
1007
+ // check it still holds the claim and mark it started. If it throws, that engine is not asked.
1008
+ beforeEngine = null,
997
1009
  }) {
998
1010
  const basePrompt = entry.prompt || buildFleetPrompt(entry.task, { worktreePath: entry.worktreePath, yolo: entry.yolo });
999
1011
  const prepFor = prepPassCache({ entry, root, clock, prepAsk, job });
@@ -1015,16 +1027,39 @@ async function dispatchEntryAttempts({
1015
1027
  // A worker whose roster line asks for prep gets its brief first.
1016
1028
  const prep = ranWith.prep ? await prepFor(ranWith.prep, scouted) : null;
1017
1029
  const prompt = withPrepBrief(scouted, prep);
1018
- const brief = captureDispatchBrief({
1019
- root,
1020
- task: entry.task,
1021
- engine: engineName,
1022
- worktreePath: entry.worktreePath,
1023
- prompt,
1024
- missionId: entry.mission_id || entry.missionId || '',
1025
- author: entry.author || '',
1026
- yolo: entry.yolo,
1027
- });
1030
+ let brief;
1031
+ try {
1032
+ brief = captureDispatchBrief({
1033
+ root,
1034
+ task: entry.task,
1035
+ engine: engineName,
1036
+ worktreePath: entry.worktreePath,
1037
+ prompt,
1038
+ missionId: entry.mission_id || entry.missionId || '',
1039
+ author: entry.author || '',
1040
+ yolo: entry.yolo,
1041
+ });
1042
+ } catch (err) {
1043
+ // Before any engine has run, a brief that cannot be written means the job never started: hand that
1044
+ // back, so the flight pauses it and lets the claim go as one that could not start. After an engine has
1045
+ // run, it fails the flight as it always did.
1046
+ if (attempts.length) throw err;
1047
+ return { not_started: 'brief_capture', detail: String(err && err.message || err) };
1048
+ }
1049
+ // An engine is about to be asked, the assigned one or a later one (a retry, a handover), and each start gets
1050
+ // the same check on the claim as it stands now: a claim can be lost between one attempt and the next. A scout
1051
+ // or prep pass before this point never counts as the build starting. A mark that cannot be saved means the
1052
+ // start did not happen, so the job is handed back as not started and the engine is not asked. A mark refused
1053
+ // because the flight no longer holds the claim (claimLost, see markClaimStarted) is handed back as such: the
1054
+ // job is someone else's now. The mark of an attempt already made stands (marking twice adds nothing).
1055
+ if (beforeEngine) {
1056
+ try {
1057
+ beforeEngine(entry);
1058
+ } catch (err) {
1059
+ if (err && err.claimLost) return { not_started: 'claim_lost', detail: String(err.message || err), ...(err.unnamed ? { unnamed: true } : {}) };
1060
+ return { not_started: 'prepare', detail: String(err && err.message || err) };
1061
+ }
1062
+ }
1028
1063
  const startedMs = clock();
1029
1064
  let runResult;
1030
1065
  try {
@@ -1045,6 +1080,8 @@ async function dispatchEntryAttempts({
1045
1080
  };
1046
1081
 
1047
1082
  const first = await runOnce(engine);
1083
+ // No engine was asked, so there is no engine health to record and no one to hand the job to.
1084
+ if (first.not_started) return first;
1048
1085
  if (!first.max_seconds && entry.maxSeconds) first.max_seconds = entry.maxSeconds;
1049
1086
  const deadEngine = detectDeadEngineDispatch(first);
1050
1087
  recordDispatchEngineHealth(first, deadEngine, root);
@@ -1069,6 +1106,9 @@ async function dispatchEntryAttempts({
1069
1106
 
1070
1107
  restaffState.used = true;
1071
1108
  const fallbackResult = await runOnce(fallback);
1109
+ // The start check refused the retry (the claim is not this run's any more): no engine was asked, so the job is
1110
+ // handed back as not started, and there is no engine health to record.
1111
+ if (fallbackResult.not_started) return fallbackResult;
1072
1112
  recordDispatchEngineHealth(fallbackResult, detectDeadEngineDispatch(fallbackResult), root);
1073
1113
  return {
1074
1114
  ...fallbackResult,
@@ -1129,6 +1169,9 @@ async function handOverStalledDispatch({ entry, engine, first, root, runOnce, jo
1129
1169
  prep: next.roster_prep || '',
1130
1170
  };
1131
1171
  const result = await runOnce(next.id, pin);
1172
+ // The start check refused this handover (see dispatchEntryAttempts): no engine was asked, so the job is
1173
+ // handed back as not started and no one else is tried.
1174
+ if (result.not_started) return result;
1132
1175
  const dead = detectDeadEngineDispatch(result);
1133
1176
  recordDispatchEngineHealth(result, dead, root);
1134
1177
  last = result;
@@ -1701,6 +1744,10 @@ module.exports = {
1701
1744
  shipPullRequest,
1702
1745
  DEAD_ENGINE_OUTPUT_PATTERNS,
1703
1746
  releasePausedClaimsWithoutWork,
1747
+ dispatchEntryWithRestaff,
1748
+ flightExitCode,
1749
+ markClaimStarted,
1750
+ releaseOwnClaim,
1704
1751
  reviewOnlyEngineEnvironment,
1705
1752
  runInReapedProcessGroup,
1706
1753
  shipWithRetry,
@@ -1722,6 +1769,9 @@ module.exports = {
1722
1769
  METHOD_KERNEL,
1723
1770
  ATRIS_BUILD_PROCESS_PREAMBLE,
1724
1771
  buildFleetPrompt,
1772
+ flightTaskName,
1773
+ formerTaskArgs,
1774
+ builtWithArgs,
1725
1775
  assertIsolatedWorktree,
1726
1776
  buildEngineCommand,
1727
1777
  isSafeLane,
@@ -2505,7 +2555,7 @@ function bindOneLapProof(worktreePath, proofRef, sourceCommit) {
2505
2555
  return sourceTree;
2506
2556
  }
2507
2557
 
2508
- function importReviewCommit(boundary, { cli, taskId, engine, expectedCommit = '', expectedTree = '', startBaseArgs = [] } = {}) {
2558
+ function importReviewCommit(boundary, { cli, taskId, ref = taskId, engine, expectedCommit = '', expectedTree = '', startBaseArgs = [] } = {}) {
2509
2559
  const fail = (stage, detail, extra = {}) => ({ ok: false, stage, detail: headTail(String(detail || '').trim()), ...extra });
2510
2560
  const runSandbox = (args) => spawnSync('git', args, {
2511
2561
  cwd: boundary.worktreePath,
@@ -2566,7 +2616,7 @@ function importReviewCommit(boundary, { cli, taskId, engine, expectedCommit = ''
2566
2616
  .map((relative) => `${baseByPath.has(relative) ? (sourceByPath.has(relative) ? 'M' : 'D') : 'A'} ${relative}`);
2567
2617
  if (!changedEntries.length) return fail('no_change', 'the verified commit has no tree change');
2568
2618
 
2569
- const started = cli(['worktree', 'start', '--agent', engine, '--task', `dispatch-${String(taskId).toLowerCase()}`, ...startBaseArgs]);
2619
+ const started = cli(['worktree', 'start', '--agent', engine, '--task', flightTaskName('dispatch', ref), ...formerTaskArgs('dispatch', ref, taskId), ...startBaseArgs]);
2570
2620
  const landingPath = (String(started && started.stdout || '').match(/next: cd (.+)/) || [])[1];
2571
2621
  if (!landingPath) return fail('worktree_start', String(started && (started.stderr || started.stdout) || 'could not create review worktree'));
2572
2622
  const worktreePath = landingPath.trim();
@@ -3218,6 +3268,9 @@ async function runFleetFlight({
3218
3268
  scoutAsk = null,
3219
3269
  clock = Date.now,
3220
3270
  prepAsk = undefined,
3271
+ // The disk check before anything is claimed (see lowDiskRefusal). Tests inject both.
3272
+ freeSpace = undefined,
3273
+ spaceGuard = undefined,
3221
3274
  } = {}) {
3222
3275
  const cli = ownCli || defaultOwnCli(root);
3223
3276
  const landTarget = resolveDispatchLandTarget(root, checkoutBase);
@@ -3267,6 +3320,12 @@ async function runFleetFlight({
3267
3320
  return flight;
3268
3321
  }
3269
3322
 
3323
+ // One look at the disk before anything is claimed or written to any job, with worktree start's own
3324
+ // check and reaping. Under its floor every start would refuse anyway, and each refused start used to
3325
+ // let a claim go that the rating then scored as a drop.
3326
+ const lowDisk = lowDiskRefusal({ root, agent: 'fleet', task: 'flight', freeSpace, spaceGuard });
3327
+ if (lowDisk) return stopForLowDisk(flight, lowDisk, staffed.map((s) => ({ task: s.task.display_id, engine: s.engine })), log);
3328
+
3270
3329
  const liveLogPath = startLiveFlight(root, receiptPath, flight);
3271
3330
 
3272
3331
  // Claim + cut a worktree per assignment, then dispatch all in parallel.
@@ -3301,8 +3360,25 @@ async function runFleetFlight({
3301
3360
  // jobs this flight claimed are in them, so a listed job it refused cannot be mistaken for one.
3302
3361
  const claimedRefs = new Map();
3303
3362
  const claimedActors = new Map();
3363
+ // The claim this flight made on each job, remembered from the claim write itself (see claimJob).
3364
+ const claims = claimLedger();
3365
+ // A claim let go before the engine ran is recorded in this process, on the claim this flight made, as one
3366
+ // that could not start. Any other claim the flight lets go (the engine was asked) is an ordinary release.
3367
+ const letGoUnstarted = ({ ref, actor, reason }) => releaseOwnClaim({ taskDb: leaseDb, db: leaseConnection, root, ref, actor, reason, claimEventId: claims.eventId(ref), neverStarted: true });
3368
+ const letGoAsked = ({ ref, actor, reason }) => releaseOwnClaim({ taskDb: leaseDb, db: leaseConnection, root, ref, actor, reason, claimEventId: claims.eventId(ref) });
3369
+ // The start-failure releases the store failed, for the closing pass to try again (see releaseUnstartedDispatchClaim).
3370
+ const releaseFailures = new Map();
3371
+ // Right before the assigned engine is asked, the claim is marked started.
3372
+ const markStarted = (entry) => markClaimStarted({
3373
+ taskDb: leaseDb,
3374
+ db: leaseConnection,
3375
+ claims,
3376
+ ref: claimedRefs.get(String(entry.task.display_id)) || entry.task.display_id,
3377
+ actor: claimedActors.get(String(entry.task.display_id)) || `fleet-${entry.engine}`,
3378
+ task: entry.task.display_id,
3379
+ });
3304
3380
  const leaseTimer = setInterval(() => {
3305
- for (const [id, holder] of leased) leaseDb.renewLease(leaseConnection, { id, holder, pid: process.pid });
3381
+ for (const [id, holder] of leased) leaseDb.renewLease(leaseConnection, { id, holder, pid: process.pid, claimEventId: claims.eventId(id) });
3306
3382
  }, 5 * 60000);
3307
3383
  leaseTimer.unref();
3308
3384
  try {
@@ -3318,7 +3394,7 @@ async function runFleetFlight({
3318
3394
  // The row the list named is the row this claims, leases, proves and lets go of: named once,
3319
3395
  // by its row id, at every step (see jobRef).
3320
3396
  const rowRef = jobRef(entry.task, entry.task.display_id);
3321
- const claimed = cli(['task', 'claim', rowRef, '--as', `fleet-${entry.engine}`, '--lease-minutes', '60', '--lease-pid', String(process.pid)]);
3397
+ const claimed = claimJob({ cli, taskDb: leaseDb, db: leaseConnection, root, ref: rowRef, actor: `fleet-${entry.engine}` });
3322
3398
  if (!claimed || claimed.status !== 0) {
3323
3399
  const detail = String(claimed && (claimed.stderr || claimed.stdout) || 'claim failed').trim().slice(0, 300);
3324
3400
  flight.paused.push({ task: entry.task.display_id, engine: entry.engine, stage: 'claim', detail });
@@ -3329,20 +3405,67 @@ async function runFleetFlight({
3329
3405
  const holder = `fleet-${entry.engine}`;
3330
3406
  claimedRefs.set(String(entry.task.display_id), rowRef);
3331
3407
  claimedActors.set(String(entry.task.display_id), holder);
3332
- const leaseSet = leaseDb.setLease(leaseConnection, { id: leaseId, holder });
3333
- if (!leaseSet && (!ownCli || leaseDb.getTask(leaseConnection, leaseId))) throw new Error('could not set claim lease');
3334
- if (leaseSet) leased.set(leaseId, holder);
3335
- const started = cli(['worktree', 'start', '--agent', entry.engine, '--task', `fleet-${String(entry.task.display_id).toLowerCase()}`, ...startBaseArgs]);
3336
- const wt = (started.stdout.match(/next: cd (.+)/) || [])[1];
3337
- if (!wt) {
3338
- flight.paused.push({ task: entry.task.display_id, stage: 'worktree_start', detail: started.stderr.slice(0, 200) });
3339
- continue;
3408
+ claims.remember(rowRef, claimed.eventId);
3409
+ // From here until the engine is asked, a job that cannot start is let go where it fails, as one that could
3410
+ // not start, so a failure later in the flight cannot leave its claim to run out as a miss.
3411
+ const cannotStart = (stage, detail) => {
3412
+ const released = releaseUnstartedDispatchClaim(cli, { taskId: entry.task.display_id, ref: rowRef, actor: holder, detail, stage, release: letGoUnstarted, failures: releaseFailures, log });
3413
+ flight.paused.push({ task: entry.task.display_id, engine: entry.engine, stage, detail, claim_released: released.released });
3414
+ };
3415
+ try {
3416
+ const leaseSet = leaseDb.setLease(leaseConnection, { id: leaseId, holder, claimEventId: claims.eventId(leaseId) });
3417
+ if (!leaseSet && (!ownCli || leaseDb.getTask(leaseConnection, leaseId))) throw new Error('could not set claim lease');
3418
+ if (leaseSet) leased.set(leaseId, holder);
3419
+ const started = cli(['worktree', 'start', '--agent', entry.engine, '--task', flightTaskName('fleet', rowRef), ...formerTaskArgs('fleet', rowRef, entry.task.display_id), ...startBaseArgs]);
3420
+ const wt = (started.stdout.match(/next: cd (.+)/) || [])[1];
3421
+ if (!wt) {
3422
+ cannotStart('worktree_start', started.stderr.slice(0, 200));
3423
+ continue;
3424
+ }
3425
+ prepared.push({ ...entry, worktreePath: wt.trim() });
3426
+ log(` ${entry.engine.padEnd(8)} building ${entry.task.display_id} in ${path.basename(wt.trim())}`);
3427
+ } catch (err) {
3428
+ cannotStart('prepare', String(err && err.message || err).slice(0, 200));
3429
+ log(` ${entry.engine.padEnd(8)} ✗ ${entry.task.display_id} prepare failed`);
3340
3430
  }
3341
- prepared.push({ ...entry, worktreePath: wt.trim() });
3342
- log(` ${entry.engine.padEnd(8)} building ${entry.task.display_id} in ${path.basename(wt.trim())}`);
3343
3431
  }
3344
3432
 
3345
- const results = await Promise.all(prepared.map((entry) =>
3433
+ // A job that never reached its engine (its brief could not be written, or its start could not be marked)
3434
+ // pauses and its claim goes back as one that could not start, here, where it failed, while the flight's
3435
+ // other jobs go on. The jobs that did start carry on.
3436
+ const letGoNotStarted = (entry, result) => {
3437
+ const taskLabel = entry.task.display_id;
3438
+ const stage = result.not_started;
3439
+ const ref = claimedRefs.get(String(taskLabel)) || taskLabel;
3440
+ if (stage === 'claim_lost') {
3441
+ // The job is another run's now (see markClaimStarted). The flight lets go of nothing, writes nothing to it, and
3442
+ // stops renewing a lease on it.
3443
+ leased.delete(ref);
3444
+ flight.paused.push({ task: taskLabel, engine: entry.engine, stage, detail: result.unnamed ? UNNAMED_CLAIM_DETAIL : LOST_CLAIM_DETAIL, worktree: entry.worktreePath, claim_released: false });
3445
+ log(result.unnamed ? unnamedClaimLine(taskLabel) : claimLostLine(taskLabel));
3446
+ return;
3447
+ }
3448
+ // Nothing in here may end the run: the other jobs still have their cleanup to come.
3449
+ let detail = result.detail;
3450
+ let released = false;
3451
+ try {
3452
+ ({ released, detail } = releaseUnstartedDispatchClaim(cli, {
3453
+ taskId: taskLabel,
3454
+ ref,
3455
+ actor: claimedActors.get(String(taskLabel)) || `fleet-${entry.engine}`,
3456
+ detail: result.detail,
3457
+ stage,
3458
+ release: letGoUnstarted,
3459
+ failures: releaseFailures,
3460
+ log,
3461
+ }));
3462
+ } catch (err) {
3463
+ detail = `${String(result.detail || '').slice(0, 120)}. claim release failed: ${String(err && err.message || err).slice(0, 80)}`;
3464
+ }
3465
+ flight.paused.push({ task: taskLabel, engine: entry.engine, stage, detail, worktree: entry.worktreePath, claim_released: released });
3466
+ log(` ${entry.engine.padEnd(8)} ✗ ${taskLabel} ${NOT_STARTED_WORDS[stage] || 'prepare'} failed`);
3467
+ };
3468
+ const settled = await Promise.all(prepared.map((entry) =>
3346
3469
  dispatchEntryWithRestaff({
3347
3470
  entry,
3348
3471
  engine: entry.engine,
@@ -3354,8 +3477,13 @@ async function runFleetFlight({
3354
3477
  clock,
3355
3478
  defaultPin: fleetPrepPin(entry.engine, root),
3356
3479
  prepAsk,
3357
- }).then((result) => ({ entry, result }))
3480
+ beforeEngine: markStarted,
3481
+ }).then((result) => {
3482
+ if (result.not_started) letGoNotStarted(entry, result);
3483
+ return { entry, result };
3484
+ })
3358
3485
  ));
3486
+ const results = settled.filter(({ result }) => !result.not_started);
3359
3487
  flight.results = results.map(({ entry, result }) => {
3360
3488
  const row = { task: entry.task.display_id, engine: result.engine || entry.engine, exitCode: result.exitCode };
3361
3489
  if (result.brief_id) row.brief_id = result.brief_id;
@@ -3448,6 +3576,8 @@ async function runFleetFlight({
3448
3576
  refByTask: claimedRefs,
3449
3577
  baseRef: landTarget,
3450
3578
  log,
3579
+ release: letGoAsked,
3580
+ failures: releaseFailures,
3451
3581
  });
3452
3582
 
3453
3583
  recordFlightRosterRuns(root, flight, results, { job: 'build', source: 'fleet' });
@@ -3462,7 +3592,7 @@ async function runFleetFlight({
3462
3592
  clearInterval(leaseTimer);
3463
3593
  for (const [id, holder] of leased) {
3464
3594
  const row = leaseDb.getTask(leaseConnection, id);
3465
- if (row && row.status !== 'claimed') leaseDb.clearLease(leaseConnection, { id, holder });
3595
+ if (row && row.status !== 'claimed') leaseDb.clearLease(leaseConnection, { id, holder, claimEventId: claims.eventId(id) });
3466
3596
  }
3467
3597
  }
3468
3598
  }
@@ -3482,7 +3612,7 @@ function releaseReason(lead, detail) {
3482
3612
  }
3483
3613
 
3484
3614
  // Pauses where the engine never got to finish its turn.
3485
- const ENGINE_DID_NOT_FINISH = new Set(['worktree_start', 'remote_quarantine', 'prepare', 'build']);
3615
+ const ENGINE_DID_NOT_FINISH = new Set(['worktree_start', 'remote_quarantine', 'prepare', 'brief_capture', 'build']);
3486
3616
 
3487
3617
  // The word a flight uses to name a job to the task command at every step: its row id when the
3488
3618
  // task command (or the task list) gave one, else the display id the flight has. The display id a
@@ -3494,6 +3624,50 @@ function jobRef(task, displayId) {
3494
3624
  return task && task.id ? String(task.id) : String(displayId);
3495
3625
  }
3496
3626
 
3627
+ // The task name a flight gives `worktree start`, which builds the build branch from it. The branch
3628
+ // carries the job's row id, lowercased, as one word, because that is what the held-pull-request
3629
+ // sweep (`atris task sweep --held`) reads to know a merged pull request is this job's own build
3630
+ // and not one a builder named on a row. A display id says nothing like that: the same folder name
3631
+ // in another place gives the same ids, and the list can be written again. `ref` is what jobRef
3632
+ // gave, so a job shown with no row id keeps its display id, and the sweep will not credit that
3633
+ // branch. Both kinds fit the 36 characters `worktree start` keeps of a name with a 26 character id.
3634
+ function flightTaskName(kind, ref) {
3635
+ return `${kind}-${String(ref).toLowerCase()}`;
3636
+ }
3637
+
3638
+ // Before the branch carried the row id it carried the display id a person reads. The flight hands
3639
+ // `worktree start` that older name as well, as --former-task, for one purpose: so start can say that
3640
+ // unfinished work may be on a branch under it. A display name is not an identity (the same folder
3641
+ // name in another place gives the same name, and the list can be written again), so start never
3642
+ // resumes a branch under the older name and is never blocked by one: it starts fresh and prints one
3643
+ // line naming the branch. Nothing is handed over when there is no display id, or when it makes the
3644
+ // same name (a job shown with no row id is named by its display id at every step). The sweep never
3645
+ // credits a branch with the older name either.
3646
+ function formerTaskArgs(kind, ref, displayId) {
3647
+ if (displayId === undefined || displayId === null || !String(displayId).trim()) return [];
3648
+ const former = flightTaskName(kind, String(displayId).trim());
3649
+ return former === flightTaskName(kind, ref) ? [] : ['--former-task', former];
3650
+ }
3651
+
3652
+ // A flight that runs for a member claims the job and hands it in under the member's name, so the
3653
+ // engines that did the building would never reach the job's history, and either one could later
3654
+ // review the work as an independent check. The hand-in also names every engine that ran a build
3655
+ // attempt, each by the name it has on a flight with no member (fleet-<engine>). The names the claim
3656
+ // and the hand-in already put on the record (`claimedAs`, `handedInAs`) are left out, so a flight
3657
+ // with no member and one or two engines names nobody extra. `engines` is the engines in the order
3658
+ // they ran. The answer is the flag with its value, or nothing to add.
3659
+ function builtWithArgs(engines, { claimedAs = '', handedInAs = '' } = {}) {
3660
+ const { normalizeActor } = require('./review-integrity');
3661
+ const onRecord = new Set([normalizeActor(claimedAs), normalizeActor(handedInAs)]);
3662
+ const names = [];
3663
+ for (const engine of Array.isArray(engines) ? engines : []) {
3664
+ const id = normalizeActor(engine);
3665
+ const name = id ? `fleet-${id}` : '';
3666
+ if (name && !onRecord.has(name) && !names.includes(name)) names.push(name);
3667
+ }
3668
+ return names.length ? ['--built-with', names.join(',')] : [];
3669
+ }
3670
+
3497
3671
  // A flight works only on jobs in its own workspace, and the store takes a row id from any
3498
3672
  // workspace, so the flight asks whose workspace the row is in before it claims anything. That is
3499
3673
  // the store's own rule for what a workspace holds (rowInWorkspace, the one its workspace listing
@@ -3506,17 +3680,225 @@ function jobOutsideWorkspace(taskDb, task, workspace) {
3506
3680
 
3507
3681
  const namesAnotherWorkspace = (displayId) => `${displayId} names a job in another workspace`;
3508
3682
 
3683
+ // What a flight does when the disk is under worktree start's floor. It has claimed nothing and written
3684
+ // nothing to any job, so it says why in one plain line (the free space and the floor, in worktree start's
3685
+ // own words) and hands back a failed flight that names the refusal. flightExitCode turns that into the
3686
+ // exit code worktree start itself gives. jobs are the rows the flight would have worked on.
3687
+ function stopForLowDisk(flight, refusal, jobs, log) {
3688
+ log(` refusing: ${refusal.message}`);
3689
+ flight.refusal = { reason: 'low_disk', detail: refusal.message, free_bytes: refusal.free, needs_bytes: refusal.needs };
3690
+ flight.paused = jobs.map((job) => ({ ...job, stage: 'low_disk', detail: refusal.message }));
3691
+ flight.status = 'failed';
3692
+ flight.finished_at = new Date().toISOString();
3693
+ return flight;
3694
+ }
3695
+
3696
+ // The exit code for a finished flight: 2 when it stopped on a low disk, as worktree start does, else 1
3697
+ // when any job paused, else 0.
3698
+ function flightExitCode(flight) {
3699
+ if (flight && flight.refusal && flight.refusal.reason === 'low_disk') return 2;
3700
+ return flight && flight.paused && flight.paused.length > 0 ? 1 : 0;
3701
+ }
3702
+
3703
+ // The two files people read, drawn again from the saved state the way the task command does after
3704
+ // every release and every claim: the task list file, and atris/TODO.md of the flight's own workspace.
3705
+ // The write is already saved, so a file that cannot be written does not undo it, and one failing does
3706
+ // not stop the other. The task command is loaded only here: this file is read on every start, and only
3707
+ // the paths that write to a job need it.
3708
+ function redrawTaskFiles({ taskDb, db, root }) {
3709
+ for (const [what, refresh] of [
3710
+ ['task list file', () => require('../commands/task').writeDefaultProjection(taskDb, db, { root })],
3711
+ ['TODO.md', () => require('../commands/task').autoRenderTodoFromDb(taskDb.workspaceRoot(root))],
3712
+ ]) {
3713
+ try { refresh(); } catch (err) {
3714
+ if (process.env.ATRIS_DEBUG) process.stderr.write(`${what} refresh failed: ${String(err?.message ?? err).replace(/[\r\n]+/g, ' ')}\n`);
3715
+ }
3716
+ }
3717
+ }
3718
+
3719
+ // Which claim this flight made on each job, by the row id it claimed. A mark or a release checks the job is
3720
+ // still held under that claim before it writes anything: another run can take a job over under the same
3721
+ // builder name (a lease that ran out, then a second flight), and a late release from this flight must not
3722
+ // land on, or stamp, a claim that belongs to someone else. The name of the claim comes from the claim write
3723
+ // itself (see claimJob) and is remembered the moment it is made; it is never looked up afterwards, because
3724
+ // a look-up cannot tell this flight's claim from one made just after it. A job claimed through the task
3725
+ // command (a stand-in task plane, a job shown with no row id) has no name to remember.
3726
+ function claimLedger() {
3727
+ const made = new Map();
3728
+ return {
3729
+ remember(ref, eventId) {
3730
+ if (ref && eventId) made.set(String(ref), String(eventId));
3731
+ },
3732
+ // The event that is this flight's claim on the job, or '' when none was remembered.
3733
+ eventId(ref) {
3734
+ return made.get(String(ref)) || '';
3735
+ },
3736
+ };
3737
+ }
3738
+
3739
+ // True when the task store holds a row under this name. It decides whether a flight may act on a job without a
3740
+ // claim to name, so a store that cannot be read counts as holding it: doubt means it may not.
3741
+ function storeHoldsJob(taskDb, db, ref) {
3742
+ if (!ref) return false;
3743
+ try {
3744
+ return Boolean(taskDb.getTask(db, ref));
3745
+ } catch {
3746
+ return true;
3747
+ }
3748
+ }
3749
+
3750
+ // How a flight claims a job. A job the task store holds a row for is claimed in the flight's own process,
3751
+ // through the store it already holds for leases, and the claim's name (its claimed event) comes from the claim
3752
+ // write itself: claimTask saves the row and the event in one transaction and answers with the event, so there is
3753
+ // no moment in which the flight has claimed a job but cannot say which claim is its own. What the task command
3754
+ // did besides the write stays: the check on the actor's name, the plain failure text, the two files people
3755
+ // read drawn again (see redrawTaskFiles), and the clearing of the claims that have run out, which every task
3756
+ // command begins with, so a job whose old claim is dead is cleared and claimed in one go. A job the store holds
3757
+ // no row for (a stand-in task plane, a job shown with no row id) has no claim to name and is claimed through the
3758
+ // task command, as it always was.
3759
+ //
3760
+ // It answers like the task command (status 0, or status 1 and why), with eventId when it claimed in process. A
3761
+ // claim that landed but came back with no name is refused: the flight cannot tell later whether the job is
3762
+ // still its own, so it builds nothing on it and leaves it alone.
3763
+ function claimJob({ cli, taskDb, db, root, ref, actor, minutes = 60 }) {
3764
+ if (!storeHoldsJob(taskDb, db, ref)) {
3765
+ return cli(['task', 'claim', ref, '--as', actor, '--lease-minutes', String(minutes), '--lease-pid', String(process.pid)]);
3766
+ }
3767
+ const check = require('./review-integrity').validateActor(actor, { root });
3768
+ if (!check.ok) {
3769
+ return {
3770
+ status: 1,
3771
+ stdout: '',
3772
+ stderr: check.reason === 'reserved_actor'
3773
+ ? `atris task claim: reserved_actor: '${actor}' is a system actor and cannot be used with --as`
3774
+ : `atris task claim: actor_not_on_roster: '${actor}' is not a workspace member or engine (actor validation is enforced)`,
3775
+ };
3776
+ }
3777
+ // The step the task command takes first, with its reach (every claim in the store that has run out) and the
3778
+ // history it writes (a claim_expired entry for each), because it is the command's own function. It never throws.
3779
+ taskDb.sweepExpiredLeases();
3780
+ let result;
3781
+ try {
3782
+ result = taskDb.claimTask(db, { id: ref, claimedBy: actor, leaseMinutes: minutes, leasePid: process.pid });
3783
+ } catch (err) {
3784
+ return { status: 1, stdout: '', stderr: `claim failed: ${String(err?.message ?? err).trim() || 'unknown'}` };
3785
+ }
3786
+ if (!result.claimed) {
3787
+ return { status: 1, stdout: '', stderr: `claim failed: ${result.reason}${result.claimed_by ? ` (held by ${result.claimed_by})` : ''}` };
3788
+ }
3789
+ redrawTaskFiles({ taskDb, db, root });
3790
+ const eventId = result.event && result.event.event_id;
3791
+ if (!eventId) return { status: 1, stdout: '', stderr: UNNAMED_CLAIM_DETAIL, unnamed: true };
3792
+ return { status: 0, stdout: `claimed ${ref} as ${actor}`, stderr: '', eventId };
3793
+ }
3794
+
3795
+ // The one plain line for a job another run has claimed since this flight made its own claim.
3796
+ const claimTakenOverLine = (task) => ` left ${task} alone: another run claimed it after this one did`;
3797
+
3798
+ // The one plain line for a job whose claim this flight no longer holds when it is about to ask the engine.
3799
+ const LOST_CLAIM_DETAIL = 'this run no longer holds the claim, so the engine was not started and the job was left alone';
3800
+ const claimLostLine = (task) => ` ${task}: ${LOST_CLAIM_DETAIL}`;
3801
+
3802
+ // The one plain line for a job the flight has claimed but cannot name the claim of: it cannot tell whether the job
3803
+ // is still its own, so it writes nothing to it, releases nothing, and starts no engine on it.
3804
+ const UNNAMED_CLAIM_DETAIL = 'this run has no record of which claim is its own, so the job was left alone';
3805
+ const unnamedClaimLine = (task) => ` ${task}: ${UNNAMED_CLAIM_DETAIL}`;
3806
+
3807
+ // What a flight calls the step a job failed at, in the one line it logs when it lets that job's claim go.
3808
+ const NOT_STARTED_WORDS = { brief_capture: 'brief capture', dispatch: 'attempt', prepare: 'prepare' };
3809
+
3810
+ // Right before an engine is asked, the first or a later one (a retry, a handover), the flight marks the claim it
3811
+ // made as started (see markBuildStarted in task-db.js). That is what keeps the could-not-start kind honest: the
3812
+ // store and the rating take it only for a claim with no mark. A job the store does not hold under this holder (a
3813
+ // stand-in task plane) has no claim to mark and goes on. A mark refused for a claim the flight remembered means
3814
+ // the claim is not the flight's any more (it was let go, or taken over, even by a run of the same name): the
3815
+ // engine is not asked, and the error says so (claimLost) for the flight to stop on, writing nothing more. A job
3816
+ // the store holds that the flight has no name for the claim of stops the same way (claimLost and unnamed).
3817
+ // A mark that cannot be saved throws like any store trouble, and the engine is not asked either.
3818
+ function markClaimStarted({ taskDb, db, claims, ref, actor, task }) {
3819
+ const claimEventId = claims.eventId(ref);
3820
+ if (!claimEventId && storeHoldsJob(taskDb, db, ref)) {
3821
+ // A job the store holds is claimed in process, so a flight with no name for its claim cannot say the job is
3822
+ // still its own: the engine is not asked, and nothing is written.
3823
+ const unnamed = new Error(`${task}: ${UNNAMED_CLAIM_DETAIL}`);
3824
+ unnamed.claimLost = true;
3825
+ unnamed.unnamed = true;
3826
+ throw unnamed;
3827
+ }
3828
+ const result = taskDb.markBuildStarted(db, { id: ref, actor, claimEventId });
3829
+ if (!result.marked && claimEventId) {
3830
+ const lost = new Error(`the claim on ${task} is no longer this run's (${result.reason})`);
3831
+ lost.claimLost = true;
3832
+ throw lost;
3833
+ }
3834
+ return result;
3835
+ }
3836
+
3837
+ // A flight lets go of a claim in its own process, on the exact claim it made, through the task store it already
3838
+ // holds for leases. ref is the row id the claim named and claimEventId the claim event the flight remembered
3839
+ // (see claimLedger): the store lets go only if the job is still held under that claim, in the same write, so a
3840
+ // job another run took over since, even under the same builder name and in the same millisecond, is never
3841
+ // released by a late call of this flight's. `atris task release` takes no kind and no claim, because any worker
3842
+ // holding a failed job could pass one to excuse its own miss; only the flights say these things, here.
3843
+ //
3844
+ // neverStarted asks for the could-not-start kind (lib/release-kind.js): the build machine could not start the
3845
+ // job, so it is nobody's miss. The store keeps that honest: it drops the kind from a claim marked started (see
3846
+ // markClaimStarted), and the release then goes through as an ordinary one, so a claim is never left held.
3847
+ //
3848
+ // It answers like the task command (status 0, or status 1 and why), with a word more: not_ours when the job is
3849
+ // not held under the flight's claim any more (nothing was written, nothing is retried, and the caller says so),
3850
+ // final when it is already let go or gone, retryable when the store failed (the closing pass tries once more),
3851
+ // unnamed when the store holds the job but the flight has no name for its claim (nothing is written and nothing
3852
+ // is retried: a release that names no claim could land on anyone's, so the job is left alone and the caller says
3853
+ // so). Null only when the store holds no row for the job (a job shown with no row id, a stand-in task plane): the
3854
+ // caller then lets it go through the task command, as a plain release, the way it always did.
3855
+ function releaseOwnClaim({ taskDb, db, root, ref, actor, reason, claimEventId = '', neverStarted = false }) {
3856
+ if (!claimEventId) {
3857
+ return storeHoldsJob(taskDb, db, ref) ? { status: 1, stdout: '', stderr: UNNAMED_CLAIM_DETAIL, unnamed: true } : null;
3858
+ }
3859
+ let result;
3860
+ try {
3861
+ result = taskDb.releaseTask(db, { id: ref, actor, reason, claimEventId, ...(neverStarted ? { kind: COULD_NOT_START } : {}) });
3862
+ } catch (err) {
3863
+ return { status: 1, stdout: '', stderr: String(err?.message ?? err).trim() || 'unknown', retryable: true };
3864
+ }
3865
+ if (!result.released) {
3866
+ const stderr = `release failed: ${result.reason}${result.claimed_by ? ` (held by ${result.claimed_by})` : ''}`;
3867
+ if (result.reason === 'claim_changed' || result.reason === 'held_by_other') return { status: 1, stdout: '', stderr, not_ours: true };
3868
+ if (result.reason === 'not_found' || String(result.reason).startsWith('already_')) return { status: 1, stdout: '', stderr, final: true };
3869
+ return { status: 1, stdout: '', stderr, retryable: true };
3870
+ }
3871
+ redrawTaskFiles({ taskDb, db, root });
3872
+ return { status: 0, stdout: 'released', stderr: '' };
3873
+ }
3874
+
3509
3875
  // A claim taken before the engine starts must not stay held if the flight
3510
3876
  // refuses or errors first. Release through the same task plane the claim used,
3511
- // naming the job by ref: the row id the claim named.
3512
- function releaseUnstartedDispatchClaim(cli, { taskId, ref = taskId, actor, detail, stage }) {
3513
- const released = cli(['task', 'release', ref, '--as', actor, '--reason', releaseReason(`paused at ${pausedStageLabel(stage)}`, detail)]);
3877
+ // naming the job by ref: the row id the claim named. release, when a flight gives
3878
+ // one, lets it go in process as a job that could not start (see releaseOwnClaim); it
3879
+ // answers null for a claim it cannot record, and the task plane lets that go. A
3880
+ // claim another run has taken over since is left alone, with one plain line. A
3881
+ // release the store failed (retryable) is noted in failures, by the job's name, for the
3882
+ // closing pass to try again.
3883
+ function releaseUnstartedDispatchClaim(cli, { taskId, ref = taskId, actor, detail, stage, release = null, failures = null, log = () => {} }) {
3884
+ const reason = releaseReason(`paused at ${pausedStageLabel(stage)}`, detail);
3885
+ const released = (release && release({ ref, actor, reason })) || cli(['task', 'release', ref, '--as', actor, '--reason', reason]);
3514
3886
  const releasedOk = Boolean(released && released.status === 0);
3887
+ const notOurs = Boolean(released && released.not_ours);
3888
+ const unnamed = Boolean(released && released.unnamed);
3889
+ if (notOurs) log(claimTakenOverLine(taskId));
3890
+ if (unnamed) log(unnamedClaimLine(taskId));
3515
3891
  const suffix = releasedOk
3516
3892
  ? 'claim released, safe to retry'
3517
- : `claim release failed: ${String(released && (released.stderr || released.stdout) || 'unknown').trim().slice(0, 80)}`;
3893
+ : notOurs
3894
+ ? 'claim left alone, another run claimed it after this one did'
3895
+ : unnamed
3896
+ ? 'claim left alone, this run has no record of which claim is its own'
3897
+ : `claim release failed: ${String(released && (released.stderr || released.stdout) || 'unknown').trim().slice(0, 80)}`;
3518
3898
  const base = String(detail || '').trim().slice(0, 200);
3519
- return { released: releasedOk, detail: base ? `${base}. ${suffix}` : suffix };
3899
+ const text = base ? `${base}. ${suffix}` : suffix;
3900
+ if (failures && released && released.retryable) failures.set(String(taskId), { ref, actor, reason, base, detail: text, release });
3901
+ return { released: releasedOk, detail: text };
3520
3902
  }
3521
3903
 
3522
3904
  // A pause with no work must not keep the task. On 2026-08-21, 49 bug tasks sat
@@ -3534,14 +3916,44 @@ function releaseUnstartedDispatchClaim(cli, { taskId, ref = taskId, actor, detai
3534
3916
  // refByTask names each claimed job by the row id the flight claimed it with, so the release
3535
3917
  // lands on that row even if the display id a person reads now means another job. A job with
3536
3918
  // no entry there is named by the display id in its pause row.
3537
- function releasePausedClaimsWithoutWork(cli, { paused, worktreeByTask, actorByTask, refByTask = new Map(), baseRef = '', log = () => {}, keepClaimsWithWork = true }) {
3919
+ //
3920
+ // release, when a flight gives one, lets a claim go in the flight's own process, on the exact claim it
3921
+ // made (see releaseOwnClaim); a job it answers null for is let go through the task command. A job another
3922
+ // run has claimed since is left alone, with one plain line, and its row says the claim was not released.
3923
+ // A release the store failed is tried once more, and a claim that still cannot be let go says so in one
3924
+ // plain line.
3925
+ //
3926
+ // failures holds the start-failure releases that failed at the point of failure (see
3927
+ // releaseUnstartedDispatchClaim): a row that says its claim was not released, with an entry there, gets its
3928
+ // release tried again here, the way it was first tried.
3929
+ function releasePausedClaimsWithoutWork(cli, { paused, worktreeByTask, actorByTask, refByTask = new Map(), baseRef = '', log = () => {}, keepClaimsWithWork = true, release = null, failures = null }) {
3930
+ const letGo = (how, { ref, actor, reason }) => (how && how({ ref, actor, reason })) || cli(['task', 'release', ref, '--as', actor, '--reason', reason]);
3931
+ const saysNo = (taskId, released) => log(` could not release the claim on ${taskId}: ${String(released && (released.stderr || released.stdout) || 'unknown').replace(/\s+/g, ' ').trim().slice(0, 80)}. it stays until its lease runs out, unless the task record is repaired first.`);
3538
3932
  for (const row of paused || []) {
3539
3933
  // No claim (the job was never found, was refused as another workspace's, or the claim
3540
- // failed), work already landed, or work waiting on a pull request for a person (the flight
3541
- // let that claim go itself when saving its proof failed).
3542
- if (row.stage === 'claim' || row.stage === 'task_lookup' || row.stage === 'other_workspace' || row.landed === true || row.held_for_review === true) continue;
3543
- if (row.claim_released !== undefined) continue;
3934
+ // failed), a claim the flight lost before its engine ran (it is the other run's now), work
3935
+ // already landed, or work waiting on a pull request for a person (the flight let that claim go
3936
+ // itself when saving its proof failed).
3937
+ if (row.stage === 'claim' || row.stage === 'task_lookup' || row.stage === 'other_workspace' || row.stage === 'claim_lost' || row.landed === true || row.held_for_review === true) continue;
3544
3938
  const taskId = String(row.task || '');
3939
+ const failed = row.claim_released === false && failures ? failures.get(taskId) : null;
3940
+ if (failed) {
3941
+ failures.delete(taskId);
3942
+ const released = letGo(failed.release, failed);
3943
+ if (released && released.status === 0) {
3944
+ row.claim_released = true;
3945
+ if (row.detail === failed.detail) row.detail = failed.base ? `${failed.base}. claim released, safe to retry` : 'claim released, safe to retry';
3946
+ log(` released claim on ${taskId} on a second try`);
3947
+ } else if (released && released.not_ours) {
3948
+ log(claimTakenOverLine(taskId));
3949
+ } else if (released && released.unnamed) {
3950
+ log(unnamedClaimLine(taskId));
3951
+ } else if (!(released && released.final)) {
3952
+ saysNo(taskId, released);
3953
+ }
3954
+ continue;
3955
+ }
3956
+ if (row.claim_released !== undefined) continue;
3545
3957
  const actor = actorByTask.get(taskId);
3546
3958
  if (!taskId || !actor) continue;
3547
3959
  const worktreePath = row.worktree || worktreeByTask.get(taskId) || '';
@@ -3553,9 +3965,19 @@ function releasePausedClaimsWithoutWork(cli, { paused, worktreeByTask, actorByTa
3553
3965
  const reason = noWork && !ENGINE_DID_NOT_FINISH.has(row.stage)
3554
3966
  ? 'engine produced no work'
3555
3967
  : releaseReason(`paused at ${pausedStageLabel(row.stage)}`, row.stage === 'build' ? row.reason || row.detail : row.detail || row.reason);
3556
- const released = cli(['task', 'release', refByTask.get(taskId) || taskId, '--as', actor, '--reason', reason]);
3968
+ const ref = refByTask.get(taskId) || taskId;
3969
+ let released = letGo(release, { ref, actor, reason });
3970
+ if (released && released.retryable) released = letGo(release, { ref, actor, reason });
3971
+ if (released && (released.not_ours || released.unnamed)) {
3972
+ row.claim_released = false;
3973
+ log(released.unnamed ? unnamedClaimLine(taskId) : claimTakenOverLine(taskId));
3974
+ continue;
3975
+ }
3557
3976
  row.claim_released = Boolean(released && released.status === 0);
3558
- if (!row.claim_released) continue;
3977
+ if (!row.claim_released) {
3978
+ if (released && released.retryable) saysNo(taskId, released);
3979
+ continue;
3980
+ }
3559
3981
  log(keepClaimsWithWork
3560
3982
  ? ` released claim on ${taskId} (engine produced no work)`
3561
3983
  : ` released claim on ${taskId} so a retry can pick it up${worktreePath ? `; any work stays in ${path.basename(worktreePath)}` : ''}`);
@@ -3582,14 +4004,26 @@ function releasePausedClaimsWithoutWork(cli, { paused, worktreeByTask, actorByTa
3582
4004
  // same ids, so it is never looked up again. With no row id there is nothing exact to
3583
4005
  // release, and nothing is.
3584
4006
  //
4007
+ // It also names the claim it made (claimEventId, see claimLedger). A job another run has claimed since is
4008
+ // left alone, with one plain line: the pull request is still open for a person, and the claim is not this
4009
+ // flight's to put the kind on. With no name for its claim, the flight cannot say the claim is its own, so it
4010
+ // writes nothing and says so in one plain line.
4011
+ //
3585
4012
  // Tried once: if the release fails too, the row says so.
3586
- function releaseClaimOfUnsavedPullRequest({ taskDb, db, root, row, rowId, actor, log = () => {} }) {
4013
+ function releaseClaimOfUnsavedPullRequest({ taskDb, db, root, row, rowId, actor, log = () => {}, claimEventId = '' }) {
3587
4014
  const reason = releaseReason(`PR ${row.pull_request.url} is open but its proof could not be saved`, row.detail);
3588
4015
  let released = false;
4016
+ let notOurs = false;
4017
+ let unnamed = false;
3589
4018
  let failure = 'unknown';
3590
4019
  try {
3591
4020
  if (!rowId) {
3592
4021
  failure = 'the flight has no row id for it';
4022
+ } else if (!claimEventId && storeHoldsJob(taskDb, db, rowId)) {
4023
+ // The store holds the job and the flight has no name for its claim: a release that names no claim
4024
+ // could land on anyone's, so the job is left alone. A row the store does not know goes on to the
4025
+ // release below, which answers not_found, as it always did.
4026
+ unnamed = true;
3593
4027
  } else {
3594
4028
  const result = taskDb.releaseTask(db, {
3595
4029
  id: rowId,
@@ -3597,9 +4031,11 @@ function releaseClaimOfUnsavedPullRequest({ taskDb, db, root, row, rowId, actor,
3597
4031
  reason,
3598
4032
  kind: PROOF_NOT_SAVED,
3599
4033
  pullRequest: row.pull_request,
4034
+ claimEventId,
3600
4035
  });
3601
4036
  if (!result.released) {
3602
4037
  failure = `release failed: ${result.reason}${result.claimed_by ? ` (held by ${result.claimed_by})` : ''}`;
4038
+ notOurs = result.reason === 'claim_changed' || result.reason === 'held_by_other';
3603
4039
  } else {
3604
4040
  released = true;
3605
4041
  // The two files people read are drawn again from the saved state, the way the
@@ -3624,7 +4060,11 @@ function releaseClaimOfUnsavedPullRequest({ taskDb, db, root, row, rowId, actor,
3624
4060
  row.claim_released = released;
3625
4061
  log(row.claim_released
3626
4062
  ? ` released the claim on ${row.task} with the pull request link as the reason`
3627
- : ` could not release the claim on ${row.task}: ${failure.replace(/\s+/g, ' ').trim().slice(0, 80)}. it stays until its lease runs out, unless the task record is repaired first.`);
4063
+ : notOurs
4064
+ ? claimTakenOverLine(row.task)
4065
+ : unnamed
4066
+ ? unnamedClaimLine(row.task)
4067
+ : ` could not release the claim on ${row.task}: ${failure.replace(/\s+/g, ' ').trim().slice(0, 80)}. it stays until its lease runs out, unless the task record is repaired first.`);
3628
4068
  return row.claim_released;
3629
4069
  }
3630
4070
 
@@ -3698,6 +4138,9 @@ async function runDispatchFlight({
3698
4138
  // Extra house rules a caller appends to the builder's brief (atris idea
3699
4139
  // tick sends the four values and the no-force-push, no-hook-skip rules).
3700
4140
  brief = '',
4141
+ // The disk check before anything is claimed (see lowDiskRefusal). Tests inject both.
4142
+ freeSpace = undefined,
4143
+ spaceGuard = undefined,
3701
4144
  } = {}) {
3702
4145
  if (!engine) throw new Error('runDispatchFlight: engine is required');
3703
4146
  if (!DISPATCH_CAPABLE.includes(engine)) {
@@ -3720,8 +4163,44 @@ async function runDispatchFlight({
3720
4163
  }
3721
4164
 
3722
4165
  const cli = ownCli || defaultOwnCli(root);
4166
+ // One look at the disk before anything is claimed or written to any job, not even the sweep of expired
4167
+ // claims below: worktree start's own check and reaping. Under its floor every start would refuse anyway,
4168
+ // and each refused start used to let a claim go that the rating then scored as a drop. A sealed review-only
4169
+ // flight builds in the temp folder, which that check leaves alone by default, so it is asked about that
4170
+ // folder with the check on.
4171
+ const sealedReview = reviewOnly && !dispatcher;
4172
+ const lowDisk = lowDiskRefusal({
4173
+ root,
4174
+ agent: engine,
4175
+ task: 'dispatch',
4176
+ freeSpace,
4177
+ spaceGuard: sealedReview ? (spaceGuard ?? true) : spaceGuard,
4178
+ ...(sealedReview ? { target: os.tmpdir() } : {}),
4179
+ });
4180
+ if (lowDisk) {
4181
+ return stopForLowDisk({
4182
+ schema: 'atris.dispatch_receipt.v1',
4183
+ at: new Date().toISOString(),
4184
+ root,
4185
+ engine,
4186
+ tasks: ids,
4187
+ results: [],
4188
+ landed: [],
4189
+ ready: [],
4190
+ held_for_review: [],
4191
+ paused: [],
4192
+ }, lowDisk, ids.map((task) => ({ task })), log);
4193
+ }
3723
4194
  const leaseDb = require('./task-db');
3724
4195
  const leaseConnection = leaseDb.open();
4196
+ // The claim this flight made on each job, remembered from the claim write itself (see claimJob).
4197
+ const claims = claimLedger();
4198
+ // A claim let go before the engine ran is recorded in this process, on the claim this flight made, as one
4199
+ // that could not start. Any other claim the flight lets go (the engine was asked) is an ordinary release.
4200
+ const letGoUnstarted = ({ ref, actor, reason }) => releaseOwnClaim({ taskDb: leaseDb, db: leaseConnection, root, ref, actor, reason, claimEventId: claims.eventId(ref), neverStarted: true });
4201
+ const letGoAsked = ({ ref, actor, reason }) => releaseOwnClaim({ taskDb: leaseDb, db: leaseConnection, root, ref, actor, reason, claimEventId: claims.eventId(ref) });
4202
+ // The start-failure releases the store failed, for the closing pass to try again (see releaseUnstartedDispatchClaim).
4203
+ const releaseFailures = new Map();
3725
4204
  // The workspace this flight works in, spelled the way the store spells it. It is read once, before
3726
4205
  // the flight writes anything into its folder.
3727
4206
  const workspace = leaseDb.workspaceRoot(root);
@@ -3774,8 +4253,17 @@ async function runDispatchFlight({
3774
4253
  // What each claimed job is called to the task command, by the id the caller gave. Only jobs this
3775
4254
  // flight claimed are in it.
3776
4255
  const claimedRefs = new Map();
4256
+ // Right before the assigned engine is asked, the claim is marked started.
4257
+ const markStarted = (entry) => markClaimStarted({
4258
+ taskDb: leaseDb,
4259
+ db: leaseConnection,
4260
+ claims,
4261
+ ref: claimedRefs.get(String(entry.taskId)) || entry.taskId,
4262
+ actor: taskActor,
4263
+ task: entry.taskId,
4264
+ });
3777
4265
  const leaseTimer = setInterval(() => {
3778
- for (const id of leasedIds) leaseDb.renewLease(leaseConnection, { id, holder: taskActor, pid: process.pid });
4266
+ for (const id of leasedIds) leaseDb.renewLease(leaseConnection, { id, holder: taskActor, pid: process.pid, claimEventId: claims.eventId(id) });
3779
4267
  }, 5 * 60000);
3780
4268
  leaseTimer.unref();
3781
4269
  try {
@@ -3799,17 +4287,18 @@ async function runDispatchFlight({
3799
4287
  // named once, by its row id, at every step (see jobRef). Log lines and the receipt keep saying
3800
4288
  // the display id.
3801
4289
  const rowRef = jobRef(task, taskId);
3802
- const claimed = cli(['task', 'claim', rowRef, '--as', taskActor, '--lease-minutes', '60', '--lease-pid', String(process.pid)]);
4290
+ const claimed = claimJob({ cli, taskDb: leaseDb, db: leaseConnection, root, ref: rowRef, actor: taskActor });
3803
4291
  if (!claimed || claimed.status !== 0) {
3804
4292
  const detail = String(claimed && (claimed.stderr || claimed.stdout) || 'claim failed').trim().slice(0, 300);
3805
4293
  flight.paused.push({ task: taskId, stage: 'claim', detail });
3806
- log(` x ${taskId} claim failed`);
4294
+ log(claimed && claimed.unnamed ? unnamedClaimLine(taskId) : ` x ${taskId} claim failed`);
3807
4295
  continue;
3808
4296
  }
3809
4297
  claimedRefs.set(String(taskId), rowRef);
4298
+ claims.remember(rowRef, claimed.eventId);
3810
4299
  try {
3811
4300
  const leaseId = rowRef;
3812
- const leased = leaseDb.setLease(leaseConnection, { id: leaseId, holder: taskActor });
4301
+ const leased = leaseDb.setLease(leaseConnection, { id: leaseId, holder: taskActor, claimEventId: claims.eventId(leaseId) });
3813
4302
  if (!leased && (!ownCli || leaseDb.getTask(leaseConnection, leaseId))) throw new Error('could not set claim lease');
3814
4303
  if (leased) leasedIds.add(leaseId);
3815
4304
  let landingWorktreePath = '';
@@ -3821,7 +4310,7 @@ async function runDispatchFlight({
3821
4310
  // earlier finished run's branch must not refuse the retry. Worktree
3822
4311
  // start resumes that branch's commits, or starts fresh when they no
3823
4312
  // longer merge cleanly (the member-path retry fix).
3824
- const started = cli(['worktree', 'start', '--agent', engine, '--task', `dispatch-${taskId.toLowerCase()}`, '--force', ...startBaseArgs]);
4313
+ const started = cli(['worktree', 'start', '--agent', engine, '--task', flightTaskName('dispatch', rowRef), ...formerTaskArgs('dispatch', rowRef, taskId), '--force', ...startBaseArgs]);
3825
4314
  const wt = (started.stdout.match(/next: cd (.+)/) || [])[1];
3826
4315
  if (!wt) {
3827
4316
  const released = releaseUnstartedDispatchClaim(cli, {
@@ -3830,6 +4319,9 @@ async function runDispatchFlight({
3830
4319
  actor: taskActor,
3831
4320
  stage: 'worktree_start',
3832
4321
  detail: String(started.stderr || '').slice(0, 200),
4322
+ release: letGoUnstarted,
4323
+ failures: releaseFailures,
4324
+ log,
3833
4325
  });
3834
4326
  flight.paused.push({ task: taskId, stage: 'worktree_start', detail: released.detail, claim_released: released.released });
3835
4327
  log(` ✗ ${taskId} worktree start failed`);
@@ -3844,6 +4336,9 @@ async function runDispatchFlight({
3844
4336
  actor: taskActor,
3845
4337
  stage: 'remote_quarantine',
3846
4338
  detail: headTail(String(remoteBoundary && remoteBoundary.detail || 'could not prepare a sealed review sandbox')),
4339
+ release: letGoUnstarted,
4340
+ failures: releaseFailures,
4341
+ log,
3847
4342
  });
3848
4343
  flight.paused.push({
3849
4344
  task: taskId,
@@ -3889,6 +4384,9 @@ async function runDispatchFlight({
3889
4384
  actor: taskActor,
3890
4385
  stage: 'prepare',
3891
4386
  detail: String(err && err.message || err).slice(0, 200),
4387
+ release: letGoUnstarted,
4388
+ failures: releaseFailures,
4389
+ log,
3892
4390
  });
3893
4391
  flight.paused.push({ task: taskId, stage: 'prepare', detail: released.detail, claim_released: released.released });
3894
4392
  log(` ✗ ${taskId} prepare failed`);
@@ -3923,7 +4421,41 @@ async function runDispatchFlight({
3923
4421
  log(` fable handoff started: receipt ${path.relative(root, receiptPath)}`);
3924
4422
  }
3925
4423
 
3926
- const results = await Promise.all(prepared.map((entry) => {
4424
+ // A job that never reached its engine (its brief could not be written, or its start could not be marked)
4425
+ // pauses and its claim goes back as one that could not start, here, where it failed, while the flight's
4426
+ // other jobs go on. The jobs that did start carry on.
4427
+ const letGoNotStarted = (entry, result) => {
4428
+ const stage = result.not_started;
4429
+ const ref = claimedRefs.get(String(entry.taskId)) || entry.taskId;
4430
+ if (stage === 'claim_lost') {
4431
+ // The job is another run's now (see markClaimStarted). The flight lets go of nothing, writes nothing to it, and
4432
+ // stops renewing a lease on it.
4433
+ leasedIds.delete(ref);
4434
+ flight.paused.push({ task: entry.taskId, stage, detail: result.unnamed ? UNNAMED_CLAIM_DETAIL : LOST_CLAIM_DETAIL, worktree: entry.worktreePath, claim_released: false });
4435
+ log(result.unnamed ? unnamedClaimLine(entry.taskId) : claimLostLine(entry.taskId));
4436
+ return;
4437
+ }
4438
+ // Nothing in here may end the run: the other jobs still have their cleanup to come.
4439
+ let detail = result.detail;
4440
+ let released = false;
4441
+ try {
4442
+ ({ released, detail } = releaseUnstartedDispatchClaim(cli, {
4443
+ taskId: entry.taskId,
4444
+ ref,
4445
+ actor: taskActor,
4446
+ stage,
4447
+ detail: result.detail,
4448
+ release: letGoUnstarted,
4449
+ failures: releaseFailures,
4450
+ log,
4451
+ }));
4452
+ } catch (err) {
4453
+ detail = `${String(result.detail || '').slice(0, 120)}. claim release failed: ${String(err && err.message || err).slice(0, 80)}`;
4454
+ }
4455
+ flight.paused.push({ task: entry.taskId, stage, detail, worktree: entry.worktreePath, claim_released: released });
4456
+ log(` ✗ ${entry.taskId} ${NOT_STARTED_WORDS[stage] || 'prepare'} failed`);
4457
+ };
4458
+ const settled = await Promise.all(prepared.map((entry) => {
3927
4459
  const startedAtMs = Date.now();
3928
4460
  return dispatchEntryWithRestaff({
3929
4461
  entry,
@@ -3937,19 +4469,22 @@ async function runDispatchFlight({
3937
4469
  defaultPin: { model, effort, maxSeconds: capSeconds(maxSeconds), ...(prep ? { prep } : {}) },
3938
4470
  job: rosterJob,
3939
4471
  prepAsk,
4472
+ beforeEngine: markStarted,
3940
4473
  }).then((result) => {
4474
+ if (result.not_started) letGoNotStarted(entry, result);
3941
4475
  const completedAtMs = Date.now();
3942
4476
  return {
3943
4477
  entry,
3944
4478
  result,
3945
4479
  startedAtMs,
3946
4480
  completedAtMs,
3947
- candidate: enforceRemoteBoundary && !detectDeadEngineDispatch(result)
4481
+ candidate: enforceRemoteBoundary && !result.not_started && !detectDeadEngineDispatch(result)
3948
4482
  ? reviewCandidateSnapshot(entry.remoteBoundary)
3949
4483
  : null,
3950
4484
  };
3951
4485
  });
3952
4486
  }));
4487
+ const results = settled.filter(({ result }) => !result.not_started);
3953
4488
  flight.results = results.map(({ entry, result, candidate, startedAtMs, completedAtMs }) => {
3954
4489
  const row = {
3955
4490
  task: entry.taskId,
@@ -4083,6 +4618,9 @@ async function runDispatchFlight({
4083
4618
  for (const { entry, result, candidate, startedAtMs } of results) {
4084
4619
  const activeEngine = result.engine || entry.engine || engine;
4085
4620
  const readyActor = explicitActor || `fleet-${activeEngine}`;
4621
+ // Every engine that ran a build attempt on this job, for the hand-in to name (see builtWithArgs).
4622
+ const ranEngines = (Array.isArray(result.attempts) ? result.attempts : []).map((attempt) => attempt && attempt.engine);
4623
+ const builtWith = builtWithArgs(ranEngines, { claimedAs: taskActor, handedInAs: readyActor });
4086
4624
  const landingEntry = { ...entry, engine: activeEngine, candidate };
4087
4625
  if (result.restaffed) log(` restaffed ${entry.taskId}: ${result.restaffed.from} -> ${result.restaffed.to} (${result.restaffed.detail || result.restaffed.reason})`);
4088
4626
  if (result.handover) log(` handed over ${entry.taskId}: ${result.handover_reason}`);
@@ -4326,6 +4864,7 @@ async function runDispatchFlight({
4326
4864
  const imported = importReviewCommit(entry.remoteBoundary, {
4327
4865
  cli,
4328
4866
  taskId: entry.taskId,
4867
+ ref: jobRef(entry.task, entry.taskId),
4329
4868
  engine: activeEngine,
4330
4869
  expectedCommit: candidate.commit,
4331
4870
  expectedTree: candidate.tree,
@@ -4435,6 +4974,7 @@ async function runDispatchFlight({
4435
4974
  '--landing', 'Operators can review the verified change before it reaches users, reducing the risk of an unreviewed release.',
4436
4975
  '--checked', `${landed.check} passed in the isolated worktree`,
4437
4976
  '--tested', 'The requested behavior passed its declared verifier.',
4977
+ ...builtWith,
4438
4978
  '--as', readyActor,
4439
4979
  ]);
4440
4980
  if (!readyResult || readyResult.status !== 0) {
@@ -4486,13 +5026,17 @@ async function runDispatchFlight({
4486
5026
  '--checked', `${landed.check} passed before the pull request was opened`,
4487
5027
  '--tested', 'The requested behavior passed its declared verifier.',
4488
5028
  ...openPullRequestFlags(landed.pull_request),
5029
+ ...builtWith,
4489
5030
  '--as', readyActor,
4490
5031
  ], { flight, landedRow: heldRow, worktree: entry.worktreePath, heldForReview: true, save: () => writeDispatchReceipt(flight, receiptPath, { ids, reviewOnly: proofOnly, enforceRemoteBoundary }) });
4491
5032
  if (!heldRecorded) {
4492
5033
  stampDispatchBrief(root, result.brief_id, 'partial', `opened a pull request for ${entry.taskId}, but task ready failed`);
4493
5034
  log(' the pull request is open, but saving its proof failed. repair the task record before starting more work.');
4494
5035
  const unsaved = flight.paused.find((row) => row.task === entry.taskId && row.stage === 'task_ready' && row.held_for_review === true);
4495
- if (unsaved) releaseClaimOfUnsavedPullRequest({ taskDb: leaseDb, db: leaseConnection, root, row: unsaved, rowId: entry.task && entry.task.id, actor: taskActor, log });
5036
+ if (unsaved) {
5037
+ const rowId = entry.task && entry.task.id;
5038
+ releaseClaimOfUnsavedPullRequest({ taskDb: leaseDb, db: leaseConnection, root, row: unsaved, rowId, actor: taskActor, log, claimEventId: claims.eventId(rowId) });
5039
+ }
4496
5040
  continue;
4497
5041
  }
4498
5042
  stampDispatchBrief(root, result.brief_id, 'pass', `held ${entry.taskId} for a person to land`);
@@ -4511,6 +5055,7 @@ async function runDispatchFlight({
4511
5055
  '--checked', `${landed.check} passed before landing`,
4512
5056
  '--tested', 'The requested behavior passed its declared verifier.',
4513
5057
  ...landedMergeFlags(landed.merge),
5058
+ ...builtWith,
4514
5059
  '--as', readyActor,
4515
5060
  ], { flight, landedRow, worktree: entry.worktreePath, save: () => writeDispatchReceipt(flight, receiptPath, { ids, reviewOnly: proofOnly, enforceRemoteBoundary }) });
4516
5061
  if (!reviewRecorded) {
@@ -4541,6 +5086,8 @@ async function runDispatchFlight({
4541
5086
  baseRef: landTarget,
4542
5087
  log,
4543
5088
  keepClaimsWithWork: proofOnly,
5089
+ release: letGoAsked,
5090
+ failures: releaseFailures,
4544
5091
  });
4545
5092
 
4546
5093
  recordFlightRosterRuns(root, flight, results, {
@@ -4567,7 +5114,7 @@ async function runDispatchFlight({
4567
5114
  clearInterval(leaseTimer);
4568
5115
  for (const id of leasedIds) {
4569
5116
  const row = leaseDb.getTask(leaseConnection, id);
4570
- if (row && row.status !== 'claimed') leaseDb.clearLease(leaseConnection, { id, holder: taskActor });
5117
+ if (row && row.status !== 'claimed') leaseDb.clearLease(leaseConnection, { id, holder: taskActor, claimEventId: claims.eventId(id) });
4571
5118
  }
4572
5119
  }
4573
5120
  }