atris 3.74.29 → 3.76.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/atris/skills/calendar/SKILL.md +9 -0
- package/atris/skills/drive/SKILL.md +7 -0
- package/atris/skills/email-agent/SKILL.md +7 -0
- package/atris/skills/magic-inbox/SKILL.md +7 -0
- package/atris/skills/slides/SKILL.md +7 -0
- package/bin/atris.js +14 -2
- package/commands/engine.js +2 -2
- package/commands/integrations.js +539 -54
- package/commands/mission.js +2 -2
- package/commands/rating.js +29 -5
- package/commands/task.js +72 -5
- package/commands/worktree.js +66 -18
- package/lib/fleet.js +602 -55
- package/lib/gm-season.js +47 -10
- package/lib/held-pull-requests.js +589 -0
- package/lib/known-commands.js +1 -1
- package/lib/pull-request-link.js +25 -0
- package/lib/release-kind.js +31 -7
- package/lib/review-integrity.js +52 -5
- package/lib/task-db.js +703 -160
- package/lib/worktree-space.js +4 -45
- package/package.json +1 -1
package/lib/fleet.js
CHANGED
|
@@ -33,7 +33,7 @@ const {
|
|
|
33
33
|
buildOneLapValidatorPrompt,
|
|
34
34
|
parseOneLapValidatorVerdict,
|
|
35
35
|
} = require('./one-lap-validator');
|
|
36
|
-
const { listWorktrees, defaultMainlineBase, gitFailureDetail, gitNetworkEnvironment } = require('../commands/worktree');
|
|
36
|
+
const { listWorktrees, defaultMainlineBase, gitFailureDetail, gitNetworkEnvironment, lowDiskRefusal } = require('../commands/worktree');
|
|
37
37
|
const { isConductorStatusLine } = require('./conductor-artifacts');
|
|
38
38
|
const { matchLessons } = require('./lesson-preflight');
|
|
39
39
|
const { sleepSync } = require('./sleep-sync');
|
|
@@ -44,7 +44,7 @@ const { buildVerifiedScoutPack, appendVerifiedScoutPack } = require('./dispatch-
|
|
|
44
44
|
const { runPrepPass, withPrepBrief, prepRecordFields } = require('./roster-prep');
|
|
45
45
|
const { mapNotesPath } = require('./map-refs');
|
|
46
46
|
const { SHARED_BRIEF_RULES } = require('./brief-rules');
|
|
47
|
-
const { PROOF_NOT_SAVED } = require('./release-kind');
|
|
47
|
+
const { PROOF_NOT_SAVED, COULD_NOT_START } = require('./release-kind');
|
|
48
48
|
const {
|
|
49
49
|
applySecretGrantEnvironment,
|
|
50
50
|
buildGatewaySupervisorScript,
|
|
@@ -952,10 +952,18 @@ async function dispatchEntryWithRestaff({
|
|
|
952
952
|
defaultPin = null,
|
|
953
953
|
job = 'build',
|
|
954
954
|
prepAsk = undefined,
|
|
955
|
+
beforeEngine = null,
|
|
955
956
|
}) {
|
|
956
957
|
const attempts = [];
|
|
957
|
-
|
|
958
|
-
|
|
958
|
+
try {
|
|
959
|
+
const result = await dispatchEntryAttempts({ entry, engine, root, dispatch, installedEngines, restaffState, scoutAsk, clock, defaultPin, attempts, job, prepAsk, beforeEngine });
|
|
960
|
+
return { ...result, attempts };
|
|
961
|
+
} catch (err) {
|
|
962
|
+
// One job's attempt blowing up (a retry brief that cannot be written, a store that throws) is that job's
|
|
963
|
+
// trouble. It comes back as a job whose attempt failed, for the flight to let its claim go where it failed,
|
|
964
|
+
// and the run goes on to its cleanup instead of ending with the other jobs' claims still held.
|
|
965
|
+
return { not_started: 'dispatch', detail: String(err && err.message || err), attempts };
|
|
966
|
+
}
|
|
959
967
|
}
|
|
960
968
|
|
|
961
969
|
// One prep pass per prep job per task: a backup whose line asks for the same
|
|
@@ -994,6 +1002,10 @@ async function dispatchEntryAttempts({
|
|
|
994
1002
|
attempts = [],
|
|
995
1003
|
job = 'build',
|
|
996
1004
|
prepAsk = undefined,
|
|
1005
|
+
// Called right before an engine is asked, every time one is: the assigned engine, a retry on another engine
|
|
1006
|
+
// after a dead one, a handover to the next worker (all after any scout, prep, and brief), so the flight can
|
|
1007
|
+
// check it still holds the claim and mark it started. If it throws, that engine is not asked.
|
|
1008
|
+
beforeEngine = null,
|
|
997
1009
|
}) {
|
|
998
1010
|
const basePrompt = entry.prompt || buildFleetPrompt(entry.task, { worktreePath: entry.worktreePath, yolo: entry.yolo });
|
|
999
1011
|
const prepFor = prepPassCache({ entry, root, clock, prepAsk, job });
|
|
@@ -1015,16 +1027,39 @@ async function dispatchEntryAttempts({
|
|
|
1015
1027
|
// A worker whose roster line asks for prep gets its brief first.
|
|
1016
1028
|
const prep = ranWith.prep ? await prepFor(ranWith.prep, scouted) : null;
|
|
1017
1029
|
const prompt = withPrepBrief(scouted, prep);
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1030
|
+
let brief;
|
|
1031
|
+
try {
|
|
1032
|
+
brief = captureDispatchBrief({
|
|
1033
|
+
root,
|
|
1034
|
+
task: entry.task,
|
|
1035
|
+
engine: engineName,
|
|
1036
|
+
worktreePath: entry.worktreePath,
|
|
1037
|
+
prompt,
|
|
1038
|
+
missionId: entry.mission_id || entry.missionId || '',
|
|
1039
|
+
author: entry.author || '',
|
|
1040
|
+
yolo: entry.yolo,
|
|
1041
|
+
});
|
|
1042
|
+
} catch (err) {
|
|
1043
|
+
// Before any engine has run, a brief that cannot be written means the job never started: hand that
|
|
1044
|
+
// back, so the flight pauses it and lets the claim go as one that could not start. After an engine has
|
|
1045
|
+
// run, it fails the flight as it always did.
|
|
1046
|
+
if (attempts.length) throw err;
|
|
1047
|
+
return { not_started: 'brief_capture', detail: String(err && err.message || err) };
|
|
1048
|
+
}
|
|
1049
|
+
// An engine is about to be asked, the assigned one or a later one (a retry, a handover), and each start gets
|
|
1050
|
+
// the same check on the claim as it stands now: a claim can be lost between one attempt and the next. A scout
|
|
1051
|
+
// or prep pass before this point never counts as the build starting. A mark that cannot be saved means the
|
|
1052
|
+
// start did not happen, so the job is handed back as not started and the engine is not asked. A mark refused
|
|
1053
|
+
// because the flight no longer holds the claim (claimLost, see markClaimStarted) is handed back as such: the
|
|
1054
|
+
// job is someone else's now. The mark of an attempt already made stands (marking twice adds nothing).
|
|
1055
|
+
if (beforeEngine) {
|
|
1056
|
+
try {
|
|
1057
|
+
beforeEngine(entry);
|
|
1058
|
+
} catch (err) {
|
|
1059
|
+
if (err && err.claimLost) return { not_started: 'claim_lost', detail: String(err.message || err), ...(err.unnamed ? { unnamed: true } : {}) };
|
|
1060
|
+
return { not_started: 'prepare', detail: String(err && err.message || err) };
|
|
1061
|
+
}
|
|
1062
|
+
}
|
|
1028
1063
|
const startedMs = clock();
|
|
1029
1064
|
let runResult;
|
|
1030
1065
|
try {
|
|
@@ -1045,6 +1080,8 @@ async function dispatchEntryAttempts({
|
|
|
1045
1080
|
};
|
|
1046
1081
|
|
|
1047
1082
|
const first = await runOnce(engine);
|
|
1083
|
+
// No engine was asked, so there is no engine health to record and no one to hand the job to.
|
|
1084
|
+
if (first.not_started) return first;
|
|
1048
1085
|
if (!first.max_seconds && entry.maxSeconds) first.max_seconds = entry.maxSeconds;
|
|
1049
1086
|
const deadEngine = detectDeadEngineDispatch(first);
|
|
1050
1087
|
recordDispatchEngineHealth(first, deadEngine, root);
|
|
@@ -1069,6 +1106,9 @@ async function dispatchEntryAttempts({
|
|
|
1069
1106
|
|
|
1070
1107
|
restaffState.used = true;
|
|
1071
1108
|
const fallbackResult = await runOnce(fallback);
|
|
1109
|
+
// The start check refused the retry (the claim is not this run's any more): no engine was asked, so the job is
|
|
1110
|
+
// handed back as not started, and there is no engine health to record.
|
|
1111
|
+
if (fallbackResult.not_started) return fallbackResult;
|
|
1072
1112
|
recordDispatchEngineHealth(fallbackResult, detectDeadEngineDispatch(fallbackResult), root);
|
|
1073
1113
|
return {
|
|
1074
1114
|
...fallbackResult,
|
|
@@ -1129,6 +1169,9 @@ async function handOverStalledDispatch({ entry, engine, first, root, runOnce, jo
|
|
|
1129
1169
|
prep: next.roster_prep || '',
|
|
1130
1170
|
};
|
|
1131
1171
|
const result = await runOnce(next.id, pin);
|
|
1172
|
+
// The start check refused this handover (see dispatchEntryAttempts): no engine was asked, so the job is
|
|
1173
|
+
// handed back as not started and no one else is tried.
|
|
1174
|
+
if (result.not_started) return result;
|
|
1132
1175
|
const dead = detectDeadEngineDispatch(result);
|
|
1133
1176
|
recordDispatchEngineHealth(result, dead, root);
|
|
1134
1177
|
last = result;
|
|
@@ -1701,6 +1744,10 @@ module.exports = {
|
|
|
1701
1744
|
shipPullRequest,
|
|
1702
1745
|
DEAD_ENGINE_OUTPUT_PATTERNS,
|
|
1703
1746
|
releasePausedClaimsWithoutWork,
|
|
1747
|
+
dispatchEntryWithRestaff,
|
|
1748
|
+
flightExitCode,
|
|
1749
|
+
markClaimStarted,
|
|
1750
|
+
releaseOwnClaim,
|
|
1704
1751
|
reviewOnlyEngineEnvironment,
|
|
1705
1752
|
runInReapedProcessGroup,
|
|
1706
1753
|
shipWithRetry,
|
|
@@ -1722,6 +1769,9 @@ module.exports = {
|
|
|
1722
1769
|
METHOD_KERNEL,
|
|
1723
1770
|
ATRIS_BUILD_PROCESS_PREAMBLE,
|
|
1724
1771
|
buildFleetPrompt,
|
|
1772
|
+
flightTaskName,
|
|
1773
|
+
formerTaskArgs,
|
|
1774
|
+
builtWithArgs,
|
|
1725
1775
|
assertIsolatedWorktree,
|
|
1726
1776
|
buildEngineCommand,
|
|
1727
1777
|
isSafeLane,
|
|
@@ -2505,7 +2555,7 @@ function bindOneLapProof(worktreePath, proofRef, sourceCommit) {
|
|
|
2505
2555
|
return sourceTree;
|
|
2506
2556
|
}
|
|
2507
2557
|
|
|
2508
|
-
function importReviewCommit(boundary, { cli, taskId, engine, expectedCommit = '', expectedTree = '', startBaseArgs = [] } = {}) {
|
|
2558
|
+
function importReviewCommit(boundary, { cli, taskId, ref = taskId, engine, expectedCommit = '', expectedTree = '', startBaseArgs = [] } = {}) {
|
|
2509
2559
|
const fail = (stage, detail, extra = {}) => ({ ok: false, stage, detail: headTail(String(detail || '').trim()), ...extra });
|
|
2510
2560
|
const runSandbox = (args) => spawnSync('git', args, {
|
|
2511
2561
|
cwd: boundary.worktreePath,
|
|
@@ -2566,7 +2616,7 @@ function importReviewCommit(boundary, { cli, taskId, engine, expectedCommit = ''
|
|
|
2566
2616
|
.map((relative) => `${baseByPath.has(relative) ? (sourceByPath.has(relative) ? 'M' : 'D') : 'A'} ${relative}`);
|
|
2567
2617
|
if (!changedEntries.length) return fail('no_change', 'the verified commit has no tree change');
|
|
2568
2618
|
|
|
2569
|
-
const started = cli(['worktree', 'start', '--agent', engine, '--task',
|
|
2619
|
+
const started = cli(['worktree', 'start', '--agent', engine, '--task', flightTaskName('dispatch', ref), ...formerTaskArgs('dispatch', ref, taskId), ...startBaseArgs]);
|
|
2570
2620
|
const landingPath = (String(started && started.stdout || '').match(/next: cd (.+)/) || [])[1];
|
|
2571
2621
|
if (!landingPath) return fail('worktree_start', String(started && (started.stderr || started.stdout) || 'could not create review worktree'));
|
|
2572
2622
|
const worktreePath = landingPath.trim();
|
|
@@ -3218,6 +3268,9 @@ async function runFleetFlight({
|
|
|
3218
3268
|
scoutAsk = null,
|
|
3219
3269
|
clock = Date.now,
|
|
3220
3270
|
prepAsk = undefined,
|
|
3271
|
+
// The disk check before anything is claimed (see lowDiskRefusal). Tests inject both.
|
|
3272
|
+
freeSpace = undefined,
|
|
3273
|
+
spaceGuard = undefined,
|
|
3221
3274
|
} = {}) {
|
|
3222
3275
|
const cli = ownCli || defaultOwnCli(root);
|
|
3223
3276
|
const landTarget = resolveDispatchLandTarget(root, checkoutBase);
|
|
@@ -3267,6 +3320,12 @@ async function runFleetFlight({
|
|
|
3267
3320
|
return flight;
|
|
3268
3321
|
}
|
|
3269
3322
|
|
|
3323
|
+
// One look at the disk before anything is claimed or written to any job, with worktree start's own
|
|
3324
|
+
// check and reaping. Under its floor every start would refuse anyway, and each refused start used to
|
|
3325
|
+
// let a claim go that the rating then scored as a drop.
|
|
3326
|
+
const lowDisk = lowDiskRefusal({ root, agent: 'fleet', task: 'flight', freeSpace, spaceGuard });
|
|
3327
|
+
if (lowDisk) return stopForLowDisk(flight, lowDisk, staffed.map((s) => ({ task: s.task.display_id, engine: s.engine })), log);
|
|
3328
|
+
|
|
3270
3329
|
const liveLogPath = startLiveFlight(root, receiptPath, flight);
|
|
3271
3330
|
|
|
3272
3331
|
// Claim + cut a worktree per assignment, then dispatch all in parallel.
|
|
@@ -3301,8 +3360,25 @@ async function runFleetFlight({
|
|
|
3301
3360
|
// jobs this flight claimed are in them, so a listed job it refused cannot be mistaken for one.
|
|
3302
3361
|
const claimedRefs = new Map();
|
|
3303
3362
|
const claimedActors = new Map();
|
|
3363
|
+
// The claim this flight made on each job, remembered from the claim write itself (see claimJob).
|
|
3364
|
+
const claims = claimLedger();
|
|
3365
|
+
// A claim let go before the engine ran is recorded in this process, on the claim this flight made, as one
|
|
3366
|
+
// that could not start. Any other claim the flight lets go (the engine was asked) is an ordinary release.
|
|
3367
|
+
const letGoUnstarted = ({ ref, actor, reason }) => releaseOwnClaim({ taskDb: leaseDb, db: leaseConnection, root, ref, actor, reason, claimEventId: claims.eventId(ref), neverStarted: true });
|
|
3368
|
+
const letGoAsked = ({ ref, actor, reason }) => releaseOwnClaim({ taskDb: leaseDb, db: leaseConnection, root, ref, actor, reason, claimEventId: claims.eventId(ref) });
|
|
3369
|
+
// The start-failure releases the store failed, for the closing pass to try again (see releaseUnstartedDispatchClaim).
|
|
3370
|
+
const releaseFailures = new Map();
|
|
3371
|
+
// Right before the assigned engine is asked, the claim is marked started.
|
|
3372
|
+
const markStarted = (entry) => markClaimStarted({
|
|
3373
|
+
taskDb: leaseDb,
|
|
3374
|
+
db: leaseConnection,
|
|
3375
|
+
claims,
|
|
3376
|
+
ref: claimedRefs.get(String(entry.task.display_id)) || entry.task.display_id,
|
|
3377
|
+
actor: claimedActors.get(String(entry.task.display_id)) || `fleet-${entry.engine}`,
|
|
3378
|
+
task: entry.task.display_id,
|
|
3379
|
+
});
|
|
3304
3380
|
const leaseTimer = setInterval(() => {
|
|
3305
|
-
for (const [id, holder] of leased) leaseDb.renewLease(leaseConnection, { id, holder, pid: process.pid });
|
|
3381
|
+
for (const [id, holder] of leased) leaseDb.renewLease(leaseConnection, { id, holder, pid: process.pid, claimEventId: claims.eventId(id) });
|
|
3306
3382
|
}, 5 * 60000);
|
|
3307
3383
|
leaseTimer.unref();
|
|
3308
3384
|
try {
|
|
@@ -3318,7 +3394,7 @@ async function runFleetFlight({
|
|
|
3318
3394
|
// The row the list named is the row this claims, leases, proves and lets go of: named once,
|
|
3319
3395
|
// by its row id, at every step (see jobRef).
|
|
3320
3396
|
const rowRef = jobRef(entry.task, entry.task.display_id);
|
|
3321
|
-
const claimed = cli
|
|
3397
|
+
const claimed = claimJob({ cli, taskDb: leaseDb, db: leaseConnection, root, ref: rowRef, actor: `fleet-${entry.engine}` });
|
|
3322
3398
|
if (!claimed || claimed.status !== 0) {
|
|
3323
3399
|
const detail = String(claimed && (claimed.stderr || claimed.stdout) || 'claim failed').trim().slice(0, 300);
|
|
3324
3400
|
flight.paused.push({ task: entry.task.display_id, engine: entry.engine, stage: 'claim', detail });
|
|
@@ -3329,20 +3405,67 @@ async function runFleetFlight({
|
|
|
3329
3405
|
const holder = `fleet-${entry.engine}`;
|
|
3330
3406
|
claimedRefs.set(String(entry.task.display_id), rowRef);
|
|
3331
3407
|
claimedActors.set(String(entry.task.display_id), holder);
|
|
3332
|
-
|
|
3333
|
-
|
|
3334
|
-
|
|
3335
|
-
const
|
|
3336
|
-
|
|
3337
|
-
|
|
3338
|
-
|
|
3339
|
-
|
|
3408
|
+
claims.remember(rowRef, claimed.eventId);
|
|
3409
|
+
// From here until the engine is asked, a job that cannot start is let go where it fails, as one that could
|
|
3410
|
+
// not start, so a failure later in the flight cannot leave its claim to run out as a miss.
|
|
3411
|
+
const cannotStart = (stage, detail) => {
|
|
3412
|
+
const released = releaseUnstartedDispatchClaim(cli, { taskId: entry.task.display_id, ref: rowRef, actor: holder, detail, stage, release: letGoUnstarted, failures: releaseFailures, log });
|
|
3413
|
+
flight.paused.push({ task: entry.task.display_id, engine: entry.engine, stage, detail, claim_released: released.released });
|
|
3414
|
+
};
|
|
3415
|
+
try {
|
|
3416
|
+
const leaseSet = leaseDb.setLease(leaseConnection, { id: leaseId, holder, claimEventId: claims.eventId(leaseId) });
|
|
3417
|
+
if (!leaseSet && (!ownCli || leaseDb.getTask(leaseConnection, leaseId))) throw new Error('could not set claim lease');
|
|
3418
|
+
if (leaseSet) leased.set(leaseId, holder);
|
|
3419
|
+
const started = cli(['worktree', 'start', '--agent', entry.engine, '--task', flightTaskName('fleet', rowRef), ...formerTaskArgs('fleet', rowRef, entry.task.display_id), ...startBaseArgs]);
|
|
3420
|
+
const wt = (started.stdout.match(/next: cd (.+)/) || [])[1];
|
|
3421
|
+
if (!wt) {
|
|
3422
|
+
cannotStart('worktree_start', started.stderr.slice(0, 200));
|
|
3423
|
+
continue;
|
|
3424
|
+
}
|
|
3425
|
+
prepared.push({ ...entry, worktreePath: wt.trim() });
|
|
3426
|
+
log(` ${entry.engine.padEnd(8)} building ${entry.task.display_id} in ${path.basename(wt.trim())}`);
|
|
3427
|
+
} catch (err) {
|
|
3428
|
+
cannotStart('prepare', String(err && err.message || err).slice(0, 200));
|
|
3429
|
+
log(` ${entry.engine.padEnd(8)} ✗ ${entry.task.display_id} prepare failed`);
|
|
3340
3430
|
}
|
|
3341
|
-
prepared.push({ ...entry, worktreePath: wt.trim() });
|
|
3342
|
-
log(` ${entry.engine.padEnd(8)} building ${entry.task.display_id} in ${path.basename(wt.trim())}`);
|
|
3343
3431
|
}
|
|
3344
3432
|
|
|
3345
|
-
|
|
3433
|
+
// A job that never reached its engine (its brief could not be written, or its start could not be marked)
|
|
3434
|
+
// pauses and its claim goes back as one that could not start, here, where it failed, while the flight's
|
|
3435
|
+
// other jobs go on. The jobs that did start carry on.
|
|
3436
|
+
const letGoNotStarted = (entry, result) => {
|
|
3437
|
+
const taskLabel = entry.task.display_id;
|
|
3438
|
+
const stage = result.not_started;
|
|
3439
|
+
const ref = claimedRefs.get(String(taskLabel)) || taskLabel;
|
|
3440
|
+
if (stage === 'claim_lost') {
|
|
3441
|
+
// The job is another run's now (see markClaimStarted). The flight lets go of nothing, writes nothing to it, and
|
|
3442
|
+
// stops renewing a lease on it.
|
|
3443
|
+
leased.delete(ref);
|
|
3444
|
+
flight.paused.push({ task: taskLabel, engine: entry.engine, stage, detail: result.unnamed ? UNNAMED_CLAIM_DETAIL : LOST_CLAIM_DETAIL, worktree: entry.worktreePath, claim_released: false });
|
|
3445
|
+
log(result.unnamed ? unnamedClaimLine(taskLabel) : claimLostLine(taskLabel));
|
|
3446
|
+
return;
|
|
3447
|
+
}
|
|
3448
|
+
// Nothing in here may end the run: the other jobs still have their cleanup to come.
|
|
3449
|
+
let detail = result.detail;
|
|
3450
|
+
let released = false;
|
|
3451
|
+
try {
|
|
3452
|
+
({ released, detail } = releaseUnstartedDispatchClaim(cli, {
|
|
3453
|
+
taskId: taskLabel,
|
|
3454
|
+
ref,
|
|
3455
|
+
actor: claimedActors.get(String(taskLabel)) || `fleet-${entry.engine}`,
|
|
3456
|
+
detail: result.detail,
|
|
3457
|
+
stage,
|
|
3458
|
+
release: letGoUnstarted,
|
|
3459
|
+
failures: releaseFailures,
|
|
3460
|
+
log,
|
|
3461
|
+
}));
|
|
3462
|
+
} catch (err) {
|
|
3463
|
+
detail = `${String(result.detail || '').slice(0, 120)}. claim release failed: ${String(err && err.message || err).slice(0, 80)}`;
|
|
3464
|
+
}
|
|
3465
|
+
flight.paused.push({ task: taskLabel, engine: entry.engine, stage, detail, worktree: entry.worktreePath, claim_released: released });
|
|
3466
|
+
log(` ${entry.engine.padEnd(8)} ✗ ${taskLabel} ${NOT_STARTED_WORDS[stage] || 'prepare'} failed`);
|
|
3467
|
+
};
|
|
3468
|
+
const settled = await Promise.all(prepared.map((entry) =>
|
|
3346
3469
|
dispatchEntryWithRestaff({
|
|
3347
3470
|
entry,
|
|
3348
3471
|
engine: entry.engine,
|
|
@@ -3354,8 +3477,13 @@ async function runFleetFlight({
|
|
|
3354
3477
|
clock,
|
|
3355
3478
|
defaultPin: fleetPrepPin(entry.engine, root),
|
|
3356
3479
|
prepAsk,
|
|
3357
|
-
|
|
3480
|
+
beforeEngine: markStarted,
|
|
3481
|
+
}).then((result) => {
|
|
3482
|
+
if (result.not_started) letGoNotStarted(entry, result);
|
|
3483
|
+
return { entry, result };
|
|
3484
|
+
})
|
|
3358
3485
|
));
|
|
3486
|
+
const results = settled.filter(({ result }) => !result.not_started);
|
|
3359
3487
|
flight.results = results.map(({ entry, result }) => {
|
|
3360
3488
|
const row = { task: entry.task.display_id, engine: result.engine || entry.engine, exitCode: result.exitCode };
|
|
3361
3489
|
if (result.brief_id) row.brief_id = result.brief_id;
|
|
@@ -3448,6 +3576,8 @@ async function runFleetFlight({
|
|
|
3448
3576
|
refByTask: claimedRefs,
|
|
3449
3577
|
baseRef: landTarget,
|
|
3450
3578
|
log,
|
|
3579
|
+
release: letGoAsked,
|
|
3580
|
+
failures: releaseFailures,
|
|
3451
3581
|
});
|
|
3452
3582
|
|
|
3453
3583
|
recordFlightRosterRuns(root, flight, results, { job: 'build', source: 'fleet' });
|
|
@@ -3462,7 +3592,7 @@ async function runFleetFlight({
|
|
|
3462
3592
|
clearInterval(leaseTimer);
|
|
3463
3593
|
for (const [id, holder] of leased) {
|
|
3464
3594
|
const row = leaseDb.getTask(leaseConnection, id);
|
|
3465
|
-
if (row && row.status !== 'claimed') leaseDb.clearLease(leaseConnection, { id, holder });
|
|
3595
|
+
if (row && row.status !== 'claimed') leaseDb.clearLease(leaseConnection, { id, holder, claimEventId: claims.eventId(id) });
|
|
3466
3596
|
}
|
|
3467
3597
|
}
|
|
3468
3598
|
}
|
|
@@ -3482,7 +3612,7 @@ function releaseReason(lead, detail) {
|
|
|
3482
3612
|
}
|
|
3483
3613
|
|
|
3484
3614
|
// Pauses where the engine never got to finish its turn.
|
|
3485
|
-
const ENGINE_DID_NOT_FINISH = new Set(['worktree_start', 'remote_quarantine', 'prepare', 'build']);
|
|
3615
|
+
const ENGINE_DID_NOT_FINISH = new Set(['worktree_start', 'remote_quarantine', 'prepare', 'brief_capture', 'build']);
|
|
3486
3616
|
|
|
3487
3617
|
// The word a flight uses to name a job to the task command at every step: its row id when the
|
|
3488
3618
|
// task command (or the task list) gave one, else the display id the flight has. The display id a
|
|
@@ -3494,6 +3624,50 @@ function jobRef(task, displayId) {
|
|
|
3494
3624
|
return task && task.id ? String(task.id) : String(displayId);
|
|
3495
3625
|
}
|
|
3496
3626
|
|
|
3627
|
+
// The task name a flight gives `worktree start`, which builds the build branch from it. The branch
|
|
3628
|
+
// carries the job's row id, lowercased, as one word, because that is what the held-pull-request
|
|
3629
|
+
// sweep (`atris task sweep --held`) reads to know a merged pull request is this job's own build
|
|
3630
|
+
// and not one a builder named on a row. A display id says nothing like that: the same folder name
|
|
3631
|
+
// in another place gives the same ids, and the list can be written again. `ref` is what jobRef
|
|
3632
|
+
// gave, so a job shown with no row id keeps its display id, and the sweep will not credit that
|
|
3633
|
+
// branch. Both kinds fit the 36 characters `worktree start` keeps of a name with a 26 character id.
|
|
3634
|
+
function flightTaskName(kind, ref) {
|
|
3635
|
+
return `${kind}-${String(ref).toLowerCase()}`;
|
|
3636
|
+
}
|
|
3637
|
+
|
|
3638
|
+
// Before the branch carried the row id it carried the display id a person reads. The flight hands
|
|
3639
|
+
// `worktree start` that older name as well, as --former-task, for one purpose: so start can say that
|
|
3640
|
+
// unfinished work may be on a branch under it. A display name is not an identity (the same folder
|
|
3641
|
+
// name in another place gives the same name, and the list can be written again), so start never
|
|
3642
|
+
// resumes a branch under the older name and is never blocked by one: it starts fresh and prints one
|
|
3643
|
+
// line naming the branch. Nothing is handed over when there is no display id, or when it makes the
|
|
3644
|
+
// same name (a job shown with no row id is named by its display id at every step). The sweep never
|
|
3645
|
+
// credits a branch with the older name either.
|
|
3646
|
+
function formerTaskArgs(kind, ref, displayId) {
|
|
3647
|
+
if (displayId === undefined || displayId === null || !String(displayId).trim()) return [];
|
|
3648
|
+
const former = flightTaskName(kind, String(displayId).trim());
|
|
3649
|
+
return former === flightTaskName(kind, ref) ? [] : ['--former-task', former];
|
|
3650
|
+
}
|
|
3651
|
+
|
|
3652
|
+
// A flight that runs for a member claims the job and hands it in under the member's name, so the
|
|
3653
|
+
// engines that did the building would never reach the job's history, and either one could later
|
|
3654
|
+
// review the work as an independent check. The hand-in also names every engine that ran a build
|
|
3655
|
+
// attempt, each by the name it has on a flight with no member (fleet-<engine>). The names the claim
|
|
3656
|
+
// and the hand-in already put on the record (`claimedAs`, `handedInAs`) are left out, so a flight
|
|
3657
|
+
// with no member and one or two engines names nobody extra. `engines` is the engines in the order
|
|
3658
|
+
// they ran. The answer is the flag with its value, or nothing to add.
|
|
3659
|
+
function builtWithArgs(engines, { claimedAs = '', handedInAs = '' } = {}) {
|
|
3660
|
+
const { normalizeActor } = require('./review-integrity');
|
|
3661
|
+
const onRecord = new Set([normalizeActor(claimedAs), normalizeActor(handedInAs)]);
|
|
3662
|
+
const names = [];
|
|
3663
|
+
for (const engine of Array.isArray(engines) ? engines : []) {
|
|
3664
|
+
const id = normalizeActor(engine);
|
|
3665
|
+
const name = id ? `fleet-${id}` : '';
|
|
3666
|
+
if (name && !onRecord.has(name) && !names.includes(name)) names.push(name);
|
|
3667
|
+
}
|
|
3668
|
+
return names.length ? ['--built-with', names.join(',')] : [];
|
|
3669
|
+
}
|
|
3670
|
+
|
|
3497
3671
|
// A flight works only on jobs in its own workspace, and the store takes a row id from any
|
|
3498
3672
|
// workspace, so the flight asks whose workspace the row is in before it claims anything. That is
|
|
3499
3673
|
// the store's own rule for what a workspace holds (rowInWorkspace, the one its workspace listing
|
|
@@ -3506,17 +3680,225 @@ function jobOutsideWorkspace(taskDb, task, workspace) {
|
|
|
3506
3680
|
|
|
3507
3681
|
const namesAnotherWorkspace = (displayId) => `${displayId} names a job in another workspace`;
|
|
3508
3682
|
|
|
3683
|
+
// What a flight does when the disk is under worktree start's floor. It has claimed nothing and written
|
|
3684
|
+
// nothing to any job, so it says why in one plain line (the free space and the floor, in worktree start's
|
|
3685
|
+
// own words) and hands back a failed flight that names the refusal. flightExitCode turns that into the
|
|
3686
|
+
// exit code worktree start itself gives. jobs are the rows the flight would have worked on.
|
|
3687
|
+
function stopForLowDisk(flight, refusal, jobs, log) {
|
|
3688
|
+
log(` refusing: ${refusal.message}`);
|
|
3689
|
+
flight.refusal = { reason: 'low_disk', detail: refusal.message, free_bytes: refusal.free, needs_bytes: refusal.needs };
|
|
3690
|
+
flight.paused = jobs.map((job) => ({ ...job, stage: 'low_disk', detail: refusal.message }));
|
|
3691
|
+
flight.status = 'failed';
|
|
3692
|
+
flight.finished_at = new Date().toISOString();
|
|
3693
|
+
return flight;
|
|
3694
|
+
}
|
|
3695
|
+
|
|
3696
|
+
// The exit code for a finished flight: 2 when it stopped on a low disk, as worktree start does, else 1
|
|
3697
|
+
// when any job paused, else 0.
|
|
3698
|
+
function flightExitCode(flight) {
|
|
3699
|
+
if (flight && flight.refusal && flight.refusal.reason === 'low_disk') return 2;
|
|
3700
|
+
return flight && flight.paused && flight.paused.length > 0 ? 1 : 0;
|
|
3701
|
+
}
|
|
3702
|
+
|
|
3703
|
+
// The two files people read, drawn again from the saved state the way the task command does after
|
|
3704
|
+
// every release and every claim: the task list file, and atris/TODO.md of the flight's own workspace.
|
|
3705
|
+
// The write is already saved, so a file that cannot be written does not undo it, and one failing does
|
|
3706
|
+
// not stop the other. The task command is loaded only here: this file is read on every start, and only
|
|
3707
|
+
// the paths that write to a job need it.
|
|
3708
|
+
function redrawTaskFiles({ taskDb, db, root }) {
|
|
3709
|
+
for (const [what, refresh] of [
|
|
3710
|
+
['task list file', () => require('../commands/task').writeDefaultProjection(taskDb, db, { root })],
|
|
3711
|
+
['TODO.md', () => require('../commands/task').autoRenderTodoFromDb(taskDb.workspaceRoot(root))],
|
|
3712
|
+
]) {
|
|
3713
|
+
try { refresh(); } catch (err) {
|
|
3714
|
+
if (process.env.ATRIS_DEBUG) process.stderr.write(`${what} refresh failed: ${String(err?.message ?? err).replace(/[\r\n]+/g, ' ')}\n`);
|
|
3715
|
+
}
|
|
3716
|
+
}
|
|
3717
|
+
}
|
|
3718
|
+
|
|
3719
|
+
// Which claim this flight made on each job, by the row id it claimed. A mark or a release checks the job is
|
|
3720
|
+
// still held under that claim before it writes anything: another run can take a job over under the same
|
|
3721
|
+
// builder name (a lease that ran out, then a second flight), and a late release from this flight must not
|
|
3722
|
+
// land on, or stamp, a claim that belongs to someone else. The name of the claim comes from the claim write
|
|
3723
|
+
// itself (see claimJob) and is remembered the moment it is made; it is never looked up afterwards, because
|
|
3724
|
+
// a look-up cannot tell this flight's claim from one made just after it. A job claimed through the task
|
|
3725
|
+
// command (a stand-in task plane, a job shown with no row id) has no name to remember.
|
|
3726
|
+
function claimLedger() {
|
|
3727
|
+
const made = new Map();
|
|
3728
|
+
return {
|
|
3729
|
+
remember(ref, eventId) {
|
|
3730
|
+
if (ref && eventId) made.set(String(ref), String(eventId));
|
|
3731
|
+
},
|
|
3732
|
+
// The event that is this flight's claim on the job, or '' when none was remembered.
|
|
3733
|
+
eventId(ref) {
|
|
3734
|
+
return made.get(String(ref)) || '';
|
|
3735
|
+
},
|
|
3736
|
+
};
|
|
3737
|
+
}
|
|
3738
|
+
|
|
3739
|
+
// True when the task store holds a row under this name. It decides whether a flight may act on a job without a
|
|
3740
|
+
// claim to name, so a store that cannot be read counts as holding it: doubt means it may not.
|
|
3741
|
+
function storeHoldsJob(taskDb, db, ref) {
|
|
3742
|
+
if (!ref) return false;
|
|
3743
|
+
try {
|
|
3744
|
+
return Boolean(taskDb.getTask(db, ref));
|
|
3745
|
+
} catch {
|
|
3746
|
+
return true;
|
|
3747
|
+
}
|
|
3748
|
+
}
|
|
3749
|
+
|
|
3750
|
+
// How a flight claims a job. A job the task store holds a row for is claimed in the flight's own process,
|
|
3751
|
+
// through the store it already holds for leases, and the claim's name (its claimed event) comes from the claim
|
|
3752
|
+
// write itself: claimTask saves the row and the event in one transaction and answers with the event, so there is
|
|
3753
|
+
// no moment in which the flight has claimed a job but cannot say which claim is its own. What the task command
|
|
3754
|
+
// did besides the write stays: the check on the actor's name, the plain failure text, the two files people
|
|
3755
|
+
// read drawn again (see redrawTaskFiles), and the clearing of the claims that have run out, which every task
|
|
3756
|
+
// command begins with, so a job whose old claim is dead is cleared and claimed in one go. A job the store holds
|
|
3757
|
+
// no row for (a stand-in task plane, a job shown with no row id) has no claim to name and is claimed through the
|
|
3758
|
+
// task command, as it always was.
|
|
3759
|
+
//
|
|
3760
|
+
// It answers like the task command (status 0, or status 1 and why), with eventId when it claimed in process. A
|
|
3761
|
+
// claim that landed but came back with no name is refused: the flight cannot tell later whether the job is
|
|
3762
|
+
// still its own, so it builds nothing on it and leaves it alone.
|
|
3763
|
+
function claimJob({ cli, taskDb, db, root, ref, actor, minutes = 60 }) {
|
|
3764
|
+
if (!storeHoldsJob(taskDb, db, ref)) {
|
|
3765
|
+
return cli(['task', 'claim', ref, '--as', actor, '--lease-minutes', String(minutes), '--lease-pid', String(process.pid)]);
|
|
3766
|
+
}
|
|
3767
|
+
const check = require('./review-integrity').validateActor(actor, { root });
|
|
3768
|
+
if (!check.ok) {
|
|
3769
|
+
return {
|
|
3770
|
+
status: 1,
|
|
3771
|
+
stdout: '',
|
|
3772
|
+
stderr: check.reason === 'reserved_actor'
|
|
3773
|
+
? `atris task claim: reserved_actor: '${actor}' is a system actor and cannot be used with --as`
|
|
3774
|
+
: `atris task claim: actor_not_on_roster: '${actor}' is not a workspace member or engine (actor validation is enforced)`,
|
|
3775
|
+
};
|
|
3776
|
+
}
|
|
3777
|
+
// The step the task command takes first, with its reach (every claim in the store that has run out) and the
|
|
3778
|
+
// history it writes (a claim_expired entry for each), because it is the command's own function. It never throws.
|
|
3779
|
+
taskDb.sweepExpiredLeases();
|
|
3780
|
+
let result;
|
|
3781
|
+
try {
|
|
3782
|
+
result = taskDb.claimTask(db, { id: ref, claimedBy: actor, leaseMinutes: minutes, leasePid: process.pid });
|
|
3783
|
+
} catch (err) {
|
|
3784
|
+
return { status: 1, stdout: '', stderr: `claim failed: ${String(err?.message ?? err).trim() || 'unknown'}` };
|
|
3785
|
+
}
|
|
3786
|
+
if (!result.claimed) {
|
|
3787
|
+
return { status: 1, stdout: '', stderr: `claim failed: ${result.reason}${result.claimed_by ? ` (held by ${result.claimed_by})` : ''}` };
|
|
3788
|
+
}
|
|
3789
|
+
redrawTaskFiles({ taskDb, db, root });
|
|
3790
|
+
const eventId = result.event && result.event.event_id;
|
|
3791
|
+
if (!eventId) return { status: 1, stdout: '', stderr: UNNAMED_CLAIM_DETAIL, unnamed: true };
|
|
3792
|
+
return { status: 0, stdout: `claimed ${ref} as ${actor}`, stderr: '', eventId };
|
|
3793
|
+
}
|
|
3794
|
+
|
|
3795
|
+
// The one plain line for a job another run has claimed since this flight made its own claim.
|
|
3796
|
+
const claimTakenOverLine = (task) => ` left ${task} alone: another run claimed it after this one did`;
|
|
3797
|
+
|
|
3798
|
+
// The one plain line for a job whose claim this flight no longer holds when it is about to ask the engine.
|
|
3799
|
+
const LOST_CLAIM_DETAIL = 'this run no longer holds the claim, so the engine was not started and the job was left alone';
|
|
3800
|
+
const claimLostLine = (task) => ` ${task}: ${LOST_CLAIM_DETAIL}`;
|
|
3801
|
+
|
|
3802
|
+
// The one plain line for a job the flight has claimed but cannot name the claim of: it cannot tell whether the job
|
|
3803
|
+
// is still its own, so it writes nothing to it, releases nothing, and starts no engine on it.
|
|
3804
|
+
const UNNAMED_CLAIM_DETAIL = 'this run has no record of which claim is its own, so the job was left alone';
|
|
3805
|
+
const unnamedClaimLine = (task) => ` ${task}: ${UNNAMED_CLAIM_DETAIL}`;
|
|
3806
|
+
|
|
3807
|
+
// What a flight calls the step a job failed at, in the one line it logs when it lets that job's claim go.
|
|
3808
|
+
const NOT_STARTED_WORDS = { brief_capture: 'brief capture', dispatch: 'attempt', prepare: 'prepare' };
|
|
3809
|
+
|
|
3810
|
+
// Right before an engine is asked, the first or a later one (a retry, a handover), the flight marks the claim it
|
|
3811
|
+
// made as started (see markBuildStarted in task-db.js). That is what keeps the could-not-start kind honest: the
|
|
3812
|
+
// store and the rating take it only for a claim with no mark. A job the store does not hold under this holder (a
|
|
3813
|
+
// stand-in task plane) has no claim to mark and goes on. A mark refused for a claim the flight remembered means
|
|
3814
|
+
// the claim is not the flight's any more (it was let go, or taken over, even by a run of the same name): the
|
|
3815
|
+
// engine is not asked, and the error says so (claimLost) for the flight to stop on, writing nothing more. A job
|
|
3816
|
+
// the store holds that the flight has no name for the claim of stops the same way (claimLost and unnamed).
|
|
3817
|
+
// A mark that cannot be saved throws like any store trouble, and the engine is not asked either.
|
|
3818
|
+
function markClaimStarted({ taskDb, db, claims, ref, actor, task }) {
|
|
3819
|
+
const claimEventId = claims.eventId(ref);
|
|
3820
|
+
if (!claimEventId && storeHoldsJob(taskDb, db, ref)) {
|
|
3821
|
+
// A job the store holds is claimed in process, so a flight with no name for its claim cannot say the job is
|
|
3822
|
+
// still its own: the engine is not asked, and nothing is written.
|
|
3823
|
+
const unnamed = new Error(`${task}: ${UNNAMED_CLAIM_DETAIL}`);
|
|
3824
|
+
unnamed.claimLost = true;
|
|
3825
|
+
unnamed.unnamed = true;
|
|
3826
|
+
throw unnamed;
|
|
3827
|
+
}
|
|
3828
|
+
const result = taskDb.markBuildStarted(db, { id: ref, actor, claimEventId });
|
|
3829
|
+
if (!result.marked && claimEventId) {
|
|
3830
|
+
const lost = new Error(`the claim on ${task} is no longer this run's (${result.reason})`);
|
|
3831
|
+
lost.claimLost = true;
|
|
3832
|
+
throw lost;
|
|
3833
|
+
}
|
|
3834
|
+
return result;
|
|
3835
|
+
}
|
|
3836
|
+
|
|
3837
|
+
// A flight lets go of a claim in its own process, on the exact claim it made, through the task store it already
|
|
3838
|
+
// holds for leases. ref is the row id the claim named and claimEventId the claim event the flight remembered
|
|
3839
|
+
// (see claimLedger): the store lets go only if the job is still held under that claim, in the same write, so a
|
|
3840
|
+
// job another run took over since, even under the same builder name and in the same millisecond, is never
|
|
3841
|
+
// released by a late call of this flight's. `atris task release` takes no kind and no claim, because any worker
|
|
3842
|
+
// holding a failed job could pass one to excuse its own miss; only the flights say these things, here.
|
|
3843
|
+
//
|
|
3844
|
+
// neverStarted asks for the could-not-start kind (lib/release-kind.js): the build machine could not start the
|
|
3845
|
+
// job, so it is nobody's miss. The store keeps that honest: it drops the kind from a claim marked started (see
|
|
3846
|
+
// markClaimStarted), and the release then goes through as an ordinary one, so a claim is never left held.
|
|
3847
|
+
//
|
|
3848
|
+
// It answers like the task command (status 0, or status 1 and why), with a word more: not_ours when the job is
|
|
3849
|
+
// not held under the flight's claim any more (nothing was written, nothing is retried, and the caller says so),
|
|
3850
|
+
// final when it is already let go or gone, retryable when the store failed (the closing pass tries once more),
|
|
3851
|
+
// unnamed when the store holds the job but the flight has no name for its claim (nothing is written and nothing
|
|
3852
|
+
// is retried: a release that names no claim could land on anyone's, so the job is left alone and the caller says
|
|
3853
|
+
// so). Null only when the store holds no row for the job (a job shown with no row id, a stand-in task plane): the
|
|
3854
|
+
// caller then lets it go through the task command, as a plain release, the way it always did.
|
|
3855
|
+
function releaseOwnClaim({ taskDb, db, root, ref, actor, reason, claimEventId = '', neverStarted = false }) {
|
|
3856
|
+
if (!claimEventId) {
|
|
3857
|
+
return storeHoldsJob(taskDb, db, ref) ? { status: 1, stdout: '', stderr: UNNAMED_CLAIM_DETAIL, unnamed: true } : null;
|
|
3858
|
+
}
|
|
3859
|
+
let result;
|
|
3860
|
+
try {
|
|
3861
|
+
result = taskDb.releaseTask(db, { id: ref, actor, reason, claimEventId, ...(neverStarted ? { kind: COULD_NOT_START } : {}) });
|
|
3862
|
+
} catch (err) {
|
|
3863
|
+
return { status: 1, stdout: '', stderr: String(err?.message ?? err).trim() || 'unknown', retryable: true };
|
|
3864
|
+
}
|
|
3865
|
+
if (!result.released) {
|
|
3866
|
+
const stderr = `release failed: ${result.reason}${result.claimed_by ? ` (held by ${result.claimed_by})` : ''}`;
|
|
3867
|
+
if (result.reason === 'claim_changed' || result.reason === 'held_by_other') return { status: 1, stdout: '', stderr, not_ours: true };
|
|
3868
|
+
if (result.reason === 'not_found' || String(result.reason).startsWith('already_')) return { status: 1, stdout: '', stderr, final: true };
|
|
3869
|
+
return { status: 1, stdout: '', stderr, retryable: true };
|
|
3870
|
+
}
|
|
3871
|
+
redrawTaskFiles({ taskDb, db, root });
|
|
3872
|
+
return { status: 0, stdout: 'released', stderr: '' };
|
|
3873
|
+
}
|
|
3874
|
+
|
|
3509
3875
|
// A claim taken before the engine starts must not stay held if the flight
|
|
3510
3876
|
// refuses or errors first. Release through the same task plane the claim used,
|
|
3511
|
-
// naming the job by ref: the row id the claim named.
|
|
3512
|
-
|
|
3513
|
-
|
|
3877
|
+
// naming the job by ref: the row id the claim named. release, when a flight gives
|
|
3878
|
+
// one, lets it go in process as a job that could not start (see releaseOwnClaim); it
|
|
3879
|
+
// answers null for a claim it cannot record, and the task plane lets that go. A
|
|
3880
|
+
// claim another run has taken over since is left alone, with one plain line. A
|
|
3881
|
+
// release the store failed (retryable) is noted in failures, by the job's name, for the
|
|
3882
|
+
// closing pass to try again.
|
|
3883
|
+
function releaseUnstartedDispatchClaim(cli, { taskId, ref = taskId, actor, detail, stage, release = null, failures = null, log = () => {} }) {
|
|
3884
|
+
const reason = releaseReason(`paused at ${pausedStageLabel(stage)}`, detail);
|
|
3885
|
+
const released = (release && release({ ref, actor, reason })) || cli(['task', 'release', ref, '--as', actor, '--reason', reason]);
|
|
3514
3886
|
const releasedOk = Boolean(released && released.status === 0);
|
|
3887
|
+
const notOurs = Boolean(released && released.not_ours);
|
|
3888
|
+
const unnamed = Boolean(released && released.unnamed);
|
|
3889
|
+
if (notOurs) log(claimTakenOverLine(taskId));
|
|
3890
|
+
if (unnamed) log(unnamedClaimLine(taskId));
|
|
3515
3891
|
const suffix = releasedOk
|
|
3516
3892
|
? 'claim released, safe to retry'
|
|
3517
|
-
:
|
|
3893
|
+
: notOurs
|
|
3894
|
+
? 'claim left alone, another run claimed it after this one did'
|
|
3895
|
+
: unnamed
|
|
3896
|
+
? 'claim left alone, this run has no record of which claim is its own'
|
|
3897
|
+
: `claim release failed: ${String(released && (released.stderr || released.stdout) || 'unknown').trim().slice(0, 80)}`;
|
|
3518
3898
|
const base = String(detail || '').trim().slice(0, 200);
|
|
3519
|
-
|
|
3899
|
+
const text = base ? `${base}. ${suffix}` : suffix;
|
|
3900
|
+
if (failures && released && released.retryable) failures.set(String(taskId), { ref, actor, reason, base, detail: text, release });
|
|
3901
|
+
return { released: releasedOk, detail: text };
|
|
3520
3902
|
}
|
|
3521
3903
|
|
|
3522
3904
|
// A pause with no work must not keep the task. On 2026-08-21, 49 bug tasks sat
|
|
@@ -3534,14 +3916,44 @@ function releaseUnstartedDispatchClaim(cli, { taskId, ref = taskId, actor, detai
|
|
|
3534
3916
|
// refByTask names each claimed job by the row id the flight claimed it with, so the release
|
|
3535
3917
|
// lands on that row even if the display id a person reads now means another job. A job with
|
|
3536
3918
|
// no entry there is named by the display id in its pause row.
|
|
3537
|
-
|
|
3919
|
+
//
|
|
3920
|
+
// release, when a flight gives one, lets a claim go in the flight's own process, on the exact claim it
|
|
3921
|
+
// made (see releaseOwnClaim); a job it answers null for is let go through the task command. A job another
|
|
3922
|
+
// run has claimed since is left alone, with one plain line, and its row says the claim was not released.
|
|
3923
|
+
// A release the store failed is tried once more, and a claim that still cannot be let go says so in one
|
|
3924
|
+
// plain line.
|
|
3925
|
+
//
|
|
3926
|
+
// failures holds the start-failure releases that failed at the point of failure (see
|
|
3927
|
+
// releaseUnstartedDispatchClaim): a row that says its claim was not released, with an entry there, gets its
|
|
3928
|
+
// release tried again here, the way it was first tried.
|
|
3929
|
+
function releasePausedClaimsWithoutWork(cli, { paused, worktreeByTask, actorByTask, refByTask = new Map(), baseRef = '', log = () => {}, keepClaimsWithWork = true, release = null, failures = null }) {
|
|
3930
|
+
const letGo = (how, { ref, actor, reason }) => (how && how({ ref, actor, reason })) || cli(['task', 'release', ref, '--as', actor, '--reason', reason]);
|
|
3931
|
+
const saysNo = (taskId, released) => log(` could not release the claim on ${taskId}: ${String(released && (released.stderr || released.stdout) || 'unknown').replace(/\s+/g, ' ').trim().slice(0, 80)}. it stays until its lease runs out, unless the task record is repaired first.`);
|
|
3538
3932
|
for (const row of paused || []) {
|
|
3539
3933
|
// No claim (the job was never found, was refused as another workspace's, or the claim
|
|
3540
|
-
// failed),
|
|
3541
|
-
//
|
|
3542
|
-
|
|
3543
|
-
if (row.
|
|
3934
|
+
// failed), a claim the flight lost before its engine ran (it is the other run's now), work
|
|
3935
|
+
// already landed, or work waiting on a pull request for a person (the flight let that claim go
|
|
3936
|
+
// itself when saving its proof failed).
|
|
3937
|
+
if (row.stage === 'claim' || row.stage === 'task_lookup' || row.stage === 'other_workspace' || row.stage === 'claim_lost' || row.landed === true || row.held_for_review === true) continue;
|
|
3544
3938
|
const taskId = String(row.task || '');
|
|
3939
|
+
const failed = row.claim_released === false && failures ? failures.get(taskId) : null;
|
|
3940
|
+
if (failed) {
|
|
3941
|
+
failures.delete(taskId);
|
|
3942
|
+
const released = letGo(failed.release, failed);
|
|
3943
|
+
if (released && released.status === 0) {
|
|
3944
|
+
row.claim_released = true;
|
|
3945
|
+
if (row.detail === failed.detail) row.detail = failed.base ? `${failed.base}. claim released, safe to retry` : 'claim released, safe to retry';
|
|
3946
|
+
log(` released claim on ${taskId} on a second try`);
|
|
3947
|
+
} else if (released && released.not_ours) {
|
|
3948
|
+
log(claimTakenOverLine(taskId));
|
|
3949
|
+
} else if (released && released.unnamed) {
|
|
3950
|
+
log(unnamedClaimLine(taskId));
|
|
3951
|
+
} else if (!(released && released.final)) {
|
|
3952
|
+
saysNo(taskId, released);
|
|
3953
|
+
}
|
|
3954
|
+
continue;
|
|
3955
|
+
}
|
|
3956
|
+
if (row.claim_released !== undefined) continue;
|
|
3545
3957
|
const actor = actorByTask.get(taskId);
|
|
3546
3958
|
if (!taskId || !actor) continue;
|
|
3547
3959
|
const worktreePath = row.worktree || worktreeByTask.get(taskId) || '';
|
|
@@ -3553,9 +3965,19 @@ function releasePausedClaimsWithoutWork(cli, { paused, worktreeByTask, actorByTa
|
|
|
3553
3965
|
const reason = noWork && !ENGINE_DID_NOT_FINISH.has(row.stage)
|
|
3554
3966
|
? 'engine produced no work'
|
|
3555
3967
|
: releaseReason(`paused at ${pausedStageLabel(row.stage)}`, row.stage === 'build' ? row.reason || row.detail : row.detail || row.reason);
|
|
3556
|
-
const
|
|
3968
|
+
const ref = refByTask.get(taskId) || taskId;
|
|
3969
|
+
let released = letGo(release, { ref, actor, reason });
|
|
3970
|
+
if (released && released.retryable) released = letGo(release, { ref, actor, reason });
|
|
3971
|
+
if (released && (released.not_ours || released.unnamed)) {
|
|
3972
|
+
row.claim_released = false;
|
|
3973
|
+
log(released.unnamed ? unnamedClaimLine(taskId) : claimTakenOverLine(taskId));
|
|
3974
|
+
continue;
|
|
3975
|
+
}
|
|
3557
3976
|
row.claim_released = Boolean(released && released.status === 0);
|
|
3558
|
-
if (!row.claim_released)
|
|
3977
|
+
if (!row.claim_released) {
|
|
3978
|
+
if (released && released.retryable) saysNo(taskId, released);
|
|
3979
|
+
continue;
|
|
3980
|
+
}
|
|
3559
3981
|
log(keepClaimsWithWork
|
|
3560
3982
|
? ` released claim on ${taskId} (engine produced no work)`
|
|
3561
3983
|
: ` released claim on ${taskId} so a retry can pick it up${worktreePath ? `; any work stays in ${path.basename(worktreePath)}` : ''}`);
|
|
@@ -3582,14 +4004,26 @@ function releasePausedClaimsWithoutWork(cli, { paused, worktreeByTask, actorByTa
|
|
|
3582
4004
|
// same ids, so it is never looked up again. With no row id there is nothing exact to
|
|
3583
4005
|
// release, and nothing is.
|
|
3584
4006
|
//
|
|
4007
|
+
// It also names the claim it made (claimEventId, see claimLedger). A job another run has claimed since is
|
|
4008
|
+
// left alone, with one plain line: the pull request is still open for a person, and the claim is not this
|
|
4009
|
+
// flight's to put the kind on. With no name for its claim, the flight cannot say the claim is its own, so it
|
|
4010
|
+
// writes nothing and says so in one plain line.
|
|
4011
|
+
//
|
|
3585
4012
|
// Tried once: if the release fails too, the row says so.
|
|
3586
|
-
function releaseClaimOfUnsavedPullRequest({ taskDb, db, root, row, rowId, actor, log = () => {} }) {
|
|
4013
|
+
function releaseClaimOfUnsavedPullRequest({ taskDb, db, root, row, rowId, actor, log = () => {}, claimEventId = '' }) {
|
|
3587
4014
|
const reason = releaseReason(`PR ${row.pull_request.url} is open but its proof could not be saved`, row.detail);
|
|
3588
4015
|
let released = false;
|
|
4016
|
+
let notOurs = false;
|
|
4017
|
+
let unnamed = false;
|
|
3589
4018
|
let failure = 'unknown';
|
|
3590
4019
|
try {
|
|
3591
4020
|
if (!rowId) {
|
|
3592
4021
|
failure = 'the flight has no row id for it';
|
|
4022
|
+
} else if (!claimEventId && storeHoldsJob(taskDb, db, rowId)) {
|
|
4023
|
+
// The store holds the job and the flight has no name for its claim: a release that names no claim
|
|
4024
|
+
// could land on anyone's, so the job is left alone. A row the store does not know goes on to the
|
|
4025
|
+
// release below, which answers not_found, as it always did.
|
|
4026
|
+
unnamed = true;
|
|
3593
4027
|
} else {
|
|
3594
4028
|
const result = taskDb.releaseTask(db, {
|
|
3595
4029
|
id: rowId,
|
|
@@ -3597,9 +4031,11 @@ function releaseClaimOfUnsavedPullRequest({ taskDb, db, root, row, rowId, actor,
|
|
|
3597
4031
|
reason,
|
|
3598
4032
|
kind: PROOF_NOT_SAVED,
|
|
3599
4033
|
pullRequest: row.pull_request,
|
|
4034
|
+
claimEventId,
|
|
3600
4035
|
});
|
|
3601
4036
|
if (!result.released) {
|
|
3602
4037
|
failure = `release failed: ${result.reason}${result.claimed_by ? ` (held by ${result.claimed_by})` : ''}`;
|
|
4038
|
+
notOurs = result.reason === 'claim_changed' || result.reason === 'held_by_other';
|
|
3603
4039
|
} else {
|
|
3604
4040
|
released = true;
|
|
3605
4041
|
// The two files people read are drawn again from the saved state, the way the
|
|
@@ -3624,7 +4060,11 @@ function releaseClaimOfUnsavedPullRequest({ taskDb, db, root, row, rowId, actor,
|
|
|
3624
4060
|
row.claim_released = released;
|
|
3625
4061
|
log(row.claim_released
|
|
3626
4062
|
? ` released the claim on ${row.task} with the pull request link as the reason`
|
|
3627
|
-
:
|
|
4063
|
+
: notOurs
|
|
4064
|
+
? claimTakenOverLine(row.task)
|
|
4065
|
+
: unnamed
|
|
4066
|
+
? unnamedClaimLine(row.task)
|
|
4067
|
+
: ` could not release the claim on ${row.task}: ${failure.replace(/\s+/g, ' ').trim().slice(0, 80)}. it stays until its lease runs out, unless the task record is repaired first.`);
|
|
3628
4068
|
return row.claim_released;
|
|
3629
4069
|
}
|
|
3630
4070
|
|
|
@@ -3698,6 +4138,9 @@ async function runDispatchFlight({
|
|
|
3698
4138
|
// Extra house rules a caller appends to the builder's brief (atris idea
|
|
3699
4139
|
// tick sends the four values and the no-force-push, no-hook-skip rules).
|
|
3700
4140
|
brief = '',
|
|
4141
|
+
// The disk check before anything is claimed (see lowDiskRefusal). Tests inject both.
|
|
4142
|
+
freeSpace = undefined,
|
|
4143
|
+
spaceGuard = undefined,
|
|
3701
4144
|
} = {}) {
|
|
3702
4145
|
if (!engine) throw new Error('runDispatchFlight: engine is required');
|
|
3703
4146
|
if (!DISPATCH_CAPABLE.includes(engine)) {
|
|
@@ -3720,8 +4163,44 @@ async function runDispatchFlight({
|
|
|
3720
4163
|
}
|
|
3721
4164
|
|
|
3722
4165
|
const cli = ownCli || defaultOwnCli(root);
|
|
4166
|
+
// One look at the disk before anything is claimed or written to any job, not even the sweep of expired
|
|
4167
|
+
// claims below: worktree start's own check and reaping. Under its floor every start would refuse anyway,
|
|
4168
|
+
// and each refused start used to let a claim go that the rating then scored as a drop. A sealed review-only
|
|
4169
|
+
// flight builds in the temp folder, which that check leaves alone by default, so it is asked about that
|
|
4170
|
+
// folder with the check on.
|
|
4171
|
+
const sealedReview = reviewOnly && !dispatcher;
|
|
4172
|
+
const lowDisk = lowDiskRefusal({
|
|
4173
|
+
root,
|
|
4174
|
+
agent: engine,
|
|
4175
|
+
task: 'dispatch',
|
|
4176
|
+
freeSpace,
|
|
4177
|
+
spaceGuard: sealedReview ? (spaceGuard ?? true) : spaceGuard,
|
|
4178
|
+
...(sealedReview ? { target: os.tmpdir() } : {}),
|
|
4179
|
+
});
|
|
4180
|
+
if (lowDisk) {
|
|
4181
|
+
return stopForLowDisk({
|
|
4182
|
+
schema: 'atris.dispatch_receipt.v1',
|
|
4183
|
+
at: new Date().toISOString(),
|
|
4184
|
+
root,
|
|
4185
|
+
engine,
|
|
4186
|
+
tasks: ids,
|
|
4187
|
+
results: [],
|
|
4188
|
+
landed: [],
|
|
4189
|
+
ready: [],
|
|
4190
|
+
held_for_review: [],
|
|
4191
|
+
paused: [],
|
|
4192
|
+
}, lowDisk, ids.map((task) => ({ task })), log);
|
|
4193
|
+
}
|
|
3723
4194
|
const leaseDb = require('./task-db');
|
|
3724
4195
|
const leaseConnection = leaseDb.open();
|
|
4196
|
+
// The claim this flight made on each job, remembered from the claim write itself (see claimJob).
|
|
4197
|
+
const claims = claimLedger();
|
|
4198
|
+
// A claim let go before the engine ran is recorded in this process, on the claim this flight made, as one
|
|
4199
|
+
// that could not start. Any other claim the flight lets go (the engine was asked) is an ordinary release.
|
|
4200
|
+
const letGoUnstarted = ({ ref, actor, reason }) => releaseOwnClaim({ taskDb: leaseDb, db: leaseConnection, root, ref, actor, reason, claimEventId: claims.eventId(ref), neverStarted: true });
|
|
4201
|
+
const letGoAsked = ({ ref, actor, reason }) => releaseOwnClaim({ taskDb: leaseDb, db: leaseConnection, root, ref, actor, reason, claimEventId: claims.eventId(ref) });
|
|
4202
|
+
// The start-failure releases the store failed, for the closing pass to try again (see releaseUnstartedDispatchClaim).
|
|
4203
|
+
const releaseFailures = new Map();
|
|
3725
4204
|
// The workspace this flight works in, spelled the way the store spells it. It is read once, before
|
|
3726
4205
|
// the flight writes anything into its folder.
|
|
3727
4206
|
const workspace = leaseDb.workspaceRoot(root);
|
|
@@ -3774,8 +4253,17 @@ async function runDispatchFlight({
|
|
|
3774
4253
|
// What each claimed job is called to the task command, by the id the caller gave. Only jobs this
|
|
3775
4254
|
// flight claimed are in it.
|
|
3776
4255
|
const claimedRefs = new Map();
|
|
4256
|
+
// Right before the assigned engine is asked, the claim is marked started.
|
|
4257
|
+
const markStarted = (entry) => markClaimStarted({
|
|
4258
|
+
taskDb: leaseDb,
|
|
4259
|
+
db: leaseConnection,
|
|
4260
|
+
claims,
|
|
4261
|
+
ref: claimedRefs.get(String(entry.taskId)) || entry.taskId,
|
|
4262
|
+
actor: taskActor,
|
|
4263
|
+
task: entry.taskId,
|
|
4264
|
+
});
|
|
3777
4265
|
const leaseTimer = setInterval(() => {
|
|
3778
|
-
for (const id of leasedIds) leaseDb.renewLease(leaseConnection, { id, holder: taskActor, pid: process.pid });
|
|
4266
|
+
for (const id of leasedIds) leaseDb.renewLease(leaseConnection, { id, holder: taskActor, pid: process.pid, claimEventId: claims.eventId(id) });
|
|
3779
4267
|
}, 5 * 60000);
|
|
3780
4268
|
leaseTimer.unref();
|
|
3781
4269
|
try {
|
|
@@ -3799,17 +4287,18 @@ async function runDispatchFlight({
|
|
|
3799
4287
|
// named once, by its row id, at every step (see jobRef). Log lines and the receipt keep saying
|
|
3800
4288
|
// the display id.
|
|
3801
4289
|
const rowRef = jobRef(task, taskId);
|
|
3802
|
-
const claimed =
|
|
4290
|
+
const claimed = claimJob({ cli, taskDb: leaseDb, db: leaseConnection, root, ref: rowRef, actor: taskActor });
|
|
3803
4291
|
if (!claimed || claimed.status !== 0) {
|
|
3804
4292
|
const detail = String(claimed && (claimed.stderr || claimed.stdout) || 'claim failed').trim().slice(0, 300);
|
|
3805
4293
|
flight.paused.push({ task: taskId, stage: 'claim', detail });
|
|
3806
|
-
log(` x ${taskId} claim failed`);
|
|
4294
|
+
log(claimed && claimed.unnamed ? unnamedClaimLine(taskId) : ` x ${taskId} claim failed`);
|
|
3807
4295
|
continue;
|
|
3808
4296
|
}
|
|
3809
4297
|
claimedRefs.set(String(taskId), rowRef);
|
|
4298
|
+
claims.remember(rowRef, claimed.eventId);
|
|
3810
4299
|
try {
|
|
3811
4300
|
const leaseId = rowRef;
|
|
3812
|
-
const leased = leaseDb.setLease(leaseConnection, { id: leaseId, holder: taskActor });
|
|
4301
|
+
const leased = leaseDb.setLease(leaseConnection, { id: leaseId, holder: taskActor, claimEventId: claims.eventId(leaseId) });
|
|
3813
4302
|
if (!leased && (!ownCli || leaseDb.getTask(leaseConnection, leaseId))) throw new Error('could not set claim lease');
|
|
3814
4303
|
if (leased) leasedIds.add(leaseId);
|
|
3815
4304
|
let landingWorktreePath = '';
|
|
@@ -3821,7 +4310,7 @@ async function runDispatchFlight({
|
|
|
3821
4310
|
// earlier finished run's branch must not refuse the retry. Worktree
|
|
3822
4311
|
// start resumes that branch's commits, or starts fresh when they no
|
|
3823
4312
|
// longer merge cleanly (the member-path retry fix).
|
|
3824
|
-
const started = cli(['worktree', 'start', '--agent', engine, '--task',
|
|
4313
|
+
const started = cli(['worktree', 'start', '--agent', engine, '--task', flightTaskName('dispatch', rowRef), ...formerTaskArgs('dispatch', rowRef, taskId), '--force', ...startBaseArgs]);
|
|
3825
4314
|
const wt = (started.stdout.match(/next: cd (.+)/) || [])[1];
|
|
3826
4315
|
if (!wt) {
|
|
3827
4316
|
const released = releaseUnstartedDispatchClaim(cli, {
|
|
@@ -3830,6 +4319,9 @@ async function runDispatchFlight({
|
|
|
3830
4319
|
actor: taskActor,
|
|
3831
4320
|
stage: 'worktree_start',
|
|
3832
4321
|
detail: String(started.stderr || '').slice(0, 200),
|
|
4322
|
+
release: letGoUnstarted,
|
|
4323
|
+
failures: releaseFailures,
|
|
4324
|
+
log,
|
|
3833
4325
|
});
|
|
3834
4326
|
flight.paused.push({ task: taskId, stage: 'worktree_start', detail: released.detail, claim_released: released.released });
|
|
3835
4327
|
log(` ✗ ${taskId} worktree start failed`);
|
|
@@ -3844,6 +4336,9 @@ async function runDispatchFlight({
|
|
|
3844
4336
|
actor: taskActor,
|
|
3845
4337
|
stage: 'remote_quarantine',
|
|
3846
4338
|
detail: headTail(String(remoteBoundary && remoteBoundary.detail || 'could not prepare a sealed review sandbox')),
|
|
4339
|
+
release: letGoUnstarted,
|
|
4340
|
+
failures: releaseFailures,
|
|
4341
|
+
log,
|
|
3847
4342
|
});
|
|
3848
4343
|
flight.paused.push({
|
|
3849
4344
|
task: taskId,
|
|
@@ -3889,6 +4384,9 @@ async function runDispatchFlight({
|
|
|
3889
4384
|
actor: taskActor,
|
|
3890
4385
|
stage: 'prepare',
|
|
3891
4386
|
detail: String(err && err.message || err).slice(0, 200),
|
|
4387
|
+
release: letGoUnstarted,
|
|
4388
|
+
failures: releaseFailures,
|
|
4389
|
+
log,
|
|
3892
4390
|
});
|
|
3893
4391
|
flight.paused.push({ task: taskId, stage: 'prepare', detail: released.detail, claim_released: released.released });
|
|
3894
4392
|
log(` ✗ ${taskId} prepare failed`);
|
|
@@ -3923,7 +4421,41 @@ async function runDispatchFlight({
|
|
|
3923
4421
|
log(` fable handoff started: receipt ${path.relative(root, receiptPath)}`);
|
|
3924
4422
|
}
|
|
3925
4423
|
|
|
3926
|
-
|
|
4424
|
+
// A job that never reached its engine (its brief could not be written, or its start could not be marked)
|
|
4425
|
+
// pauses and its claim goes back as one that could not start, here, where it failed, while the flight's
|
|
4426
|
+
// other jobs go on. The jobs that did start carry on.
|
|
4427
|
+
const letGoNotStarted = (entry, result) => {
|
|
4428
|
+
const stage = result.not_started;
|
|
4429
|
+
const ref = claimedRefs.get(String(entry.taskId)) || entry.taskId;
|
|
4430
|
+
if (stage === 'claim_lost') {
|
|
4431
|
+
// The job is another run's now (see markClaimStarted). The flight lets go of nothing, writes nothing to it, and
|
|
4432
|
+
// stops renewing a lease on it.
|
|
4433
|
+
leasedIds.delete(ref);
|
|
4434
|
+
flight.paused.push({ task: entry.taskId, stage, detail: result.unnamed ? UNNAMED_CLAIM_DETAIL : LOST_CLAIM_DETAIL, worktree: entry.worktreePath, claim_released: false });
|
|
4435
|
+
log(result.unnamed ? unnamedClaimLine(entry.taskId) : claimLostLine(entry.taskId));
|
|
4436
|
+
return;
|
|
4437
|
+
}
|
|
4438
|
+
// Nothing in here may end the run: the other jobs still have their cleanup to come.
|
|
4439
|
+
let detail = result.detail;
|
|
4440
|
+
let released = false;
|
|
4441
|
+
try {
|
|
4442
|
+
({ released, detail } = releaseUnstartedDispatchClaim(cli, {
|
|
4443
|
+
taskId: entry.taskId,
|
|
4444
|
+
ref,
|
|
4445
|
+
actor: taskActor,
|
|
4446
|
+
stage,
|
|
4447
|
+
detail: result.detail,
|
|
4448
|
+
release: letGoUnstarted,
|
|
4449
|
+
failures: releaseFailures,
|
|
4450
|
+
log,
|
|
4451
|
+
}));
|
|
4452
|
+
} catch (err) {
|
|
4453
|
+
detail = `${String(result.detail || '').slice(0, 120)}. claim release failed: ${String(err && err.message || err).slice(0, 80)}`;
|
|
4454
|
+
}
|
|
4455
|
+
flight.paused.push({ task: entry.taskId, stage, detail, worktree: entry.worktreePath, claim_released: released });
|
|
4456
|
+
log(` ✗ ${entry.taskId} ${NOT_STARTED_WORDS[stage] || 'prepare'} failed`);
|
|
4457
|
+
};
|
|
4458
|
+
const settled = await Promise.all(prepared.map((entry) => {
|
|
3927
4459
|
const startedAtMs = Date.now();
|
|
3928
4460
|
return dispatchEntryWithRestaff({
|
|
3929
4461
|
entry,
|
|
@@ -3937,19 +4469,22 @@ async function runDispatchFlight({
|
|
|
3937
4469
|
defaultPin: { model, effort, maxSeconds: capSeconds(maxSeconds), ...(prep ? { prep } : {}) },
|
|
3938
4470
|
job: rosterJob,
|
|
3939
4471
|
prepAsk,
|
|
4472
|
+
beforeEngine: markStarted,
|
|
3940
4473
|
}).then((result) => {
|
|
4474
|
+
if (result.not_started) letGoNotStarted(entry, result);
|
|
3941
4475
|
const completedAtMs = Date.now();
|
|
3942
4476
|
return {
|
|
3943
4477
|
entry,
|
|
3944
4478
|
result,
|
|
3945
4479
|
startedAtMs,
|
|
3946
4480
|
completedAtMs,
|
|
3947
|
-
candidate: enforceRemoteBoundary && !detectDeadEngineDispatch(result)
|
|
4481
|
+
candidate: enforceRemoteBoundary && !result.not_started && !detectDeadEngineDispatch(result)
|
|
3948
4482
|
? reviewCandidateSnapshot(entry.remoteBoundary)
|
|
3949
4483
|
: null,
|
|
3950
4484
|
};
|
|
3951
4485
|
});
|
|
3952
4486
|
}));
|
|
4487
|
+
const results = settled.filter(({ result }) => !result.not_started);
|
|
3953
4488
|
flight.results = results.map(({ entry, result, candidate, startedAtMs, completedAtMs }) => {
|
|
3954
4489
|
const row = {
|
|
3955
4490
|
task: entry.taskId,
|
|
@@ -4083,6 +4618,9 @@ async function runDispatchFlight({
|
|
|
4083
4618
|
for (const { entry, result, candidate, startedAtMs } of results) {
|
|
4084
4619
|
const activeEngine = result.engine || entry.engine || engine;
|
|
4085
4620
|
const readyActor = explicitActor || `fleet-${activeEngine}`;
|
|
4621
|
+
// Every engine that ran a build attempt on this job, for the hand-in to name (see builtWithArgs).
|
|
4622
|
+
const ranEngines = (Array.isArray(result.attempts) ? result.attempts : []).map((attempt) => attempt && attempt.engine);
|
|
4623
|
+
const builtWith = builtWithArgs(ranEngines, { claimedAs: taskActor, handedInAs: readyActor });
|
|
4086
4624
|
const landingEntry = { ...entry, engine: activeEngine, candidate };
|
|
4087
4625
|
if (result.restaffed) log(` restaffed ${entry.taskId}: ${result.restaffed.from} -> ${result.restaffed.to} (${result.restaffed.detail || result.restaffed.reason})`);
|
|
4088
4626
|
if (result.handover) log(` handed over ${entry.taskId}: ${result.handover_reason}`);
|
|
@@ -4326,6 +4864,7 @@ async function runDispatchFlight({
|
|
|
4326
4864
|
const imported = importReviewCommit(entry.remoteBoundary, {
|
|
4327
4865
|
cli,
|
|
4328
4866
|
taskId: entry.taskId,
|
|
4867
|
+
ref: jobRef(entry.task, entry.taskId),
|
|
4329
4868
|
engine: activeEngine,
|
|
4330
4869
|
expectedCommit: candidate.commit,
|
|
4331
4870
|
expectedTree: candidate.tree,
|
|
@@ -4435,6 +4974,7 @@ async function runDispatchFlight({
|
|
|
4435
4974
|
'--landing', 'Operators can review the verified change before it reaches users, reducing the risk of an unreviewed release.',
|
|
4436
4975
|
'--checked', `${landed.check} passed in the isolated worktree`,
|
|
4437
4976
|
'--tested', 'The requested behavior passed its declared verifier.',
|
|
4977
|
+
...builtWith,
|
|
4438
4978
|
'--as', readyActor,
|
|
4439
4979
|
]);
|
|
4440
4980
|
if (!readyResult || readyResult.status !== 0) {
|
|
@@ -4486,13 +5026,17 @@ async function runDispatchFlight({
|
|
|
4486
5026
|
'--checked', `${landed.check} passed before the pull request was opened`,
|
|
4487
5027
|
'--tested', 'The requested behavior passed its declared verifier.',
|
|
4488
5028
|
...openPullRequestFlags(landed.pull_request),
|
|
5029
|
+
...builtWith,
|
|
4489
5030
|
'--as', readyActor,
|
|
4490
5031
|
], { flight, landedRow: heldRow, worktree: entry.worktreePath, heldForReview: true, save: () => writeDispatchReceipt(flight, receiptPath, { ids, reviewOnly: proofOnly, enforceRemoteBoundary }) });
|
|
4491
5032
|
if (!heldRecorded) {
|
|
4492
5033
|
stampDispatchBrief(root, result.brief_id, 'partial', `opened a pull request for ${entry.taskId}, but task ready failed`);
|
|
4493
5034
|
log(' the pull request is open, but saving its proof failed. repair the task record before starting more work.');
|
|
4494
5035
|
const unsaved = flight.paused.find((row) => row.task === entry.taskId && row.stage === 'task_ready' && row.held_for_review === true);
|
|
4495
|
-
if (unsaved)
|
|
5036
|
+
if (unsaved) {
|
|
5037
|
+
const rowId = entry.task && entry.task.id;
|
|
5038
|
+
releaseClaimOfUnsavedPullRequest({ taskDb: leaseDb, db: leaseConnection, root, row: unsaved, rowId, actor: taskActor, log, claimEventId: claims.eventId(rowId) });
|
|
5039
|
+
}
|
|
4496
5040
|
continue;
|
|
4497
5041
|
}
|
|
4498
5042
|
stampDispatchBrief(root, result.brief_id, 'pass', `held ${entry.taskId} for a person to land`);
|
|
@@ -4511,6 +5055,7 @@ async function runDispatchFlight({
|
|
|
4511
5055
|
'--checked', `${landed.check} passed before landing`,
|
|
4512
5056
|
'--tested', 'The requested behavior passed its declared verifier.',
|
|
4513
5057
|
...landedMergeFlags(landed.merge),
|
|
5058
|
+
...builtWith,
|
|
4514
5059
|
'--as', readyActor,
|
|
4515
5060
|
], { flight, landedRow, worktree: entry.worktreePath, save: () => writeDispatchReceipt(flight, receiptPath, { ids, reviewOnly: proofOnly, enforceRemoteBoundary }) });
|
|
4516
5061
|
if (!reviewRecorded) {
|
|
@@ -4541,6 +5086,8 @@ async function runDispatchFlight({
|
|
|
4541
5086
|
baseRef: landTarget,
|
|
4542
5087
|
log,
|
|
4543
5088
|
keepClaimsWithWork: proofOnly,
|
|
5089
|
+
release: letGoAsked,
|
|
5090
|
+
failures: releaseFailures,
|
|
4544
5091
|
});
|
|
4545
5092
|
|
|
4546
5093
|
recordFlightRosterRuns(root, flight, results, {
|
|
@@ -4567,7 +5114,7 @@ async function runDispatchFlight({
|
|
|
4567
5114
|
clearInterval(leaseTimer);
|
|
4568
5115
|
for (const id of leasedIds) {
|
|
4569
5116
|
const row = leaseDb.getTask(leaseConnection, id);
|
|
4570
|
-
if (row && row.status !== 'claimed') leaseDb.clearLease(leaseConnection, { id, holder: taskActor });
|
|
5117
|
+
if (row && row.status !== 'claimed') leaseDb.clearLease(leaseConnection, { id, holder: taskActor, claimEventId: claims.eventId(id) });
|
|
4571
5118
|
}
|
|
4572
5119
|
}
|
|
4573
5120
|
}
|