atris 3.74.29 → 3.75.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/atris/skills/calendar/SKILL.md +9 -0
- package/atris/skills/drive/SKILL.md +7 -0
- package/atris/skills/email-agent/SKILL.md +7 -0
- package/atris/skills/magic-inbox/SKILL.md +7 -0
- package/atris/skills/slides/SKILL.md +7 -0
- package/bin/atris.js +14 -2
- package/commands/engine.js +2 -2
- package/commands/integrations.js +539 -54
- package/commands/mission.js +2 -2
- package/commands/rating.js +14 -4
- package/commands/worktree.js +39 -12
- package/lib/fleet.js +545 -52
- package/lib/gm-season.js +31 -9
- package/lib/known-commands.js +1 -1
- package/lib/release-kind.js +31 -7
- package/lib/task-db.js +185 -26
- package/lib/worktree-space.js +4 -45
- package/package.json +1 -1
package/lib/fleet.js
CHANGED
|
@@ -33,7 +33,7 @@ const {
|
|
|
33
33
|
buildOneLapValidatorPrompt,
|
|
34
34
|
parseOneLapValidatorVerdict,
|
|
35
35
|
} = require('./one-lap-validator');
|
|
36
|
-
const { listWorktrees, defaultMainlineBase, gitFailureDetail, gitNetworkEnvironment } = require('../commands/worktree');
|
|
36
|
+
const { listWorktrees, defaultMainlineBase, gitFailureDetail, gitNetworkEnvironment, lowDiskRefusal } = require('../commands/worktree');
|
|
37
37
|
const { isConductorStatusLine } = require('./conductor-artifacts');
|
|
38
38
|
const { matchLessons } = require('./lesson-preflight');
|
|
39
39
|
const { sleepSync } = require('./sleep-sync');
|
|
@@ -44,7 +44,7 @@ const { buildVerifiedScoutPack, appendVerifiedScoutPack } = require('./dispatch-
|
|
|
44
44
|
const { runPrepPass, withPrepBrief, prepRecordFields } = require('./roster-prep');
|
|
45
45
|
const { mapNotesPath } = require('./map-refs');
|
|
46
46
|
const { SHARED_BRIEF_RULES } = require('./brief-rules');
|
|
47
|
-
const { PROOF_NOT_SAVED } = require('./release-kind');
|
|
47
|
+
const { PROOF_NOT_SAVED, COULD_NOT_START } = require('./release-kind');
|
|
48
48
|
const {
|
|
49
49
|
applySecretGrantEnvironment,
|
|
50
50
|
buildGatewaySupervisorScript,
|
|
@@ -952,10 +952,18 @@ async function dispatchEntryWithRestaff({
|
|
|
952
952
|
defaultPin = null,
|
|
953
953
|
job = 'build',
|
|
954
954
|
prepAsk = undefined,
|
|
955
|
+
beforeEngine = null,
|
|
955
956
|
}) {
|
|
956
957
|
const attempts = [];
|
|
957
|
-
|
|
958
|
-
|
|
958
|
+
try {
|
|
959
|
+
const result = await dispatchEntryAttempts({ entry, engine, root, dispatch, installedEngines, restaffState, scoutAsk, clock, defaultPin, attempts, job, prepAsk, beforeEngine });
|
|
960
|
+
return { ...result, attempts };
|
|
961
|
+
} catch (err) {
|
|
962
|
+
// One job's attempt blowing up (a retry brief that cannot be written, a store that throws) is that job's
|
|
963
|
+
// trouble. It comes back as a job whose attempt failed, for the flight to let its claim go where it failed,
|
|
964
|
+
// and the run goes on to its cleanup instead of ending with the other jobs' claims still held.
|
|
965
|
+
return { not_started: 'dispatch', detail: String(err && err.message || err), attempts };
|
|
966
|
+
}
|
|
959
967
|
}
|
|
960
968
|
|
|
961
969
|
// One prep pass per prep job per task: a backup whose line asks for the same
|
|
@@ -994,6 +1002,10 @@ async function dispatchEntryAttempts({
|
|
|
994
1002
|
attempts = [],
|
|
995
1003
|
job = 'build',
|
|
996
1004
|
prepAsk = undefined,
|
|
1005
|
+
// Called right before an engine is asked, every time one is: the assigned engine, a retry on another engine
|
|
1006
|
+
// after a dead one, a handover to the next worker (all after any scout, prep, and brief), so the flight can
|
|
1007
|
+
// check it still holds the claim and mark it started. If it throws, that engine is not asked.
|
|
1008
|
+
beforeEngine = null,
|
|
997
1009
|
}) {
|
|
998
1010
|
const basePrompt = entry.prompt || buildFleetPrompt(entry.task, { worktreePath: entry.worktreePath, yolo: entry.yolo });
|
|
999
1011
|
const prepFor = prepPassCache({ entry, root, clock, prepAsk, job });
|
|
@@ -1015,16 +1027,39 @@ async function dispatchEntryAttempts({
|
|
|
1015
1027
|
// A worker whose roster line asks for prep gets its brief first.
|
|
1016
1028
|
const prep = ranWith.prep ? await prepFor(ranWith.prep, scouted) : null;
|
|
1017
1029
|
const prompt = withPrepBrief(scouted, prep);
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1030
|
+
let brief;
|
|
1031
|
+
try {
|
|
1032
|
+
brief = captureDispatchBrief({
|
|
1033
|
+
root,
|
|
1034
|
+
task: entry.task,
|
|
1035
|
+
engine: engineName,
|
|
1036
|
+
worktreePath: entry.worktreePath,
|
|
1037
|
+
prompt,
|
|
1038
|
+
missionId: entry.mission_id || entry.missionId || '',
|
|
1039
|
+
author: entry.author || '',
|
|
1040
|
+
yolo: entry.yolo,
|
|
1041
|
+
});
|
|
1042
|
+
} catch (err) {
|
|
1043
|
+
// Before any engine has run, a brief that cannot be written means the job never started: hand that
|
|
1044
|
+
// back, so the flight pauses it and lets the claim go as one that could not start. After an engine has
|
|
1045
|
+
// run, it fails the flight as it always did.
|
|
1046
|
+
if (attempts.length) throw err;
|
|
1047
|
+
return { not_started: 'brief_capture', detail: String(err && err.message || err) };
|
|
1048
|
+
}
|
|
1049
|
+
// An engine is about to be asked, the assigned one or a later one (a retry, a handover), and each start gets
|
|
1050
|
+
// the same check on the claim as it stands now: a claim can be lost between one attempt and the next. A scout
|
|
1051
|
+
// or prep pass before this point never counts as the build starting. A mark that cannot be saved means the
|
|
1052
|
+
// start did not happen, so the job is handed back as not started and the engine is not asked. A mark refused
|
|
1053
|
+
// because the flight no longer holds the claim (claimLost, see markClaimStarted) is handed back as such: the
|
|
1054
|
+
// job is someone else's now. The mark of an attempt already made stands (marking twice adds nothing).
|
|
1055
|
+
if (beforeEngine) {
|
|
1056
|
+
try {
|
|
1057
|
+
beforeEngine(entry);
|
|
1058
|
+
} catch (err) {
|
|
1059
|
+
if (err && err.claimLost) return { not_started: 'claim_lost', detail: String(err.message || err), ...(err.unnamed ? { unnamed: true } : {}) };
|
|
1060
|
+
return { not_started: 'prepare', detail: String(err && err.message || err) };
|
|
1061
|
+
}
|
|
1062
|
+
}
|
|
1028
1063
|
const startedMs = clock();
|
|
1029
1064
|
let runResult;
|
|
1030
1065
|
try {
|
|
@@ -1045,6 +1080,8 @@ async function dispatchEntryAttempts({
|
|
|
1045
1080
|
};
|
|
1046
1081
|
|
|
1047
1082
|
const first = await runOnce(engine);
|
|
1083
|
+
// No engine was asked, so there is no engine health to record and no one to hand the job to.
|
|
1084
|
+
if (first.not_started) return first;
|
|
1048
1085
|
if (!first.max_seconds && entry.maxSeconds) first.max_seconds = entry.maxSeconds;
|
|
1049
1086
|
const deadEngine = detectDeadEngineDispatch(first);
|
|
1050
1087
|
recordDispatchEngineHealth(first, deadEngine, root);
|
|
@@ -1069,6 +1106,9 @@ async function dispatchEntryAttempts({
|
|
|
1069
1106
|
|
|
1070
1107
|
restaffState.used = true;
|
|
1071
1108
|
const fallbackResult = await runOnce(fallback);
|
|
1109
|
+
// The start check refused the retry (the claim is not this run's any more): no engine was asked, so the job is
|
|
1110
|
+
// handed back as not started, and there is no engine health to record.
|
|
1111
|
+
if (fallbackResult.not_started) return fallbackResult;
|
|
1072
1112
|
recordDispatchEngineHealth(fallbackResult, detectDeadEngineDispatch(fallbackResult), root);
|
|
1073
1113
|
return {
|
|
1074
1114
|
...fallbackResult,
|
|
@@ -1129,6 +1169,9 @@ async function handOverStalledDispatch({ entry, engine, first, root, runOnce, jo
|
|
|
1129
1169
|
prep: next.roster_prep || '',
|
|
1130
1170
|
};
|
|
1131
1171
|
const result = await runOnce(next.id, pin);
|
|
1172
|
+
// The start check refused this handover (see dispatchEntryAttempts): no engine was asked, so the job is
|
|
1173
|
+
// handed back as not started and no one else is tried.
|
|
1174
|
+
if (result.not_started) return result;
|
|
1132
1175
|
const dead = detectDeadEngineDispatch(result);
|
|
1133
1176
|
recordDispatchEngineHealth(result, dead, root);
|
|
1134
1177
|
last = result;
|
|
@@ -1701,6 +1744,10 @@ module.exports = {
|
|
|
1701
1744
|
shipPullRequest,
|
|
1702
1745
|
DEAD_ENGINE_OUTPUT_PATTERNS,
|
|
1703
1746
|
releasePausedClaimsWithoutWork,
|
|
1747
|
+
dispatchEntryWithRestaff,
|
|
1748
|
+
flightExitCode,
|
|
1749
|
+
markClaimStarted,
|
|
1750
|
+
releaseOwnClaim,
|
|
1704
1751
|
reviewOnlyEngineEnvironment,
|
|
1705
1752
|
runInReapedProcessGroup,
|
|
1706
1753
|
shipWithRetry,
|
|
@@ -3218,6 +3265,9 @@ async function runFleetFlight({
|
|
|
3218
3265
|
scoutAsk = null,
|
|
3219
3266
|
clock = Date.now,
|
|
3220
3267
|
prepAsk = undefined,
|
|
3268
|
+
// The disk check before anything is claimed (see lowDiskRefusal). Tests inject both.
|
|
3269
|
+
freeSpace = undefined,
|
|
3270
|
+
spaceGuard = undefined,
|
|
3221
3271
|
} = {}) {
|
|
3222
3272
|
const cli = ownCli || defaultOwnCli(root);
|
|
3223
3273
|
const landTarget = resolveDispatchLandTarget(root, checkoutBase);
|
|
@@ -3267,6 +3317,12 @@ async function runFleetFlight({
|
|
|
3267
3317
|
return flight;
|
|
3268
3318
|
}
|
|
3269
3319
|
|
|
3320
|
+
// One look at the disk before anything is claimed or written to any job, with worktree start's own
|
|
3321
|
+
// check and reaping. Under its floor every start would refuse anyway, and each refused start used to
|
|
3322
|
+
// let a claim go that the rating then scored as a drop.
|
|
3323
|
+
const lowDisk = lowDiskRefusal({ root, agent: 'fleet', task: 'flight', freeSpace, spaceGuard });
|
|
3324
|
+
if (lowDisk) return stopForLowDisk(flight, lowDisk, staffed.map((s) => ({ task: s.task.display_id, engine: s.engine })), log);
|
|
3325
|
+
|
|
3270
3326
|
const liveLogPath = startLiveFlight(root, receiptPath, flight);
|
|
3271
3327
|
|
|
3272
3328
|
// Claim + cut a worktree per assignment, then dispatch all in parallel.
|
|
@@ -3301,8 +3357,25 @@ async function runFleetFlight({
|
|
|
3301
3357
|
// jobs this flight claimed are in them, so a listed job it refused cannot be mistaken for one.
|
|
3302
3358
|
const claimedRefs = new Map();
|
|
3303
3359
|
const claimedActors = new Map();
|
|
3360
|
+
// The claim this flight made on each job, remembered from the claim write itself (see claimJob).
|
|
3361
|
+
const claims = claimLedger();
|
|
3362
|
+
// A claim let go before the engine ran is recorded in this process, on the claim this flight made, as one
|
|
3363
|
+
// that could not start. Any other claim the flight lets go (the engine was asked) is an ordinary release.
|
|
3364
|
+
const letGoUnstarted = ({ ref, actor, reason }) => releaseOwnClaim({ taskDb: leaseDb, db: leaseConnection, root, ref, actor, reason, claimEventId: claims.eventId(ref), neverStarted: true });
|
|
3365
|
+
const letGoAsked = ({ ref, actor, reason }) => releaseOwnClaim({ taskDb: leaseDb, db: leaseConnection, root, ref, actor, reason, claimEventId: claims.eventId(ref) });
|
|
3366
|
+
// The start-failure releases the store failed, for the closing pass to try again (see releaseUnstartedDispatchClaim).
|
|
3367
|
+
const releaseFailures = new Map();
|
|
3368
|
+
// Right before the assigned engine is asked, the claim is marked started.
|
|
3369
|
+
const markStarted = (entry) => markClaimStarted({
|
|
3370
|
+
taskDb: leaseDb,
|
|
3371
|
+
db: leaseConnection,
|
|
3372
|
+
claims,
|
|
3373
|
+
ref: claimedRefs.get(String(entry.task.display_id)) || entry.task.display_id,
|
|
3374
|
+
actor: claimedActors.get(String(entry.task.display_id)) || `fleet-${entry.engine}`,
|
|
3375
|
+
task: entry.task.display_id,
|
|
3376
|
+
});
|
|
3304
3377
|
const leaseTimer = setInterval(() => {
|
|
3305
|
-
for (const [id, holder] of leased) leaseDb.renewLease(leaseConnection, { id, holder, pid: process.pid });
|
|
3378
|
+
for (const [id, holder] of leased) leaseDb.renewLease(leaseConnection, { id, holder, pid: process.pid, claimEventId: claims.eventId(id) });
|
|
3306
3379
|
}, 5 * 60000);
|
|
3307
3380
|
leaseTimer.unref();
|
|
3308
3381
|
try {
|
|
@@ -3318,7 +3391,7 @@ async function runFleetFlight({
|
|
|
3318
3391
|
// The row the list named is the row this claims, leases, proves and lets go of: named once,
|
|
3319
3392
|
// by its row id, at every step (see jobRef).
|
|
3320
3393
|
const rowRef = jobRef(entry.task, entry.task.display_id);
|
|
3321
|
-
const claimed = cli
|
|
3394
|
+
const claimed = claimJob({ cli, taskDb: leaseDb, db: leaseConnection, root, ref: rowRef, actor: `fleet-${entry.engine}` });
|
|
3322
3395
|
if (!claimed || claimed.status !== 0) {
|
|
3323
3396
|
const detail = String(claimed && (claimed.stderr || claimed.stdout) || 'claim failed').trim().slice(0, 300);
|
|
3324
3397
|
flight.paused.push({ task: entry.task.display_id, engine: entry.engine, stage: 'claim', detail });
|
|
@@ -3329,20 +3402,67 @@ async function runFleetFlight({
|
|
|
3329
3402
|
const holder = `fleet-${entry.engine}`;
|
|
3330
3403
|
claimedRefs.set(String(entry.task.display_id), rowRef);
|
|
3331
3404
|
claimedActors.set(String(entry.task.display_id), holder);
|
|
3332
|
-
|
|
3333
|
-
|
|
3334
|
-
|
|
3335
|
-
const
|
|
3336
|
-
|
|
3337
|
-
|
|
3338
|
-
|
|
3339
|
-
|
|
3405
|
+
claims.remember(rowRef, claimed.eventId);
|
|
3406
|
+
// From here until the engine is asked, a job that cannot start is let go where it fails, as one that could
|
|
3407
|
+
// not start, so a failure later in the flight cannot leave its claim to run out as a miss.
|
|
3408
|
+
const cannotStart = (stage, detail) => {
|
|
3409
|
+
const released = releaseUnstartedDispatchClaim(cli, { taskId: entry.task.display_id, ref: rowRef, actor: holder, detail, stage, release: letGoUnstarted, failures: releaseFailures, log });
|
|
3410
|
+
flight.paused.push({ task: entry.task.display_id, engine: entry.engine, stage, detail, claim_released: released.released });
|
|
3411
|
+
};
|
|
3412
|
+
try {
|
|
3413
|
+
const leaseSet = leaseDb.setLease(leaseConnection, { id: leaseId, holder, claimEventId: claims.eventId(leaseId) });
|
|
3414
|
+
if (!leaseSet && (!ownCli || leaseDb.getTask(leaseConnection, leaseId))) throw new Error('could not set claim lease');
|
|
3415
|
+
if (leaseSet) leased.set(leaseId, holder);
|
|
3416
|
+
const started = cli(['worktree', 'start', '--agent', entry.engine, '--task', `fleet-${String(entry.task.display_id).toLowerCase()}`, ...startBaseArgs]);
|
|
3417
|
+
const wt = (started.stdout.match(/next: cd (.+)/) || [])[1];
|
|
3418
|
+
if (!wt) {
|
|
3419
|
+
cannotStart('worktree_start', started.stderr.slice(0, 200));
|
|
3420
|
+
continue;
|
|
3421
|
+
}
|
|
3422
|
+
prepared.push({ ...entry, worktreePath: wt.trim() });
|
|
3423
|
+
log(` ${entry.engine.padEnd(8)} building ${entry.task.display_id} in ${path.basename(wt.trim())}`);
|
|
3424
|
+
} catch (err) {
|
|
3425
|
+
cannotStart('prepare', String(err && err.message || err).slice(0, 200));
|
|
3426
|
+
log(` ${entry.engine.padEnd(8)} ✗ ${entry.task.display_id} prepare failed`);
|
|
3340
3427
|
}
|
|
3341
|
-
prepared.push({ ...entry, worktreePath: wt.trim() });
|
|
3342
|
-
log(` ${entry.engine.padEnd(8)} building ${entry.task.display_id} in ${path.basename(wt.trim())}`);
|
|
3343
3428
|
}
|
|
3344
3429
|
|
|
3345
|
-
|
|
3430
|
+
// A job that never reached its engine (its brief could not be written, or its start could not be marked)
|
|
3431
|
+
// pauses and its claim goes back as one that could not start, here, where it failed, while the flight's
|
|
3432
|
+
// other jobs go on. The jobs that did start carry on.
|
|
3433
|
+
const letGoNotStarted = (entry, result) => {
|
|
3434
|
+
const taskLabel = entry.task.display_id;
|
|
3435
|
+
const stage = result.not_started;
|
|
3436
|
+
const ref = claimedRefs.get(String(taskLabel)) || taskLabel;
|
|
3437
|
+
if (stage === 'claim_lost') {
|
|
3438
|
+
// The job is another run's now (see markClaimStarted). The flight lets go of nothing, writes nothing to it, and
|
|
3439
|
+
// stops renewing a lease on it.
|
|
3440
|
+
leased.delete(ref);
|
|
3441
|
+
flight.paused.push({ task: taskLabel, engine: entry.engine, stage, detail: result.unnamed ? UNNAMED_CLAIM_DETAIL : LOST_CLAIM_DETAIL, worktree: entry.worktreePath, claim_released: false });
|
|
3442
|
+
log(result.unnamed ? unnamedClaimLine(taskLabel) : claimLostLine(taskLabel));
|
|
3443
|
+
return;
|
|
3444
|
+
}
|
|
3445
|
+
// Nothing in here may end the run: the other jobs still have their cleanup to come.
|
|
3446
|
+
let detail = result.detail;
|
|
3447
|
+
let released = false;
|
|
3448
|
+
try {
|
|
3449
|
+
({ released, detail } = releaseUnstartedDispatchClaim(cli, {
|
|
3450
|
+
taskId: taskLabel,
|
|
3451
|
+
ref,
|
|
3452
|
+
actor: claimedActors.get(String(taskLabel)) || `fleet-${entry.engine}`,
|
|
3453
|
+
detail: result.detail,
|
|
3454
|
+
stage,
|
|
3455
|
+
release: letGoUnstarted,
|
|
3456
|
+
failures: releaseFailures,
|
|
3457
|
+
log,
|
|
3458
|
+
}));
|
|
3459
|
+
} catch (err) {
|
|
3460
|
+
detail = `${String(result.detail || '').slice(0, 120)}. claim release failed: ${String(err && err.message || err).slice(0, 80)}`;
|
|
3461
|
+
}
|
|
3462
|
+
flight.paused.push({ task: taskLabel, engine: entry.engine, stage, detail, worktree: entry.worktreePath, claim_released: released });
|
|
3463
|
+
log(` ${entry.engine.padEnd(8)} ✗ ${taskLabel} ${NOT_STARTED_WORDS[stage] || 'prepare'} failed`);
|
|
3464
|
+
};
|
|
3465
|
+
const settled = await Promise.all(prepared.map((entry) =>
|
|
3346
3466
|
dispatchEntryWithRestaff({
|
|
3347
3467
|
entry,
|
|
3348
3468
|
engine: entry.engine,
|
|
@@ -3354,8 +3474,13 @@ async function runFleetFlight({
|
|
|
3354
3474
|
clock,
|
|
3355
3475
|
defaultPin: fleetPrepPin(entry.engine, root),
|
|
3356
3476
|
prepAsk,
|
|
3357
|
-
|
|
3477
|
+
beforeEngine: markStarted,
|
|
3478
|
+
}).then((result) => {
|
|
3479
|
+
if (result.not_started) letGoNotStarted(entry, result);
|
|
3480
|
+
return { entry, result };
|
|
3481
|
+
})
|
|
3358
3482
|
));
|
|
3483
|
+
const results = settled.filter(({ result }) => !result.not_started);
|
|
3359
3484
|
flight.results = results.map(({ entry, result }) => {
|
|
3360
3485
|
const row = { task: entry.task.display_id, engine: result.engine || entry.engine, exitCode: result.exitCode };
|
|
3361
3486
|
if (result.brief_id) row.brief_id = result.brief_id;
|
|
@@ -3448,6 +3573,8 @@ async function runFleetFlight({
|
|
|
3448
3573
|
refByTask: claimedRefs,
|
|
3449
3574
|
baseRef: landTarget,
|
|
3450
3575
|
log,
|
|
3576
|
+
release: letGoAsked,
|
|
3577
|
+
failures: releaseFailures,
|
|
3451
3578
|
});
|
|
3452
3579
|
|
|
3453
3580
|
recordFlightRosterRuns(root, flight, results, { job: 'build', source: 'fleet' });
|
|
@@ -3462,7 +3589,7 @@ async function runFleetFlight({
|
|
|
3462
3589
|
clearInterval(leaseTimer);
|
|
3463
3590
|
for (const [id, holder] of leased) {
|
|
3464
3591
|
const row = leaseDb.getTask(leaseConnection, id);
|
|
3465
|
-
if (row && row.status !== 'claimed') leaseDb.clearLease(leaseConnection, { id, holder });
|
|
3592
|
+
if (row && row.status !== 'claimed') leaseDb.clearLease(leaseConnection, { id, holder, claimEventId: claims.eventId(id) });
|
|
3466
3593
|
}
|
|
3467
3594
|
}
|
|
3468
3595
|
}
|
|
@@ -3482,7 +3609,7 @@ function releaseReason(lead, detail) {
|
|
|
3482
3609
|
}
|
|
3483
3610
|
|
|
3484
3611
|
// Pauses where the engine never got to finish its turn.
|
|
3485
|
-
const ENGINE_DID_NOT_FINISH = new Set(['worktree_start', 'remote_quarantine', 'prepare', 'build']);
|
|
3612
|
+
const ENGINE_DID_NOT_FINISH = new Set(['worktree_start', 'remote_quarantine', 'prepare', 'brief_capture', 'build']);
|
|
3486
3613
|
|
|
3487
3614
|
// The word a flight uses to name a job to the task command at every step: its row id when the
|
|
3488
3615
|
// task command (or the task list) gave one, else the display id the flight has. The display id a
|
|
@@ -3506,17 +3633,225 @@ function jobOutsideWorkspace(taskDb, task, workspace) {
|
|
|
3506
3633
|
|
|
3507
3634
|
const namesAnotherWorkspace = (displayId) => `${displayId} names a job in another workspace`;
|
|
3508
3635
|
|
|
3636
|
+
// What a flight does when the disk is under worktree start's floor. It has claimed nothing and written
|
|
3637
|
+
// nothing to any job, so it says why in one plain line (the free space and the floor, in worktree start's
|
|
3638
|
+
// own words) and hands back a failed flight that names the refusal. flightExitCode turns that into the
|
|
3639
|
+
// exit code worktree start itself gives. jobs are the rows the flight would have worked on.
|
|
3640
|
+
function stopForLowDisk(flight, refusal, jobs, log) {
|
|
3641
|
+
log(` refusing: ${refusal.message}`);
|
|
3642
|
+
flight.refusal = { reason: 'low_disk', detail: refusal.message, free_bytes: refusal.free, needs_bytes: refusal.needs };
|
|
3643
|
+
flight.paused = jobs.map((job) => ({ ...job, stage: 'low_disk', detail: refusal.message }));
|
|
3644
|
+
flight.status = 'failed';
|
|
3645
|
+
flight.finished_at = new Date().toISOString();
|
|
3646
|
+
return flight;
|
|
3647
|
+
}
|
|
3648
|
+
|
|
3649
|
+
// The exit code for a finished flight: 2 when it stopped on a low disk, as worktree start does, else 1
|
|
3650
|
+
// when any job paused, else 0.
|
|
3651
|
+
function flightExitCode(flight) {
|
|
3652
|
+
if (flight && flight.refusal && flight.refusal.reason === 'low_disk') return 2;
|
|
3653
|
+
return flight && flight.paused && flight.paused.length > 0 ? 1 : 0;
|
|
3654
|
+
}
|
|
3655
|
+
|
|
3656
|
+
// The two files people read, drawn again from the saved state the way the task command does after
|
|
3657
|
+
// every release and every claim: the task list file, and atris/TODO.md of the flight's own workspace.
|
|
3658
|
+
// The write is already saved, so a file that cannot be written does not undo it, and one failing does
|
|
3659
|
+
// not stop the other. The task command is loaded only here: this file is read on every start, and only
|
|
3660
|
+
// the paths that write to a job need it.
|
|
3661
|
+
function redrawTaskFiles({ taskDb, db, root }) {
|
|
3662
|
+
for (const [what, refresh] of [
|
|
3663
|
+
['task list file', () => require('../commands/task').writeDefaultProjection(taskDb, db, { root })],
|
|
3664
|
+
['TODO.md', () => require('../commands/task').autoRenderTodoFromDb(taskDb.workspaceRoot(root))],
|
|
3665
|
+
]) {
|
|
3666
|
+
try { refresh(); } catch (err) {
|
|
3667
|
+
if (process.env.ATRIS_DEBUG) process.stderr.write(`${what} refresh failed: ${String(err?.message ?? err).replace(/[\r\n]+/g, ' ')}\n`);
|
|
3668
|
+
}
|
|
3669
|
+
}
|
|
3670
|
+
}
|
|
3671
|
+
|
|
3672
|
+
// Which claim this flight made on each job, by the row id it claimed. A mark or a release checks the job is
|
|
3673
|
+
// still held under that claim before it writes anything: another run can take a job over under the same
|
|
3674
|
+
// builder name (a lease that ran out, then a second flight), and a late release from this flight must not
|
|
3675
|
+
// land on, or stamp, a claim that belongs to someone else. The name of the claim comes from the claim write
|
|
3676
|
+
// itself (see claimJob) and is remembered the moment it is made; it is never looked up afterwards, because
|
|
3677
|
+
// a look-up cannot tell this flight's claim from one made just after it. A job claimed through the task
|
|
3678
|
+
// command (a stand-in task plane, a job shown with no row id) has no name to remember.
|
|
3679
|
+
function claimLedger() {
|
|
3680
|
+
const made = new Map();
|
|
3681
|
+
return {
|
|
3682
|
+
remember(ref, eventId) {
|
|
3683
|
+
if (ref && eventId) made.set(String(ref), String(eventId));
|
|
3684
|
+
},
|
|
3685
|
+
// The event that is this flight's claim on the job, or '' when none was remembered.
|
|
3686
|
+
eventId(ref) {
|
|
3687
|
+
return made.get(String(ref)) || '';
|
|
3688
|
+
},
|
|
3689
|
+
};
|
|
3690
|
+
}
|
|
3691
|
+
|
|
3692
|
+
// True when the task store holds a row under this name. It decides whether a flight may act on a job without a
|
|
3693
|
+
// claim to name, so a store that cannot be read counts as holding it: doubt means it may not.
|
|
3694
|
+
function storeHoldsJob(taskDb, db, ref) {
|
|
3695
|
+
if (!ref) return false;
|
|
3696
|
+
try {
|
|
3697
|
+
return Boolean(taskDb.getTask(db, ref));
|
|
3698
|
+
} catch {
|
|
3699
|
+
return true;
|
|
3700
|
+
}
|
|
3701
|
+
}
|
|
3702
|
+
|
|
3703
|
+
// How a flight claims a job. A job the task store holds a row for is claimed in the flight's own process,
|
|
3704
|
+
// through the store it already holds for leases, and the claim's name (its claimed event) comes from the claim
|
|
3705
|
+
// write itself: claimTask saves the row and the event in one transaction and answers with the event, so there is
|
|
3706
|
+
// no moment in which the flight has claimed a job but cannot say which claim is its own. What the task command
|
|
3707
|
+
// did besides the write stays: the check on the actor's name, the plain failure text, the two files people
|
|
3708
|
+
// read drawn again (see redrawTaskFiles), and the clearing of the claims that have run out, which every task
|
|
3709
|
+
// command begins with, so a job whose old claim is dead is cleared and claimed in one go. A job the store holds
|
|
3710
|
+
// no row for (a stand-in task plane, a job shown with no row id) has no claim to name and is claimed through the
|
|
3711
|
+
// task command, as it always was.
|
|
3712
|
+
//
|
|
3713
|
+
// It answers like the task command (status 0, or status 1 and why), with eventId when it claimed in process. A
|
|
3714
|
+
// claim that landed but came back with no name is refused: the flight cannot tell later whether the job is
|
|
3715
|
+
// still its own, so it builds nothing on it and leaves it alone.
|
|
3716
|
+
function claimJob({ cli, taskDb, db, root, ref, actor, minutes = 60 }) {
|
|
3717
|
+
if (!storeHoldsJob(taskDb, db, ref)) {
|
|
3718
|
+
return cli(['task', 'claim', ref, '--as', actor, '--lease-minutes', String(minutes), '--lease-pid', String(process.pid)]);
|
|
3719
|
+
}
|
|
3720
|
+
const check = require('./review-integrity').validateActor(actor, { root });
|
|
3721
|
+
if (!check.ok) {
|
|
3722
|
+
return {
|
|
3723
|
+
status: 1,
|
|
3724
|
+
stdout: '',
|
|
3725
|
+
stderr: check.reason === 'reserved_actor'
|
|
3726
|
+
? `atris task claim: reserved_actor: '${actor}' is a system actor and cannot be used with --as`
|
|
3727
|
+
: `atris task claim: actor_not_on_roster: '${actor}' is not a workspace member or engine (actor validation is enforced)`,
|
|
3728
|
+
};
|
|
3729
|
+
}
|
|
3730
|
+
// The step the task command takes first, with its reach (every claim in the store that has run out) and the
|
|
3731
|
+
// history it writes (a claim_expired entry for each), because it is the command's own function. It never throws.
|
|
3732
|
+
taskDb.sweepExpiredLeases();
|
|
3733
|
+
let result;
|
|
3734
|
+
try {
|
|
3735
|
+
result = taskDb.claimTask(db, { id: ref, claimedBy: actor, leaseMinutes: minutes, leasePid: process.pid });
|
|
3736
|
+
} catch (err) {
|
|
3737
|
+
return { status: 1, stdout: '', stderr: `claim failed: ${String(err?.message ?? err).trim() || 'unknown'}` };
|
|
3738
|
+
}
|
|
3739
|
+
if (!result.claimed) {
|
|
3740
|
+
return { status: 1, stdout: '', stderr: `claim failed: ${result.reason}${result.claimed_by ? ` (held by ${result.claimed_by})` : ''}` };
|
|
3741
|
+
}
|
|
3742
|
+
redrawTaskFiles({ taskDb, db, root });
|
|
3743
|
+
const eventId = result.event && result.event.event_id;
|
|
3744
|
+
if (!eventId) return { status: 1, stdout: '', stderr: UNNAMED_CLAIM_DETAIL, unnamed: true };
|
|
3745
|
+
return { status: 0, stdout: `claimed ${ref} as ${actor}`, stderr: '', eventId };
|
|
3746
|
+
}
|
|
3747
|
+
|
|
3748
|
+
// The one plain line for a job another run has claimed since this flight made its own claim.
|
|
3749
|
+
const claimTakenOverLine = (task) => ` left ${task} alone: another run claimed it after this one did`;
|
|
3750
|
+
|
|
3751
|
+
// The one plain line for a job whose claim this flight no longer holds when it is about to ask the engine.
|
|
3752
|
+
const LOST_CLAIM_DETAIL = 'this run no longer holds the claim, so the engine was not started and the job was left alone';
|
|
3753
|
+
const claimLostLine = (task) => ` ${task}: ${LOST_CLAIM_DETAIL}`;
|
|
3754
|
+
|
|
3755
|
+
// The one plain line for a job the flight has claimed but cannot name the claim of: it cannot tell whether the job
|
|
3756
|
+
// is still its own, so it writes nothing to it, releases nothing, and starts no engine on it.
|
|
3757
|
+
const UNNAMED_CLAIM_DETAIL = 'this run has no record of which claim is its own, so the job was left alone';
|
|
3758
|
+
const unnamedClaimLine = (task) => ` ${task}: ${UNNAMED_CLAIM_DETAIL}`;
|
|
3759
|
+
|
|
3760
|
+
// What a flight calls the step a job failed at, in the one line it logs when it lets that job's claim go.
|
|
3761
|
+
const NOT_STARTED_WORDS = { brief_capture: 'brief capture', dispatch: 'attempt', prepare: 'prepare' };
|
|
3762
|
+
|
|
3763
|
+
// Right before an engine is asked, the first or a later one (a retry, a handover), the flight marks the claim it
|
|
3764
|
+
// made as started (see markBuildStarted in task-db.js). That is what keeps the could-not-start kind honest: the
|
|
3765
|
+
// store and the rating take it only for a claim with no mark. A job the store does not hold under this holder (a
|
|
3766
|
+
// stand-in task plane) has no claim to mark and goes on. A mark refused for a claim the flight remembered means
|
|
3767
|
+
// the claim is not the flight's any more (it was let go, or taken over, even by a run of the same name): the
|
|
3768
|
+
// engine is not asked, and the error says so (claimLost) for the flight to stop on, writing nothing more. A job
|
|
3769
|
+
// the store holds that the flight has no name for the claim of stops the same way (claimLost and unnamed).
|
|
3770
|
+
// A mark that cannot be saved throws like any store trouble, and the engine is not asked either.
|
|
3771
|
+
function markClaimStarted({ taskDb, db, claims, ref, actor, task }) {
|
|
3772
|
+
const claimEventId = claims.eventId(ref);
|
|
3773
|
+
if (!claimEventId && storeHoldsJob(taskDb, db, ref)) {
|
|
3774
|
+
// A job the store holds is claimed in process, so a flight with no name for its claim cannot say the job is
|
|
3775
|
+
// still its own: the engine is not asked, and nothing is written.
|
|
3776
|
+
const unnamed = new Error(`${task}: ${UNNAMED_CLAIM_DETAIL}`);
|
|
3777
|
+
unnamed.claimLost = true;
|
|
3778
|
+
unnamed.unnamed = true;
|
|
3779
|
+
throw unnamed;
|
|
3780
|
+
}
|
|
3781
|
+
const result = taskDb.markBuildStarted(db, { id: ref, actor, claimEventId });
|
|
3782
|
+
if (!result.marked && claimEventId) {
|
|
3783
|
+
const lost = new Error(`the claim on ${task} is no longer this run's (${result.reason})`);
|
|
3784
|
+
lost.claimLost = true;
|
|
3785
|
+
throw lost;
|
|
3786
|
+
}
|
|
3787
|
+
return result;
|
|
3788
|
+
}
|
|
3789
|
+
|
|
3790
|
+
// A flight lets go of a claim in its own process, on the exact claim it made, through the task store it already
|
|
3791
|
+
// holds for leases. ref is the row id the claim named and claimEventId the claim event the flight remembered
|
|
3792
|
+
// (see claimLedger): the store lets go only if the job is still held under that claim, in the same write, so a
|
|
3793
|
+
// job another run took over since, even under the same builder name and in the same millisecond, is never
|
|
3794
|
+
// released by a late call of this flight's. `atris task release` takes no kind and no claim, because any worker
|
|
3795
|
+
// holding a failed job could pass one to excuse its own miss; only the flights say these things, here.
|
|
3796
|
+
//
|
|
3797
|
+
// neverStarted asks for the could-not-start kind (lib/release-kind.js): the build machine could not start the
|
|
3798
|
+
// job, so it is nobody's miss. The store keeps that honest: it drops the kind from a claim marked started (see
|
|
3799
|
+
// markClaimStarted), and the release then goes through as an ordinary one, so a claim is never left held.
|
|
3800
|
+
//
|
|
3801
|
+
// It answers like the task command (status 0, or status 1 and why), with a word more: not_ours when the job is
|
|
3802
|
+
// not held under the flight's claim any more (nothing was written, nothing is retried, and the caller says so),
|
|
3803
|
+
// final when it is already let go or gone, retryable when the store failed (the closing pass tries once more),
|
|
3804
|
+
// unnamed when the store holds the job but the flight has no name for its claim (nothing is written and nothing
|
|
3805
|
+
// is retried: a release that names no claim could land on anyone's, so the job is left alone and the caller says
|
|
3806
|
+
// so). Null only when the store holds no row for the job (a job shown with no row id, a stand-in task plane): the
|
|
3807
|
+
// caller then lets it go through the task command, as a plain release, the way it always did.
|
|
3808
|
+
function releaseOwnClaim({ taskDb, db, root, ref, actor, reason, claimEventId = '', neverStarted = false }) {
|
|
3809
|
+
if (!claimEventId) {
|
|
3810
|
+
return storeHoldsJob(taskDb, db, ref) ? { status: 1, stdout: '', stderr: UNNAMED_CLAIM_DETAIL, unnamed: true } : null;
|
|
3811
|
+
}
|
|
3812
|
+
let result;
|
|
3813
|
+
try {
|
|
3814
|
+
result = taskDb.releaseTask(db, { id: ref, actor, reason, claimEventId, ...(neverStarted ? { kind: COULD_NOT_START } : {}) });
|
|
3815
|
+
} catch (err) {
|
|
3816
|
+
return { status: 1, stdout: '', stderr: String(err?.message ?? err).trim() || 'unknown', retryable: true };
|
|
3817
|
+
}
|
|
3818
|
+
if (!result.released) {
|
|
3819
|
+
const stderr = `release failed: ${result.reason}${result.claimed_by ? ` (held by ${result.claimed_by})` : ''}`;
|
|
3820
|
+
if (result.reason === 'claim_changed' || result.reason === 'held_by_other') return { status: 1, stdout: '', stderr, not_ours: true };
|
|
3821
|
+
if (result.reason === 'not_found' || String(result.reason).startsWith('already_')) return { status: 1, stdout: '', stderr, final: true };
|
|
3822
|
+
return { status: 1, stdout: '', stderr, retryable: true };
|
|
3823
|
+
}
|
|
3824
|
+
redrawTaskFiles({ taskDb, db, root });
|
|
3825
|
+
return { status: 0, stdout: 'released', stderr: '' };
|
|
3826
|
+
}
|
|
3827
|
+
|
|
3509
3828
|
// A claim taken before the engine starts must not stay held if the flight
|
|
3510
3829
|
// refuses or errors first. Release through the same task plane the claim used,
|
|
3511
|
-
// naming the job by ref: the row id the claim named.
|
|
3512
|
-
|
|
3513
|
-
|
|
3830
|
+
// naming the job by ref: the row id the claim named. release, when a flight gives
|
|
3831
|
+
// one, lets it go in process as a job that could not start (see releaseOwnClaim); it
|
|
3832
|
+
// answers null for a claim it cannot record, and the task plane lets that go. A
|
|
3833
|
+
// claim another run has taken over since is left alone, with one plain line. A
|
|
3834
|
+
// release the store failed (retryable) is noted in failures, by the job's name, for the
|
|
3835
|
+
// closing pass to try again.
|
|
3836
|
+
function releaseUnstartedDispatchClaim(cli, { taskId, ref = taskId, actor, detail, stage, release = null, failures = null, log = () => {} }) {
|
|
3837
|
+
const reason = releaseReason(`paused at ${pausedStageLabel(stage)}`, detail);
|
|
3838
|
+
const released = (release && release({ ref, actor, reason })) || cli(['task', 'release', ref, '--as', actor, '--reason', reason]);
|
|
3514
3839
|
const releasedOk = Boolean(released && released.status === 0);
|
|
3840
|
+
const notOurs = Boolean(released && released.not_ours);
|
|
3841
|
+
const unnamed = Boolean(released && released.unnamed);
|
|
3842
|
+
if (notOurs) log(claimTakenOverLine(taskId));
|
|
3843
|
+
if (unnamed) log(unnamedClaimLine(taskId));
|
|
3515
3844
|
const suffix = releasedOk
|
|
3516
3845
|
? 'claim released, safe to retry'
|
|
3517
|
-
:
|
|
3846
|
+
: notOurs
|
|
3847
|
+
? 'claim left alone, another run claimed it after this one did'
|
|
3848
|
+
: unnamed
|
|
3849
|
+
? 'claim left alone, this run has no record of which claim is its own'
|
|
3850
|
+
: `claim release failed: ${String(released && (released.stderr || released.stdout) || 'unknown').trim().slice(0, 80)}`;
|
|
3518
3851
|
const base = String(detail || '').trim().slice(0, 200);
|
|
3519
|
-
|
|
3852
|
+
const text = base ? `${base}. ${suffix}` : suffix;
|
|
3853
|
+
if (failures && released && released.retryable) failures.set(String(taskId), { ref, actor, reason, base, detail: text, release });
|
|
3854
|
+
return { released: releasedOk, detail: text };
|
|
3520
3855
|
}
|
|
3521
3856
|
|
|
3522
3857
|
// A pause with no work must not keep the task. On 2026-08-21, 49 bug tasks sat
|
|
@@ -3534,14 +3869,44 @@ function releaseUnstartedDispatchClaim(cli, { taskId, ref = taskId, actor, detai
|
|
|
3534
3869
|
// refByTask names each claimed job by the row id the flight claimed it with, so the release
|
|
3535
3870
|
// lands on that row even if the display id a person reads now means another job. A job with
|
|
3536
3871
|
// no entry there is named by the display id in its pause row.
|
|
3537
|
-
|
|
3872
|
+
//
|
|
3873
|
+
// release, when a flight gives one, lets a claim go in the flight's own process, on the exact claim it
|
|
3874
|
+
// made (see releaseOwnClaim); a job it answers null for is let go through the task command. A job another
|
|
3875
|
+
// run has claimed since is left alone, with one plain line, and its row says the claim was not released.
|
|
3876
|
+
// A release the store failed is tried once more, and a claim that still cannot be let go says so in one
|
|
3877
|
+
// plain line.
|
|
3878
|
+
//
|
|
3879
|
+
// failures holds the start-failure releases that failed at the point of failure (see
|
|
3880
|
+
// releaseUnstartedDispatchClaim): a row that says its claim was not released, with an entry there, gets its
|
|
3881
|
+
// release tried again here, the way it was first tried.
|
|
3882
|
+
function releasePausedClaimsWithoutWork(cli, { paused, worktreeByTask, actorByTask, refByTask = new Map(), baseRef = '', log = () => {}, keepClaimsWithWork = true, release = null, failures = null }) {
|
|
3883
|
+
const letGo = (how, { ref, actor, reason }) => (how && how({ ref, actor, reason })) || cli(['task', 'release', ref, '--as', actor, '--reason', reason]);
|
|
3884
|
+
const saysNo = (taskId, released) => log(` could not release the claim on ${taskId}: ${String(released && (released.stderr || released.stdout) || 'unknown').replace(/\s+/g, ' ').trim().slice(0, 80)}. it stays until its lease runs out, unless the task record is repaired first.`);
|
|
3538
3885
|
for (const row of paused || []) {
|
|
3539
3886
|
// No claim (the job was never found, was refused as another workspace's, or the claim
|
|
3540
|
-
// failed),
|
|
3541
|
-
//
|
|
3542
|
-
|
|
3543
|
-
if (row.
|
|
3887
|
+
// failed), a claim the flight lost before its engine ran (it is the other run's now), work
|
|
3888
|
+
// already landed, or work waiting on a pull request for a person (the flight let that claim go
|
|
3889
|
+
// itself when saving its proof failed).
|
|
3890
|
+
if (row.stage === 'claim' || row.stage === 'task_lookup' || row.stage === 'other_workspace' || row.stage === 'claim_lost' || row.landed === true || row.held_for_review === true) continue;
|
|
3544
3891
|
const taskId = String(row.task || '');
|
|
3892
|
+
const failed = row.claim_released === false && failures ? failures.get(taskId) : null;
|
|
3893
|
+
if (failed) {
|
|
3894
|
+
failures.delete(taskId);
|
|
3895
|
+
const released = letGo(failed.release, failed);
|
|
3896
|
+
if (released && released.status === 0) {
|
|
3897
|
+
row.claim_released = true;
|
|
3898
|
+
if (row.detail === failed.detail) row.detail = failed.base ? `${failed.base}. claim released, safe to retry` : 'claim released, safe to retry';
|
|
3899
|
+
log(` released claim on ${taskId} on a second try`);
|
|
3900
|
+
} else if (released && released.not_ours) {
|
|
3901
|
+
log(claimTakenOverLine(taskId));
|
|
3902
|
+
} else if (released && released.unnamed) {
|
|
3903
|
+
log(unnamedClaimLine(taskId));
|
|
3904
|
+
} else if (!(released && released.final)) {
|
|
3905
|
+
saysNo(taskId, released);
|
|
3906
|
+
}
|
|
3907
|
+
continue;
|
|
3908
|
+
}
|
|
3909
|
+
if (row.claim_released !== undefined) continue;
|
|
3545
3910
|
const actor = actorByTask.get(taskId);
|
|
3546
3911
|
if (!taskId || !actor) continue;
|
|
3547
3912
|
const worktreePath = row.worktree || worktreeByTask.get(taskId) || '';
|
|
@@ -3553,9 +3918,19 @@ function releasePausedClaimsWithoutWork(cli, { paused, worktreeByTask, actorByTa
|
|
|
3553
3918
|
const reason = noWork && !ENGINE_DID_NOT_FINISH.has(row.stage)
|
|
3554
3919
|
? 'engine produced no work'
|
|
3555
3920
|
: releaseReason(`paused at ${pausedStageLabel(row.stage)}`, row.stage === 'build' ? row.reason || row.detail : row.detail || row.reason);
|
|
3556
|
-
const
|
|
3921
|
+
const ref = refByTask.get(taskId) || taskId;
|
|
3922
|
+
let released = letGo(release, { ref, actor, reason });
|
|
3923
|
+
if (released && released.retryable) released = letGo(release, { ref, actor, reason });
|
|
3924
|
+
if (released && (released.not_ours || released.unnamed)) {
|
|
3925
|
+
row.claim_released = false;
|
|
3926
|
+
log(released.unnamed ? unnamedClaimLine(taskId) : claimTakenOverLine(taskId));
|
|
3927
|
+
continue;
|
|
3928
|
+
}
|
|
3557
3929
|
row.claim_released = Boolean(released && released.status === 0);
|
|
3558
|
-
if (!row.claim_released)
|
|
3930
|
+
if (!row.claim_released) {
|
|
3931
|
+
if (released && released.retryable) saysNo(taskId, released);
|
|
3932
|
+
continue;
|
|
3933
|
+
}
|
|
3559
3934
|
log(keepClaimsWithWork
|
|
3560
3935
|
? ` released claim on ${taskId} (engine produced no work)`
|
|
3561
3936
|
: ` released claim on ${taskId} so a retry can pick it up${worktreePath ? `; any work stays in ${path.basename(worktreePath)}` : ''}`);
|
|
@@ -3582,14 +3957,26 @@ function releasePausedClaimsWithoutWork(cli, { paused, worktreeByTask, actorByTa
|
|
|
3582
3957
|
// same ids, so it is never looked up again. With no row id there is nothing exact to
|
|
3583
3958
|
// release, and nothing is.
|
|
3584
3959
|
//
|
|
3960
|
+
// It also names the claim it made (claimEventId, see claimLedger). A job another run has claimed since is
|
|
3961
|
+
// left alone, with one plain line: the pull request is still open for a person, and the claim is not this
|
|
3962
|
+
// flight's to put the kind on. With no name for its claim, the flight cannot say the claim is its own, so it
|
|
3963
|
+
// writes nothing and says so in one plain line.
|
|
3964
|
+
//
|
|
3585
3965
|
// Tried once: if the release fails too, the row says so.
|
|
3586
|
-
function releaseClaimOfUnsavedPullRequest({ taskDb, db, root, row, rowId, actor, log = () => {} }) {
|
|
3966
|
+
function releaseClaimOfUnsavedPullRequest({ taskDb, db, root, row, rowId, actor, log = () => {}, claimEventId = '' }) {
|
|
3587
3967
|
const reason = releaseReason(`PR ${row.pull_request.url} is open but its proof could not be saved`, row.detail);
|
|
3588
3968
|
let released = false;
|
|
3969
|
+
let notOurs = false;
|
|
3970
|
+
let unnamed = false;
|
|
3589
3971
|
let failure = 'unknown';
|
|
3590
3972
|
try {
|
|
3591
3973
|
if (!rowId) {
|
|
3592
3974
|
failure = 'the flight has no row id for it';
|
|
3975
|
+
} else if (!claimEventId && storeHoldsJob(taskDb, db, rowId)) {
|
|
3976
|
+
// The store holds the job and the flight has no name for its claim: a release that names no claim
|
|
3977
|
+
// could land on anyone's, so the job is left alone. A row the store does not know goes on to the
|
|
3978
|
+
// release below, which answers not_found, as it always did.
|
|
3979
|
+
unnamed = true;
|
|
3593
3980
|
} else {
|
|
3594
3981
|
const result = taskDb.releaseTask(db, {
|
|
3595
3982
|
id: rowId,
|
|
@@ -3597,9 +3984,11 @@ function releaseClaimOfUnsavedPullRequest({ taskDb, db, root, row, rowId, actor,
|
|
|
3597
3984
|
reason,
|
|
3598
3985
|
kind: PROOF_NOT_SAVED,
|
|
3599
3986
|
pullRequest: row.pull_request,
|
|
3987
|
+
claimEventId,
|
|
3600
3988
|
});
|
|
3601
3989
|
if (!result.released) {
|
|
3602
3990
|
failure = `release failed: ${result.reason}${result.claimed_by ? ` (held by ${result.claimed_by})` : ''}`;
|
|
3991
|
+
notOurs = result.reason === 'claim_changed' || result.reason === 'held_by_other';
|
|
3603
3992
|
} else {
|
|
3604
3993
|
released = true;
|
|
3605
3994
|
// The two files people read are drawn again from the saved state, the way the
|
|
@@ -3624,7 +4013,11 @@ function releaseClaimOfUnsavedPullRequest({ taskDb, db, root, row, rowId, actor,
|
|
|
3624
4013
|
row.claim_released = released;
|
|
3625
4014
|
log(row.claim_released
|
|
3626
4015
|
? ` released the claim on ${row.task} with the pull request link as the reason`
|
|
3627
|
-
:
|
|
4016
|
+
: notOurs
|
|
4017
|
+
? claimTakenOverLine(row.task)
|
|
4018
|
+
: unnamed
|
|
4019
|
+
? unnamedClaimLine(row.task)
|
|
4020
|
+
: ` could not release the claim on ${row.task}: ${failure.replace(/\s+/g, ' ').trim().slice(0, 80)}. it stays until its lease runs out, unless the task record is repaired first.`);
|
|
3628
4021
|
return row.claim_released;
|
|
3629
4022
|
}
|
|
3630
4023
|
|
|
@@ -3698,6 +4091,9 @@ async function runDispatchFlight({
|
|
|
3698
4091
|
// Extra house rules a caller appends to the builder's brief (atris idea
|
|
3699
4092
|
// tick sends the four values and the no-force-push, no-hook-skip rules).
|
|
3700
4093
|
brief = '',
|
|
4094
|
+
// The disk check before anything is claimed (see lowDiskRefusal). Tests inject both.
|
|
4095
|
+
freeSpace = undefined,
|
|
4096
|
+
spaceGuard = undefined,
|
|
3701
4097
|
} = {}) {
|
|
3702
4098
|
if (!engine) throw new Error('runDispatchFlight: engine is required');
|
|
3703
4099
|
if (!DISPATCH_CAPABLE.includes(engine)) {
|
|
@@ -3720,8 +4116,44 @@ async function runDispatchFlight({
|
|
|
3720
4116
|
}
|
|
3721
4117
|
|
|
3722
4118
|
const cli = ownCli || defaultOwnCli(root);
|
|
4119
|
+
// One look at the disk before anything is claimed or written to any job, not even the sweep of expired
|
|
4120
|
+
// claims below: worktree start's own check and reaping. Under its floor every start would refuse anyway,
|
|
4121
|
+
// and each refused start used to let a claim go that the rating then scored as a drop. A sealed review-only
|
|
4122
|
+
// flight builds in the temp folder, which that check leaves alone by default, so it is asked about that
|
|
4123
|
+
// folder with the check on.
|
|
4124
|
+
const sealedReview = reviewOnly && !dispatcher;
|
|
4125
|
+
const lowDisk = lowDiskRefusal({
|
|
4126
|
+
root,
|
|
4127
|
+
agent: engine,
|
|
4128
|
+
task: 'dispatch',
|
|
4129
|
+
freeSpace,
|
|
4130
|
+
spaceGuard: sealedReview ? (spaceGuard ?? true) : spaceGuard,
|
|
4131
|
+
...(sealedReview ? { target: os.tmpdir() } : {}),
|
|
4132
|
+
});
|
|
4133
|
+
if (lowDisk) {
|
|
4134
|
+
return stopForLowDisk({
|
|
4135
|
+
schema: 'atris.dispatch_receipt.v1',
|
|
4136
|
+
at: new Date().toISOString(),
|
|
4137
|
+
root,
|
|
4138
|
+
engine,
|
|
4139
|
+
tasks: ids,
|
|
4140
|
+
results: [],
|
|
4141
|
+
landed: [],
|
|
4142
|
+
ready: [],
|
|
4143
|
+
held_for_review: [],
|
|
4144
|
+
paused: [],
|
|
4145
|
+
}, lowDisk, ids.map((task) => ({ task })), log);
|
|
4146
|
+
}
|
|
3723
4147
|
const leaseDb = require('./task-db');
|
|
3724
4148
|
const leaseConnection = leaseDb.open();
|
|
4149
|
+
// The claim this flight made on each job, remembered from the claim write itself (see claimJob).
|
|
4150
|
+
const claims = claimLedger();
|
|
4151
|
+
// A claim let go before the engine ran is recorded in this process, on the claim this flight made, as one
|
|
4152
|
+
// that could not start. Any other claim the flight lets go (the engine was asked) is an ordinary release.
|
|
4153
|
+
const letGoUnstarted = ({ ref, actor, reason }) => releaseOwnClaim({ taskDb: leaseDb, db: leaseConnection, root, ref, actor, reason, claimEventId: claims.eventId(ref), neverStarted: true });
|
|
4154
|
+
const letGoAsked = ({ ref, actor, reason }) => releaseOwnClaim({ taskDb: leaseDb, db: leaseConnection, root, ref, actor, reason, claimEventId: claims.eventId(ref) });
|
|
4155
|
+
// The start-failure releases the store failed, for the closing pass to try again (see releaseUnstartedDispatchClaim).
|
|
4156
|
+
const releaseFailures = new Map();
|
|
3725
4157
|
// The workspace this flight works in, spelled the way the store spells it. It is read once, before
|
|
3726
4158
|
// the flight writes anything into its folder.
|
|
3727
4159
|
const workspace = leaseDb.workspaceRoot(root);
|
|
@@ -3774,8 +4206,17 @@ async function runDispatchFlight({
|
|
|
3774
4206
|
// What each claimed job is called to the task command, by the id the caller gave. Only jobs this
|
|
3775
4207
|
// flight claimed are in it.
|
|
3776
4208
|
const claimedRefs = new Map();
|
|
4209
|
+
// Right before the assigned engine is asked, the claim is marked started.
|
|
4210
|
+
const markStarted = (entry) => markClaimStarted({
|
|
4211
|
+
taskDb: leaseDb,
|
|
4212
|
+
db: leaseConnection,
|
|
4213
|
+
claims,
|
|
4214
|
+
ref: claimedRefs.get(String(entry.taskId)) || entry.taskId,
|
|
4215
|
+
actor: taskActor,
|
|
4216
|
+
task: entry.taskId,
|
|
4217
|
+
});
|
|
3777
4218
|
const leaseTimer = setInterval(() => {
|
|
3778
|
-
for (const id of leasedIds) leaseDb.renewLease(leaseConnection, { id, holder: taskActor, pid: process.pid });
|
|
4219
|
+
for (const id of leasedIds) leaseDb.renewLease(leaseConnection, { id, holder: taskActor, pid: process.pid, claimEventId: claims.eventId(id) });
|
|
3779
4220
|
}, 5 * 60000);
|
|
3780
4221
|
leaseTimer.unref();
|
|
3781
4222
|
try {
|
|
@@ -3799,17 +4240,18 @@ async function runDispatchFlight({
|
|
|
3799
4240
|
// named once, by its row id, at every step (see jobRef). Log lines and the receipt keep saying
|
|
3800
4241
|
// the display id.
|
|
3801
4242
|
const rowRef = jobRef(task, taskId);
|
|
3802
|
-
const claimed =
|
|
4243
|
+
const claimed = claimJob({ cli, taskDb: leaseDb, db: leaseConnection, root, ref: rowRef, actor: taskActor });
|
|
3803
4244
|
if (!claimed || claimed.status !== 0) {
|
|
3804
4245
|
const detail = String(claimed && (claimed.stderr || claimed.stdout) || 'claim failed').trim().slice(0, 300);
|
|
3805
4246
|
flight.paused.push({ task: taskId, stage: 'claim', detail });
|
|
3806
|
-
log(` x ${taskId} claim failed`);
|
|
4247
|
+
log(claimed && claimed.unnamed ? unnamedClaimLine(taskId) : ` x ${taskId} claim failed`);
|
|
3807
4248
|
continue;
|
|
3808
4249
|
}
|
|
3809
4250
|
claimedRefs.set(String(taskId), rowRef);
|
|
4251
|
+
claims.remember(rowRef, claimed.eventId);
|
|
3810
4252
|
try {
|
|
3811
4253
|
const leaseId = rowRef;
|
|
3812
|
-
const leased = leaseDb.setLease(leaseConnection, { id: leaseId, holder: taskActor });
|
|
4254
|
+
const leased = leaseDb.setLease(leaseConnection, { id: leaseId, holder: taskActor, claimEventId: claims.eventId(leaseId) });
|
|
3813
4255
|
if (!leased && (!ownCli || leaseDb.getTask(leaseConnection, leaseId))) throw new Error('could not set claim lease');
|
|
3814
4256
|
if (leased) leasedIds.add(leaseId);
|
|
3815
4257
|
let landingWorktreePath = '';
|
|
@@ -3830,6 +4272,9 @@ async function runDispatchFlight({
|
|
|
3830
4272
|
actor: taskActor,
|
|
3831
4273
|
stage: 'worktree_start',
|
|
3832
4274
|
detail: String(started.stderr || '').slice(0, 200),
|
|
4275
|
+
release: letGoUnstarted,
|
|
4276
|
+
failures: releaseFailures,
|
|
4277
|
+
log,
|
|
3833
4278
|
});
|
|
3834
4279
|
flight.paused.push({ task: taskId, stage: 'worktree_start', detail: released.detail, claim_released: released.released });
|
|
3835
4280
|
log(` ✗ ${taskId} worktree start failed`);
|
|
@@ -3844,6 +4289,9 @@ async function runDispatchFlight({
|
|
|
3844
4289
|
actor: taskActor,
|
|
3845
4290
|
stage: 'remote_quarantine',
|
|
3846
4291
|
detail: headTail(String(remoteBoundary && remoteBoundary.detail || 'could not prepare a sealed review sandbox')),
|
|
4292
|
+
release: letGoUnstarted,
|
|
4293
|
+
failures: releaseFailures,
|
|
4294
|
+
log,
|
|
3847
4295
|
});
|
|
3848
4296
|
flight.paused.push({
|
|
3849
4297
|
task: taskId,
|
|
@@ -3889,6 +4337,9 @@ async function runDispatchFlight({
|
|
|
3889
4337
|
actor: taskActor,
|
|
3890
4338
|
stage: 'prepare',
|
|
3891
4339
|
detail: String(err && err.message || err).slice(0, 200),
|
|
4340
|
+
release: letGoUnstarted,
|
|
4341
|
+
failures: releaseFailures,
|
|
4342
|
+
log,
|
|
3892
4343
|
});
|
|
3893
4344
|
flight.paused.push({ task: taskId, stage: 'prepare', detail: released.detail, claim_released: released.released });
|
|
3894
4345
|
log(` ✗ ${taskId} prepare failed`);
|
|
@@ -3923,7 +4374,41 @@ async function runDispatchFlight({
|
|
|
3923
4374
|
log(` fable handoff started: receipt ${path.relative(root, receiptPath)}`);
|
|
3924
4375
|
}
|
|
3925
4376
|
|
|
3926
|
-
|
|
4377
|
+
// A job that never reached its engine (its brief could not be written, or its start could not be marked)
|
|
4378
|
+
// pauses and its claim goes back as one that could not start, here, where it failed, while the flight's
|
|
4379
|
+
// other jobs go on. The jobs that did start carry on.
|
|
4380
|
+
const letGoNotStarted = (entry, result) => {
|
|
4381
|
+
const stage = result.not_started;
|
|
4382
|
+
const ref = claimedRefs.get(String(entry.taskId)) || entry.taskId;
|
|
4383
|
+
if (stage === 'claim_lost') {
|
|
4384
|
+
// The job is another run's now (see markClaimStarted). The flight lets go of nothing, writes nothing to it, and
|
|
4385
|
+
// stops renewing a lease on it.
|
|
4386
|
+
leasedIds.delete(ref);
|
|
4387
|
+
flight.paused.push({ task: entry.taskId, stage, detail: result.unnamed ? UNNAMED_CLAIM_DETAIL : LOST_CLAIM_DETAIL, worktree: entry.worktreePath, claim_released: false });
|
|
4388
|
+
log(result.unnamed ? unnamedClaimLine(entry.taskId) : claimLostLine(entry.taskId));
|
|
4389
|
+
return;
|
|
4390
|
+
}
|
|
4391
|
+
// Nothing in here may end the run: the other jobs still have their cleanup to come.
|
|
4392
|
+
let detail = result.detail;
|
|
4393
|
+
let released = false;
|
|
4394
|
+
try {
|
|
4395
|
+
({ released, detail } = releaseUnstartedDispatchClaim(cli, {
|
|
4396
|
+
taskId: entry.taskId,
|
|
4397
|
+
ref,
|
|
4398
|
+
actor: taskActor,
|
|
4399
|
+
stage,
|
|
4400
|
+
detail: result.detail,
|
|
4401
|
+
release: letGoUnstarted,
|
|
4402
|
+
failures: releaseFailures,
|
|
4403
|
+
log,
|
|
4404
|
+
}));
|
|
4405
|
+
} catch (err) {
|
|
4406
|
+
detail = `${String(result.detail || '').slice(0, 120)}. claim release failed: ${String(err && err.message || err).slice(0, 80)}`;
|
|
4407
|
+
}
|
|
4408
|
+
flight.paused.push({ task: entry.taskId, stage, detail, worktree: entry.worktreePath, claim_released: released });
|
|
4409
|
+
log(` ✗ ${entry.taskId} ${NOT_STARTED_WORDS[stage] || 'prepare'} failed`);
|
|
4410
|
+
};
|
|
4411
|
+
const settled = await Promise.all(prepared.map((entry) => {
|
|
3927
4412
|
const startedAtMs = Date.now();
|
|
3928
4413
|
return dispatchEntryWithRestaff({
|
|
3929
4414
|
entry,
|
|
@@ -3937,19 +4422,22 @@ async function runDispatchFlight({
|
|
|
3937
4422
|
defaultPin: { model, effort, maxSeconds: capSeconds(maxSeconds), ...(prep ? { prep } : {}) },
|
|
3938
4423
|
job: rosterJob,
|
|
3939
4424
|
prepAsk,
|
|
4425
|
+
beforeEngine: markStarted,
|
|
3940
4426
|
}).then((result) => {
|
|
4427
|
+
if (result.not_started) letGoNotStarted(entry, result);
|
|
3941
4428
|
const completedAtMs = Date.now();
|
|
3942
4429
|
return {
|
|
3943
4430
|
entry,
|
|
3944
4431
|
result,
|
|
3945
4432
|
startedAtMs,
|
|
3946
4433
|
completedAtMs,
|
|
3947
|
-
candidate: enforceRemoteBoundary && !detectDeadEngineDispatch(result)
|
|
4434
|
+
candidate: enforceRemoteBoundary && !result.not_started && !detectDeadEngineDispatch(result)
|
|
3948
4435
|
? reviewCandidateSnapshot(entry.remoteBoundary)
|
|
3949
4436
|
: null,
|
|
3950
4437
|
};
|
|
3951
4438
|
});
|
|
3952
4439
|
}));
|
|
4440
|
+
const results = settled.filter(({ result }) => !result.not_started);
|
|
3953
4441
|
flight.results = results.map(({ entry, result, candidate, startedAtMs, completedAtMs }) => {
|
|
3954
4442
|
const row = {
|
|
3955
4443
|
task: entry.taskId,
|
|
@@ -4492,7 +4980,10 @@ async function runDispatchFlight({
|
|
|
4492
4980
|
stampDispatchBrief(root, result.brief_id, 'partial', `opened a pull request for ${entry.taskId}, but task ready failed`);
|
|
4493
4981
|
log(' the pull request is open, but saving its proof failed. repair the task record before starting more work.');
|
|
4494
4982
|
const unsaved = flight.paused.find((row) => row.task === entry.taskId && row.stage === 'task_ready' && row.held_for_review === true);
|
|
4495
|
-
if (unsaved)
|
|
4983
|
+
if (unsaved) {
|
|
4984
|
+
const rowId = entry.task && entry.task.id;
|
|
4985
|
+
releaseClaimOfUnsavedPullRequest({ taskDb: leaseDb, db: leaseConnection, root, row: unsaved, rowId, actor: taskActor, log, claimEventId: claims.eventId(rowId) });
|
|
4986
|
+
}
|
|
4496
4987
|
continue;
|
|
4497
4988
|
}
|
|
4498
4989
|
stampDispatchBrief(root, result.brief_id, 'pass', `held ${entry.taskId} for a person to land`);
|
|
@@ -4541,6 +5032,8 @@ async function runDispatchFlight({
|
|
|
4541
5032
|
baseRef: landTarget,
|
|
4542
5033
|
log,
|
|
4543
5034
|
keepClaimsWithWork: proofOnly,
|
|
5035
|
+
release: letGoAsked,
|
|
5036
|
+
failures: releaseFailures,
|
|
4544
5037
|
});
|
|
4545
5038
|
|
|
4546
5039
|
recordFlightRosterRuns(root, flight, results, {
|
|
@@ -4567,7 +5060,7 @@ async function runDispatchFlight({
|
|
|
4567
5060
|
clearInterval(leaseTimer);
|
|
4568
5061
|
for (const id of leasedIds) {
|
|
4569
5062
|
const row = leaseDb.getTask(leaseConnection, id);
|
|
4570
|
-
if (row && row.status !== 'claimed') leaseDb.clearLease(leaseConnection, { id, holder: taskActor });
|
|
5063
|
+
if (row && row.status !== 'claimed') leaseDb.clearLease(leaseConnection, { id, holder: taskActor, claimEventId: claims.eventId(id) });
|
|
4571
5064
|
}
|
|
4572
5065
|
}
|
|
4573
5066
|
}
|