@bridge4dev/runner 0.52.0 → 0.54.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/claude.js +111 -33
- package/dist/adapters/codex-protocol.d.ts +11 -0
- package/dist/adapters/codex-protocol.js +41 -3
- package/dist/adapters/codex.js +14 -26
- package/dist/adapters/types.d.ts +45 -1
- package/dist/adapters/types.js +53 -0
- package/dist/checkpoints.d.ts +62 -0
- package/dist/checkpoints.js +50 -1
- package/dist/git.d.ts +64 -0
- package/dist/git.js +487 -36
- package/dist/gitops.d.ts +5 -0
- package/dist/gitops.js +7 -8
- package/dist/host-load.d.ts +156 -0
- package/dist/host-load.js +223 -0
- package/dist/index.js +190 -40
- package/dist/policy.d.ts +38 -0
- package/dist/policy.js +228 -7
- package/dist/process-priority.d.ts +55 -0
- package/dist/process-priority.js +99 -0
- package/dist/protocol.d.ts +42 -20
- package/dist/recipe-schema.d.ts +6 -6
- package/dist/self-update.js +43 -2
- package/dist/service-unit.d.ts +232 -10
- package/dist/service-unit.js +372 -43
- package/dist/session-cage.d.ts +297 -0
- package/dist/session-cage.js +755 -0
- package/dist/supervisor.d.ts +156 -3
- package/dist/supervisor.js +351 -32
- package/dist/systemd-memory.d.ts +35 -0
- package/dist/systemd-memory.js +115 -0
- package/dist/verify.js +28 -2
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +1 -1
package/dist/supervisor.js
CHANGED
|
@@ -8,7 +8,7 @@ import { classifyFailure, isRepeatOfSameFailure, MAX_RETRIES_PER_SESSION, retryD
|
|
|
8
8
|
import { evaluateRecipeCommand, maskSecrets, maskString } from './policy.js';
|
|
9
9
|
import { agentPromptSizeLabel, inspectAgentPrompt, quotePath, readAgentPrompt, } from './agent-prompt.js';
|
|
10
10
|
import { JournalStore } from './journal.js';
|
|
11
|
-
import { deleteSessionBranch, ensurePreviewWorktree, ensureSessionWorktree, prepareDirectWorkspace, previewWorktreePath, removePreviewWorktree, removeSessionWorktree, repoKeyFor, sessionWorktreePath, validateWorkspacePath, } from './git.js';
|
|
11
|
+
import { deleteSessionBranch, ensurePreviewWorktree, ensureSessionWorktree, prepareDirectWorkspace, previewWorktreePath, removePreviewWorktree, removeSessionWorktree, repoKeyFor, sessionWorktreePath, validateWorkspacePath, WorktreePrepareError, } from './git.js';
|
|
12
12
|
import { readRecipeProposal } from './recipe.js';
|
|
13
13
|
import { RECIPE_STEP_NAMES, parseProjectRecipe } from './recipe-schema.js';
|
|
14
14
|
import { proposeCommitMessage } from './commit-message.js';
|
|
@@ -25,8 +25,9 @@ import { pruneNativeClaudeVersions } from './agent-cleanup.js';
|
|
|
25
25
|
import { agentByDbValue } from './agent-registry.js';
|
|
26
26
|
import { invalidateAgentVersions, measureAgentVersions, } from './agent-versions.js';
|
|
27
27
|
import { rememberWorkspacePath } from './environment.js';
|
|
28
|
+
import { hostLoadChangedEnough, hostLoadHeartbeatDue, readHostLoad, HOST_LOAD_HEARTBEAT_MS, HOST_LOAD_SAMPLE_INTERVAL_MS, } from './host-load.js';
|
|
28
29
|
import { composeMessageWithAttachments, saveAttachments, } from './attachments.js';
|
|
29
|
-
import { applyRewind, createCheckpoint, dropCheckpoints, listCheckpoints, previewRewind, pruneCheckpoints, } from './checkpoints.js';
|
|
30
|
+
import { applyRewind, createCheckpoint, dropCheckpoints, listCheckpoints, MAX_BUSY_SESSIONS, previewRewind, pruneCheckpoints, } from './checkpoints.js';
|
|
30
31
|
import { DeliverMessageArgsSchema, QuestionAnswerArgsSchema } from './protocol.js';
|
|
31
32
|
import { availableModes, MODE_REFUSED_TEXT } from './adapters/types.js';
|
|
32
33
|
/** Refusals shared by every checkpoint command (ticket #126). */
|
|
@@ -209,6 +210,21 @@ export class Supervisor {
|
|
|
209
210
|
this.agentCleanupFirstTimer.unref?.();
|
|
210
211
|
this.agentCleanupTimer = setInterval(() => this.sweepOldAgentVersions(), cleanupEvery);
|
|
211
212
|
this.agentCleanupTimer.unref?.();
|
|
213
|
+
/**
|
|
214
|
+
* The machine's own load, sampled on a timer for the reason the seat report
|
|
215
|
+
* is: nothing on this side is an EVENT. Load is not something the runner
|
|
216
|
+
* does, it is something that happens to it — a `pnpm build` a person
|
|
217
|
+
* started over ssh, a neighbour container, this machine's own three agents
|
|
218
|
+
* — and there is no call site to hook. A tick that re-reads the truth is
|
|
219
|
+
* the only shape that cannot be forgotten.
|
|
220
|
+
*
|
|
221
|
+
* Nearly free, and quiet by default: two small files are read, and
|
|
222
|
+
* `publishHostLoad` returns without touching the socket unless the number
|
|
223
|
+
* actually moved (or the heartbeat came due).
|
|
224
|
+
*/
|
|
225
|
+
this.hostLoadHeartbeatMs = opts.hostLoadHeartbeatMs ?? HOST_LOAD_HEARTBEAT_MS;
|
|
226
|
+
this.hostLoadTimer = setInterval(() => this.publishHostLoad(), opts.hostLoadSampleMs ?? HOST_LOAD_SAMPLE_INTERVAL_MS);
|
|
227
|
+
this.hostLoadTimer.unref?.();
|
|
212
228
|
}
|
|
213
229
|
/** How often the agent versions are re-derived. See the constructor. */
|
|
214
230
|
static AGENT_VERSIONS_INTERVAL_MS = 60 * 60 * 1_000;
|
|
@@ -572,6 +588,53 @@ export class Supervisor {
|
|
|
572
588
|
this.lastPublishedSlots = fingerprint;
|
|
573
589
|
}
|
|
574
590
|
}
|
|
591
|
+
/** The heartbeat window actually used — the constant, or a test's own. */
|
|
592
|
+
hostLoadHeartbeatMs;
|
|
593
|
+
hostLoadTimer;
|
|
594
|
+
/** The last measurement the API actually took from us, or `null` for «nothing yet». */
|
|
595
|
+
lastPublishedHostLoad = null;
|
|
596
|
+
/** When that frame went out, by this machine's clock. `0` = never. */
|
|
597
|
+
lastHostLoadSentAt = 0;
|
|
598
|
+
/**
|
|
599
|
+
* Tell the API what this machine's load looks like — when it is worth telling.
|
|
600
|
+
*
|
|
601
|
+
* Three ways a frame goes out, and each covers a hole the others leave:
|
|
602
|
+
*
|
|
603
|
+
* - the value MOVED past a threshold (`hostLoadChangedEnough`) — the reason
|
|
604
|
+
* the frame exists, and the only one that makes the card timely;
|
|
605
|
+
* - the heartbeat came due — an idle machine still has to say «still here,
|
|
606
|
+
* still idle», because the API expires the measurement after two minutes
|
|
607
|
+
* and silence would otherwise turn «nothing is happening» into «we have no
|
|
608
|
+
* idea» on a perfectly healthy card;
|
|
609
|
+
* - nothing was ever sent on this connection (`hello_ack` clears the memory)
|
|
610
|
+
* — a fresh socket knows nothing about what the last one was told.
|
|
611
|
+
*
|
|
612
|
+
* `null` from the sampler is a complete answer: not Linux, `/proc` masked, a
|
|
613
|
+
* kernel too old for `MemAvailable`. Nothing is sent and nothing is logged —
|
|
614
|
+
* a machine that cannot measure itself leaves the card saying «no data»,
|
|
615
|
+
* which is exactly what is true.
|
|
616
|
+
*
|
|
617
|
+
* Recorded ONLY when the socket took it, same as `publishSlots`: a frame
|
|
618
|
+
* dropped by a dead socket must not be remembered as sent, or a machine whose
|
|
619
|
+
* load never moves again would go silent until the heartbeat — and on a
|
|
620
|
+
* reconnect the API would have no measurement at all while this side believed
|
|
621
|
+
* it had one.
|
|
622
|
+
*/
|
|
623
|
+
publishHostLoad() {
|
|
624
|
+
const sample = (this.opts.readHostLoad ?? readHostLoad)();
|
|
625
|
+
if (!sample)
|
|
626
|
+
return;
|
|
627
|
+
const now = Date.now();
|
|
628
|
+
const sinceLastSent = now - this.lastHostLoadSentAt;
|
|
629
|
+
// A backwards clock step is «due», not «early» — see `hostLoadHeartbeatDue`.
|
|
630
|
+
const heartbeatDue = hostLoadHeartbeatDue(sinceLastSent, this.hostLoadHeartbeatMs);
|
|
631
|
+
if (!heartbeatDue && !hostLoadChangedEnough(this.lastPublishedHostLoad, sample))
|
|
632
|
+
return;
|
|
633
|
+
if (this.ws.send({ type: 'host_load', ...sample })) {
|
|
634
|
+
this.lastPublishedHostLoad = sample;
|
|
635
|
+
this.lastHostLoadSentAt = now;
|
|
636
|
+
}
|
|
637
|
+
}
|
|
575
638
|
async onFrame(frame) {
|
|
576
639
|
switch (frame.type) {
|
|
577
640
|
case 'hello_ack':
|
|
@@ -580,9 +643,17 @@ export class Supervisor {
|
|
|
580
643
|
// because it is only true while the socket is. Forget what we told the
|
|
581
644
|
// old one so the first tick after this reconnect actually sends.
|
|
582
645
|
this.lastPublishedSlots = '';
|
|
646
|
+
// Same rule for the load: the API keeps the measurement in Redis beside
|
|
647
|
+
// the socket, with a TTL shorter than our heartbeat, so a machine that
|
|
648
|
+
// has just come back has no load on its card at all. Forget what the
|
|
649
|
+
// last connection was told and say it again immediately — a card that
|
|
650
|
+
// is right only after five minutes of silence is not right.
|
|
651
|
+
this.lastPublishedHostLoad = null;
|
|
652
|
+
this.lastHostLoadSentAt = 0;
|
|
583
653
|
this.setMaxSessions(frame.maxSessions);
|
|
584
654
|
await this.reconcile(frame.sessions);
|
|
585
655
|
this.publishSlots();
|
|
656
|
+
this.publishHostLoad();
|
|
586
657
|
// A build that finished while the socket was down has its verdict
|
|
587
658
|
// sitting on disk. This is the moment it can be delivered.
|
|
588
659
|
this.flushVerifyReports();
|
|
@@ -839,10 +910,7 @@ export class Supervisor {
|
|
|
839
910
|
running.baseBranch = prepared.baseBranch;
|
|
840
911
|
}
|
|
841
912
|
catch (error) {
|
|
842
|
-
this.
|
|
843
|
-
errorMessage: `Failed to prepare git worktree: ${maskSecretText(error)}`,
|
|
844
|
-
});
|
|
845
|
-
this.sessions.delete(descriptor.id);
|
|
913
|
+
this.workspacePrepareFailed(running, error, 'Failed to prepare git worktree');
|
|
846
914
|
return;
|
|
847
915
|
}
|
|
848
916
|
// A session that is already past STARTING (re-sent because this runner
|
|
@@ -922,6 +990,50 @@ export class Supervisor {
|
|
|
922
990
|
...(descriptor.branchPlan ? { plan: descriptor.branchPlan } : {}),
|
|
923
991
|
}));
|
|
924
992
|
}
|
|
993
|
+
/**
|
|
994
|
+
* The working folder could not be built — say so in the feed, pin the fork
|
|
995
|
+
* point, and let the session go (#360).
|
|
996
|
+
*
|
|
997
|
+
* Both doors into `prepareWorkspace` end here. Three things happen in an
|
|
998
|
+
* order that is not free:
|
|
999
|
+
*
|
|
1000
|
+
* 1. **The fork point is recorded first.** `git.ts` resolves it before it
|
|
1001
|
+
* writes anything, precisely so that a failure still knows it. Without it
|
|
1002
|
+
* the API keeps `base_sha = null` for the session's whole life, and
|
|
1003
|
+
* «Continue» can never prove the leftover branch is the empty one this
|
|
1004
|
+
* session created — which is the trap the ticket is about.
|
|
1005
|
+
* 2. **The feed event goes out before the map entry is dropped**, because
|
|
1006
|
+
* `sendEvent` needs it. Until now neither door sent one: a failed start
|
|
1007
|
+
* was visible only as an `errorMessage` on the session row, and pressing
|
|
1008
|
+
* «Continue» wipes that.
|
|
1009
|
+
* 3. **Then the status, then the entry.** A FAILED session is over, and a
|
|
1010
|
+
* leftover entry would hold one of the runner's few slots.
|
|
1011
|
+
*/
|
|
1012
|
+
workspacePrepareFailed(running, error, prefix) {
|
|
1013
|
+
const { descriptor } = running;
|
|
1014
|
+
if (error instanceof WorktreePrepareError) {
|
|
1015
|
+
if (error.baseSha && !running.baseSha)
|
|
1016
|
+
running.baseSha = error.baseSha;
|
|
1017
|
+
if (error.baseBranch && !running.baseBranch)
|
|
1018
|
+
running.baseBranch = error.baseBranch;
|
|
1019
|
+
}
|
|
1020
|
+
const message = maskSecretText(error);
|
|
1021
|
+
log.error('supervisor: the working folder could not be prepared', {
|
|
1022
|
+
sessionId: descriptor.id,
|
|
1023
|
+
workMode: descriptor.workMode ?? 'BRANCH',
|
|
1024
|
+
code: error instanceof WorktreePrepareError ? error.code : 'unknown',
|
|
1025
|
+
error: message,
|
|
1026
|
+
});
|
|
1027
|
+
this.sendEvent(running, 'error', {
|
|
1028
|
+
message,
|
|
1029
|
+
// Not an enum on either side (`RunnerEventPayloadSchema` is a record, and
|
|
1030
|
+
// the dashboard carries whatever code it is given), so this stays a plain
|
|
1031
|
+
// string and a new one costs nobody a deployment.
|
|
1032
|
+
code: error instanceof WorktreePrepareError ? error.code : 'worktree_prepare_failed',
|
|
1033
|
+
});
|
|
1034
|
+
this.reportStatus(descriptor.id, 'FAILED', { errorMessage: `${prefix}: ${message}` });
|
|
1035
|
+
this.sessions.delete(descriptor.id);
|
|
1036
|
+
}
|
|
925
1037
|
/**
|
|
926
1038
|
* Spin the adapter up — for a fresh session, a resume-on-next-message, or a
|
|
927
1039
|
* free CHAT session with no prompt at all (the agent boots, reports its
|
|
@@ -1040,6 +1152,10 @@ export class Supervisor {
|
|
|
1040
1152
|
// (`resolveGitPolicy` gives both the restrictive reading), and passing it
|
|
1041
1153
|
// unconditionally keeps one code path instead of two.
|
|
1042
1154
|
gitPolicy: gitPolicyOf(descriptor),
|
|
1155
|
+
// #361 п. 5: whether this session shares its folder with everybody else.
|
|
1156
|
+
// Rides down for the same reason as `gitPolicy` — it lands in
|
|
1157
|
+
// `PolicyContext` and it decides one line of the system prompt.
|
|
1158
|
+
...(descriptor.workMode ? { workMode: descriptor.workMode } : {}),
|
|
1043
1159
|
mode: running.mode,
|
|
1044
1160
|
...(running.model ? { model: running.model } : {}),
|
|
1045
1161
|
...(running.effort ? { effort: running.effort } : {}),
|
|
@@ -1726,7 +1842,8 @@ export class Supervisor {
|
|
|
1726
1842
|
/**
|
|
1727
1843
|
* Make room for one more agent process.
|
|
1728
1844
|
*
|
|
1729
|
-
* Up to `maxSessions` agents run side by side
|
|
1845
|
+
* Up to `maxSessions` agents run side by side — since session 16 most of them
|
|
1846
|
+
* share the project folder rather than each having a worktree of its own. Over
|
|
1730
1847
|
* that, idle-but-resumable sessions (REVIEW / WAITING_INPUT) are parked —
|
|
1731
1848
|
* their provider session survives on disk and relaunches on the next message,
|
|
1732
1849
|
* so parking costs context nothing. Only mid-turn sessions (RUNNING /
|
|
@@ -3106,17 +3223,42 @@ export class Supervisor {
|
|
|
3106
3223
|
* Three reasons it declines, and each of them is a state in which a snapshot
|
|
3107
3224
|
* would be a lie rather than a restore point:
|
|
3108
3225
|
* - the machine's owner switched checkpoints off;
|
|
3109
|
-
* -
|
|
3226
|
+
* - THIS session is mid-turn, so its own tree is being written as we read it;
|
|
3110
3227
|
* - a repo-mutating command holds the repository.
|
|
3228
|
+
*
|
|
3229
|
+
* «A neighbour in the same folder is working» used to be a fourth reason, and
|
|
3230
|
+
* it was the wrong one (#310): in DIRECT mode the folder is shared by design,
|
|
3231
|
+
* so that rule silently switched restore points off for everybody the moment
|
|
3232
|
+
* a second session opened. The neighbours are recorded on the point instead —
|
|
3233
|
+
* the conversation can always be rewound to it, the files cannot.
|
|
3234
|
+
*
|
|
3235
|
+
* Every refusal is now audible. A restore point that was never taken is
|
|
3236
|
+
* invisible until the day somebody reaches for it, and «the button is not
|
|
3237
|
+
* there» is not a sentence anybody can act on.
|
|
3111
3238
|
*/
|
|
3112
3239
|
async captureCheckpoint(running, kind, messageSeq) {
|
|
3113
3240
|
const worktreePath = running.worktreePath;
|
|
3114
|
-
if (!worktreePath
|
|
3241
|
+
if (!worktreePath)
|
|
3242
|
+
return;
|
|
3243
|
+
if (this.opts.checkpointsEnabled === false) {
|
|
3244
|
+
// Once per session: this is a machine-wide setting, not an event.
|
|
3245
|
+
if (!running.checkpointsOffSaid) {
|
|
3246
|
+
running.checkpointsOffSaid = true;
|
|
3247
|
+
this.sendEvent(running, 'notice', {
|
|
3248
|
+
level: 'info',
|
|
3249
|
+
text: 'Restore points are switched off on this dev server, so this session cannot be rewound. A server administrator turns them back on.',
|
|
3250
|
+
});
|
|
3251
|
+
}
|
|
3115
3252
|
return;
|
|
3116
|
-
|
|
3253
|
+
}
|
|
3254
|
+
if (kind === 'TURN' && this.isSessionMidTurn(running)) {
|
|
3255
|
+
this.noticeOncePerTurn(running, 'checkpoint-self-busy', 'No restore point was taken for this step: this session was still answering when it was due.');
|
|
3117
3256
|
return;
|
|
3118
|
-
|
|
3257
|
+
}
|
|
3258
|
+
if (await this.isRepoLocked(worktreePath)) {
|
|
3259
|
+
this.noticeOncePerTurn(running, 'checkpoint-repo-locked', 'No restore point was taken for this step: another git command holds this repository. It will be taken on the next one.');
|
|
3119
3260
|
return;
|
|
3261
|
+
}
|
|
3120
3262
|
// The anchor and the conversation it names travel TOGETHER: a point
|
|
3121
3263
|
// recorded against an older provider session stays usable, because the
|
|
3122
3264
|
// rewind resumes the session the point names rather than whichever one the
|
|
@@ -3128,6 +3270,10 @@ export class Supervisor {
|
|
|
3128
3270
|
kind,
|
|
3129
3271
|
...(messageSeq === undefined ? {} : { messageSeq }),
|
|
3130
3272
|
...(anchor ? { agentAnchor: anchor.anchor, agentSession: anchor.agentSession } : {}),
|
|
3273
|
+
// A getter, not a value: taking the snapshot is not instantaneous, and
|
|
3274
|
+
// «who was working while it was taken» is a question about the whole of
|
|
3275
|
+
// that interval (#310).
|
|
3276
|
+
busySessions: () => this.busyNeighbours(running),
|
|
3131
3277
|
});
|
|
3132
3278
|
if (this.isStale(running))
|
|
3133
3279
|
return;
|
|
@@ -3140,6 +3286,16 @@ export class Supervisor {
|
|
|
3140
3286
|
text: 'This working tree is too large to take a restore point — rewind is unavailable for this step. Check that build output is in .gitignore.',
|
|
3141
3287
|
});
|
|
3142
3288
|
}
|
|
3289
|
+
else {
|
|
3290
|
+
// Not a repository, or git refused: every time, because unlike the two
|
|
3291
|
+
// conditions above this one does not clear itself.
|
|
3292
|
+
this.sendEvent(running, 'notice', {
|
|
3293
|
+
level: 'warn',
|
|
3294
|
+
text: result.reason === 'not-a-repo'
|
|
3295
|
+
? 'No restore point could be taken: this folder is not a git repository, so there is nothing to rewind to.'
|
|
3296
|
+
: 'No restore point could be taken for this step, so it cannot be rewound. The session itself is unaffected.',
|
|
3297
|
+
});
|
|
3298
|
+
}
|
|
3143
3299
|
return;
|
|
3144
3300
|
}
|
|
3145
3301
|
this.sendEvent(running, 'checkpoint', {
|
|
@@ -3150,8 +3306,36 @@ export class Supervisor {
|
|
|
3150
3306
|
canRewindContext: Boolean(result.record.agentAnchor),
|
|
3151
3307
|
...(messageSeq === undefined ? {} : { messageSeq }),
|
|
3152
3308
|
...(result.skippedFiles.length ? { skippedFiles: result.skippedFiles.slice(0, 20) } : {}),
|
|
3309
|
+
// Only when non-empty (#310, gotcha 433): on this path the DASHBOARD
|
|
3310
|
+
// reads an empty list as «the folder was quiet», which is the opposite
|
|
3311
|
+
// convention to the rewind preview, where the presence of the key is the
|
|
3312
|
+
// warning. Two paths, two conventions, both deliberate.
|
|
3313
|
+
...(result.record.busySessions?.length
|
|
3314
|
+
? { busySessions: result.record.busySessions.slice(0, MAX_BUSY_SESSIONS) }
|
|
3315
|
+
: {}),
|
|
3153
3316
|
});
|
|
3154
3317
|
}
|
|
3318
|
+
/**
|
|
3319
|
+
* Say something once per BUSY PERIOD, not once per message (#310).
|
|
3320
|
+
*
|
|
3321
|
+
* A folder held by a neighbour stays held for minutes, and a session mid-turn
|
|
3322
|
+
* can be sent three follow-up notes inside one answer. Keying this on the
|
|
3323
|
+
* message seq would have counted each of those as its own turn and said the
|
|
3324
|
+
* same sentence three times — the noise the frequency policy exists to
|
|
3325
|
+
* prevent. The set is cleared when the session next comes to rest
|
|
3326
|
+
* (`reportStatus`), which is exactly when the reason stops being true.
|
|
3327
|
+
*
|
|
3328
|
+
* A SET of keys, not the last one said: two different reasons can both come
|
|
3329
|
+
* up inside one period, and remembering only the most recent would let them
|
|
3330
|
+
* take turns re-announcing each other.
|
|
3331
|
+
*/
|
|
3332
|
+
noticeOncePerTurn(running, key, text) {
|
|
3333
|
+
running.noticesThisTurn ??= new Set();
|
|
3334
|
+
if (running.noticesThisTurn.has(key))
|
|
3335
|
+
return;
|
|
3336
|
+
running.noticesThisTurn.add(key);
|
|
3337
|
+
this.sendEvent(running, 'notice', { level: 'info', text });
|
|
3338
|
+
}
|
|
3155
3339
|
/**
|
|
3156
3340
|
* Deliver messages that raced session start (already journaled).
|
|
3157
3341
|
*
|
|
@@ -3225,7 +3409,7 @@ export class Supervisor {
|
|
|
3225
3409
|
return;
|
|
3226
3410
|
}
|
|
3227
3411
|
const open = running.stopCycle;
|
|
3228
|
-
if (open &&
|
|
3412
|
+
if (open && Supervisor.stopInFlight(running)) {
|
|
3229
3413
|
// #357 part 2, and invariant 2 of the pause plan. Ten frames arrive while
|
|
3230
3414
|
// the CLI is waiting out a provider retry — from a phone and a laptop at
|
|
3231
3415
|
// once, or from a person pressing a button that gave them no sign it had
|
|
@@ -3624,7 +3808,7 @@ export class Supervisor {
|
|
|
3624
3808
|
// could not tell a tool RESULT from a tool starting, so the tail of the stop
|
|
3625
3809
|
// read as a turn beginning.
|
|
3626
3810
|
const cycle = running.stopCycle;
|
|
3627
|
-
if (cycle &&
|
|
3811
|
+
if (cycle && Supervisor.stopInFlight(running)) {
|
|
3628
3812
|
log.info('supervisor: output under a pause belongs to the stop in flight', {
|
|
3629
3813
|
sessionId: running.descriptor.id,
|
|
3630
3814
|
phase: cycle.phase,
|
|
@@ -3746,12 +3930,17 @@ export class Supervisor {
|
|
|
3746
3930
|
}
|
|
3747
3931
|
await this.applyLiveSettings(running, model, mode, effort);
|
|
3748
3932
|
}
|
|
3749
|
-
/**
|
|
3933
|
+
/**
|
|
3934
|
+
* Is a turn (or a question the agent is parked on) in flight right now?
|
|
3935
|
+
*
|
|
3936
|
+
* The same status list as `holdsTheTree` plus one term: an open question
|
|
3937
|
+
* parks the session without any of those statuses, and a live settings change
|
|
3938
|
+
* must not land under it. Reading the shared array rather than spelling the
|
|
3939
|
+
* statuses out again is the whole point of Р8 — a fourth waiting status has
|
|
3940
|
+
* to arrive in one place, not four.
|
|
3941
|
+
*/
|
|
3750
3942
|
isMidTurn(running) {
|
|
3751
|
-
return (running.lastReported
|
|
3752
|
-
running.lastReported === 'STARTING' ||
|
|
3753
|
-
running.lastReported === 'WAITING_PERMISSION' ||
|
|
3754
|
-
running.openQuestions.size > 0);
|
|
3943
|
+
return (Supervisor.MID_TURN_STATUSES.includes(running.lastReported) || running.openQuestions.size > 0);
|
|
3755
3944
|
}
|
|
3756
3945
|
/** The three live setters, in the order that lets an explicit pick win. */
|
|
3757
3946
|
async applyLiveSettings(running, model, mode, effort) {
|
|
@@ -4096,10 +4285,12 @@ export class Supervisor {
|
|
|
4096
4285
|
running.baseBranch = prepared.baseBranch;
|
|
4097
4286
|
}
|
|
4098
4287
|
catch (error) {
|
|
4099
|
-
|
|
4100
|
-
|
|
4101
|
-
|
|
4102
|
-
|
|
4288
|
+
// The same treatment as the start door (#360). Two doors doing one
|
|
4289
|
+
// thing is how the pause bug of #373 stayed half-fixed for a
|
|
4290
|
+
// release; this one used to mask its text differently AND emit no
|
|
4291
|
+
// feed event at all, so a session that could not be restored went
|
|
4292
|
+
// FAILED with nothing to read anywhere.
|
|
4293
|
+
this.workspacePrepareFailed(running, error, 'Failed to restore session worktree');
|
|
4103
4294
|
continue;
|
|
4104
4295
|
}
|
|
4105
4296
|
/**
|
|
@@ -4253,7 +4444,24 @@ export class Supervisor {
|
|
|
4253
4444
|
}
|
|
4254
4445
|
// ─── Commands ──────────────────────────────────────────────────────
|
|
4255
4446
|
async runCommand(frame) {
|
|
4256
|
-
|
|
4447
|
+
// How long a command actually took on THIS machine. Nothing measured it
|
|
4448
|
+
// before, which is why «the branch probe timed out after 4 s» could never be
|
|
4449
|
+
// answered with «it takes 9 s on that server» or «it never arrived» — the
|
|
4450
|
+
// two failures #316 turned out to be. One line, at INFO, per command.
|
|
4451
|
+
const startedAt = Date.now();
|
|
4452
|
+
let answered = false;
|
|
4453
|
+
const reply = (result) => {
|
|
4454
|
+
if (!answered) {
|
|
4455
|
+
answered = true;
|
|
4456
|
+
log.info('supervisor: command handled', {
|
|
4457
|
+
command: frame.name,
|
|
4458
|
+
...(frame.sessionId ? { sessionId: frame.sessionId } : {}),
|
|
4459
|
+
elapsedMs: Date.now() - startedAt,
|
|
4460
|
+
ok: result.ok,
|
|
4461
|
+
});
|
|
4462
|
+
}
|
|
4463
|
+
this.ws.send({ type: 'command_result', requestId: frame.requestId, ...result });
|
|
4464
|
+
};
|
|
4257
4465
|
try {
|
|
4258
4466
|
switch (frame.name) {
|
|
4259
4467
|
case 'validate_path': {
|
|
@@ -4446,7 +4654,9 @@ export class Supervisor {
|
|
|
4446
4654
|
if (!running.worktreePath) {
|
|
4447
4655
|
return void reply({ ok: false, error: 'The session has no working folder yet' });
|
|
4448
4656
|
}
|
|
4449
|
-
|
|
4657
|
+
// #310: this session's own turn, not the folder's. A neighbour
|
|
4658
|
+
// working next door marks the point; it does not forbid it.
|
|
4659
|
+
if (this.isSessionMidTurn(running)) {
|
|
4450
4660
|
return void reply({ ok: false, error: AGENT_BUSY });
|
|
4451
4661
|
}
|
|
4452
4662
|
const before = (await listCheckpoints(running.worktreePath, running.descriptor.id))
|
|
@@ -4477,6 +4687,12 @@ export class Supervisor {
|
|
|
4477
4687
|
// conversation is still on disk.
|
|
4478
4688
|
canRewindContext: Boolean(item.agentAnchor),
|
|
4479
4689
|
...(item.messageSeq === undefined ? {} : { messageSeq: item.messageSeq }),
|
|
4690
|
+
// The mark travels with the LIST as well as with the live event
|
|
4691
|
+
// (#310): the list is how a reloaded page rebuilds state, and a
|
|
4692
|
+
// mark that only ever rode the event would be lost there.
|
|
4693
|
+
...(item.busySessions?.length
|
|
4694
|
+
? { busySessions: item.busySessions.slice(0, MAX_BUSY_SESSIONS) }
|
|
4695
|
+
: {}),
|
|
4480
4696
|
})),
|
|
4481
4697
|
},
|
|
4482
4698
|
});
|
|
@@ -4507,8 +4723,18 @@ export class Supervisor {
|
|
|
4507
4723
|
if (ordinal === null || !running.worktreePath) {
|
|
4508
4724
|
return void reply({ ok: false, error: 'ordinal is required' });
|
|
4509
4725
|
}
|
|
4726
|
+
// The folder check STAYS here: this one writes files, and files are
|
|
4727
|
+
// shared. What changes is the sentence — `AGENT_BUSY` talks about
|
|
4728
|
+
// «the agent», meaning this session's own, and names nobody. When it
|
|
4729
|
+
// is a neighbour holding the folder, say so (#310).
|
|
4510
4730
|
if (this.isWorktreeBusy(running.worktreePath)) {
|
|
4511
|
-
|
|
4731
|
+
const neighbours = this.busyNeighbours(running);
|
|
4732
|
+
return void reply({
|
|
4733
|
+
ok: false,
|
|
4734
|
+
error: neighbours.length > 0
|
|
4735
|
+
? `Another session in this folder is still working (${neighbours.join(', ')}), so the files cannot be put back until it stops. Only the conversation can be rewound.`
|
|
4736
|
+
: AGENT_BUSY,
|
|
4737
|
+
});
|
|
4512
4738
|
}
|
|
4513
4739
|
const confirmRaw = frame.args?.['confirmDeletes'];
|
|
4514
4740
|
const confirmDeletes = Array.isArray(confirmRaw)
|
|
@@ -4522,6 +4748,10 @@ export class Supervisor {
|
|
|
4522
4748
|
sessionId: running.descriptor.id,
|
|
4523
4749
|
ordinal,
|
|
4524
4750
|
confirmDeletes,
|
|
4751
|
+
// The safety point is a restore point like any other and gets the
|
|
4752
|
+
// same question asked of it. Non-empty only through the #373
|
|
4753
|
+
// window — the folder check above has already passed.
|
|
4754
|
+
busySessions: () => this.busyNeighbours(running),
|
|
4525
4755
|
...(typeof expectedTreeOid === 'string' && expectedTreeOid
|
|
4526
4756
|
? { expectedTreeOid }
|
|
4527
4757
|
: {}),
|
|
@@ -4532,12 +4762,20 @@ export class Supervisor {
|
|
|
4532
4762
|
createdAt: result.safety.createdAt,
|
|
4533
4763
|
fileCount: result.safety.fileCount,
|
|
4534
4764
|
canRewindContext: Boolean(result.safety.agentAnchor),
|
|
4765
|
+
...(result.safety.busySessions?.length
|
|
4766
|
+
? { busySessions: result.safety.busySessions.slice(0, MAX_BUSY_SESSIONS) }
|
|
4767
|
+
: {}),
|
|
4535
4768
|
// This frame IS the browser's offer of «Undo rewind» — the point it
|
|
4536
4769
|
// names belongs to no message, so there is nowhere else the offer
|
|
4537
4770
|
// could come from. Withheld when the rewind was itself an undo:
|
|
4538
4771
|
// going back again is a redo, and a button that quietly changes
|
|
4539
4772
|
// meaning after one press is worse than no button.
|
|
4540
|
-
|
|
4773
|
+
//
|
|
4774
|
+
// Also withheld when the safety point itself came out marked (#310).
|
|
4775
|
+
// That happens through the #373 window — a neighbour woke between
|
|
4776
|
+
// the folder check and the snapshot — and `applyRewind` refuses a
|
|
4777
|
+
// marked point, so the offer would be a button that cannot work.
|
|
4778
|
+
undoable: result.rewoundToKind !== 'SAFETY' && !(result.safety.busySessions?.length ?? 0),
|
|
4541
4779
|
});
|
|
4542
4780
|
this.sendEvent(running, 'system_note', {
|
|
4543
4781
|
code: 'files_rewound',
|
|
@@ -4553,7 +4791,11 @@ export class Supervisor {
|
|
|
4553
4791
|
if (ordinal === null || !running.worktreePath) {
|
|
4554
4792
|
return void reply({ ok: false, error: 'ordinal is required' });
|
|
4555
4793
|
}
|
|
4556
|
-
|
|
4794
|
+
// #310: the conversation can be rewound whatever the neighbours are
|
|
4795
|
+
// doing — it touches no file of theirs. This session's OWN unfinished
|
|
4796
|
+
// turn still has to stop first: the rewind parks and relaunches this
|
|
4797
|
+
// very process, and the button is hidden mid-turn anyway.
|
|
4798
|
+
if (this.isSessionMidTurn(running)) {
|
|
4557
4799
|
return void reply({ ok: false, error: AGENT_BUSY });
|
|
4558
4800
|
}
|
|
4559
4801
|
const items = await listCheckpoints(running.worktreePath, running.descriptor.id);
|
|
@@ -5457,19 +5699,90 @@ export class Supervisor {
|
|
|
5457
5699
|
return 'sessionId is required';
|
|
5458
5700
|
return this.sessions.get(sessionId) ?? 'Unknown session';
|
|
5459
5701
|
}
|
|
5702
|
+
/**
|
|
5703
|
+
* The statuses in which a session is holding its working tree (#310).
|
|
5704
|
+
*
|
|
5705
|
+
* One array, read by all three predicates below. There used to be one
|
|
5706
|
+
* literal, then two would have been needed, and a third would have been
|
|
5707
|
+
* written the day somebody added a waiting state — which is how «the agent
|
|
5708
|
+
* is busy» and «the folder is busy» come to disagree about what busy means.
|
|
5709
|
+
* (`rewindActionsLive` in the dashboard is a fourth reader and a deliberate
|
|
5710
|
+
* mirror: the runner does not depend on `@devbridge/shared`.)
|
|
5711
|
+
*/
|
|
5712
|
+
static MID_TURN_STATUSES = [
|
|
5713
|
+
'STARTING',
|
|
5714
|
+
'RUNNING',
|
|
5715
|
+
'WAITING_PERMISSION',
|
|
5716
|
+
];
|
|
5717
|
+
/**
|
|
5718
|
+
* Is a stop still in flight over this session's live process?
|
|
5719
|
+
*
|
|
5720
|
+
* Three terms, written out in three places before this: an open cycle, that
|
|
5721
|
+
* cycle owning the process that is running NOW, and the cycle not yet
|
|
5722
|
+
* settled. #373 is what makes it load-bearing — the resting status is
|
|
5723
|
+
* published before the process dies, so this is the difference between «the
|
|
5724
|
+
* session is quiet» and «the session is still writing files».
|
|
5725
|
+
*/
|
|
5726
|
+
static stopInFlight(running) {
|
|
5727
|
+
const cycle = running.stopCycle;
|
|
5728
|
+
return cycle !== undefined && ownsProcess(cycle, running.session) && !cycle.settled;
|
|
5729
|
+
}
|
|
5730
|
+
static holdsTheTree(running) {
|
|
5731
|
+
return Boolean(running.session) && Supervisor.MID_TURN_STATUSES.includes(running.lastReported);
|
|
5732
|
+
}
|
|
5460
5733
|
/** A session actively mid-turn in this worktree — git writes must wait. */
|
|
5461
5734
|
isWorktreeBusy(worktreePath) {
|
|
5462
5735
|
for (const running of this.sessions.values()) {
|
|
5463
|
-
if (running.worktreePath !== worktreePath
|
|
5736
|
+
if (running.worktreePath !== worktreePath)
|
|
5464
5737
|
continue;
|
|
5465
|
-
if (running
|
|
5466
|
-
running.lastReported === 'RUNNING' ||
|
|
5467
|
-
running.lastReported === 'WAITING_PERMISSION') {
|
|
5738
|
+
if (Supervisor.holdsTheTree(running))
|
|
5468
5739
|
return true;
|
|
5469
|
-
}
|
|
5470
5740
|
}
|
|
5471
5741
|
return false;
|
|
5472
5742
|
}
|
|
5743
|
+
/**
|
|
5744
|
+
* Is THIS session mid-turn? (#310)
|
|
5745
|
+
*
|
|
5746
|
+
* The question `isWorktreeBusy` was answering in three places where the right
|
|
5747
|
+
* question was this one. They are the same question only when a folder holds
|
|
5748
|
+
* exactly one session — and DIRECT mode, the default since session 16, is
|
|
5749
|
+
* precisely the arrangement in which it holds several. A restore point is a
|
|
5750
|
+
* snapshot of this session's own conversation; a neighbour typing in the same
|
|
5751
|
+
* folder is a reason to MARK it, not to refuse to take it.
|
|
5752
|
+
*/
|
|
5753
|
+
isSessionMidTurn(running) {
|
|
5754
|
+
return Supervisor.holdsTheTree(running);
|
|
5755
|
+
}
|
|
5756
|
+
/**
|
|
5757
|
+
* Everybody else who is working in this folder right now (#310).
|
|
5758
|
+
*
|
|
5759
|
+
* Includes a neighbour whose stop is still running: after #373 the resting
|
|
5760
|
+
* status is published BEFORE the process is actually gone, so a session that
|
|
5761
|
+
* reports IDLE while its stop cycle still owns a live process is still
|
|
5762
|
+
* writing files. Reading `lastReported` alone would call that folder quiet.
|
|
5763
|
+
*/
|
|
5764
|
+
busyNeighbours(running) {
|
|
5765
|
+
const worktreePath = running.worktreePath;
|
|
5766
|
+
if (!worktreePath)
|
|
5767
|
+
return [];
|
|
5768
|
+
const ids = [];
|
|
5769
|
+
for (const other of this.sessions.values()) {
|
|
5770
|
+
// By id as well as by identity: a session that was re-started under a
|
|
5771
|
+
// higher epoch is a NEW entry for the SAME session, and it must not end
|
|
5772
|
+
// up marking its own restore point with its own id.
|
|
5773
|
+
if (other === running ||
|
|
5774
|
+
other.descriptor.id === running.descriptor.id ||
|
|
5775
|
+
other.worktreePath !== worktreePath) {
|
|
5776
|
+
continue;
|
|
5777
|
+
}
|
|
5778
|
+
if (Supervisor.holdsTheTree(other) || Supervisor.stopInFlight(other)) {
|
|
5779
|
+
ids.push(other.descriptor.id);
|
|
5780
|
+
}
|
|
5781
|
+
if (ids.length >= MAX_BUSY_SESSIONS)
|
|
5782
|
+
break;
|
|
5783
|
+
}
|
|
5784
|
+
return ids;
|
|
5785
|
+
}
|
|
5473
5786
|
// ─── Outbound helpers ──────────────────────────────────────────────
|
|
5474
5787
|
// Hard cap far below the API's 128KB Zod limit — an oversized payload would
|
|
5475
5788
|
// be rejected forever and wedge the journal (QA-96 F2).
|
|
@@ -5511,6 +5824,11 @@ export class Supervisor {
|
|
|
5511
5824
|
const running = this.sessions.get(sessionId);
|
|
5512
5825
|
if (running) {
|
|
5513
5826
|
running.lastReported = status;
|
|
5827
|
+
// Coming to rest ends the busy period the once-per-turn notices were
|
|
5828
|
+
// limited to: the next one is about a new answer and deserves saying.
|
|
5829
|
+
if (!Supervisor.MID_TURN_STATUSES.includes(status)) {
|
|
5830
|
+
delete running.noticesThisTurn;
|
|
5831
|
+
}
|
|
5514
5832
|
this.syncBudgetClock(running);
|
|
5515
5833
|
}
|
|
5516
5834
|
// The fork point rides along on the first frame after the branch was
|
|
@@ -5558,6 +5876,7 @@ export class Supervisor {
|
|
|
5558
5876
|
/** Graceful daemon shutdown: kill agents, keep sessions resumable server-side. */
|
|
5559
5877
|
shutdown() {
|
|
5560
5878
|
clearInterval(this.slotsTimer);
|
|
5879
|
+
clearInterval(this.hostLoadTimer);
|
|
5561
5880
|
clearInterval(this.agentVersionsTimer);
|
|
5562
5881
|
clearTimeout(this.agentCleanupFirstTimer);
|
|
5563
5882
|
clearInterval(this.agentCleanupTimer);
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { type CgroupMemory, type MemoryFacts } from './service-unit.js';
|
|
2
|
+
/**
|
|
3
|
+
* Read the unit's group, never the CLI's own `/proc/self` group. A missing
|
|
4
|
+
* reading is not an empty service: a timed-out user bus can belong to a busy
|
|
5
|
+
* machine. Raw `memory.current` deliberately survives beside the split, so the
|
|
6
|
+
* safety floor never depends on an estimate of how much can be reclaimed.
|
|
7
|
+
*/
|
|
8
|
+
export declare function unitMemoryReading(output: string, cgroupRoot?: string): CgroupMemory | null;
|
|
9
|
+
/**
|
|
10
|
+
* Both units, measured through systemd, with the filesystem underneath.
|
|
11
|
+
*
|
|
12
|
+
* One helper so `doctor`, `doctor --fix`, `install-service` and the daemon's
|
|
13
|
+
* hourly re-measure cannot drift apart in what they measure — which is how the
|
|
14
|
+
* CLI once computed a ceiling 34 % away from the daemon's and the two rewrote
|
|
15
|
+
* the file forever.
|
|
16
|
+
*
|
|
17
|
+
* The fallback is not a second opinion, it is the same reading taken another
|
|
18
|
+
* way, and it is safe for every caller precisely because each reader
|
|
19
|
+
* self-identifies: `readOwnCgroupMemory` hands back null for any process that
|
|
20
|
+
* is not the service itself, so the daemon measures itself when the bus is
|
|
21
|
+
* unhappy and a CLI never mistakes its own `session-N.scope` for the service.
|
|
22
|
+
* Without it, one slow `systemctl` meant the daemon wrote no policy at all —
|
|
23
|
+
* for an hour, on the overloaded machine this policy exists to protect.
|
|
24
|
+
*/
|
|
25
|
+
export declare function readMemoryFactsFromSystemd(): Promise<MemoryReadings>;
|
|
26
|
+
export interface MemoryReadings {
|
|
27
|
+
facts: MemoryFacts | null;
|
|
28
|
+
sessionsUsageBytes: number | null;
|
|
29
|
+
}
|
|
30
|
+
/** The half of the above that has no bus in it, so it can be tested. */
|
|
31
|
+
export declare function memoryReadings(service: CgroupMemory | null, sessions: CgroupMemory | null, fromFilesystem?: {
|
|
32
|
+
own: () => CgroupMemory | null;
|
|
33
|
+
sessions: () => CgroupMemory | null;
|
|
34
|
+
}): MemoryReadings;
|
|
35
|
+
//# sourceMappingURL=systemd-memory.d.ts.map
|