@bridge4dev/runner 0.37.0 → 0.39.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-binary.d.ts +28 -0
- package/dist/agent-binary.js +99 -0
- package/dist/index.js +108 -12
- package/dist/self-update.js +49 -1
- package/dist/service-unit.d.ts +122 -6
- package/dist/service-unit.js +300 -16
- package/dist/supervisor.d.ts +51 -3
- package/dist/supervisor.js +435 -244
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/ws-client.d.ts +28 -0
- package/dist/ws-client.js +33 -4
- package/package.json +1 -1
package/dist/supervisor.js
CHANGED
|
@@ -51,6 +51,7 @@ function gitPolicyOf(descriptor) {
|
|
|
51
51
|
: {}),
|
|
52
52
|
};
|
|
53
53
|
}
|
|
54
|
+
const LAUNCH_REFUSED = { ok: false, reason: 'refused' };
|
|
54
55
|
export class Supervisor {
|
|
55
56
|
ws;
|
|
56
57
|
opts;
|
|
@@ -367,7 +368,11 @@ export class Supervisor {
|
|
|
367
368
|
await this.captureCheckpoint(running, 'TURN', 0);
|
|
368
369
|
if (this.isStale(running))
|
|
369
370
|
return;
|
|
370
|
-
|
|
371
|
+
// A launch that failed has already said so and reported a status the
|
|
372
|
+
// person can act on. Flushing the queue into it would only walk the same
|
|
373
|
+
// failure again, once per waiting message (ticket #225).
|
|
374
|
+
if (!this.launchAgent(running, composeInitialPrompt(descriptor), null).ok)
|
|
375
|
+
return;
|
|
371
376
|
}
|
|
372
377
|
else {
|
|
373
378
|
if (descriptor.epoch > 0) {
|
|
@@ -425,15 +430,23 @@ export class Supervisor {
|
|
|
425
430
|
* free CHAT session with no prompt at all (the agent boots, reports its
|
|
426
431
|
* capabilities and waits for the first message).
|
|
427
432
|
*
|
|
428
|
-
* Returns
|
|
429
|
-
*
|
|
430
|
-
*
|
|
433
|
+
* Returns what came of it: a caller holding a user message needs to know,
|
|
434
|
+
* because anything but `ok` means the message has to stay queued rather than
|
|
435
|
+
* be marked delivered (session 9).
|
|
436
|
+
*
|
|
437
|
+
* NEVER throws (ticket #225). Everything from reading the project prompt to
|
|
438
|
+
* the adapter's own constructor runs inside one guard, because the caller
|
|
439
|
+
* chain above cannot tell the difference between «did not start» and
|
|
440
|
+
* «threw»: the message path swallows the exception into a log line, and the
|
|
441
|
+
* reconnect path lets it abort the restore of every OTHER session on the
|
|
442
|
+
* machine. A launch that fails is a state this session reports, not an
|
|
443
|
+
* exception somebody else has to remember to catch.
|
|
431
444
|
*/
|
|
432
445
|
launchAgent(running, prompt, resumeId) {
|
|
433
446
|
const { descriptor } = running;
|
|
434
447
|
const adapter = this.opts.adapters[descriptor.agent];
|
|
435
448
|
if (!adapter || !running.worktreePath || !running.branch)
|
|
436
|
-
return
|
|
449
|
+
return LAUNCH_REFUSED;
|
|
437
450
|
// An exhausted USD budget must not relaunch $0.01-floor processes (QA-96 F4).
|
|
438
451
|
// Codex reports no cost at all, so its costUsd never leaves 0 — gating on it
|
|
439
452
|
// would be a limit that can never fire while the UI shows $0.00. Those
|
|
@@ -455,7 +468,7 @@ export class Supervisor {
|
|
|
455
468
|
errorMessage: `Session budget ($${descriptor.workspace.budgetUsd}) is exhausted`,
|
|
456
469
|
});
|
|
457
470
|
this.sessions.delete(descriptor.id);
|
|
458
|
-
return
|
|
471
|
+
return LAUNCH_REFUSED;
|
|
459
472
|
}
|
|
460
473
|
// The time budget is already spent: relaunching would burn a process for
|
|
461
474
|
// nothing and immediately stop again.
|
|
@@ -464,7 +477,7 @@ export class Supervisor {
|
|
|
464
477
|
level: 'warn',
|
|
465
478
|
text: 'The time budget is used up — press «Continue» to give the agent more time.',
|
|
466
479
|
});
|
|
467
|
-
return
|
|
480
|
+
return LAUNCH_REFUSED;
|
|
468
481
|
}
|
|
469
482
|
running.lastPrompt = prompt;
|
|
470
483
|
// Facts about this session only, plus the one file the project named. The
|
|
@@ -477,49 +490,60 @@ export class Supervisor {
|
|
|
477
490
|
// «loaded» nor «unchanged».
|
|
478
491
|
const rewinding = Boolean(running.rewindAnchor?.agentSession);
|
|
479
492
|
const resuming = rewinding || Boolean(resumeId);
|
|
480
|
-
|
|
481
|
-
|
|
493
|
+
// Held so the guard below can put it back: a rewind the person asked for
|
|
494
|
+
// must not be silently forgotten because the process that was going to
|
|
495
|
+
// apply it never started (ticket #225). The feed has already told them the
|
|
496
|
+
// conversation was cut.
|
|
482
497
|
const rewind = running.rewindAnchor;
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
? {}
|
|
501
|
-
:
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
? {
|
|
521
|
-
|
|
522
|
-
|
|
498
|
+
try {
|
|
499
|
+
const agentPrompt = this.resolveAgentPrompt(running, resuming && !rewinding);
|
|
500
|
+
const workspaceContext = composeWorkspaceContext(descriptor, agentPrompt?.text);
|
|
501
|
+
delete running.rewindAnchor;
|
|
502
|
+
// A rewind resumes the conversation the POINT names, which is not always
|
|
503
|
+
// the one this session is on now: rewinding twice, or rewinding the first
|
|
504
|
+
// message after a rewind, both reach back into the thread the fork came
|
|
505
|
+
// from. Its transcript is still on disk — that is what makes the point
|
|
506
|
+
// usable at all.
|
|
507
|
+
const resumeTarget = rewind ? rewind.agentSession : resumeId;
|
|
508
|
+
running.session = adapter.startSession({
|
|
509
|
+
sessionId: descriptor.id,
|
|
510
|
+
cwd: running.worktreePath,
|
|
511
|
+
...(prompt ? { prompt } : {}),
|
|
512
|
+
...(workspaceContext ? { workspaceContext } : {}),
|
|
513
|
+
// Only when it was actually read: layer 1 must refuse writes to the file
|
|
514
|
+
// this process was given, not to a path it was merely told about.
|
|
515
|
+
...(agentPrompt ? { agentPromptFile: agentPrompt.absPath } : {}),
|
|
516
|
+
trustMode: descriptor.workspace.trustMode,
|
|
517
|
+
...(descriptor.workspace.agentAutoCommit === undefined
|
|
518
|
+
? {}
|
|
519
|
+
: { agentAutoCommit: descriptor.workspace.agentAutoCommit }),
|
|
520
|
+
// Session 18. Always present, even when every field inside it is absent:
|
|
521
|
+
// an absent OBJECT and an object of absent fields resolve identically
|
|
522
|
+
// (`resolveGitPolicy` gives both the restrictive reading), and passing it
|
|
523
|
+
// unconditionally keeps one code path instead of two.
|
|
524
|
+
gitPolicy: gitPolicyOf(descriptor),
|
|
525
|
+
mode: running.mode,
|
|
526
|
+
...(running.model ? { model: running.model } : {}),
|
|
527
|
+
...(running.effort ? { effort: running.effort } : {}),
|
|
528
|
+
...(resumeTarget ? { resumeProviderSessionId: resumeTarget } : {}),
|
|
529
|
+
// Ticket #126: a conversation rewind takes effect exactly here, on the
|
|
530
|
+
// next process this session starts. Consumed rather than kept — a rewind
|
|
531
|
+
// is one event, not a standing setting, and re-applying it on a later
|
|
532
|
+
// relaunch would silently throw away everything said since.
|
|
533
|
+
...(rewind ? { resumeAtAnchor: rewind.anchor } : {}),
|
|
534
|
+
// Descriptor MCP (auto-issued per-workspace key) wins over config.toml.
|
|
535
|
+
...((descriptor.mcp ?? this.opts.mcp) ? { mcp: descriptor.mcp ?? this.opts.mcp } : {}),
|
|
536
|
+
// The SDK budget is per-process; hand the RESIDUAL session budget down.
|
|
537
|
+
...(reportsCost(descriptor.agent) && descriptor.workspace.budgetUsd !== null
|
|
538
|
+
? { maxBudgetUsd: Math.max(0.01, descriptor.workspace.budgetUsd - running.costBaseUsd) }
|
|
539
|
+
: {}),
|
|
540
|
+
});
|
|
541
|
+
}
|
|
542
|
+
catch (error) {
|
|
543
|
+
if (rewind)
|
|
544
|
+
running.rewindAnchor = rewind;
|
|
545
|
+
return this.launchCrashed(running, error);
|
|
546
|
+
}
|
|
523
547
|
// No prompt → nothing is running yet: the agent is up and waiting for the
|
|
524
548
|
// user's first message (free CHAT session). reportStatus drives the budget
|
|
525
549
|
// clock, so this call is also what starts (or does not start) billing.
|
|
@@ -527,6 +551,8 @@ export class Supervisor {
|
|
|
527
551
|
branch: running.branch,
|
|
528
552
|
worktreePath: running.worktreePath,
|
|
529
553
|
});
|
|
554
|
+
// The object exists; the PROCESS still has to prove it does (ticket #225).
|
|
555
|
+
this.watchForFirstSignOfLife(running);
|
|
530
556
|
// Ticket #196: a Stop or a pause that arrived while this process was coming
|
|
531
557
|
// up was dropped on the floor — `interruptSession` returns early when there
|
|
532
558
|
// is no adapter yet, and the window covers preparing the worktree and
|
|
@@ -541,7 +567,99 @@ export class Supervisor {
|
|
|
541
567
|
}));
|
|
542
568
|
}
|
|
543
569
|
void this.pumpEvents(running);
|
|
544
|
-
return true;
|
|
570
|
+
return { ok: true };
|
|
571
|
+
}
|
|
572
|
+
/**
|
|
573
|
+
* The agent process could not be started at all (ticket #225).
|
|
574
|
+
*
|
|
575
|
+
* Three things have to happen here, and until this ticket none of them did:
|
|
576
|
+
* the reason is said WHERE THE PERSON IS LOOKING (an `error` event is the
|
|
577
|
+
* feed's red line), the session stops claiming to be working, and the stack
|
|
578
|
+
* reaches journald for whoever has to fix the machine. The status is the
|
|
579
|
+
* honest one for a session with no process — the agent is not running, and a
|
|
580
|
+
* session left in `RUNNING` shows a stop button for a turn that does not
|
|
581
|
+
* exist.
|
|
582
|
+
*
|
|
583
|
+
* Never rethrows: this IS the handling. The caller gets `crashed` and decides
|
|
584
|
+
* what to do with the message it was holding.
|
|
585
|
+
*/
|
|
586
|
+
launchCrashed(running, error) {
|
|
587
|
+
const { descriptor } = running;
|
|
588
|
+
running.session = null;
|
|
589
|
+
log.error('supervisor: the agent process could not be started', {
|
|
590
|
+
sessionId: descriptor.id,
|
|
591
|
+
agent: descriptor.agent,
|
|
592
|
+
error: error instanceof Error ? (error.stack ?? error.message) : String(error),
|
|
593
|
+
});
|
|
594
|
+
const reason = maskSecretText(error);
|
|
595
|
+
this.sendEvent(running, 'error', {
|
|
596
|
+
message: `${AGENT_LABELS[descriptor.agent] ?? descriptor.agent} could not be started on this server: ${reason}`,
|
|
597
|
+
});
|
|
598
|
+
// A session that never began is FAILED — the same answer the neighbouring
|
|
599
|
+
// startup failures (no adapter, no worktree) already give. Everything else
|
|
600
|
+
// goes back to waiting for a human, which is the state «press Continue» is
|
|
601
|
+
// meaningful in; REVIEW keeps its row in the dashboard.
|
|
602
|
+
const status = running.lastReported === 'STARTING'
|
|
603
|
+
? 'FAILED'
|
|
604
|
+
: running.lastReported === 'REVIEW'
|
|
605
|
+
? 'REVIEW'
|
|
606
|
+
: 'WAITING_INPUT';
|
|
607
|
+
this.reportStatus(descriptor.id, status, {
|
|
608
|
+
costUsd: running.costUsd,
|
|
609
|
+
activeMs: running.activeMs,
|
|
610
|
+
errorMessage: `Agent process failed to start: ${reason}`,
|
|
611
|
+
});
|
|
612
|
+
// A FAILED session is over, and an entry left in the map would hold one of
|
|
613
|
+
// the runner's few slots for a process that never existed — `ensureCapacity`
|
|
614
|
+
// counts entries, not processes. Dropped exactly like the other startup
|
|
615
|
+
// failures do it. The parked states keep their entry on purpose: that is
|
|
616
|
+
// what «Продолжить» picks back up.
|
|
617
|
+
if (status === 'FAILED')
|
|
618
|
+
this.sessions.delete(descriptor.id);
|
|
619
|
+
return { ok: false, reason: 'crashed' };
|
|
620
|
+
}
|
|
621
|
+
/** How long a freshly started agent may say nothing before we say so. */
|
|
622
|
+
static STARTUP_SILENCE_MS = 60_000;
|
|
623
|
+
/**
|
|
624
|
+
* Watch for the first word out of a process we just started (ticket #225).
|
|
625
|
+
*
|
|
626
|
+
* «The adapter object exists» is not «the agent is running». A CLI that hangs
|
|
627
|
+
* before its first frame — a stuck hook, an MCP server that never answers, a
|
|
628
|
+
* transcript it cannot read — produces no events, no error and no exit, and
|
|
629
|
+
* the session sits in `RUNNING` forever. On a healthy launch the first event
|
|
630
|
+
* arrives in about two seconds, so a minute of silence is not a slow start,
|
|
631
|
+
* it is something worth saying out loud.
|
|
632
|
+
*
|
|
633
|
+
* Says it and stops there: no kill. A long conversation has the right to boot
|
|
634
|
+
* slowly, and killing it would cost the person the very turn they are waiting
|
|
635
|
+
* for.
|
|
636
|
+
*/
|
|
637
|
+
watchForFirstSignOfLife(running) {
|
|
638
|
+
this.clearStartupWatch(running);
|
|
639
|
+
running.heardFromAgent = false;
|
|
640
|
+
const silentMs = this.opts.startupSilenceMs ?? Supervisor.STARTUP_SILENCE_MS;
|
|
641
|
+
const timer = setTimeout(() => {
|
|
642
|
+
delete running.startupTimer;
|
|
643
|
+
if (running.heardFromAgent || !running.session || this.isStale(running))
|
|
644
|
+
return;
|
|
645
|
+
log.warn('supervisor: the agent process has said nothing since it started', {
|
|
646
|
+
sessionId: running.descriptor.id,
|
|
647
|
+
silentMs,
|
|
648
|
+
});
|
|
649
|
+
this.sendEvent(running, 'notice', {
|
|
650
|
+
level: 'warn',
|
|
651
|
+
text: 'The agent process started but has not said a word for a minute. It may still be loading a long conversation. If nothing happens, press «Stop» and then «Continue» — and if that does not help either, this server needs a look.',
|
|
652
|
+
});
|
|
653
|
+
}, silentMs);
|
|
654
|
+
timer.unref?.();
|
|
655
|
+
running.startupTimer = timer;
|
|
656
|
+
}
|
|
657
|
+
/** The process spoke, or went away — either way the watch is over. */
|
|
658
|
+
clearStartupWatch(running) {
|
|
659
|
+
if (running.startupTimer) {
|
|
660
|
+
clearTimeout(running.startupTimer);
|
|
661
|
+
delete running.startupTimer;
|
|
662
|
+
}
|
|
545
663
|
}
|
|
546
664
|
/**
|
|
547
665
|
* The project's own prompt file, read fresh for THIS agent process.
|
|
@@ -794,6 +912,9 @@ export class Supervisor {
|
|
|
794
912
|
delete running.activeSince;
|
|
795
913
|
}
|
|
796
914
|
this.clearBudgetTimers(running);
|
|
915
|
+
// The process this watch was armed for is gone; whatever it did or did not
|
|
916
|
+
// say, there is nothing left to wait for (ticket #225).
|
|
917
|
+
this.clearStartupWatch(running);
|
|
797
918
|
// Stale-resume recovery: relaunch once without a resume id.
|
|
798
919
|
if (running.freshRetry && !running.stopRequested) {
|
|
799
920
|
const { prompt } = running.freshRetry;
|
|
@@ -851,7 +972,8 @@ export class Supervisor {
|
|
|
851
972
|
this.sendEvent(running, 'settings', { mode });
|
|
852
973
|
// Empty prompt: the agent boots, reports its capabilities and waits, the
|
|
853
974
|
// same as a free CHAT session. It must NOT start a turn of its own here.
|
|
854
|
-
|
|
975
|
+
const relaunched = this.launchAgent(running, '', running.descriptor.providerSessionId);
|
|
976
|
+
if (relaunched.ok) {
|
|
855
977
|
// Anything typed during the park window is waiting on disk (see
|
|
856
978
|
// `deliverMessage`), and the new process is the one that can take it.
|
|
857
979
|
this.flushPendingMessages(running);
|
|
@@ -863,12 +985,17 @@ export class Supervisor {
|
|
|
863
985
|
}
|
|
864
986
|
return;
|
|
865
987
|
}
|
|
866
|
-
// The agent did not start — an exhausted budget
|
|
867
|
-
// session stays parked and resumable rather than
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
988
|
+
// The agent did not start — an exhausted budget, or a launch that crashed
|
|
989
|
+
// (ticket #225). The session stays parked and resumable rather than
|
|
990
|
+
// silently disappearing. A crash has already reported its own status and
|
|
991
|
+
// reason; re-reporting `statusForReport` here would overwrite them with
|
|
992
|
+
// the state the session was in before it failed.
|
|
993
|
+
if (relaunched.reason === 'refused') {
|
|
994
|
+
this.reportStatus(descriptor.id, statusForReport(running), {
|
|
995
|
+
costUsd: running.costUsd,
|
|
996
|
+
activeMs: running.activeMs,
|
|
997
|
+
});
|
|
998
|
+
}
|
|
872
999
|
this.drainSessionsWaitingForCapacity();
|
|
873
1000
|
return;
|
|
874
1001
|
}
|
|
@@ -1153,6 +1280,15 @@ export class Supervisor {
|
|
|
1153
1280
|
}
|
|
1154
1281
|
forwardEvent(running, event) {
|
|
1155
1282
|
const { descriptor } = running;
|
|
1283
|
+
// Ticket #225: ANY event is the process proving it came up — its own
|
|
1284
|
+
// capabilities probe answers within about two seconds of a healthy launch,
|
|
1285
|
+
// long before the agent says anything a person would read. That is a
|
|
1286
|
+
// deliberately weaker bar than «the agent is working» below: what this
|
|
1287
|
+
// watch is for is a CLI that never boots at all.
|
|
1288
|
+
if (!running.heardFromAgent) {
|
|
1289
|
+
running.heardFromAgent = true;
|
|
1290
|
+
this.clearStartupWatch(running);
|
|
1291
|
+
}
|
|
1156
1292
|
// Anything below that is the agent talking means the agent is working. Read
|
|
1157
1293
|
// before the switch so every such case gets it, including the ones added
|
|
1158
1294
|
// after this line was written.
|
|
@@ -1724,11 +1860,13 @@ export class Supervisor {
|
|
|
1724
1860
|
// A person typing into the session is the clearest signal that the work is
|
|
1725
1861
|
// back on track, so the automatic-continuation allowance starts over.
|
|
1726
1862
|
clearAutoResume(running.descriptor.id);
|
|
1727
|
-
if (this.launchAgent(running, text, running.descriptor.providerSessionId)) {
|
|
1863
|
+
if (this.launchAgent(running, text, running.descriptor.providerSessionId).ok) {
|
|
1728
1864
|
settle();
|
|
1729
1865
|
return;
|
|
1730
1866
|
}
|
|
1731
|
-
// The agent did not start (an exhausted budget
|
|
1867
|
+
// The agent did not start (an exhausted budget, or a launch that crashed —
|
|
1868
|
+
// ticket #225: it used to be swallowed into a log line, and the words the
|
|
1869
|
+
// person typed were retired from disk as though an agent had them). The
|
|
1732
1870
|
// instruction stays on disk, so «Продолжить» — which is what raises the
|
|
1733
1871
|
// budget — carries it to the agent instead of dropping it.
|
|
1734
1872
|
this.requeue(running, held, text, originSeq);
|
|
@@ -2167,210 +2305,263 @@ export class Supervisor {
|
|
|
2167
2305
|
}
|
|
2168
2306
|
}
|
|
2169
2307
|
// Redeliver unacked events for every persisted journal (at-least-once).
|
|
2308
|
+
//
|
|
2309
|
+
// Per file, because one that cannot be read must not cost the whole
|
|
2310
|
+
// reconnect (ticket #225): this runs BEFORE a single session is looked at,
|
|
2311
|
+
// so a throw here means nothing is restored at all — no launch, no status,
|
|
2312
|
+
// no note, on every reconnect for as long as the file stays broken.
|
|
2170
2313
|
for (const sessionId of this.journals.persistedSessionIds()) {
|
|
2171
|
-
|
|
2172
|
-
|
|
2173
|
-
|
|
2174
|
-
|
|
2314
|
+
try {
|
|
2315
|
+
const journal = this.journals.open(sessionId);
|
|
2316
|
+
for (const event of journal.unacked()) {
|
|
2317
|
+
this.ws.send({
|
|
2318
|
+
type: 'event',
|
|
2319
|
+
sessionId,
|
|
2320
|
+
seq: event.seq,
|
|
2321
|
+
eventType: event.eventType,
|
|
2322
|
+
payload: event.payload,
|
|
2323
|
+
});
|
|
2324
|
+
}
|
|
2325
|
+
}
|
|
2326
|
+
catch (error) {
|
|
2327
|
+
log.error('supervisor: journal could not be replayed', {
|
|
2175
2328
|
sessionId,
|
|
2176
|
-
|
|
2177
|
-
eventType: event.eventType,
|
|
2178
|
-
payload: event.payload,
|
|
2329
|
+
error: String(error),
|
|
2179
2330
|
});
|
|
2180
2331
|
}
|
|
2181
2332
|
}
|
|
2182
2333
|
for (const descriptor of descriptors) {
|
|
2183
|
-
//
|
|
2184
|
-
//
|
|
2185
|
-
|
|
2186
|
-
|
|
2187
|
-
|
|
2188
|
-
|
|
2189
|
-
|
|
2190
|
-
//
|
|
2191
|
-
//
|
|
2192
|
-
|
|
2193
|
-
|
|
2194
|
-
|
|
2195
|
-
|
|
2196
|
-
|
|
2197
|
-
|
|
2198
|
-
|
|
2199
|
-
|
|
2200
|
-
|
|
2201
|
-
|
|
2202
|
-
// was down never arrived at all, because `session_pause` is
|
|
2203
|
-
// fire-and-forget with no outbox behind it.
|
|
2204
|
-
//
|
|
2205
|
-
// Both directions matter: the row may have gained a clock (hold now) or
|
|
2206
|
-
// lost one (release and send what was held). `applyPause` does both, and
|
|
2207
|
-
// it runs BEFORE `flushSessionOutbox` arrives from the API side.
|
|
2208
|
-
await this.applyPause(descriptor.id, descriptor.pausedUntil ?? null);
|
|
2209
|
-
this.reportStatus(descriptor.id, statusForReport(tracked), {
|
|
2210
|
-
costUsd: tracked.costUsd,
|
|
2211
|
-
...(tracked.branch ? { branch: tracked.branch } : {}),
|
|
2212
|
-
...(tracked.worktreePath ? { worktreePath: tracked.worktreePath } : {}),
|
|
2213
|
-
...(tracked.descriptor.providerSessionId
|
|
2214
|
-
? { providerSessionId: tracked.descriptor.providerSessionId }
|
|
2215
|
-
: {}),
|
|
2216
|
-
});
|
|
2217
|
-
continue;
|
|
2218
|
-
}
|
|
2219
|
-
// Session that went terminal while we were offline: the journal keeps
|
|
2220
|
-
// the last reported status — replay it instead of resurrecting the
|
|
2221
|
-
// session as resumable (QA-96 F1).
|
|
2222
|
-
if (this.journals.exists(descriptor.id)) {
|
|
2223
|
-
const journal = this.journals.open(descriptor.id);
|
|
2224
|
-
const last = journal.lastStatus;
|
|
2225
|
-
// Only replay a terminal status from THIS life of the session. A
|
|
2226
|
-
// resumed session carries a higher epoch, and replaying the FAILED it
|
|
2227
|
-
// was resumed from would kill it again the moment the runner reconnects.
|
|
2228
|
-
// A journal written by a runner from before session 13 could hold a
|
|
2229
|
-
// `DONE` — it is no longer a status this runner may report, and
|
|
2230
|
-
// replaying one would close the session for the user. Drop it and let
|
|
2231
|
-
// the session be picked back up like any other.
|
|
2232
|
-
if (last &&
|
|
2233
|
-
isTerminal(last.status) &&
|
|
2234
|
-
last.status !== 'DONE' &&
|
|
2235
|
-
(last.epoch ?? 0) >= descriptor.epoch) {
|
|
2236
|
-
this.ws.send({
|
|
2237
|
-
type: 'session_status',
|
|
2238
|
-
sessionId: descriptor.id,
|
|
2239
|
-
status: last.status,
|
|
2240
|
-
...(last.extra ?? {}),
|
|
2241
|
-
// Guarded above to be >= the descriptor's epoch, so the API keeps it.
|
|
2242
|
-
...(last.epoch === undefined ? {} : { epoch: last.epoch }),
|
|
2243
|
-
});
|
|
2244
|
-
if (journal.unacked().length === 0) {
|
|
2245
|
-
this.journals.closeAndDelete(descriptor.id);
|
|
2334
|
+
// One session must never cost the others their restore (ticket #225).
|
|
2335
|
+
// Everything below runs per descriptor, and until this guard existed a
|
|
2336
|
+
// single throw — a launch that could not start, a worktree that moved —
|
|
2337
|
+
// left the loop entirely: every session AFTER the failing one stayed
|
|
2338
|
+
// unrestored, on every reconnect, with nothing said anywhere. The blast
|
|
2339
|
+
// radius of a broken session is now that session.
|
|
2340
|
+
try {
|
|
2341
|
+
// Live local session: statuses are fire-and-forget on the wire, so a
|
|
2342
|
+
// status reached while the WS was down is re-reported here (QA-96 F1).
|
|
2343
|
+
const tracked = this.sessions.get(descriptor.id);
|
|
2344
|
+
if (tracked) {
|
|
2345
|
+
// The session was resumed server-side while this runner was offline, and
|
|
2346
|
+
// the local copy is that same work. Adopt the new epoch BEFORE reporting:
|
|
2347
|
+
// the API drops frames stamped with an older one, so keeping ours would
|
|
2348
|
+
// make every status this session ever sends invisible — it would sit in
|
|
2349
|
+
// "waiting" while the agent worked.
|
|
2350
|
+
if (descriptor.epoch > tracked.epoch) {
|
|
2351
|
+
tracked.epoch = descriptor.epoch;
|
|
2352
|
+
tracked.descriptor = { ...tracked.descriptor, epoch: descriptor.epoch };
|
|
2246
2353
|
}
|
|
2354
|
+
// Ticket #196, QA-149 MAJOR-1. The pause is re-established HERE, and
|
|
2355
|
+
// this is the case that matters most: a dropped socket leaves the agent
|
|
2356
|
+
// process running, so «reconnect» is precisely when a session is
|
|
2357
|
+
// `tracked`. The first cut of #196 read `pausedUntil` only in the two
|
|
2358
|
+
// constructors of a NEW `RunningSession`, which meant it survived a
|
|
2359
|
+
// runner RESTART and not a reconnect — and a pause set while the socket
|
|
2360
|
+
// was down never arrived at all, because `session_pause` is
|
|
2361
|
+
// fire-and-forget with no outbox behind it.
|
|
2362
|
+
//
|
|
2363
|
+
// Both directions matter: the row may have gained a clock (hold now) or
|
|
2364
|
+
// lost one (release and send what was held). `applyPause` does both, and
|
|
2365
|
+
// it runs BEFORE `flushSessionOutbox` arrives from the API side.
|
|
2366
|
+
await this.applyPause(descriptor.id, descriptor.pausedUntil ?? null);
|
|
2367
|
+
this.reportStatus(descriptor.id, statusForReport(tracked), {
|
|
2368
|
+
costUsd: tracked.costUsd,
|
|
2369
|
+
...(tracked.branch ? { branch: tracked.branch } : {}),
|
|
2370
|
+
...(tracked.worktreePath ? { worktreePath: tracked.worktreePath } : {}),
|
|
2371
|
+
...(tracked.descriptor.providerSessionId
|
|
2372
|
+
? { providerSessionId: tracked.descriptor.providerSessionId }
|
|
2373
|
+
: {}),
|
|
2374
|
+
});
|
|
2247
2375
|
continue;
|
|
2248
2376
|
}
|
|
2249
|
-
|
|
2250
|
-
|
|
2251
|
-
|
|
2252
|
-
|
|
2253
|
-
|
|
2254
|
-
|
|
2255
|
-
|
|
2256
|
-
|
|
2257
|
-
|
|
2258
|
-
|
|
2259
|
-
|
|
2260
|
-
|
|
2261
|
-
session
|
|
2262
|
-
|
|
2263
|
-
|
|
2264
|
-
|
|
2265
|
-
|
|
2266
|
-
|
|
2267
|
-
|
|
2268
|
-
|
|
2269
|
-
|
|
2270
|
-
|
|
2271
|
-
|
|
2272
|
-
|
|
2273
|
-
|
|
2274
|
-
|
|
2275
|
-
|
|
2276
|
-
|
|
2277
|
-
|
|
2278
|
-
|
|
2279
|
-
lastPrompt: '',
|
|
2280
|
-
};
|
|
2281
|
-
running.journal.ensureSeqAbove(descriptor.lastSeq);
|
|
2282
|
-
// Anything the API handed us before the daemon stopped (session 9).
|
|
2283
|
-
running.pendingMessages.push(...running.journal.pending());
|
|
2284
|
-
this.sessions.set(descriptor.id, running);
|
|
2285
|
-
try {
|
|
2286
|
-
// A session being restored after a runner restart already has its
|
|
2287
|
-
// branch, so the API sends `CONTINUE` — the NEW guard inside would
|
|
2288
|
-
// otherwise fire on the runner's own previous work.
|
|
2289
|
-
const prepared = await this.prepareWorkspace(descriptor);
|
|
2290
|
-
running.branch = prepared.branch;
|
|
2291
|
-
running.worktreePath = prepared.worktreePath;
|
|
2292
|
-
if (prepared.baseSha)
|
|
2293
|
-
running.baseSha = prepared.baseSha;
|
|
2294
|
-
if (prepared.baseBranch)
|
|
2295
|
-
running.baseBranch = prepared.baseBranch;
|
|
2377
|
+
// Session that went terminal while we were offline: the journal keeps
|
|
2378
|
+
// the last reported status — replay it instead of resurrecting the
|
|
2379
|
+
// session as resumable (QA-96 F1).
|
|
2380
|
+
if (this.journals.exists(descriptor.id)) {
|
|
2381
|
+
const journal = this.journals.open(descriptor.id);
|
|
2382
|
+
const last = journal.lastStatus;
|
|
2383
|
+
// Only replay a terminal status from THIS life of the session. A
|
|
2384
|
+
// resumed session carries a higher epoch, and replaying the FAILED it
|
|
2385
|
+
// was resumed from would kill it again the moment the runner reconnects.
|
|
2386
|
+
// A journal written by a runner from before session 13 could hold a
|
|
2387
|
+
// `DONE` — it is no longer a status this runner may report, and
|
|
2388
|
+
// replaying one would close the session for the user. Drop it and let
|
|
2389
|
+
// the session be picked back up like any other.
|
|
2390
|
+
if (last &&
|
|
2391
|
+
isTerminal(last.status) &&
|
|
2392
|
+
last.status !== 'DONE' &&
|
|
2393
|
+
(last.epoch ?? 0) >= descriptor.epoch) {
|
|
2394
|
+
this.ws.send({
|
|
2395
|
+
type: 'session_status',
|
|
2396
|
+
sessionId: descriptor.id,
|
|
2397
|
+
status: last.status,
|
|
2398
|
+
...(last.extra ?? {}),
|
|
2399
|
+
// Guarded above to be >= the descriptor's epoch, so the API keeps it.
|
|
2400
|
+
...(last.epoch === undefined ? {} : { epoch: last.epoch }),
|
|
2401
|
+
});
|
|
2402
|
+
if (journal.unacked().length === 0) {
|
|
2403
|
+
this.journals.closeAndDelete(descriptor.id);
|
|
2404
|
+
}
|
|
2405
|
+
continue;
|
|
2406
|
+
}
|
|
2296
2407
|
}
|
|
2297
|
-
|
|
2298
|
-
this.
|
|
2299
|
-
errorMessage: `Failed to restore session worktree: ${String(error instanceof Error ? error.message : error).slice(0, 500)}`,
|
|
2300
|
-
});
|
|
2301
|
-
this.sessions.delete(descriptor.id);
|
|
2302
|
-
continue;
|
|
2408
|
+
if (descriptor.status === 'STARTING') {
|
|
2409
|
+
await this.startSession(descriptor);
|
|
2303
2410
|
}
|
|
2304
|
-
|
|
2305
|
-
|
|
2306
|
-
|
|
2307
|
-
|
|
2308
|
-
|
|
2309
|
-
|
|
2310
|
-
|
|
2311
|
-
|
|
2312
|
-
|
|
2313
|
-
|
|
2314
|
-
|
|
2315
|
-
|
|
2316
|
-
|
|
2317
|
-
|
|
2318
|
-
|
|
2319
|
-
|
|
2320
|
-
|
|
2321
|
-
|
|
2322
|
-
|
|
2323
|
-
|
|
2324
|
-
|
|
2325
|
-
|
|
2326
|
-
|
|
2327
|
-
|
|
2328
|
-
|
|
2329
|
-
|
|
2330
|
-
|
|
2331
|
-
|
|
2332
|
-
|
|
2333
|
-
|
|
2334
|
-
|
|
2335
|
-
|
|
2336
|
-
|
|
2337
|
-
|
|
2338
|
-
|
|
2339
|
-
|
|
2340
|
-
|
|
2341
|
-
|
|
2342
|
-
|
|
2343
|
-
|
|
2344
|
-
|
|
2345
|
-
|
|
2346
|
-
|
|
2347
|
-
|
|
2348
|
-
|
|
2411
|
+
else if (descriptor.providerSessionId) {
|
|
2412
|
+
// Runner restarted mid-session. The provider session is resumable —
|
|
2413
|
+
// park it until the user sends the next instruction. The map entry is
|
|
2414
|
+
// registered BEFORE the worktree await so a racing message is
|
|
2415
|
+
// buffered instead of dropped (QA-96 F3).
|
|
2416
|
+
const running = {
|
|
2417
|
+
descriptor,
|
|
2418
|
+
journal: this.journals.open(descriptor.id),
|
|
2419
|
+
session: null,
|
|
2420
|
+
lastReported: descriptor.status === 'REVIEW' ? 'REVIEW' : 'WAITING_INPUT',
|
|
2421
|
+
costUsd: descriptor.costUsd,
|
|
2422
|
+
costBaseUsd: descriptor.costUsd,
|
|
2423
|
+
stopRequested: false,
|
|
2424
|
+
parkRequested: false,
|
|
2425
|
+
pendingMessages: [],
|
|
2426
|
+
// Seed from the API, not 0: a runner restart used to hand the session
|
|
2427
|
+
// a full fresh budget silently.
|
|
2428
|
+
activeMs: descriptor.activeMsBase,
|
|
2429
|
+
extraBudgetMinutes: descriptor.extraBudgetMinutes,
|
|
2430
|
+
epoch: descriptor.epoch,
|
|
2431
|
+
openQuestions: new Set(),
|
|
2432
|
+
answeredAsks: new Set(),
|
|
2433
|
+
...pausedUntilOf(descriptor),
|
|
2434
|
+
mode: descriptor.mode,
|
|
2435
|
+
...(descriptor.model ? { model: descriptor.model } : {}),
|
|
2436
|
+
...(descriptor.effort ? { effort: descriptor.effort } : {}),
|
|
2437
|
+
lastPrompt: '',
|
|
2438
|
+
};
|
|
2439
|
+
running.journal.ensureSeqAbove(descriptor.lastSeq);
|
|
2440
|
+
// Anything the API handed us before the daemon stopped (session 9).
|
|
2441
|
+
running.pendingMessages.push(...running.journal.pending());
|
|
2442
|
+
this.sessions.set(descriptor.id, running);
|
|
2443
|
+
try {
|
|
2444
|
+
// A session being restored after a runner restart already has its
|
|
2445
|
+
// branch, so the API sends `CONTINUE` — the NEW guard inside would
|
|
2446
|
+
// otherwise fire on the runner's own previous work.
|
|
2447
|
+
const prepared = await this.prepareWorkspace(descriptor);
|
|
2448
|
+
running.branch = prepared.branch;
|
|
2449
|
+
running.worktreePath = prepared.worktreePath;
|
|
2450
|
+
if (prepared.baseSha)
|
|
2451
|
+
running.baseSha = prepared.baseSha;
|
|
2452
|
+
if (prepared.baseBranch)
|
|
2453
|
+
running.baseBranch = prepared.baseBranch;
|
|
2454
|
+
}
|
|
2455
|
+
catch (error) {
|
|
2456
|
+
this.reportStatus(descriptor.id, 'FAILED', {
|
|
2457
|
+
errorMessage: `Failed to restore session worktree: ${String(error instanceof Error ? error.message : error).slice(0, 500)}`,
|
|
2458
|
+
});
|
|
2459
|
+
this.sessions.delete(descriptor.id);
|
|
2349
2460
|
continue;
|
|
2350
2461
|
}
|
|
2351
|
-
|
|
2352
|
-
|
|
2462
|
+
/**
|
|
2463
|
+
* Was a turn actually in flight when the process died?
|
|
2464
|
+
*
|
|
2465
|
+
* `RUNNING` and `WAITING_PERMISSION` are the mid-turn statuses; the
|
|
2466
|
+
* others mean the agent was already waiting for a human, and there is
|
|
2467
|
+
* nothing to continue. REVIEW is deliberately excluded — the work is
|
|
2468
|
+
* finished and waiting to be looked at.
|
|
2469
|
+
*/
|
|
2470
|
+
const wasMidTurn = descriptor.status === 'RUNNING' || descriptor.status === 'WAITING_PERMISSION';
|
|
2471
|
+
const resumeId = descriptor.providerSessionId;
|
|
2472
|
+
// Ticket #177: `resumeId` is required, not merely nice to have. Without
|
|
2473
|
+
// it the relaunch starts a FRESH conversation, and `AUTO_RESUME_PROMPT`
|
|
2474
|
+
// — "continue from where you stopped, re-check what you were in the
|
|
2475
|
+
// middle of" — would be addressed to an agent that remembers none of
|
|
2476
|
+
// it. A process killed before it reported its session id (the SIGABRT
|
|
2477
|
+
// this ticket came from) leaves the row in exactly that state.
|
|
2478
|
+
// Ticket #196: a paused session is never continued automatically. The
|
|
2479
|
+
// row still says RUNNING — a pause interrupts the turn but is not a
|
|
2480
|
+
// status — so without this the reconnect would read «mid-turn» and
|
|
2481
|
+
// relaunch the agent with «continue from where you stopped», which is
|
|
2482
|
+
// the exact opposite of what the clock was set for.
|
|
2483
|
+
const willContinue = wasMidTurn &&
|
|
2484
|
+
!Supervisor.isPaused(running) &&
|
|
2485
|
+
Boolean(resumeId) &&
|
|
2486
|
+
claimAutoResume(descriptor.id);
|
|
2487
|
+
// The note stays either way (owner's call): an interruption is a fact
|
|
2488
|
+
// about the session and must not disappear just because we recovered
|
|
2489
|
+
// from it. Only the instruction at the end changes — telling someone to
|
|
2490
|
+
// send a message while the agent is already working again would be a lie.
|
|
2353
2491
|
this.sendEvent(running, 'system_note', {
|
|
2354
|
-
text:
|
|
2492
|
+
text: willContinue
|
|
2493
|
+
? 'Runner reconnected. The session was resumed — continuing the interrupted turn.'
|
|
2494
|
+
: resumeId
|
|
2495
|
+
? 'Runner reconnected. The session was resumed — send a message to continue.'
|
|
2496
|
+
: 'Runner reconnected, but the agent never got as far as naming its conversation, so it starts this one over. Your files, your branch and everything above are untouched.',
|
|
2355
2497
|
});
|
|
2498
|
+
if (willContinue) {
|
|
2499
|
+
// Resumed through the PROVIDER session, so the agent keeps its whole
|
|
2500
|
+
// conversation; the prompt is only the nudge a human would otherwise
|
|
2501
|
+
// have to type. Exactly what «продолжай» did by hand — no new class of
|
|
2502
|
+
// risk, and the same ceiling protects against a crash loop doing it
|
|
2503
|
+
// forever (see `auto-resume.ts`).
|
|
2504
|
+
const continued = this.launchAgent(running, AUTO_RESUME_PROMPT, resumeId);
|
|
2505
|
+
if (continued.ok) {
|
|
2506
|
+
this.reportStatus(descriptor.id, 'RUNNING', {});
|
|
2507
|
+
this.flushPendingMessages(running);
|
|
2508
|
+
continue;
|
|
2509
|
+
}
|
|
2510
|
+
// Could not start — an exhausted budget, or a launch that crashed
|
|
2511
|
+
// (ticket #225: this is the exact line the incident died on, and the
|
|
2512
|
+
// throw took the WHOLE restore loop with it). Fall through to the old
|
|
2513
|
+
// behaviour and say so honestly; a crash has already put its own
|
|
2514
|
+
// reason in the feed, so this note would only repeat it.
|
|
2515
|
+
if (continued.reason === 'refused') {
|
|
2516
|
+
this.sendEvent(running, 'system_note', {
|
|
2517
|
+
text: 'Could not continue automatically — send a message to pick the work back up.',
|
|
2518
|
+
});
|
|
2519
|
+
}
|
|
2520
|
+
}
|
|
2521
|
+
// REVIEW stays REVIEW (WAITING_INPUT is not reachable from it);
|
|
2522
|
+
// mid-turn statuses are downgraded to "waiting for the user".
|
|
2523
|
+
if (descriptor.status !== 'REVIEW') {
|
|
2524
|
+
this.reportStatus(descriptor.id, 'WAITING_INPUT', {});
|
|
2525
|
+
}
|
|
2526
|
+
this.flushPendingMessages(running);
|
|
2356
2527
|
}
|
|
2357
|
-
|
|
2358
|
-
|
|
2359
|
-
|
|
2360
|
-
|
|
2528
|
+
else if (descriptor.status === 'WAITING_INPUT') {
|
|
2529
|
+
// A session that never had a turn (a free session still waiting for
|
|
2530
|
+
// its first message) has nothing to resume — just bring the agent back
|
|
2531
|
+
// up and keep waiting, instead of failing the session.
|
|
2532
|
+
await this.startSession({ ...descriptor, status: 'STARTING', prompt: '' });
|
|
2533
|
+
}
|
|
2534
|
+
else {
|
|
2535
|
+
this.reportStatus(descriptor.id, 'FAILED', {
|
|
2536
|
+
errorMessage: 'Runner restarted and this session cannot be resumed',
|
|
2537
|
+
});
|
|
2361
2538
|
}
|
|
2362
|
-
this.flushPendingMessages(running);
|
|
2363
|
-
}
|
|
2364
|
-
else if (descriptor.status === 'WAITING_INPUT') {
|
|
2365
|
-
// A session that never had a turn (a free session still waiting for
|
|
2366
|
-
// its first message) has nothing to resume — just bring the agent back
|
|
2367
|
-
// up and keep waiting, instead of failing the session.
|
|
2368
|
-
await this.startSession({ ...descriptor, status: 'STARTING', prompt: '' });
|
|
2369
2539
|
}
|
|
2370
|
-
|
|
2371
|
-
|
|
2372
|
-
|
|
2540
|
+
catch (error) {
|
|
2541
|
+
log.error('supervisor: session could not be restored after reconnect', {
|
|
2542
|
+
sessionId: descriptor.id,
|
|
2543
|
+
error: error instanceof Error ? (error.stack ?? error.message) : String(error),
|
|
2373
2544
|
});
|
|
2545
|
+
// Said on the session it belongs to, not only in journald. FAILED is
|
|
2546
|
+
// the honest word: this runner is not going to run it as things stand,
|
|
2547
|
+
// and «Продолжить» is what asks it to try again.
|
|
2548
|
+
//
|
|
2549
|
+
// Guarded in turn, and not out of superstition: reporting a status
|
|
2550
|
+
// WRITES to that session's journal, so the most likely reason the body
|
|
2551
|
+
// above failed — this session's own file — would fail the handler the
|
|
2552
|
+
// same way and take the loop with it after all. The other sessions
|
|
2553
|
+
// matter more than this one's status frame.
|
|
2554
|
+
try {
|
|
2555
|
+
this.reportStatus(descriptor.id, 'FAILED', {
|
|
2556
|
+
errorMessage: `Runner could not restore this session: ${maskSecretText(error)}`,
|
|
2557
|
+
});
|
|
2558
|
+
}
|
|
2559
|
+
catch (reportError) {
|
|
2560
|
+
log.error('supervisor: could not even report the failed restore', {
|
|
2561
|
+
sessionId: descriptor.id,
|
|
2562
|
+
error: String(reportError),
|
|
2563
|
+
});
|
|
2564
|
+
}
|
|
2374
2565
|
}
|
|
2375
2566
|
}
|
|
2376
2567
|
// Redelivery is done — anything still on disk from long-finished sessions
|