@bridge4dev/runner 0.37.0 → 0.39.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -51,6 +51,7 @@ function gitPolicyOf(descriptor) {
51
51
  : {}),
52
52
  };
53
53
  }
54
+ const LAUNCH_REFUSED = { ok: false, reason: 'refused' };
54
55
  export class Supervisor {
55
56
  ws;
56
57
  opts;
@@ -367,7 +368,11 @@ export class Supervisor {
367
368
  await this.captureCheckpoint(running, 'TURN', 0);
368
369
  if (this.isStale(running))
369
370
  return;
370
- this.launchAgent(running, composeInitialPrompt(descriptor), null);
371
+ // A launch that failed has already said so and reported a status the
372
+ // person can act on. Flushing the queue into it would only walk the same
373
+ // failure again, once per waiting message (ticket #225).
374
+ if (!this.launchAgent(running, composeInitialPrompt(descriptor), null).ok)
375
+ return;
371
376
  }
372
377
  else {
373
378
  if (descriptor.epoch > 0) {
@@ -425,15 +430,23 @@ export class Supervisor {
425
430
  * free CHAT session with no prompt at all (the agent boots, reports its
426
431
  * capabilities and waits for the first message).
427
432
  *
428
- * Returns whether an agent process actually started: a caller holding a user
429
- * message needs to know, because a refused launch means the message has to
430
- * stay queued rather than be marked delivered (session 9).
433
+ * Returns what came of it: a caller holding a user message needs to know,
434
+ * because anything but `ok` means the message has to stay queued rather than
435
+ * be marked delivered (session 9).
436
+ *
437
+ * NEVER throws (ticket #225). Everything from reading the project prompt to
438
+ * the adapter's own constructor runs inside one guard, because the caller
439
+ * chain above cannot tell the difference between «did not start» and
440
+ * «threw»: the message path swallows the exception into a log line, and the
441
+ * reconnect path lets it abort the restore of every OTHER session on the
442
+ * machine. A launch that fails is a state this session reports, not an
443
+ * exception somebody else has to remember to catch.
431
444
  */
432
445
  launchAgent(running, prompt, resumeId) {
433
446
  const { descriptor } = running;
434
447
  const adapter = this.opts.adapters[descriptor.agent];
435
448
  if (!adapter || !running.worktreePath || !running.branch)
436
- return false;
449
+ return LAUNCH_REFUSED;
437
450
  // An exhausted USD budget must not relaunch $0.01-floor processes (QA-96 F4).
438
451
  // Codex reports no cost at all, so its costUsd never leaves 0 — gating on it
439
452
  // would be a limit that can never fire while the UI shows $0.00. Those
@@ -455,7 +468,7 @@ export class Supervisor {
455
468
  errorMessage: `Session budget ($${descriptor.workspace.budgetUsd}) is exhausted`,
456
469
  });
457
470
  this.sessions.delete(descriptor.id);
458
- return false;
471
+ return LAUNCH_REFUSED;
459
472
  }
460
473
  // The time budget is already spent: relaunching would burn a process for
461
474
  // nothing and immediately stop again.
@@ -464,7 +477,7 @@ export class Supervisor {
464
477
  level: 'warn',
465
478
  text: 'The time budget is used up — press «Continue» to give the agent more time.',
466
479
  });
467
- return false;
480
+ return LAUNCH_REFUSED;
468
481
  }
469
482
  running.lastPrompt = prompt;
470
483
  // Facts about this session only, plus the one file the project named. The
@@ -477,49 +490,60 @@ export class Supervisor {
477
490
  // «loaded» nor «unchanged».
478
491
  const rewinding = Boolean(running.rewindAnchor?.agentSession);
479
492
  const resuming = rewinding || Boolean(resumeId);
480
- const agentPrompt = this.resolveAgentPrompt(running, resuming && !rewinding);
481
- const workspaceContext = composeWorkspaceContext(descriptor, agentPrompt?.text);
493
+ // Held so the guard below can put it back: a rewind the person asked for
494
+ // must not be silently forgotten because the process that was going to
495
+ // apply it never started (ticket #225). The feed has already told them the
496
+ // conversation was cut.
482
497
  const rewind = running.rewindAnchor;
483
- delete running.rewindAnchor;
484
- // A rewind resumes the conversation the POINT names, which is not always
485
- // the one this session is on now: rewinding twice, or rewinding the first
486
- // message after a rewind, both reach back into the thread the fork came
487
- // from. Its transcript is still on disk — that is what makes the point
488
- // usable at all.
489
- const resumeTarget = rewind ? rewind.agentSession : resumeId;
490
- running.session = adapter.startSession({
491
- sessionId: descriptor.id,
492
- cwd: running.worktreePath,
493
- ...(prompt ? { prompt } : {}),
494
- ...(workspaceContext ? { workspaceContext } : {}),
495
- // Only when it was actually read: layer 1 must refuse writes to the file
496
- // this process was given, not to a path it was merely told about.
497
- ...(agentPrompt ? { agentPromptFile: agentPrompt.absPath } : {}),
498
- trustMode: descriptor.workspace.trustMode,
499
- ...(descriptor.workspace.agentAutoCommit === undefined
500
- ? {}
501
- : { agentAutoCommit: descriptor.workspace.agentAutoCommit }),
502
- // Session 18. Always present, even when every field inside it is absent:
503
- // an absent OBJECT and an object of absent fields resolve identically
504
- // (`resolveGitPolicy` gives both the restrictive reading), and passing it
505
- // unconditionally keeps one code path instead of two.
506
- gitPolicy: gitPolicyOf(descriptor),
507
- mode: running.mode,
508
- ...(running.model ? { model: running.model } : {}),
509
- ...(running.effort ? { effort: running.effort } : {}),
510
- ...(resumeTarget ? { resumeProviderSessionId: resumeTarget } : {}),
511
- // Ticket #126: a conversation rewind takes effect exactly here, on the
512
- // next process this session starts. Consumed rather than kept — a rewind
513
- // is one event, not a standing setting, and re-applying it on a later
514
- // relaunch would silently throw away everything said since.
515
- ...(rewind ? { resumeAtAnchor: rewind.anchor } : {}),
516
- // Descriptor MCP (auto-issued per-workspace key) wins over config.toml.
517
- ...((descriptor.mcp ?? this.opts.mcp) ? { mcp: descriptor.mcp ?? this.opts.mcp } : {}),
518
- // The SDK budget is per-process; hand the RESIDUAL session budget down.
519
- ...(reportsCost(descriptor.agent) && descriptor.workspace.budgetUsd !== null
520
- ? { maxBudgetUsd: Math.max(0.01, descriptor.workspace.budgetUsd - running.costBaseUsd) }
521
- : {}),
522
- });
498
+ try {
499
+ const agentPrompt = this.resolveAgentPrompt(running, resuming && !rewinding);
500
+ const workspaceContext = composeWorkspaceContext(descriptor, agentPrompt?.text);
501
+ delete running.rewindAnchor;
502
+ // A rewind resumes the conversation the POINT names, which is not always
503
+ // the one this session is on now: rewinding twice, or rewinding the first
504
+ // message after a rewind, both reach back into the thread the fork came
505
+ // from. Its transcript is still on disk — that is what makes the point
506
+ // usable at all.
507
+ const resumeTarget = rewind ? rewind.agentSession : resumeId;
508
+ running.session = adapter.startSession({
509
+ sessionId: descriptor.id,
510
+ cwd: running.worktreePath,
511
+ ...(prompt ? { prompt } : {}),
512
+ ...(workspaceContext ? { workspaceContext } : {}),
513
+ // Only when it was actually read: layer 1 must refuse writes to the file
514
+ // this process was given, not to a path it was merely told about.
515
+ ...(agentPrompt ? { agentPromptFile: agentPrompt.absPath } : {}),
516
+ trustMode: descriptor.workspace.trustMode,
517
+ ...(descriptor.workspace.agentAutoCommit === undefined
518
+ ? {}
519
+ : { agentAutoCommit: descriptor.workspace.agentAutoCommit }),
520
+ // Session 18. Always present, even when every field inside it is absent:
521
+ // an absent OBJECT and an object of absent fields resolve identically
522
+ // (`resolveGitPolicy` gives both the restrictive reading), and passing it
523
+ // unconditionally keeps one code path instead of two.
524
+ gitPolicy: gitPolicyOf(descriptor),
525
+ mode: running.mode,
526
+ ...(running.model ? { model: running.model } : {}),
527
+ ...(running.effort ? { effort: running.effort } : {}),
528
+ ...(resumeTarget ? { resumeProviderSessionId: resumeTarget } : {}),
529
+ // Ticket #126: a conversation rewind takes effect exactly here, on the
530
+ // next process this session starts. Consumed rather than kept — a rewind
531
+ // is one event, not a standing setting, and re-applying it on a later
532
+ // relaunch would silently throw away everything said since.
533
+ ...(rewind ? { resumeAtAnchor: rewind.anchor } : {}),
534
+ // Descriptor MCP (auto-issued per-workspace key) wins over config.toml.
535
+ ...((descriptor.mcp ?? this.opts.mcp) ? { mcp: descriptor.mcp ?? this.opts.mcp } : {}),
536
+ // The SDK budget is per-process; hand the RESIDUAL session budget down.
537
+ ...(reportsCost(descriptor.agent) && descriptor.workspace.budgetUsd !== null
538
+ ? { maxBudgetUsd: Math.max(0.01, descriptor.workspace.budgetUsd - running.costBaseUsd) }
539
+ : {}),
540
+ });
541
+ }
542
+ catch (error) {
543
+ if (rewind)
544
+ running.rewindAnchor = rewind;
545
+ return this.launchCrashed(running, error);
546
+ }
523
547
  // No prompt → nothing is running yet: the agent is up and waiting for the
524
548
  // user's first message (free CHAT session). reportStatus drives the budget
525
549
  // clock, so this call is also what starts (or does not start) billing.
@@ -527,6 +551,8 @@ export class Supervisor {
527
551
  branch: running.branch,
528
552
  worktreePath: running.worktreePath,
529
553
  });
554
+ // The object exists; the PROCESS still has to prove it does (ticket #225).
555
+ this.watchForFirstSignOfLife(running);
530
556
  // Ticket #196: a Stop or a pause that arrived while this process was coming
531
557
  // up was dropped on the floor — `interruptSession` returns early when there
532
558
  // is no adapter yet, and the window covers preparing the worktree and
@@ -541,7 +567,99 @@ export class Supervisor {
541
567
  }));
542
568
  }
543
569
  void this.pumpEvents(running);
544
- return true;
570
+ return { ok: true };
571
+ }
572
+ /**
573
+ * The agent process could not be started at all (ticket #225).
574
+ *
575
+ * Three things have to happen here, and until this ticket none of them did:
576
+ * the reason is said WHERE THE PERSON IS LOOKING (an `error` event is the
577
+ * feed's red line), the session stops claiming to be working, and the stack
578
+ * reaches journald for whoever has to fix the machine. The status is the
579
+ * honest one for a session with no process — the agent is not running, and a
580
+ * session left in `RUNNING` shows a stop button for a turn that does not
581
+ * exist.
582
+ *
583
+ * Never rethrows: this IS the handling. The caller gets `crashed` and decides
584
+ * what to do with the message it was holding.
585
+ */
586
+ launchCrashed(running, error) {
587
+ const { descriptor } = running;
588
+ running.session = null;
589
+ log.error('supervisor: the agent process could not be started', {
590
+ sessionId: descriptor.id,
591
+ agent: descriptor.agent,
592
+ error: error instanceof Error ? (error.stack ?? error.message) : String(error),
593
+ });
594
+ const reason = maskSecretText(error);
595
+ this.sendEvent(running, 'error', {
596
+ message: `${AGENT_LABELS[descriptor.agent] ?? descriptor.agent} could not be started on this server: ${reason}`,
597
+ });
598
+ // A session that never began is FAILED — the same answer the neighbouring
599
+ // startup failures (no adapter, no worktree) already give. Everything else
600
+ // goes back to waiting for a human, which is the state «press Continue» is
601
+ // meaningful in; REVIEW keeps its row in the dashboard.
602
+ const status = running.lastReported === 'STARTING'
603
+ ? 'FAILED'
604
+ : running.lastReported === 'REVIEW'
605
+ ? 'REVIEW'
606
+ : 'WAITING_INPUT';
607
+ this.reportStatus(descriptor.id, status, {
608
+ costUsd: running.costUsd,
609
+ activeMs: running.activeMs,
610
+ errorMessage: `Agent process failed to start: ${reason}`,
611
+ });
612
+ // A FAILED session is over, and an entry left in the map would hold one of
613
+ // the runner's few slots for a process that never existed — `ensureCapacity`
614
+ // counts entries, not processes. Dropped exactly like the other startup
615
+ // failures do it. The parked states keep their entry on purpose: that is
616
+ // what «Продолжить» picks back up.
617
+ if (status === 'FAILED')
618
+ this.sessions.delete(descriptor.id);
619
+ return { ok: false, reason: 'crashed' };
620
+ }
621
+ /** How long a freshly started agent may say nothing before we say so. */
622
+ static STARTUP_SILENCE_MS = 60_000;
623
+ /**
624
+ * Watch for the first word out of a process we just started (ticket #225).
625
+ *
626
+ * «The adapter object exists» is not «the agent is running». A CLI that hangs
627
+ * before its first frame — a stuck hook, an MCP server that never answers, a
628
+ * transcript it cannot read — produces no events, no error and no exit, and
629
+ * the session sits in `RUNNING` forever. On a healthy launch the first event
630
+ * arrives in about two seconds, so a minute of silence is not a slow start,
631
+ * it is something worth saying out loud.
632
+ *
633
+ * Says it and stops there: no kill. A long conversation has the right to boot
634
+ * slowly, and killing it would cost the person the very turn they are waiting
635
+ * for.
636
+ */
637
+ watchForFirstSignOfLife(running) {
638
+ this.clearStartupWatch(running);
639
+ running.heardFromAgent = false;
640
+ const silentMs = this.opts.startupSilenceMs ?? Supervisor.STARTUP_SILENCE_MS;
641
+ const timer = setTimeout(() => {
642
+ delete running.startupTimer;
643
+ if (running.heardFromAgent || !running.session || this.isStale(running))
644
+ return;
645
+ log.warn('supervisor: the agent process has said nothing since it started', {
646
+ sessionId: running.descriptor.id,
647
+ silentMs,
648
+ });
649
+ this.sendEvent(running, 'notice', {
650
+ level: 'warn',
651
+ text: 'The agent process started but has not said a word for a minute. It may still be loading a long conversation. If nothing happens, press «Stop» and then «Continue» — and if that does not help either, this server needs a look.',
652
+ });
653
+ }, silentMs);
654
+ timer.unref?.();
655
+ running.startupTimer = timer;
656
+ }
657
+ /** The process spoke, or went away — either way the watch is over. */
658
+ clearStartupWatch(running) {
659
+ if (running.startupTimer) {
660
+ clearTimeout(running.startupTimer);
661
+ delete running.startupTimer;
662
+ }
545
663
  }
546
664
  /**
547
665
  * The project's own prompt file, read fresh for THIS agent process.
@@ -794,6 +912,9 @@ export class Supervisor {
794
912
  delete running.activeSince;
795
913
  }
796
914
  this.clearBudgetTimers(running);
915
+ // The process this watch was armed for is gone; whatever it did or did not
916
+ // say, there is nothing left to wait for (ticket #225).
917
+ this.clearStartupWatch(running);
797
918
  // Stale-resume recovery: relaunch once without a resume id.
798
919
  if (running.freshRetry && !running.stopRequested) {
799
920
  const { prompt } = running.freshRetry;
@@ -851,7 +972,8 @@ export class Supervisor {
851
972
  this.sendEvent(running, 'settings', { mode });
852
973
  // Empty prompt: the agent boots, reports its capabilities and waits, the
853
974
  // same as a free CHAT session. It must NOT start a turn of its own here.
854
- if (this.launchAgent(running, '', running.descriptor.providerSessionId)) {
975
+ const relaunched = this.launchAgent(running, '', running.descriptor.providerSessionId);
976
+ if (relaunched.ok) {
855
977
  // Anything typed during the park window is waiting on disk (see
856
978
  // `deliverMessage`), and the new process is the one that can take it.
857
979
  this.flushPendingMessages(running);
@@ -863,12 +985,17 @@ export class Supervisor {
863
985
  }
864
986
  return;
865
987
  }
866
- // The agent did not start — an exhausted budget is the only way here. The
867
- // session stays parked and resumable rather than silently disappearing.
868
- this.reportStatus(descriptor.id, statusForReport(running), {
869
- costUsd: running.costUsd,
870
- activeMs: running.activeMs,
871
- });
988
+ // The agent did not start — an exhausted budget, or a launch that crashed
989
+ // (ticket #225). The session stays parked and resumable rather than
990
+ // silently disappearing. A crash has already reported its own status and
991
+ // reason; re-reporting `statusForReport` here would overwrite them with
992
+ // the state the session was in before it failed.
993
+ if (relaunched.reason === 'refused') {
994
+ this.reportStatus(descriptor.id, statusForReport(running), {
995
+ costUsd: running.costUsd,
996
+ activeMs: running.activeMs,
997
+ });
998
+ }
872
999
  this.drainSessionsWaitingForCapacity();
873
1000
  return;
874
1001
  }
@@ -1153,6 +1280,15 @@ export class Supervisor {
1153
1280
  }
1154
1281
  forwardEvent(running, event) {
1155
1282
  const { descriptor } = running;
1283
+ // Ticket #225: ANY event is the process proving it came up — its own
1284
+ // capabilities probe answers within about two seconds of a healthy launch,
1285
+ // long before the agent says anything a person would read. That is a
1286
+ // deliberately weaker bar than «the agent is working» below: what this
1287
+ // watch is for is a CLI that never boots at all.
1288
+ if (!running.heardFromAgent) {
1289
+ running.heardFromAgent = true;
1290
+ this.clearStartupWatch(running);
1291
+ }
1156
1292
  // Anything below that is the agent talking means the agent is working. Read
1157
1293
  // before the switch so every such case gets it, including the ones added
1158
1294
  // after this line was written.
@@ -1724,11 +1860,13 @@ export class Supervisor {
1724
1860
  // A person typing into the session is the clearest signal that the work is
1725
1861
  // back on track, so the automatic-continuation allowance starts over.
1726
1862
  clearAutoResume(running.descriptor.id);
1727
- if (this.launchAgent(running, text, running.descriptor.providerSessionId)) {
1863
+ if (this.launchAgent(running, text, running.descriptor.providerSessionId).ok) {
1728
1864
  settle();
1729
1865
  return;
1730
1866
  }
1731
- // The agent did not start (an exhausted budget is the only way here). The
1867
+ // The agent did not start (an exhausted budget, or a launch that crashed —
1868
+ // ticket #225: it used to be swallowed into a log line, and the words the
1869
+ // person typed were retired from disk as though an agent had them). The
1732
1870
  // instruction stays on disk, so «Продолжить» — which is what raises the
1733
1871
  // budget — carries it to the agent instead of dropping it.
1734
1872
  this.requeue(running, held, text, originSeq);
@@ -2167,210 +2305,263 @@ export class Supervisor {
2167
2305
  }
2168
2306
  }
2169
2307
  // Redeliver unacked events for every persisted journal (at-least-once).
2308
+ //
2309
+ // Per file, because one that cannot be read must not cost the whole
2310
+ // reconnect (ticket #225): this runs BEFORE a single session is looked at,
2311
+ // so a throw here means nothing is restored at all — no launch, no status,
2312
+ // no note, on every reconnect for as long as the file stays broken.
2170
2313
  for (const sessionId of this.journals.persistedSessionIds()) {
2171
- const journal = this.journals.open(sessionId);
2172
- for (const event of journal.unacked()) {
2173
- this.ws.send({
2174
- type: 'event',
2314
+ try {
2315
+ const journal = this.journals.open(sessionId);
2316
+ for (const event of journal.unacked()) {
2317
+ this.ws.send({
2318
+ type: 'event',
2319
+ sessionId,
2320
+ seq: event.seq,
2321
+ eventType: event.eventType,
2322
+ payload: event.payload,
2323
+ });
2324
+ }
2325
+ }
2326
+ catch (error) {
2327
+ log.error('supervisor: journal could not be replayed', {
2175
2328
  sessionId,
2176
- seq: event.seq,
2177
- eventType: event.eventType,
2178
- payload: event.payload,
2329
+ error: String(error),
2179
2330
  });
2180
2331
  }
2181
2332
  }
2182
2333
  for (const descriptor of descriptors) {
2183
- // Live local session: statuses are fire-and-forget on the wire, so a
2184
- // status reached while the WS was down is re-reported here (QA-96 F1).
2185
- const tracked = this.sessions.get(descriptor.id);
2186
- if (tracked) {
2187
- // The session was resumed server-side while this runner was offline, and
2188
- // the local copy is that same work. Adopt the new epoch BEFORE reporting:
2189
- // the API drops frames stamped with an older one, so keeping ours would
2190
- // make every status this session ever sends invisible — it would sit in
2191
- // "waiting" while the agent worked.
2192
- if (descriptor.epoch > tracked.epoch) {
2193
- tracked.epoch = descriptor.epoch;
2194
- tracked.descriptor = { ...tracked.descriptor, epoch: descriptor.epoch };
2195
- }
2196
- // Ticket #196, QA-149 MAJOR-1. The pause is re-established HERE, and
2197
- // this is the case that matters most: a dropped socket leaves the agent
2198
- // process running, so «reconnect» is precisely when a session is
2199
- // `tracked`. The first cut of #196 read `pausedUntil` only in the two
2200
- // constructors of a NEW `RunningSession`, which meant it survived a
2201
- // runner RESTART and not a reconnect — and a pause set while the socket
2202
- // was down never arrived at all, because `session_pause` is
2203
- // fire-and-forget with no outbox behind it.
2204
- //
2205
- // Both directions matter: the row may have gained a clock (hold now) or
2206
- // lost one (release and send what was held). `applyPause` does both, and
2207
- // it runs BEFORE `flushSessionOutbox` arrives from the API side.
2208
- await this.applyPause(descriptor.id, descriptor.pausedUntil ?? null);
2209
- this.reportStatus(descriptor.id, statusForReport(tracked), {
2210
- costUsd: tracked.costUsd,
2211
- ...(tracked.branch ? { branch: tracked.branch } : {}),
2212
- ...(tracked.worktreePath ? { worktreePath: tracked.worktreePath } : {}),
2213
- ...(tracked.descriptor.providerSessionId
2214
- ? { providerSessionId: tracked.descriptor.providerSessionId }
2215
- : {}),
2216
- });
2217
- continue;
2218
- }
2219
- // Session that went terminal while we were offline: the journal keeps
2220
- // the last reported status — replay it instead of resurrecting the
2221
- // session as resumable (QA-96 F1).
2222
- if (this.journals.exists(descriptor.id)) {
2223
- const journal = this.journals.open(descriptor.id);
2224
- const last = journal.lastStatus;
2225
- // Only replay a terminal status from THIS life of the session. A
2226
- // resumed session carries a higher epoch, and replaying the FAILED it
2227
- // was resumed from would kill it again the moment the runner reconnects.
2228
- // A journal written by a runner from before session 13 could hold a
2229
- // `DONE` — it is no longer a status this runner may report, and
2230
- // replaying one would close the session for the user. Drop it and let
2231
- // the session be picked back up like any other.
2232
- if (last &&
2233
- isTerminal(last.status) &&
2234
- last.status !== 'DONE' &&
2235
- (last.epoch ?? 0) >= descriptor.epoch) {
2236
- this.ws.send({
2237
- type: 'session_status',
2238
- sessionId: descriptor.id,
2239
- status: last.status,
2240
- ...(last.extra ?? {}),
2241
- // Guarded above to be >= the descriptor's epoch, so the API keeps it.
2242
- ...(last.epoch === undefined ? {} : { epoch: last.epoch }),
2243
- });
2244
- if (journal.unacked().length === 0) {
2245
- this.journals.closeAndDelete(descriptor.id);
2334
+ // One session must never cost the others their restore (ticket #225).
2335
+ // Everything below runs per descriptor, and until this guard existed a
2336
+ // single throw — a launch that could not start, a worktree that moved —
2337
+ // left the loop entirely: every session AFTER the failing one stayed
2338
+ // unrestored, on every reconnect, with nothing said anywhere. The blast
2339
+ // radius of a broken session is now that session.
2340
+ try {
2341
+ // Live local session: statuses are fire-and-forget on the wire, so a
2342
+ // status reached while the WS was down is re-reported here (QA-96 F1).
2343
+ const tracked = this.sessions.get(descriptor.id);
2344
+ if (tracked) {
2345
+ // The session was resumed server-side while this runner was offline, and
2346
+ // the local copy is that same work. Adopt the new epoch BEFORE reporting:
2347
+ // the API drops frames stamped with an older one, so keeping ours would
2348
+ // make every status this session ever sends invisible — it would sit in
2349
+ // "waiting" while the agent worked.
2350
+ if (descriptor.epoch > tracked.epoch) {
2351
+ tracked.epoch = descriptor.epoch;
2352
+ tracked.descriptor = { ...tracked.descriptor, epoch: descriptor.epoch };
2246
2353
  }
2354
+ // Ticket #196, QA-149 MAJOR-1. The pause is re-established HERE, and
2355
+ // this is the case that matters most: a dropped socket leaves the agent
2356
+ // process running, so «reconnect» is precisely when a session is
2357
+ // `tracked`. The first cut of #196 read `pausedUntil` only in the two
2358
+ // constructors of a NEW `RunningSession`, which meant it survived a
2359
+ // runner RESTART and not a reconnect — and a pause set while the socket
2360
+ // was down never arrived at all, because `session_pause` is
2361
+ // fire-and-forget with no outbox behind it.
2362
+ //
2363
+ // Both directions matter: the row may have gained a clock (hold now) or
2364
+ // lost one (release and send what was held). `applyPause` does both, and
2365
+ // it runs BEFORE `flushSessionOutbox` arrives from the API side.
2366
+ await this.applyPause(descriptor.id, descriptor.pausedUntil ?? null);
2367
+ this.reportStatus(descriptor.id, statusForReport(tracked), {
2368
+ costUsd: tracked.costUsd,
2369
+ ...(tracked.branch ? { branch: tracked.branch } : {}),
2370
+ ...(tracked.worktreePath ? { worktreePath: tracked.worktreePath } : {}),
2371
+ ...(tracked.descriptor.providerSessionId
2372
+ ? { providerSessionId: tracked.descriptor.providerSessionId }
2373
+ : {}),
2374
+ });
2247
2375
  continue;
2248
2376
  }
2249
- }
2250
- if (descriptor.status === 'STARTING') {
2251
- await this.startSession(descriptor);
2252
- }
2253
- else if (descriptor.providerSessionId) {
2254
- // Runner restarted mid-session. The provider session is resumable —
2255
- // park it until the user sends the next instruction. The map entry is
2256
- // registered BEFORE the worktree await so a racing message is
2257
- // buffered instead of dropped (QA-96 F3).
2258
- const running = {
2259
- descriptor,
2260
- journal: this.journals.open(descriptor.id),
2261
- session: null,
2262
- lastReported: descriptor.status === 'REVIEW' ? 'REVIEW' : 'WAITING_INPUT',
2263
- costUsd: descriptor.costUsd,
2264
- costBaseUsd: descriptor.costUsd,
2265
- stopRequested: false,
2266
- parkRequested: false,
2267
- pendingMessages: [],
2268
- // Seed from the API, not 0: a runner restart used to hand the session
2269
- // a full fresh budget silently.
2270
- activeMs: descriptor.activeMsBase,
2271
- extraBudgetMinutes: descriptor.extraBudgetMinutes,
2272
- epoch: descriptor.epoch,
2273
- openQuestions: new Set(),
2274
- answeredAsks: new Set(),
2275
- ...pausedUntilOf(descriptor),
2276
- mode: descriptor.mode,
2277
- ...(descriptor.model ? { model: descriptor.model } : {}),
2278
- ...(descriptor.effort ? { effort: descriptor.effort } : {}),
2279
- lastPrompt: '',
2280
- };
2281
- running.journal.ensureSeqAbove(descriptor.lastSeq);
2282
- // Anything the API handed us before the daemon stopped (session 9).
2283
- running.pendingMessages.push(...running.journal.pending());
2284
- this.sessions.set(descriptor.id, running);
2285
- try {
2286
- // A session being restored after a runner restart already has its
2287
- // branch, so the API sends `CONTINUE` — the NEW guard inside would
2288
- // otherwise fire on the runner's own previous work.
2289
- const prepared = await this.prepareWorkspace(descriptor);
2290
- running.branch = prepared.branch;
2291
- running.worktreePath = prepared.worktreePath;
2292
- if (prepared.baseSha)
2293
- running.baseSha = prepared.baseSha;
2294
- if (prepared.baseBranch)
2295
- running.baseBranch = prepared.baseBranch;
2377
+ // Session that went terminal while we were offline: the journal keeps
2378
+ // the last reported status — replay it instead of resurrecting the
2379
+ // session as resumable (QA-96 F1).
2380
+ if (this.journals.exists(descriptor.id)) {
2381
+ const journal = this.journals.open(descriptor.id);
2382
+ const last = journal.lastStatus;
2383
+ // Only replay a terminal status from THIS life of the session. A
2384
+ // resumed session carries a higher epoch, and replaying the FAILED it
2385
+ // was resumed from would kill it again the moment the runner reconnects.
2386
+ // A journal written by a runner from before session 13 could hold a
2387
+ // `DONE` — it is no longer a status this runner may report, and
2388
+ // replaying one would close the session for the user. Drop it and let
2389
+ // the session be picked back up like any other.
2390
+ if (last &&
2391
+ isTerminal(last.status) &&
2392
+ last.status !== 'DONE' &&
2393
+ (last.epoch ?? 0) >= descriptor.epoch) {
2394
+ this.ws.send({
2395
+ type: 'session_status',
2396
+ sessionId: descriptor.id,
2397
+ status: last.status,
2398
+ ...(last.extra ?? {}),
2399
+ // Guarded above to be >= the descriptor's epoch, so the API keeps it.
2400
+ ...(last.epoch === undefined ? {} : { epoch: last.epoch }),
2401
+ });
2402
+ if (journal.unacked().length === 0) {
2403
+ this.journals.closeAndDelete(descriptor.id);
2404
+ }
2405
+ continue;
2406
+ }
2296
2407
  }
2297
- catch (error) {
2298
- this.reportStatus(descriptor.id, 'FAILED', {
2299
- errorMessage: `Failed to restore session worktree: ${String(error instanceof Error ? error.message : error).slice(0, 500)}`,
2300
- });
2301
- this.sessions.delete(descriptor.id);
2302
- continue;
2408
+ if (descriptor.status === 'STARTING') {
2409
+ await this.startSession(descriptor);
2303
2410
  }
2304
- /**
2305
- * Was a turn actually in flight when the process died?
2306
- *
2307
- * `RUNNING` and `WAITING_PERMISSION` are the mid-turn statuses; the
2308
- * others mean the agent was already waiting for a human, and there is
2309
- * nothing to continue. REVIEW is deliberately excluded — the work is
2310
- * finished and waiting to be looked at.
2311
- */
2312
- const wasMidTurn = descriptor.status === 'RUNNING' || descriptor.status === 'WAITING_PERMISSION';
2313
- const resumeId = descriptor.providerSessionId;
2314
- // Ticket #177: `resumeId` is required, not merely nice to have. Without
2315
- // it the relaunch starts a FRESH conversation, and `AUTO_RESUME_PROMPT`
2316
- // — "continue from where you stopped, re-check what you were in the
2317
- // middle of" — would be addressed to an agent that remembers none of
2318
- // it. A process killed before it reported its session id (the SIGABRT
2319
- // this ticket came from) leaves the row in exactly that state.
2320
- // Ticket #196: a paused session is never continued automatically. The
2321
- // row still says RUNNING — a pause interrupts the turn but is not a
2322
- // status — so without this the reconnect would read «mid-turn» and
2323
- // relaunch the agent with «continue from where you stopped», which is
2324
- // the exact opposite of what the clock was set for.
2325
- const willContinue = wasMidTurn &&
2326
- !Supervisor.isPaused(running) &&
2327
- Boolean(resumeId) &&
2328
- claimAutoResume(descriptor.id);
2329
- // The note stays either way (owner's call): an interruption is a fact
2330
- // about the session and must not disappear just because we recovered
2331
- // from it. Only the instruction at the end changes — telling someone to
2332
- // send a message while the agent is already working again would be a lie.
2333
- this.sendEvent(running, 'system_note', {
2334
- text: willContinue
2335
- ? 'Runner reconnected. The session was resumed — continuing the interrupted turn.'
2336
- : resumeId
2337
- ? 'Runner reconnected. The session was resumed — send a message to continue.'
2338
- : 'Runner reconnected, but the agent never got as far as naming its conversation, so it starts this one over. Your files, your branch and everything above are untouched.',
2339
- });
2340
- if (willContinue) {
2341
- // Resumed through the PROVIDER session, so the agent keeps its whole
2342
- // conversation; the prompt is only the nudge a human would otherwise
2343
- // have to type. Exactly what «продолжай» did by hand — no new class of
2344
- // risk, and the same ceiling protects against a crash loop doing it
2345
- // forever (see `auto-resume.ts`).
2346
- if (this.launchAgent(running, AUTO_RESUME_PROMPT, resumeId)) {
2347
- this.reportStatus(descriptor.id, 'RUNNING', {});
2348
- this.flushPendingMessages(running);
2411
+ else if (descriptor.providerSessionId) {
2412
+ // Runner restarted mid-session. The provider session is resumable —
2413
+ // park it until the user sends the next instruction. The map entry is
2414
+ // registered BEFORE the worktree await so a racing message is
2415
+ // buffered instead of dropped (QA-96 F3).
2416
+ const running = {
2417
+ descriptor,
2418
+ journal: this.journals.open(descriptor.id),
2419
+ session: null,
2420
+ lastReported: descriptor.status === 'REVIEW' ? 'REVIEW' : 'WAITING_INPUT',
2421
+ costUsd: descriptor.costUsd,
2422
+ costBaseUsd: descriptor.costUsd,
2423
+ stopRequested: false,
2424
+ parkRequested: false,
2425
+ pendingMessages: [],
2426
+ // Seed from the API, not 0: a runner restart used to hand the session
2427
+ // a full fresh budget silently.
2428
+ activeMs: descriptor.activeMsBase,
2429
+ extraBudgetMinutes: descriptor.extraBudgetMinutes,
2430
+ epoch: descriptor.epoch,
2431
+ openQuestions: new Set(),
2432
+ answeredAsks: new Set(),
2433
+ ...pausedUntilOf(descriptor),
2434
+ mode: descriptor.mode,
2435
+ ...(descriptor.model ? { model: descriptor.model } : {}),
2436
+ ...(descriptor.effort ? { effort: descriptor.effort } : {}),
2437
+ lastPrompt: '',
2438
+ };
2439
+ running.journal.ensureSeqAbove(descriptor.lastSeq);
2440
+ // Anything the API handed us before the daemon stopped (session 9).
2441
+ running.pendingMessages.push(...running.journal.pending());
2442
+ this.sessions.set(descriptor.id, running);
2443
+ try {
2444
+ // A session being restored after a runner restart already has its
2445
+ // branch, so the API sends `CONTINUE` — the NEW guard inside would
2446
+ // otherwise fire on the runner's own previous work.
2447
+ const prepared = await this.prepareWorkspace(descriptor);
2448
+ running.branch = prepared.branch;
2449
+ running.worktreePath = prepared.worktreePath;
2450
+ if (prepared.baseSha)
2451
+ running.baseSha = prepared.baseSha;
2452
+ if (prepared.baseBranch)
2453
+ running.baseBranch = prepared.baseBranch;
2454
+ }
2455
+ catch (error) {
2456
+ this.reportStatus(descriptor.id, 'FAILED', {
2457
+ errorMessage: `Failed to restore session worktree: ${String(error instanceof Error ? error.message : error).slice(0, 500)}`,
2458
+ });
2459
+ this.sessions.delete(descriptor.id);
2349
2460
  continue;
2350
2461
  }
2351
- // Could not start (an exhausted budget is the only way here). Fall
2352
- // through to the old behaviour and say so honestly.
2462
+ /**
2463
+ * Was a turn actually in flight when the process died?
2464
+ *
2465
+ * `RUNNING` and `WAITING_PERMISSION` are the mid-turn statuses; the
2466
+ * others mean the agent was already waiting for a human, and there is
2467
+ * nothing to continue. REVIEW is deliberately excluded — the work is
2468
+ * finished and waiting to be looked at.
2469
+ */
2470
+ const wasMidTurn = descriptor.status === 'RUNNING' || descriptor.status === 'WAITING_PERMISSION';
2471
+ const resumeId = descriptor.providerSessionId;
2472
+ // Ticket #177: `resumeId` is required, not merely nice to have. Without
2473
+ // it the relaunch starts a FRESH conversation, and `AUTO_RESUME_PROMPT`
2474
+ // — "continue from where you stopped, re-check what you were in the
2475
+ // middle of" — would be addressed to an agent that remembers none of
2476
+ // it. A process killed before it reported its session id (the SIGABRT
2477
+ // this ticket came from) leaves the row in exactly that state.
2478
+ // Ticket #196: a paused session is never continued automatically. The
2479
+ // row still says RUNNING — a pause interrupts the turn but is not a
2480
+ // status — so without this the reconnect would read «mid-turn» and
2481
+ // relaunch the agent with «continue from where you stopped», which is
2482
+ // the exact opposite of what the clock was set for.
2483
+ const willContinue = wasMidTurn &&
2484
+ !Supervisor.isPaused(running) &&
2485
+ Boolean(resumeId) &&
2486
+ claimAutoResume(descriptor.id);
2487
+ // The note stays either way (owner's call): an interruption is a fact
2488
+ // about the session and must not disappear just because we recovered
2489
+ // from it. Only the instruction at the end changes — telling someone to
2490
+ // send a message while the agent is already working again would be a lie.
2353
2491
  this.sendEvent(running, 'system_note', {
2354
- text: 'Could not continue automatically — send a message to pick the work back up.',
2492
+ text: willContinue
2493
+ ? 'Runner reconnected. The session was resumed — continuing the interrupted turn.'
2494
+ : resumeId
2495
+ ? 'Runner reconnected. The session was resumed — send a message to continue.'
2496
+ : 'Runner reconnected, but the agent never got as far as naming its conversation, so it starts this one over. Your files, your branch and everything above are untouched.',
2355
2497
  });
2498
+ if (willContinue) {
2499
+ // Resumed through the PROVIDER session, so the agent keeps its whole
2500
+ // conversation; the prompt is only the nudge a human would otherwise
2501
+ // have to type. Exactly what «продолжай» did by hand — no new class of
2502
+ // risk, and the same ceiling protects against a crash loop doing it
2503
+ // forever (see `auto-resume.ts`).
2504
+ const continued = this.launchAgent(running, AUTO_RESUME_PROMPT, resumeId);
2505
+ if (continued.ok) {
2506
+ this.reportStatus(descriptor.id, 'RUNNING', {});
2507
+ this.flushPendingMessages(running);
2508
+ continue;
2509
+ }
2510
+ // Could not start — an exhausted budget, or a launch that crashed
2511
+ // (ticket #225: this is the exact line the incident died on, and the
2512
+ // throw took the WHOLE restore loop with it). Fall through to the old
2513
+ // behaviour and say so honestly; a crash has already put its own
2514
+ // reason in the feed, so this note would only repeat it.
2515
+ if (continued.reason === 'refused') {
2516
+ this.sendEvent(running, 'system_note', {
2517
+ text: 'Could not continue automatically — send a message to pick the work back up.',
2518
+ });
2519
+ }
2520
+ }
2521
+ // REVIEW stays REVIEW (WAITING_INPUT is not reachable from it);
2522
+ // mid-turn statuses are downgraded to "waiting for the user".
2523
+ if (descriptor.status !== 'REVIEW') {
2524
+ this.reportStatus(descriptor.id, 'WAITING_INPUT', {});
2525
+ }
2526
+ this.flushPendingMessages(running);
2356
2527
  }
2357
- // REVIEW stays REVIEW (WAITING_INPUT is not reachable from it);
2358
- // mid-turn statuses are downgraded to "waiting for the user".
2359
- if (descriptor.status !== 'REVIEW') {
2360
- this.reportStatus(descriptor.id, 'WAITING_INPUT', {});
2528
+ else if (descriptor.status === 'WAITING_INPUT') {
2529
+ // A session that never had a turn (a free session still waiting for
2530
+ // its first message) has nothing to resume — just bring the agent back
2531
+ // up and keep waiting, instead of failing the session.
2532
+ await this.startSession({ ...descriptor, status: 'STARTING', prompt: '' });
2533
+ }
2534
+ else {
2535
+ this.reportStatus(descriptor.id, 'FAILED', {
2536
+ errorMessage: 'Runner restarted and this session cannot be resumed',
2537
+ });
2361
2538
  }
2362
- this.flushPendingMessages(running);
2363
- }
2364
- else if (descriptor.status === 'WAITING_INPUT') {
2365
- // A session that never had a turn (a free session still waiting for
2366
- // its first message) has nothing to resume — just bring the agent back
2367
- // up and keep waiting, instead of failing the session.
2368
- await this.startSession({ ...descriptor, status: 'STARTING', prompt: '' });
2369
2539
  }
2370
- else {
2371
- this.reportStatus(descriptor.id, 'FAILED', {
2372
- errorMessage: 'Runner restarted and this session cannot be resumed',
2540
+ catch (error) {
2541
+ log.error('supervisor: session could not be restored after reconnect', {
2542
+ sessionId: descriptor.id,
2543
+ error: error instanceof Error ? (error.stack ?? error.message) : String(error),
2373
2544
  });
2545
+ // Said on the session it belongs to, not only in journald. FAILED is
2546
+ // the honest word: this runner is not going to run it as things stand,
2547
+ // and «Продолжить» is what asks it to try again.
2548
+ //
2549
+ // Guarded in turn, and not out of superstition: reporting a status
2550
+ // WRITES to that session's journal, so the most likely reason the body
2551
+ // above failed — this session's own file — would fail the handler the
2552
+ // same way and take the loop with it after all. The other sessions
2553
+ // matter more than this one's status frame.
2554
+ try {
2555
+ this.reportStatus(descriptor.id, 'FAILED', {
2556
+ errorMessage: `Runner could not restore this session: ${maskSecretText(error)}`,
2557
+ });
2558
+ }
2559
+ catch (reportError) {
2560
+ log.error('supervisor: could not even report the failed restore', {
2561
+ sessionId: descriptor.id,
2562
+ error: String(reportError),
2563
+ });
2564
+ }
2374
2565
  }
2375
2566
  }
2376
2567
  // Redelivery is done — anything still on disk from long-finished sessions