@rynx-ai/runtime 0.1.11-beta.2 → 0.1.11-beta.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -22,7 +22,9 @@ import { fileURLToPath } from "node:url";
22
22
  import { AgentRuntimeError, } from "@rynx-ai/core";
23
23
  import { listRuntimeModels } from "../models-catalog.js";
24
24
  import { probeRuntimeStatus } from "../runtime-status.js";
25
+ import { terminateTmuxServer, tmuxHasAttachedClient, tmuxWindowActivityAt, } from "../terminal/tmux.js";
25
26
  import { fromWireError, } from "./protocol.js";
27
+ import { isManagedNativeProvider } from "./startup-policy.js";
26
28
  import { StdioRunnerTransport } from "./transport.js";
27
29
  /** Routing key for the shared capability runner (slash-command RPCs). */
28
30
  const CAP_KEY = "__cap__";
@@ -32,10 +34,27 @@ const STDERR_TAIL_LINES = 40;
32
34
  const DEFAULT_SHUTDOWN_GRACE_MS = 5_000;
33
35
  /** Time allowed for exit after SIGKILL before shutdown reports failure. */
34
36
  const DEFAULT_SHUTDOWN_KILL_GRACE_MS = 5_000;
35
- /** Setup-pane launch should acknowledge quickly; never pin its HTTP request. */
36
- const DEFAULT_LIVE_START_TIMEOUT_MS = 10_000;
37
- /** Thread readiness may legitimately wait through Provider startup. */
37
+ /** Setup-pane launch includes the app-server readiness phase but does not wait
38
+ * for native thread discovery. Keep enough headroom around the app-server's
39
+ * own 10s gate so scheduling/IPC overhead cannot win the same deadline. */
40
+ const DEFAULT_LIVE_START_TIMEOUT_MS = 30_000;
41
+ /** Legacy fallback for a provider without a native startup policy. */
38
42
  const DEFAULT_LIVE_READY_TIMEOUT_MS = 25_000;
43
+ /** Claude has no app-server bridge and keeps a parent-owned SessionStart
44
+ * deadline. Codex-lineage startup instead acknowledges pane/observer startup
45
+ * and lets thread discovery race injection. */
46
+ const DEFAULT_NATIVE_LIVE_START_TIMEOUT_MS = 60_000;
47
+ /** A submitted TUI command should become a mirrored turn or native rotation
48
+ * quickly. If it does not, the runner is fenced by a verified process-tree
49
+ * shutdown before maintenance may treat the submission as settled. */
50
+ const DEFAULT_TERMINAL_INPUT_HANDOFF_TIMEOUT_MS = 30_000;
51
+ /** Retry transient terminal-server cleanup without spinning forever. */
52
+ const DEFAULT_TERMINAL_INPUT_CLEANUP_RETRY_MS = 1_000;
53
+ const MAX_TERMINAL_INPUT_CLEANUP_RETRY_MS = 30_000;
54
+ /** Full idle window before an inactive native pane becomes reapable. */
55
+ const DEFAULT_NATIVE_PANE_IDLE_TTL_MS = 60 * 60_000;
56
+ /** tmux output this recent independently proves that a native pane is busy. */
57
+ const DEFAULT_NATIVE_PANE_OUTPUT_BUSY_WINDOW_MS = 120_000;
39
58
  /** Keep per-Session spawn context small enough to remain an environment handoff,
40
59
  * not an unbounded transport. */
41
60
  const MAX_SESSION_CONTEXT_ENV_ENTRIES = 32;
@@ -136,9 +155,13 @@ export class RunnerManager {
136
155
  sessionStore;
137
156
  runnerEntry;
138
157
  idleTtlMs;
158
+ nativePaneIdleTtlMs;
159
+ nativePaneOutputBusyWindowMs;
139
160
  spawn;
140
161
  childEnv;
141
162
  sessionContextProvider;
163
+ admissionOpen;
164
+ admissionReserve;
142
165
  now;
143
166
  defaultRuntime;
144
167
  handles = new Map();
@@ -148,8 +171,18 @@ export class RunnerManager {
148
171
  reapTimer;
149
172
  shutdownGraceMs;
150
173
  shutdownKillGraceMs;
174
+ signalChild;
175
+ terminateTerminalServer;
176
+ terminalWindowActivityAt;
177
+ terminalHasAttachedClient;
178
+ wallNow;
151
179
  liveStartTimeoutMs;
152
180
  liveReadyTimeoutMs;
181
+ nativeLiveStartTimeoutMs;
182
+ liveInterruptTimeoutMs;
183
+ terminalInputHandoffTimeoutMs;
184
+ terminalInputCleanupRetryMs;
185
+ reapPromise = null;
153
186
  stopping = false;
154
187
  stopPromise;
155
188
  /** Sink for mirrored {@link SessionEvent}s from every session's forwarder. */
@@ -175,23 +208,39 @@ export class RunnerManager {
175
208
  forkOperations = new Map();
176
209
  forkBufferedMessages = new Map();
177
210
  forkBufferedTerminalInputs = new Map();
211
+ /** Owner TUI submissions accepted by the parent but not yet represented by a
212
+ * mirrored response or a durably published native rotation. */
213
+ terminalInputHandoffs = new Map();
178
214
  constructor(opts) {
179
215
  this.config = opts.config;
180
216
  this.sessionStore = opts.sessionStore;
181
217
  this.runnerEntry = opts.runnerEntry ?? defaultRunnerEntry();
182
218
  this.idleTtlMs = opts.idleTtlMs ?? 300_000;
219
+ this.nativePaneIdleTtlMs = opts.nativePaneIdleTtlMs ?? DEFAULT_NATIVE_PANE_IDLE_TTL_MS;
220
+ this.nativePaneOutputBusyWindowMs = Math.max(0, opts.nativePaneOutputBusyWindowMs ?? DEFAULT_NATIVE_PANE_OUTPUT_BUSY_WINDOW_MS);
183
221
  this.spawn = opts.spawn ?? nodeSpawn;
184
222
  this.childEnv = opts.childEnv ?? {};
185
223
  this.sessionContextProvider = opts.sessionContextProvider;
224
+ this.admissionOpen = opts.admissionOpen ?? (() => true);
225
+ this.admissionReserve = opts.admissionReserve;
186
226
  this.now = opts.now ?? (() => Date.now());
187
- this.defaultRuntime = opts.config.AGENT_RUNTIME ?? "codex";
227
+ this.wallNow = opts.wallNow ?? (() => Date.now());
228
+ this.defaultRuntime = opts.config.DEFAULT_RUNTIME ?? "codex";
188
229
  this.shutdownGraceMs = Math.max(0, opts.shutdownGraceMs ?? DEFAULT_SHUTDOWN_GRACE_MS);
189
230
  this.shutdownKillGraceMs = Math.max(0, opts.shutdownKillGraceMs ?? DEFAULT_SHUTDOWN_KILL_GRACE_MS);
231
+ this.signalChild = opts.signalChild ?? signalRunnerChild;
232
+ this.terminateTerminalServer = opts.terminateTerminalServer ?? terminateTmuxServer;
233
+ this.terminalWindowActivityAt = opts.terminalWindowActivityAt ?? tmuxWindowActivityAt;
234
+ this.terminalHasAttachedClient = opts.terminalHasAttachedClient ?? tmuxHasAttachedClient;
190
235
  this.liveStartTimeoutMs = Math.max(1, opts.liveStartTimeoutMs ?? DEFAULT_LIVE_START_TIMEOUT_MS);
191
236
  this.liveReadyTimeoutMs = Math.max(1, opts.liveReadyTimeoutMs ?? DEFAULT_LIVE_READY_TIMEOUT_MS);
237
+ this.nativeLiveStartTimeoutMs = Math.max(1, opts.nativeLiveStartTimeoutMs ?? DEFAULT_NATIVE_LIVE_START_TIMEOUT_MS);
238
+ this.liveInterruptTimeoutMs = Math.max(1, opts.liveInterruptTimeoutMs ?? 10_000);
239
+ this.terminalInputHandoffTimeoutMs = Math.max(1, opts.terminalInputHandoffTimeoutMs ?? DEFAULT_TERMINAL_INPUT_HANDOFF_TIMEOUT_MS);
240
+ this.terminalInputCleanupRetryMs = Math.max(1, opts.terminalInputCleanupRetryMs ?? DEFAULT_TERMINAL_INPUT_CLEANUP_RETRY_MS);
192
241
  const reapIntervalMs = opts.reapIntervalMs ?? 60_000;
193
242
  if (reapIntervalMs > 0) {
194
- this.reapTimer = setInterval(() => this.reapIdle(), reapIntervalMs);
243
+ this.reapTimer = setInterval(() => void this.reapIdle(), reapIntervalMs);
195
244
  this.reapTimer.unref?.();
196
245
  }
197
246
  else {
@@ -216,6 +265,7 @@ export class RunnerManager {
216
265
  * and attach.
217
266
  */
218
267
  openLiveTerminal(localThreadId, opts) {
268
+ this.assertAdmissionOpen();
219
269
  if (this.forkReservations.has(localThreadId)) {
220
270
  throw new TerminalOpenError("Session fork is still being committed", "terminal_not_live");
221
271
  }
@@ -228,12 +278,38 @@ export class RunnerManager {
228
278
  openTerminalOnHandle(handle, localThreadId, opts) {
229
279
  handle.lastUsedAt = this.now();
230
280
  const attachId = randomUUID();
281
+ const inputTracker = {
282
+ buffer: "",
283
+ previousWasCarriageReturn: false,
284
+ bracketedPaste: false,
285
+ pendingEscape: "",
286
+ };
231
287
  const terminal = new ManagedTerminal(attachId, (msg) => {
288
+ if (msg.t === "term.input")
289
+ handle.lastUsedAt = this.now();
290
+ const handoffs = msg.t === "term.input" && opts.role === "owner"
291
+ ? this.beginTerminalInputHandoffs(handle, this.currentTerminalSessionId(handle, localThreadId), trackedTerminalSubmissions(inputTracker, Buffer.from(msg.dataB64, "base64").toString("utf8")))
292
+ : [];
293
+ const send = () => {
294
+ try {
295
+ handle.transport.send(msg);
296
+ }
297
+ catch (error) {
298
+ for (const handoff of handoffs) {
299
+ this.finishTerminalInputHandoff(handle, handoff);
300
+ }
301
+ throw error;
302
+ }
303
+ };
232
304
  const sourceReservation = this.sourceForkReservations.get(localThreadId);
233
305
  if (msg.t === "term.input" && sourceReservation) {
234
306
  void sourceReservation.then(() => {
235
307
  if (!handle.dead)
236
- handle.transport.send(msg);
308
+ send();
309
+ }).catch((error) => {
310
+ void this.terminateHandle(handle, `terminal input delivery failed: ${errorMessage(error)}`).catch((terminationError) => {
311
+ logTerminationFailure(handle, terminationError);
312
+ });
237
313
  });
238
314
  return;
239
315
  }
@@ -246,7 +322,7 @@ export class RunnerManager {
246
322
  return;
247
323
  }
248
324
  }
249
- handle.transport.send(msg);
325
+ send();
250
326
  }, () => handle.terminals.delete(attachId));
251
327
  handle.terminals.set(attachId, terminal);
252
328
  handle.transport.send({
@@ -284,20 +360,171 @@ export class RunnerManager {
284
360
  onRotate(listener) {
285
361
  this.rotateListener = listener;
286
362
  }
363
+ /** Accepted TUI submissions that have not crossed into an observable runtime
364
+ * state. The daemon folds this into runningTurns after closing admission. */
365
+ pendingTerminalInputCount() {
366
+ let count = 0;
367
+ for (const handoffs of this.terminalInputHandoffs.values()) {
368
+ count += handoffs.length;
369
+ }
370
+ return count;
371
+ }
372
+ beginTerminalInputHandoffs(handle, sessionId, kinds) {
373
+ if (kinds.length === 0)
374
+ return [];
375
+ const created = [];
376
+ try {
377
+ for (const kind of kinds) {
378
+ const handoff = {
379
+ sessionId,
380
+ kind,
381
+ reservation: this.reserveAdmission(),
382
+ timer: undefined,
383
+ };
384
+ handoff.timer = setTimeout(() => {
385
+ this.expireTerminalInputHandoffs(handle);
386
+ }, this.terminalInputHandoffTimeoutMs);
387
+ handoff.timer.unref?.();
388
+ const handoffs = this.terminalInputHandoffs.get(handle);
389
+ if (handoffs)
390
+ handoffs.push(handoff);
391
+ else
392
+ this.terminalInputHandoffs.set(handle, [handoff]);
393
+ created.push(handoff);
394
+ }
395
+ return created;
396
+ }
397
+ catch (error) {
398
+ for (const handoff of created) {
399
+ this.finishTerminalInputHandoff(handle, handoff);
400
+ }
401
+ throw error;
402
+ }
403
+ }
404
+ settleTerminalInputHandoff(handle, sessionId, kind) {
405
+ const handoff = this.terminalInputHandoffs
406
+ .get(handle)
407
+ ?.find((candidate) => candidate.sessionId === sessionId);
408
+ // Runtime observations settle accepted input in submission order. In
409
+ // particular, a response from the source Session while /clear or /fork is
410
+ // still publishing must not skip that rotation and release a later turn
411
+ // which has not yet been rebound or delivered.
412
+ if (handoff?.kind === kind)
413
+ this.finishTerminalInputHandoff(handle, handoff);
414
+ }
415
+ settleTerminalRotationHandoff(handle, sourceSessionId, targetSessionId) {
416
+ const handoffs = this.terminalInputHandoffs.get(handle);
417
+ const rotationIndex = handoffs?.findIndex((handoff) => handoff.sessionId === sourceSessionId && handoff.kind === "rotation") ?? -1;
418
+ if (!handoffs || rotationIndex < 0)
419
+ return;
420
+ // Turns submitted before /clear or /fork are superseded once the rotation
421
+ // is published. Inputs accepted afterwards belong to the transferred pane
422
+ // and must follow it to the target Session.
423
+ const rotation = handoffs[rotationIndex];
424
+ for (const handoff of [...handoffs.slice(0, rotationIndex)]) {
425
+ if (handoff.sessionId === sourceSessionId && handoff.kind === "turn") {
426
+ this.finishTerminalInputHandoff(handle, handoff);
427
+ }
428
+ }
429
+ if (rotation)
430
+ this.finishTerminalInputHandoff(handle, rotation);
431
+ for (const handoff of this.terminalInputHandoffs.get(handle) ?? []) {
432
+ if (handoff.sessionId === sourceSessionId)
433
+ handoff.sessionId = targetSessionId;
434
+ }
435
+ }
436
+ currentTerminalSessionId(handle, fallback) {
437
+ for (const [sessionId, candidate] of this.handles) {
438
+ if (candidate === handle && this.liveSessionKeys.has(sessionId))
439
+ return sessionId;
440
+ }
441
+ return fallback;
442
+ }
443
+ finishTerminalInputHandoff(handle, handoff) {
444
+ if (handoff.timer) {
445
+ clearTimeout(handoff.timer);
446
+ handoff.timer = undefined;
447
+ }
448
+ handoff.reservation?.release();
449
+ handoff.reservation = undefined;
450
+ const handoffs = this.terminalInputHandoffs.get(handle);
451
+ if (!handoffs)
452
+ return;
453
+ const index = handoffs.indexOf(handoff);
454
+ if (index >= 0)
455
+ handoffs.splice(index, 1);
456
+ if (handoffs.length === 0)
457
+ this.terminalInputHandoffs.delete(handle);
458
+ }
459
+ finishAllTerminalInputHandoffs(handle) {
460
+ if (handle.terminalCleanupRetryTimer) {
461
+ clearTimeout(handle.terminalCleanupRetryTimer);
462
+ delete handle.terminalCleanupRetryTimer;
463
+ }
464
+ handle.terminalCleanupRetryFailures = 0;
465
+ for (const handoff of [...(this.terminalInputHandoffs.get(handle) ?? [])]) {
466
+ this.finishTerminalInputHandoff(handle, handoff);
467
+ }
468
+ }
469
+ preserveTerminalInputHandoffsAsActivity(handle) {
470
+ for (const handoff of this.terminalInputHandoffs.get(handle) ?? []) {
471
+ if (handoff.timer) {
472
+ clearTimeout(handoff.timer);
473
+ handoff.timer = undefined;
474
+ }
475
+ handoff.reservation?.release();
476
+ handoff.reservation = undefined;
477
+ }
478
+ }
479
+ scheduleTerminalInputCleanupRetry(handle) {
480
+ if (handle.terminalCleanupRetryTimer ||
481
+ !this.terminalInputHandoffs.has(handle))
482
+ return;
483
+ const exponent = Math.min(handle.terminalCleanupRetryFailures - 1, 10);
484
+ const delay = Math.min(this.terminalInputCleanupRetryMs * (2 ** Math.max(0, exponent)), MAX_TERMINAL_INPUT_CLEANUP_RETRY_MS);
485
+ handle.terminalCleanupRetryTimer = setTimeout(() => {
486
+ delete handle.terminalCleanupRetryTimer;
487
+ if (!this.terminalInputHandoffs.has(handle))
488
+ return;
489
+ void this.terminateHandle(handle, "retrying unproven terminal input cleanup").catch((error) => {
490
+ logTerminationFailure(handle, error);
491
+ });
492
+ }, delay);
493
+ handle.terminalCleanupRetryTimer.unref?.();
494
+ }
495
+ expireTerminalInputHandoffs(handle) {
496
+ const handoffs = this.terminalInputHandoffs.get(handle);
497
+ if (!handoffs?.length)
498
+ return;
499
+ for (const handoff of handoffs) {
500
+ if (handoff.timer) {
501
+ clearTimeout(handoff.timer);
502
+ handoff.timer = undefined;
503
+ }
504
+ }
505
+ void this.terminateHandle(handle, "accepted terminal input did not become observable before maintenance timeout").catch((error) => {
506
+ logTerminationFailure(handle, error);
507
+ });
508
+ }
287
509
  /**
288
510
  * Eagerly bring up a session's codex-native live view (persistent forwarder +
289
511
  * detached `codex --remote` TUI) in its runner child, spawning the runner if
290
- * needed. Idempotent. Resolves true once the codex thread is bound; false for a
291
- * non-codex / non-live session (the caller then uses the normal run path).
512
+ * needed. Idempotent. For Codex-lineage fresh sessions, resolves after the
513
+ * app-server/observer and pane are launched; thread discovery continues in
514
+ * parallel with injection. Other providers retain their readiness gate.
292
515
  */
293
516
  ensureLiveSession(localThreadId, opts) {
294
- return this.requestLiveSession(localThreadId, opts, true);
517
+ const admission = this.reserveAdmission();
518
+ return this.requestLiveSession(localThreadId, opts, true)
519
+ .finally(() => admission.release());
295
520
  }
296
521
  /** Start the Provider pane without waiting for login/onboarding to create a
297
522
  * native thread. This is the setup-terminal gate; callers may attach as soon
298
523
  * as it resolves, while normal message delivery still uses ensureLiveSession. */
299
524
  startLiveSession(localThreadId, opts) {
300
- return this.requestLiveSession(localThreadId, opts, false);
525
+ const admission = this.reserveAdmission();
526
+ return this.requestLiveSession(localThreadId, opts, false)
527
+ .finally(() => admission.release());
301
528
  }
302
529
  requestLiveSession(localThreadId, opts, waitForReady, allowReservedForkTarget = false) {
303
530
  const sourceReservation = allowReservedForkTarget
@@ -323,12 +550,25 @@ export class RunnerManager {
323
550
  this.liveSessionKeys.add(localThreadId);
324
551
  const reqId = randomUUID();
325
552
  return new Promise((resolve) => {
326
- const timeoutMs = waitForReady ? this.liveReadyTimeoutMs : this.liveStartTimeoutMs;
553
+ const timeoutMs = !waitForReady
554
+ ? this.liveStartTimeoutMs
555
+ : opts.execution.provider === "claude"
556
+ ? this.nativeLiveStartTimeoutMs
557
+ : isManagedNativeProvider(opts.execution.provider)
558
+ ? this.liveStartTimeoutMs
559
+ : this.liveReadyTimeoutMs;
327
560
  const timeout = setTimeout(() => {
328
561
  if (!handle.live.delete(reqId))
329
562
  return;
330
563
  const finishTimeout = () => {
331
- const reason = `runner did not acknowledge ${waitForReady ? "live readiness" : "terminal start"} within ${timeoutMs}ms`;
564
+ const provider = opts.execution.provider;
565
+ const reason = !waitForReady
566
+ ? `runner did not acknowledge terminal start within ${timeoutMs}ms`
567
+ : provider === "claude"
568
+ ? `Claude SessionStart was not observed within ${Math.round(timeoutMs / 1_000)}s; the runner process was reset before retry`
569
+ : provider === "codex" || provider === "traex"
570
+ ? `${provider === "traex" ? "Traex" : "Codex"} app-server/observer/Terminal launch was not acknowledged within ${Math.round(timeoutMs / 1_000)}s; the runner process was reset before retry`
571
+ : `runner did not acknowledge live readiness within ${timeoutMs}ms`;
332
572
  this.liveErrors.set(localThreadId, reason);
333
573
  this.liveSessionKeys.delete(localThreadId);
334
574
  // A child that cannot answer a bounded control round-trip is unsafe
@@ -395,13 +635,18 @@ export class RunnerManager {
395
635
  * the session has no live forwarder (caller falls back to the run path).
396
636
  */
397
637
  injectMessage(localThreadId, input) {
638
+ const admission = this.reserveAdmission();
639
+ return this.injectMessageAdmitted(localThreadId, input)
640
+ .finally(() => admission.release());
641
+ }
642
+ injectMessageAdmitted(localThreadId, input) {
398
643
  const sourceReservation = this.sourceForkReservations.get(localThreadId);
399
644
  if (sourceReservation) {
400
- return sourceReservation.then(() => this.injectMessage(localThreadId, input));
645
+ return sourceReservation.then(() => this.injectMessageAdmitted(localThreadId, input));
401
646
  }
402
647
  const reservation = this.forkReservations.get(localThreadId);
403
648
  if (reservation) {
404
- return reservation.then(() => this.injectMessage(localThreadId, input));
649
+ return reservation.then(() => this.injectMessageAdmitted(localThreadId, input));
405
650
  }
406
651
  const handle = this.getOrSpawn(localThreadId);
407
652
  handle.lastUsedAt = this.now();
@@ -410,7 +655,7 @@ export class RunnerManager {
410
655
  handle.live.set(reqId, (res) => {
411
656
  const outcome = res.outcome ?? "failed";
412
657
  const finish = () => {
413
- if (outcome === "injected") {
658
+ if (outcome === "injected" || outcome === "steered") {
414
659
  this.liveErrors.delete(localThreadId);
415
660
  }
416
661
  else {
@@ -444,7 +689,16 @@ export class RunnerManager {
444
689
  handle.lastUsedAt = this.now();
445
690
  const reqId = randomUUID();
446
691
  return new Promise((resolve) => {
447
- handle.live.set(reqId, (res) => resolve(res.ok ?? false));
692
+ const timeout = setTimeout(() => {
693
+ if (!handle.live.delete(reqId))
694
+ return;
695
+ resolve(false);
696
+ }, this.liveInterruptTimeoutMs);
697
+ timeout.unref?.();
698
+ handle.live.set(reqId, (res) => {
699
+ clearTimeout(timeout);
700
+ resolve(res.ok ?? false);
701
+ });
448
702
  handle.transport.send({ t: "live.interrupt", reqId, localThreadId });
449
703
  });
450
704
  }
@@ -488,6 +742,7 @@ export class RunnerManager {
488
742
  return this.forwardCap("clearGoal", [localThreadId], localThreadId);
489
743
  }
490
744
  forkSession(currentLocalThreadId, newLocalThreadId, options) {
745
+ this.assertAdmissionOpen();
491
746
  if (currentLocalThreadId === newLocalThreadId) {
492
747
  return Promise.resolve({
493
748
  ok: false,
@@ -573,7 +828,7 @@ export class RunnerManager {
573
828
  }
574
829
  /** Stop the runner bound to one session (if any), rejecting its in-flight work. */
575
830
  stopRunner(localThreadId) {
576
- const handle = this.handles.get(localThreadId);
831
+ const handle = this.handleForCleanup(localThreadId);
577
832
  if (!handle) {
578
833
  return;
579
834
  }
@@ -581,6 +836,15 @@ export class RunnerManager {
581
836
  logTerminationFailure(handle, error);
582
837
  });
583
838
  }
839
+ /** Force-stop and join the runner bound to one Session. Unlike stopRunner,
840
+ * completion proves that the child process has exited. */
841
+ async terminateLiveSession(localThreadId) {
842
+ const handle = this.handleForCleanup(localThreadId);
843
+ if (!handle)
844
+ return false;
845
+ await this.terminateHandle(handle, "runner force-stopped");
846
+ return true;
847
+ }
584
848
  /** Stop every runner and join all child exits. Idempotent across concurrent calls. */
585
849
  stop() {
586
850
  if (this.stopPromise)
@@ -589,9 +853,12 @@ export class RunnerManager {
589
853
  if (this.reapTimer) {
590
854
  clearInterval(this.reapTimer);
591
855
  }
592
- const children = [...this.childHandles];
593
- this.stopPromise = (async () => {
594
- const results = await Promise.allSettled(children.map((handle) => this.terminateHandle(handle, "runner manager stopped")));
856
+ const children = new Set([
857
+ ...this.childHandles,
858
+ ...this.terminalInputHandoffs.keys(),
859
+ ]);
860
+ const attempt = (async () => {
861
+ const results = await Promise.allSettled([...children].map((handle) => this.terminateHandle(handle, "runner manager stopped")));
595
862
  this.handles.clear();
596
863
  this.liveSessionKeys.clear();
597
864
  this.liveErrors.clear();
@@ -606,9 +873,19 @@ export class RunnerManager {
606
873
  }
607
874
  this.childHandles.clear();
608
875
  })();
609
- return this.stopPromise;
876
+ this.stopPromise = attempt;
877
+ void attempt.catch(() => {
878
+ if (this.stopPromise === attempt)
879
+ this.stopPromise = undefined;
880
+ });
881
+ return attempt;
610
882
  }
611
883
  // ── internals ──────────────────────────────────────────────────────────────
884
+ handleForCleanup(localThreadId) {
885
+ return this.handles.get(localThreadId) ??
886
+ [...this.terminalInputHandoffs].find(([handle, handoffs]) => handle.key === localThreadId ||
887
+ handoffs.some((handoff) => handoff.sessionId === localThreadId))?.[0];
888
+ }
612
889
  /** Forward a capability to a runner child. Session-less caps (listModels/status)
613
890
  * use the shared `CAP_KEY` child; per-thread caps pass the session's key so they
614
891
  * run on that session's child (which owns its private CODEX_HOME). */
@@ -622,23 +899,47 @@ export class RunnerManager {
622
899
  });
623
900
  }
624
901
  getOrSpawn(key, allowReservedForkTarget = false) {
625
- if (this.stopping) {
626
- throw new AgentRuntimeError("runner manager is stopped", 503, "runner_stopped");
627
- }
628
- if (key !== CAP_KEY &&
629
- this.forkReservations.has(key) &&
630
- !allowReservedForkTarget) {
631
- throw new AgentRuntimeError("Session fork is still being committed", 409, "session_fork_pending");
632
- }
633
- this.reapIdle();
634
- const existing = this.handles.get(key);
635
- if (existing && !existing.dead) {
636
- existing.lastUsedAt = this.now();
637
- return existing;
638
- }
639
- const handle = this.spawnHandle(key);
640
- this.handles.set(key, handle);
641
- return handle;
902
+ const admission = this.reserveAdmission();
903
+ try {
904
+ if (this.stopping) {
905
+ throw new AgentRuntimeError("runner manager is stopped", 503, "runner_stopped");
906
+ }
907
+ if (key !== CAP_KEY &&
908
+ this.forkReservations.has(key) &&
909
+ !allowReservedForkTarget) {
910
+ throw new AgentRuntimeError("Session fork is still being committed", 409, "session_fork_pending");
911
+ }
912
+ const existing = this.handles.get(key);
913
+ if (existing && !existing.dead) {
914
+ existing.lastUsedAt = this.now();
915
+ return existing;
916
+ }
917
+ const handle = this.spawnHandle(key);
918
+ this.handles.set(key, handle);
919
+ return handle;
920
+ }
921
+ finally {
922
+ admission.release();
923
+ }
924
+ }
925
+ assertAdmissionOpen() {
926
+ const reservation = this.admissionReserve?.();
927
+ if (reservation) {
928
+ reservation.release();
929
+ return;
930
+ }
931
+ if (!this.admissionReserve && this.admissionOpen())
932
+ return;
933
+ throw new AgentRuntimeError("daemon maintenance is in progress", 503, "daemon_maintenance");
934
+ }
935
+ reserveAdmission() {
936
+ const reservation = this.admissionReserve?.();
937
+ if (reservation)
938
+ return reservation;
939
+ if (!this.admissionReserve && this.admissionOpen()) {
940
+ return { release() { } };
941
+ }
942
+ throw new AgentRuntimeError("daemon maintenance is in progress", 503, "daemon_maintenance");
642
943
  }
643
944
  async performManagedFork(currentLocalThreadId, newLocalThreadId, options) {
644
945
  const existingTarget = await this.sessionStore.get(newLocalThreadId);
@@ -749,7 +1050,11 @@ export class RunnerManager {
749
1050
  if (handle.dead)
750
1051
  return;
751
1052
  if (message.t === "mirror") {
1053
+ this.observeHandleRuntimeEvent(handle, message.event);
752
1054
  this.mirrorListener?.(message.sessionId, message.event);
1055
+ if (message.event.type === "response.created") {
1056
+ this.settleTerminalInputHandoff(handle, message.sessionId, "turn");
1057
+ }
753
1058
  return;
754
1059
  }
755
1060
  this.deliverRotateMessage(handle, message);
@@ -806,8 +1111,14 @@ export class RunnerManager {
806
1111
  };
807
1112
  void Promise.resolve(this.rotateListener?.(rotation))
808
1113
  .then(() => {
1114
+ // The server's rotation listener resolves only after the target Session
1115
+ // is published. Release the terminal admission handoff afterwards so a
1116
+ // maintenance snapshot cannot observe a gap with neither source work
1117
+ // nor the target Session.
1118
+ this.settleTerminalRotationHandoff(handle, message.from, message.to);
809
1119
  if (handle.dead)
810
1120
  return;
1121
+ handle.activeResponseIds.clear();
811
1122
  this.handles.set(message.to, handle);
812
1123
  this.liveSessionKeys.add(message.to);
813
1124
  this.liveOptions.set(message.to, {
@@ -845,10 +1156,12 @@ export class RunnerManager {
845
1156
  const sessionContext = key === CAP_KEY
846
1157
  ? undefined
847
1158
  : this.openSessionContext(key);
1159
+ const processGroup = process.platform !== "win32";
848
1160
  let child;
849
1161
  try {
850
1162
  child = this.spawn(process.execPath, args, {
851
1163
  stdio: ["pipe", "pipe", "pipe"],
1164
+ detached: processGroup,
852
1165
  // `RYNX_RUNNER_SESSION` = the child's session key (localThreadId, or `__cap__`
853
1166
  // for the shared capability child) so its LocalAgentHost scopes the private
854
1167
  // CODEX_HOME to this session. It is deliberately applied last.
@@ -872,8 +1185,11 @@ export class RunnerManager {
872
1185
  transport,
873
1186
  stderr: [],
874
1187
  lastUsedAt: this.now(),
1188
+ activeResponseIds: new Set(),
875
1189
  dead: false,
876
1190
  completion,
1191
+ terminalCleanupRetryFailures: 0,
1192
+ processGroup,
877
1193
  ...(sessionContext ? { sessionContext } : {}),
878
1194
  caps: new Map(),
879
1195
  terminals: new Map(),
@@ -936,7 +1252,14 @@ export class RunnerManager {
936
1252
  return;
937
1253
  if (this.bufferForkTargetMessage(handle, msg))
938
1254
  return;
1255
+ this.observeHandleRuntimeEvent(handle, msg.event);
939
1256
  this.mirrorListener?.(msg.sessionId, msg.event);
1257
+ if (msg.event.type === "response.created") {
1258
+ // The listener projects response.created into SessionRuntimeIndex
1259
+ // synchronously. Only then may the accepted terminal reservation
1260
+ // drain into a maintenance activity snapshot.
1261
+ this.settleTerminalInputHandoff(handle, msg.sessionId, "turn");
1262
+ }
940
1263
  return;
941
1264
  case "rotate": {
942
1265
  if (handle.dead)
@@ -965,7 +1288,7 @@ export class RunnerManager {
965
1288
  }
966
1289
  }
967
1290
  /** Mark a handle dead and reject every pending run/cap with the exit reason. */
968
- failHandle(handle, reason) {
1291
+ failHandle(handle, reason, opts = {}) {
969
1292
  if (handle.dead) {
970
1293
  return;
971
1294
  }
@@ -984,6 +1307,9 @@ export class RunnerManager {
984
1307
  }
985
1308
  const tail = handle.stderr.join("\n");
986
1309
  const message = tail ? `${reason}\n--- runner log tail ---\n${tail}` : reason;
1310
+ if (opts.failActiveResponses !== false) {
1311
+ this.failActiveResponses(handle, "runner_crashed", message);
1312
+ }
987
1313
  const error = new AgentRuntimeError(message, 500, "runner_crashed");
988
1314
  for (const pending of handle.caps.values()) {
989
1315
  pending.reject(error);
@@ -1002,7 +1328,11 @@ export class RunnerManager {
1002
1328
  terminateHandle(handle, reason) {
1003
1329
  if (handle.termination)
1004
1330
  return handle.termination;
1005
- handle.termination = (async () => {
1331
+ if (handle.terminalCleanupRetryTimer) {
1332
+ clearTimeout(handle.terminalCleanupRetryTimer);
1333
+ delete handle.terminalCleanupRetryTimer;
1334
+ }
1335
+ const attempt = (async () => {
1006
1336
  if (!handle.dead) {
1007
1337
  try {
1008
1338
  handle.transport.send({ t: "shutdown" });
@@ -1010,38 +1340,189 @@ export class RunnerManager {
1010
1340
  catch {
1011
1341
  // The process signal below remains the authoritative shutdown path.
1012
1342
  }
1013
- this.failHandle(handle, reason);
1343
+ this.failHandle(handle, reason, { failActiveResponses: false });
1014
1344
  }
1015
- signalChild(handle.child, "SIGTERM");
1016
- if (!(await waitForChildExit(handle.completion, this.shutdownGraceMs))) {
1017
- signalChild(handle.child, "SIGKILL");
1018
- if (!(await waitForChildExit(handle.completion, this.shutdownKillGraceMs))) {
1019
- throw new Error(`runner child ${handle.child.pid ?? handle.key} did not exit after SIGKILL`);
1020
- }
1345
+ this.signalChild(handle.child, "SIGTERM", handle.processGroup);
1346
+ let childExited = await waitForChildExit(handle.completion, this.shutdownGraceMs);
1347
+ if (!childExited) {
1348
+ this.signalChild(handle.child, "SIGKILL", handle.processGroup);
1349
+ childExited = await waitForChildExit(handle.completion, this.shutdownKillGraceMs);
1350
+ }
1351
+ const terminalStopped = handle.key === CAP_KEY
1352
+ ? true
1353
+ : await Promise.resolve(this.terminateTerminalServer(`${handle.key}-main`)).catch(() => false);
1354
+ if (!childExited) {
1355
+ throw new Error(`runner child ${handle.child.pid ?? handle.key} did not exit after SIGKILL`);
1356
+ }
1357
+ if (!terminalStopped) {
1358
+ throw new Error(`runner terminal ${handle.key}-main remained live after child exit`);
1021
1359
  }
1022
1360
  this.childHandles.delete(handle);
1023
1361
  })();
1024
- return handle.termination;
1362
+ handle.termination = attempt;
1363
+ void attempt.then(() => {
1364
+ this.finishAllTerminalInputHandoffs(handle);
1365
+ }, () => {
1366
+ if (handle.termination === attempt)
1367
+ delete handle.termination;
1368
+ handle.terminalCleanupRetryFailures += 1;
1369
+ // A failed cleanup must not pin closeAndDrain forever. Keep the
1370
+ // submission in the daemon activity snapshot, but release its gate
1371
+ // reservation so maintenance returns a structured busy result.
1372
+ this.preserveTerminalInputHandoffsAsActivity(handle);
1373
+ this.scheduleTerminalInputCleanupRetry(handle);
1374
+ });
1375
+ return attempt;
1025
1376
  }
1026
1377
  reapIdle() {
1378
+ if (this.reapPromise)
1379
+ return this.reapPromise;
1380
+ const attempt = this.reapIdleOnce();
1381
+ this.reapPromise = attempt;
1382
+ const clear = () => {
1383
+ if (this.reapPromise === attempt)
1384
+ this.reapPromise = null;
1385
+ };
1386
+ void attempt.then(clear, clear);
1387
+ return attempt;
1388
+ }
1389
+ async reapIdleOnce() {
1027
1390
  const now = this.now();
1028
1391
  for (const [key, handle] of this.handles) {
1029
- if (handle.caps.size > 0 || handle.terminals.size > 0) {
1392
+ if (this.isManagedNativeHandle(handle)) {
1393
+ await this.reapNativePane(handle, now);
1030
1394
  continue;
1031
1395
  }
1032
- // A live codex forwarder must outlive terminal detach / idle turns — it is
1033
- // the session's single event writer, so never reap a session that has one.
1034
- if (this.liveSessionKeys.has(key)) {
1396
+ if (handle.caps.size > 0 || handle.terminals.size > 0 || handle.live.size > 0) {
1035
1397
  continue;
1036
1398
  }
1037
- if (now - handle.lastUsedAt < this.idleTtlMs) {
1399
+ const idleForMs = now - handle.lastUsedAt;
1400
+ if (idleForMs < this.idleTtlMs) {
1038
1401
  continue;
1039
1402
  }
1040
- void this.terminateHandle(handle, "idle runner reaped").catch((error) => {
1403
+ const hasActiveResponse = handle.activeResponseIds.size > 0;
1404
+ if (hasActiveResponse)
1405
+ continue;
1406
+ const reason = this.liveSessionKeys.has(key)
1407
+ ? "idle live runner reaped"
1408
+ : "idle runner reaped";
1409
+ void this.terminateHandle(handle, reason).catch((error) => {
1041
1410
  logTerminationFailure(handle, error);
1042
1411
  });
1043
1412
  }
1044
1413
  }
1414
+ /** Provider-authoritative turn state is the lifecycle truth; attached clients
1415
+ * and tmux's own activity clock are independent busy evidence. There is no
1416
+ * "user was inactive for an hour" override for an active turn. */
1417
+ async reapNativePane(handle, now) {
1418
+ if (this.nativePaneIdleTtlMs <= 0 || handle.dead)
1419
+ return;
1420
+ if (await this.isNativePaneBusy(handle)) {
1421
+ handle.nativePaneLastBusyAt = now;
1422
+ return;
1423
+ }
1424
+ const lastBusyAt = handle.nativePaneLastBusyAt;
1425
+ if (lastBusyAt === undefined) {
1426
+ // First idle observation starts a full grace window.
1427
+ handle.nativePaneLastBusyAt = now;
1428
+ return;
1429
+ }
1430
+ if (now - lastBusyAt < this.nativePaneIdleTtlMs)
1431
+ return;
1432
+ // Close the select→reap race: activity may begin after classification but
1433
+ // before teardown.
1434
+ if (await this.isNativePaneBusy(handle)) {
1435
+ handle.nativePaneLastBusyAt = this.now();
1436
+ return;
1437
+ }
1438
+ // A failed teardown must re-arm on the next scan instead of retrying on
1439
+ // every sweep forever.
1440
+ delete handle.nativePaneLastBusyAt;
1441
+ await this.terminateHandle(handle, "idle native pane reaped").catch((error) => logTerminationFailure(handle, error));
1442
+ }
1443
+ async isNativePaneBusy(handle) {
1444
+ if (handle.activeResponseIds.size > 0 ||
1445
+ handle.caps.size > 0 ||
1446
+ handle.live.size > 0 ||
1447
+ handle.terminals.size > 0) {
1448
+ return true;
1449
+ }
1450
+ const terminalName = `${handle.key}-main`;
1451
+ try {
1452
+ if (await this.terminalHasAttachedClient(terminalName))
1453
+ return true;
1454
+ }
1455
+ catch {
1456
+ // A failed probe contributes no busy evidence; check activity next.
1457
+ }
1458
+ let activityAt = null;
1459
+ try {
1460
+ activityAt = await this.terminalWindowActivityAt(terminalName);
1461
+ }
1462
+ catch {
1463
+ // A failed probe contributes no busy evidence.
1464
+ }
1465
+ return activityAt !== null &&
1466
+ this.wallNow() - activityAt * 1_000 < this.nativePaneOutputBusyWindowMs;
1467
+ }
1468
+ isManagedNativeHandle(handle) {
1469
+ const sessionId = this.currentTerminalSessionId(handle, handle.key);
1470
+ const provider = this.liveOptions.get(sessionId)?.execution.provider;
1471
+ return isManagedNativeProvider(provider);
1472
+ }
1473
+ failActiveResponses(handle, code, message) {
1474
+ const responseIds = [...handle.activeResponseIds];
1475
+ handle.activeResponseIds.clear();
1476
+ const sessionId = this.currentTerminalSessionId(handle, handle.key);
1477
+ for (const responseId of responseIds) {
1478
+ this.mirrorListener?.(sessionId, {
1479
+ type: "response.failed",
1480
+ responseId,
1481
+ error: {
1482
+ source: "execution",
1483
+ code,
1484
+ message,
1485
+ },
1486
+ });
1487
+ }
1488
+ }
1489
+ /** Track the provider-authoritative response lifecycle. Native-pane busy
1490
+ * classification consumes this level directly; output volume is separately
1491
+ * grounded in tmux's own activity clock. */
1492
+ observeHandleRuntimeEvent(handle, event) {
1493
+ switch (event.type) {
1494
+ case "response.created":
1495
+ case "response.output_text.delta":
1496
+ case "response.reasoning_summary_text.delta":
1497
+ case "response.function_call_output.delta":
1498
+ case "response.output_item.done":
1499
+ handle.activeResponseIds.add(event.responseId);
1500
+ return;
1501
+ case "session.interaction.requested":
1502
+ handle.activeResponseIds.add(event.responseId);
1503
+ return;
1504
+ case "response.completed":
1505
+ case "response.failed":
1506
+ handle.activeResponseIds.delete(event.responseId);
1507
+ return;
1508
+ case "session.status":
1509
+ if (event.status === "running" && event.responseId) {
1510
+ handle.activeResponseIds.add(event.responseId);
1511
+ }
1512
+ else if (event.status !== "running") {
1513
+ if (event.responseId)
1514
+ handle.activeResponseIds.delete(event.responseId);
1515
+ else
1516
+ handle.activeResponseIds.clear();
1517
+ }
1518
+ return;
1519
+ case "session.rotated":
1520
+ handle.activeResponseIds.clear();
1521
+ return;
1522
+ default:
1523
+ return;
1524
+ }
1525
+ }
1045
1526
  openSessionContext(sessionId) {
1046
1527
  if (!this.sessionContextProvider)
1047
1528
  return undefined;
@@ -1060,6 +1541,70 @@ export class RunnerManager {
1060
1541
  }
1061
1542
  }
1062
1543
  }
1544
+ function trackedTerminalSubmissions(tracker, data) {
1545
+ const pasteStart = "\u001b[200~";
1546
+ const pasteEnd = "\u001b[201~";
1547
+ const input = tracker.pendingEscape + data;
1548
+ tracker.pendingEscape = "";
1549
+ const submissions = [];
1550
+ for (let index = 0; index < input.length; index += 1) {
1551
+ const character = input[index] ?? "";
1552
+ if (character === "\u001b") {
1553
+ const remaining = input.slice(index);
1554
+ if (remaining.startsWith(pasteStart)) {
1555
+ tracker.bracketedPaste = true;
1556
+ tracker.previousWasCarriageReturn = false;
1557
+ index += pasteStart.length - 1;
1558
+ continue;
1559
+ }
1560
+ if (remaining.startsWith(pasteEnd)) {
1561
+ tracker.bracketedPaste = false;
1562
+ tracker.previousWasCarriageReturn = false;
1563
+ index += pasteEnd.length - 1;
1564
+ continue;
1565
+ }
1566
+ if (pasteStart.startsWith(remaining) || pasteEnd.startsWith(remaining)) {
1567
+ tracker.pendingEscape = remaining;
1568
+ break;
1569
+ }
1570
+ }
1571
+ if (tracker.bracketedPaste) {
1572
+ tracker.buffer += character;
1573
+ tracker.previousWasCarriageReturn = false;
1574
+ continue;
1575
+ }
1576
+ if (character === "\n" && tracker.previousWasCarriageReturn) {
1577
+ tracker.previousWasCarriageReturn = false;
1578
+ continue;
1579
+ }
1580
+ tracker.previousWasCarriageReturn = character === "\r";
1581
+ if (character === "\r" || character === "\n") {
1582
+ const command = tracker.buffer.trim();
1583
+ tracker.buffer = "";
1584
+ if (command.length > 0 &&
1585
+ (command.includes("\u001b") ||
1586
+ !command.startsWith("/") ||
1587
+ /^\/(?:clear|fork)(?:\s|$)/u.test(command))) {
1588
+ submissions.push(/^\/(?:clear|fork)(?:\s|$)/u.test(command) ? "rotation" : "turn");
1589
+ }
1590
+ continue;
1591
+ }
1592
+ if (character === "\b" || character === "\u007f") {
1593
+ tracker.buffer = tracker.buffer.slice(0, -1);
1594
+ continue;
1595
+ }
1596
+ if (character === "\u0015" || character === "\u0003") {
1597
+ tracker.buffer = "";
1598
+ continue;
1599
+ }
1600
+ // Printable text and tabs are enough to distinguish work-producing prompts
1601
+ // from local TUI slash commands. Other control sequences are ignored.
1602
+ if (character === "\u001b" || character === "\t" || character >= " ") {
1603
+ tracker.buffer += character;
1604
+ }
1605
+ }
1606
+ return submissions;
1607
+ }
1063
1608
  function validateSessionContext(context) {
1064
1609
  if (!context || typeof context !== "object") {
1065
1610
  throw new Error("Session context environment provider must return an object");
@@ -1164,9 +1709,18 @@ function observeChildCompletion(child) {
1164
1709
  child.once("exit", onExit);
1165
1710
  });
1166
1711
  }
1167
- function signalChild(child, signal) {
1712
+ function signalRunnerChild(child, signal, processGroup) {
1168
1713
  if (childHasExited(child))
1169
1714
  return;
1715
+ if (processGroup && child.pid) {
1716
+ try {
1717
+ process.kill(-child.pid, signal);
1718
+ return;
1719
+ }
1720
+ catch {
1721
+ // The group may have exited between the completion check and the signal.
1722
+ }
1723
+ }
1170
1724
  try {
1171
1725
  child.kill(signal);
1172
1726
  }