@ours.network/fleet 0.17.1 → 0.17.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. package/README.md +38 -2
  2. package/dist/application/role-removal-service.js +1 -1
  3. package/dist/application/session-control.d.ts +14 -10
  4. package/dist/application/session-control.js +14 -3
  5. package/dist/atomic-file.d.ts +7 -1
  6. package/dist/atomic-file.js +33 -5
  7. package/dist/build-info.json +10 -0
  8. package/dist/capabilities.d.ts +20 -0
  9. package/dist/capabilities.js +21 -0
  10. package/dist/cli.js +98 -10
  11. package/dist/config.d.ts +9 -2
  12. package/dist/config.js +16 -2
  13. package/dist/creation.d.ts +16 -0
  14. package/dist/creation.js +28 -0
  15. package/dist/docs.d.ts +1 -1
  16. package/dist/docs.js +70 -4
  17. package/dist/doctor.d.ts +5 -0
  18. package/dist/doctor.js +87 -2
  19. package/dist/harness/acp-agent.d.ts +3 -0
  20. package/dist/harness/acp-agent.js +4 -1
  21. package/dist/harness/codex-app-server-proxy.d.ts +4 -0
  22. package/dist/harness/codex-app-server-proxy.js +133 -0
  23. package/dist/harness/codex.js +79 -11
  24. package/dist/index.d.ts +2 -1
  25. package/dist/index.js +1 -0
  26. package/dist/loops/manager.d.ts +42 -1
  27. package/dist/loops/manager.js +115 -16
  28. package/dist/loops/state.d.ts +46 -2
  29. package/dist/loops/state.js +81 -3
  30. package/dist/monitor.d.ts +21 -0
  31. package/dist/monitor.js +42 -0
  32. package/dist/ops.d.ts +6 -0
  33. package/dist/ops.js +46 -1
  34. package/dist/owner-channel/channel.d.ts +18 -2
  35. package/dist/owner-channel/channel.js +146 -2
  36. package/dist/owner-channel/commands.d.ts +2 -2
  37. package/dist/owner-channel/commands.js +7 -2
  38. package/dist/owner-channel/notices.d.ts +2 -0
  39. package/dist/owner-channel/notices.js +3 -0
  40. package/dist/provenance.d.ts +77 -0
  41. package/dist/provenance.js +283 -0
  42. package/dist/runner.d.ts +7 -1
  43. package/dist/runner.js +100 -14
  44. package/dist/session/acp.d.ts +40 -4
  45. package/dist/session/acp.js +157 -30
  46. package/dist/session/arbiter.d.ts +28 -2
  47. package/dist/session/arbiter.js +75 -4
  48. package/dist/session/control.js +12 -6
  49. package/dist/session/event-log.d.ts +109 -0
  50. package/dist/session/event-log.js +247 -0
  51. package/dist/session/events.d.ts +21 -0
  52. package/dist/session/events.js +105 -26
  53. package/dist/session/tmux.d.ts +3 -2
  54. package/dist/session/tmux.js +2 -0
  55. package/dist/session/types.d.ts +39 -2
  56. package/dist/session/types.js +11 -1
  57. package/dist/spawn.d.ts +3 -3
  58. package/dist/spawn.js +40 -14
  59. package/dist/temp-lifecycle.d.ts +62 -0
  60. package/dist/temp-lifecycle.js +437 -0
  61. package/package.json +5 -3
@@ -7,8 +7,9 @@ import * as acp from '@agentclientprotocol/sdk';
7
7
  import { normalizeSessionUpdate } from './conversation-normalizer.js';
8
8
  import { ConversationEventStore } from './conversation-store.js';
9
9
  import { SessionEvents } from './events.js';
10
- import { SessionControlError, classifyChildExit, turnResult } from './types.js';
10
+ import { ACP_CANCEL_DEADLINE_EXCEEDED, SessionControlError, classifyChildExit, turnResult, } from './types.js';
11
11
  const CANCEL_SETTLE_GRACE_MS = 15_000;
12
+ const CANCEL_TERMINATE_GRACE_MS = 5_000;
12
13
  /** A permission no human answered is eventually a decision nobody made. */
13
14
  const PERMISSION_TIMEOUT_MS = 10 * 60_000;
14
15
  /** Spec §4.3: 10-15 s before a vanished controller triggers the unattended policy. */
@@ -214,6 +215,15 @@ export class AcpSession {
214
215
  /** Armed when the last controller detaches; unattended policy applies on fire. */
215
216
  controllerGrace;
216
217
  cancelEscalation;
218
+ cancelForceKill;
219
+ cancelRecoveryReason;
220
+ /**
221
+ * Rejects the moment the adapter process is gone. Every in-flight ACP request
222
+ * races it, so a dead adapter can never leave a turn — and therefore a
223
+ * scheduled run's `activeRunId` or an admission claim — unsettled forever.
224
+ */
225
+ terminated;
226
+ terminate;
217
227
  /** ACP-authenticated in-flight calls, including independently reserved permissions. */
218
228
  activeToolCalls = new Map();
219
229
  toolBoundaryWaiters = new Set();
@@ -228,16 +238,29 @@ export class AcpSession {
228
238
  roleId: options.name, log: line => options.log(`[${options.name}] ${line}`),
229
239
  });
230
240
  this.sessionFile = join(options.stateDir, '.acp-session-id');
241
+ this.terminated = new Promise((_resolve, reject) => { this.terminate = reject; });
242
+ // Nothing awaits this promise until a request races it; an unobserved
243
+ // rejection here must never take the whole runner down.
244
+ this.terminated.catch(() => undefined);
231
245
  child.stderr.on('data', chunk => options.log(`[${options.name}] acp: ${String(chunk).trimEnd()}`));
232
246
  child.once('exit', (code, signal) => {
247
+ if (this.cancelForceKill)
248
+ clearTimeout(this.cancelForceKill);
249
+ this.cancelForceKill = undefined;
233
250
  // Record the child's real exit code/signal. The tmux path can only see a
234
251
  // shell's `$?`; here the truth is available, so keep it.
235
- this.exit = classifyChildExit(code, signal);
252
+ const classified = classifyChildExit(code, signal);
253
+ this.exit = this.cancelRecoveryReason
254
+ ? { ...classified, detail: `${this.cancelRecoveryReason}; ${classified.detail}` }
255
+ : classified;
236
256
  options.log(`[${options.name}] acp: agent exited (${code ?? signal ?? 'unknown'})`);
237
257
  if (this.readiness !== 'failed') {
238
258
  this.readiness = 'failed';
239
259
  this.lastError = `ACP agent ${this.exit.detail}`;
240
260
  }
261
+ // A request whose peer no longer exists will never answer. Fail it here
262
+ // rather than trusting the transport to notice the closed stream.
263
+ this.terminate(new SessionControlError('offline', `ACP agent ${this.exit.detail}`));
241
264
  this.events.emit('state', { status: 'failed', text: this.lastError });
242
265
  this.conversation.appendSafe({
243
266
  kind: 'session.state', sessionGeneration: this.sessionGeneration,
@@ -319,7 +342,10 @@ export class AcpSession {
319
342
  }
320
343
  }
321
344
  isAlive() {
322
- return this.child.exitCode === null && !this.child.killed;
345
+ // child.killed only means kill() successfully SENT a signal. A process may
346
+ // ignore SIGTERM and remain alive. exitCode or signalCode is the actual
347
+ // terminal fact (signal exits deliberately leave exitCode null).
348
+ return this.child.exitCode === null && (this.child.signalCode ?? null) === null;
323
349
  }
324
350
  snapshot() {
325
351
  return {
@@ -425,8 +451,28 @@ export class AcpSession {
425
451
  });
426
452
  }
427
453
  /**
428
- * Monitor-only safe-boundary delivery. Steering is the interruption: this
429
- * path never calls session/cancel and never resolves a pending permission.
454
+ * Steering is an optional admission fast path, not the only safe way to
455
+ * deliver a wake. Codex can reject `_session/steering` while a long-running
456
+ * turn is between tools. Queue one ordinary, non-cancelling prompt in that
457
+ * case and wait for its terminal result. This keeps the monitor's cursor
458
+ * uncommitted until the wake really runs and, critically, keeps one rejected
459
+ * steering response from becoming a tight replay loop.
460
+ */
461
+ async steerOrQueueWake(text, options) {
462
+ const steered = await this.steerPrompt(text);
463
+ if (steered.accepted || steered.detail !== 'ACP steering failed'
464
+ || this.closing || !this.isAlive())
465
+ return steered;
466
+ const queued = await this.submitPrompt(text, { ...options, interrupt: false, steer: false });
467
+ return {
468
+ ...queued,
469
+ detail: `steering rejected; queued delivery ${queued.detail ?? queued.outcome}`,
470
+ };
471
+ }
472
+ /**
473
+ * Monitor-only safe-boundary delivery. Steering is the preferred live
474
+ * insertion and rejected steering is queued: this path never calls
475
+ * session/cancel and never resolves a pending permission.
430
476
  */
431
477
  async submitPromptAfterTool(text, options = {}) {
432
478
  if (this.closing || !this.isAlive())
@@ -443,7 +489,7 @@ export class AcpSession {
443
489
  }
444
490
  if (initialToolCount === 0) {
445
491
  this.recordAfterToolDelivery('direct', 0, 0);
446
- const result = await this.steerPrompt(text);
492
+ const result = await this.steerOrQueueWake(text, options);
447
493
  return { ...result, safeBoundary: { state: 'direct', waitedMs: 0, activeToolCount: 0 } };
448
494
  }
449
495
  this.recordAfterToolDelivery('deferred', initialToolCount, 0);
@@ -464,7 +510,7 @@ export class AcpSession {
464
510
  const state = atBoundary ? 'after_tool' : 'timeout';
465
511
  const remainingToolCount = this.activeToolCalls.size;
466
512
  this.recordAfterToolDelivery(state, remainingToolCount, waitedMs);
467
- const result = await this.steerPrompt(text);
513
+ const result = await this.steerOrQueueWake(text, options);
468
514
  return {
469
515
  ...result,
470
516
  safeBoundary: { state, waitedMs, activeToolCount: remainingToolCount },
@@ -476,7 +522,9 @@ export class AcpSession {
476
522
  * for it is what turned a busy agent into a timeout and then into "dead".
477
523
  */
478
524
  async queuePrompt(text, options = {}) {
479
- if (!this.sessionId || !this.isAlive())
525
+ if (this.cancelRecoveryReason)
526
+ throw new SessionControlError('control-unavailable', 'ACP adapter restart is in progress after the cancellation deadline', ACP_CANCEL_DEADLINE_EXCEEDED);
527
+ if (this.closing || !this.sessionId || !this.isAlive())
480
528
  throw new SessionControlError('offline', this.lastError ?? 'ACP session is offline');
481
529
  if (options.interrupt)
482
530
  await this.cancelActive(options.interruptSource ?? 'local-console');
@@ -570,8 +618,24 @@ export class AcpSession {
570
618
  throw error;
571
619
  }
572
620
  }
621
+ /**
622
+ * Explicit cancellation on behalf of a human or an operator. Forced recovery
623
+ * is reported as an outcome, never as a thrown failure: by the time this
624
+ * resolves the turn is over either way, and only the durable-ingress path
625
+ * (`queuePrompt({ interrupt: true })`) needs the typed error, because only it
626
+ * still owes an undelivered message a replay.
627
+ */
573
628
  async interrupt(source = 'local-console') {
574
- await this.cancelActive(source);
629
+ try {
630
+ await this.cancelActive(source);
631
+ return { state: 'settled' };
632
+ }
633
+ catch (error) {
634
+ if (error instanceof SessionControlError
635
+ && error.reasonCode === ACP_CANCEL_DEADLINE_EXCEEDED)
636
+ return { state: 'forced', reasonCode: ACP_CANCEL_DEADLINE_EXCEEDED };
637
+ throw error;
638
+ }
575
639
  }
576
640
  async cancelActive(source) {
577
641
  if (!this.sessionId)
@@ -594,21 +658,66 @@ export class AcpSession {
594
658
  acpSessionId: this.sessionId, promptId: active.id, turnId: active.id,
595
659
  payload: { cancellationSource: source },
596
660
  });
597
- if (this.cancelEscalation)
598
- clearTimeout(this.cancelEscalation);
599
- const turnId = active.id;
661
+ }
662
+ for (const [permissionId, pending] of [...this.pendingPermissions])
663
+ this.settlePendingAutomatically(permissionId, pending, 'cancelled', undefined, 'the turn was cancelled while this request was pending');
664
+ if (active && this.activeTurn === active) {
665
+ active.cancellationWait ??= this.awaitCancellationSettlement(active);
666
+ await active.cancellationWait;
667
+ }
668
+ }
669
+ /**
670
+ * Do not admit work behind a turn whose adapter may already require restart.
671
+ * A cooperative adapter settles this promise immediately through runPrompt's
672
+ * finally block. A stubborn adapter receives SIGTERM at the deadline and
673
+ * SIGKILL after one more bounded grace; callers get a typed recovery error so
674
+ * durable ingress can leave the next request replayable for the resumed run.
675
+ */
676
+ awaitCancellationSettlement(active) {
677
+ const settleMs = this.options.cancelGraceMs ?? CANCEL_SETTLE_GRACE_MS;
678
+ const deadline = new Promise((resolve, reject) => {
600
679
  this.cancelEscalation = setTimeout(() => {
601
- if (this.activeTurn?.id !== turnId || !this.isAlive())
680
+ // Both bail-outs must SETTLE the race. Returning silently once left the
681
+ // caller — and the arbiter's exclusive tail behind it — awaiting a
682
+ // promise nothing would ever resolve.
683
+ if (this.activeTurn !== active) {
684
+ resolve();
602
685
  return;
603
- this.lastError = 'ACP turn ignored cancellation; restarting adapter';
604
- this.options.log(`[${this.options.name}] ${this.lastError}`);
605
- this.events.emit('error', { turnId, origin: active.origin, text: this.lastError });
686
+ }
687
+ if (!this.isAlive()) {
688
+ resolve();
689
+ return;
690
+ }
691
+ active.cancellationDeadlineExceeded = true;
692
+ this.cancelRecoveryReason = ACP_CANCEL_DEADLINE_EXCEEDED;
693
+ this.readiness = 'failed';
694
+ this.lastError = `${ACP_CANCEL_DEADLINE_EXCEEDED}: ACP turn did not settle within ${settleMs}ms`;
695
+ this.options.log(`[${this.options.name}] ${this.lastError}; restarting adapter with resume`);
696
+ this.events.emit('error', {
697
+ turnId: active.id, origin: active.origin, status: ACP_CANCEL_DEADLINE_EXCEEDED,
698
+ text: this.lastError,
699
+ });
700
+ this.conversation.appendSafe({
701
+ kind: 'session.state', sessionGeneration: this.sessionGeneration,
702
+ acpSessionId: this.sessionId,
703
+ promptId: active.id, turnId: active.id,
704
+ payload: { status: 'failed', detail: this.lastError },
705
+ });
606
706
  this.child.kill('SIGTERM');
607
- }, this.options.cancelGraceMs ?? CANCEL_SETTLE_GRACE_MS);
707
+ const terminateMs = this.options.cancelTerminateGraceMs ?? CANCEL_TERMINATE_GRACE_MS;
708
+ this.cancelForceKill = setTimeout(() => {
709
+ if (this.child.exitCode === null) {
710
+ this.options.log(`[${this.options.name}] ${ACP_CANCEL_DEADLINE_EXCEEDED}: `
711
+ + `adapter ignored SIGTERM for ${terminateMs}ms; sending SIGKILL`);
712
+ this.child.kill('SIGKILL');
713
+ }
714
+ }, terminateMs);
715
+ this.cancelForceKill.unref?.();
716
+ reject(new SessionControlError('control-unavailable', 'ACP adapter restart is in progress after the cancellation deadline', ACP_CANCEL_DEADLINE_EXCEEDED));
717
+ }, settleMs);
608
718
  this.cancelEscalation.unref?.();
609
- }
610
- for (const [permissionId, pending] of [...this.pendingPermissions])
611
- this.settlePendingAutomatically(permissionId, pending, 'cancelled', undefined, 'the turn was cancelled while this request was pending');
719
+ });
720
+ return Promise.race([active.settled, deadline]);
612
721
  }
613
722
  respondPermission(permissionId, optionId) {
614
723
  const pending = this.pendingPermissions.get(permissionId);
@@ -740,6 +849,9 @@ export class AcpSession {
740
849
  if (this.cancelEscalation)
741
850
  clearTimeout(this.cancelEscalation);
742
851
  this.cancelEscalation = undefined;
852
+ if (this.cancelForceKill)
853
+ clearTimeout(this.cancelForceKill);
854
+ this.cancelForceKill = undefined;
743
855
  if (this.controllerGrace)
744
856
  clearTimeout(this.controllerGrace);
745
857
  this.controllerGrace = undefined;
@@ -752,6 +864,8 @@ export class AcpSession {
752
864
  this.connection.close();
753
865
  if (this.isAlive())
754
866
  this.child.kill('SIGTERM');
867
+ // The transport is gone: nothing still awaiting an ACP answer can get one.
868
+ this.terminate(new SessionControlError('offline', 'the ACP session was closed'));
755
869
  this.conversation.appendSafe({
756
870
  kind: 'session.state', sessionGeneration: this.sessionGeneration,
757
871
  acpSessionId: this.sessionId, payload: { status: 'offline' },
@@ -839,7 +953,9 @@ export class AcpSession {
839
953
  if (!this.sessionId || !this.isAlive())
840
954
  return turnResult(false, 'failed', this.lastError ?? 'ACP session is offline');
841
955
  this.readiness = 'running';
842
- this.activeTurn = { id: turnId, output: '', origin };
956
+ let settle;
957
+ const settled = new Promise(resolve => { settle = resolve; });
958
+ this.activeTurn = { id: turnId, output: '', origin, settled, settle };
843
959
  this.events.emit('state', { turnId, status: 'running', origin });
844
960
  this.conversation.appendSafe({
845
961
  kind: 'prompt.started', sessionGeneration: this.sessionGeneration,
@@ -847,10 +963,13 @@ export class AcpSession {
847
963
  source: conversationSource(origin).source, payload: {},
848
964
  });
849
965
  try {
850
- const response = await this.connection.agent.request(acp.methods.agent.session.prompt, {
851
- sessionId: this.sessionId,
852
- prompt: promptContentBlocks(text, origin),
853
- });
966
+ const response = await Promise.race([
967
+ this.connection.agent.request(acp.methods.agent.session.prompt, {
968
+ sessionId: this.sessionId,
969
+ prompt: promptContentBlocks(text, origin),
970
+ }),
971
+ this.terminated,
972
+ ]);
854
973
  this.readiness = 'idle';
855
974
  const cancellationSource = this.activeTurn?.id === turnId
856
975
  ? this.activeTurn.cancellationSource : undefined;
@@ -872,7 +991,11 @@ export class AcpSession {
872
991
  return turnResult(true, classifyStopReason(response.stopReason), response.stopReason, this.activeTurn?.id === turnId ? this.activeTurn.output : undefined, this.activeTurn?.id === turnId ? this.activeTurn.cancellationSource : undefined);
873
992
  }
874
993
  catch (error) {
875
- const detail = error?.message ?? String(error);
994
+ const escalated = this.activeTurn?.id === turnId
995
+ && this.activeTurn.cancellationDeadlineExceeded;
996
+ const detail = escalated
997
+ ? this.lastError ?? ACP_CANCEL_DEADLINE_EXCEEDED
998
+ : error?.message ?? String(error);
876
999
  this.lastError = origin?.kind === 'scheduled-loop' ? 'scheduled-loop turn failed' : detail;
877
1000
  this.readiness = this.isAlive() ? 'idle' : 'failed';
878
1001
  this.events.emit('error', {
@@ -891,6 +1014,7 @@ export class AcpSession {
891
1014
  finally {
892
1015
  this.releaseAllTools();
893
1016
  if (this.activeTurn?.id === turnId) {
1017
+ this.activeTurn.settle();
894
1018
  if (this.cancelEscalation)
895
1019
  clearTimeout(this.cancelEscalation);
896
1020
  this.cancelEscalation = undefined;
@@ -902,10 +1026,13 @@ export class AcpSession {
902
1026
  if (!this.sessionId || !this.isAlive())
903
1027
  return turnResult(false, 'failed', this.lastError ?? 'ACP session is offline');
904
1028
  try {
905
- const response = await this.connection.agent.request('_session/steering', {
906
- sessionId: this.sessionId,
907
- prompt: [{ type: 'text', text }],
908
- });
1029
+ const response = await Promise.race([
1030
+ this.connection.agent.request('_session/steering', {
1031
+ sessionId: this.sessionId,
1032
+ prompt: [{ type: 'text', text }],
1033
+ }),
1034
+ this.terminated,
1035
+ ]);
909
1036
  if (response.outcome === 'failed')
910
1037
  return turnResult(false, 'failed', 'ACP steering failed');
911
1038
  return turnResult(true, 'inconclusive', response.outcome);
@@ -1,4 +1,4 @@
1
- import type { ConversationHandlePage, ExitRecord, PromptOrigin, QueuedPrompt, SessionEvent, SessionHandle, SessionSnapshot, SubmitPromptOptions, TurnCancellationSource, TurnResult } from './types.js';
1
+ import type { ConversationHandlePage, ExitRecord, InterruptOutcome, PromptOrigin, QueuedPrompt, SessionEvent, SessionHandle, SessionSnapshot, SubmitPromptOptions, TurnCancellationSource, TurnResult } from './types.js';
2
2
  import type { ConversationEventV1, ConversationSnapshot, PromptReceipt, SubmitPromptCommand } from './conversation-types.js';
3
3
  export type ScheduledAttempt = {
4
4
  state: 'started';
@@ -22,9 +22,35 @@ export declare class RoleTurnArbiter implements SessionHandle {
22
22
  private tail;
23
23
  private unsettled;
24
24
  private stopping;
25
+ /** Bumped by `retireStalledAdmission`; every claim carries the one it was made under. */
26
+ private generation;
27
+ /** Resolves the instant the CURRENT generation is retired. */
28
+ private retirement;
25
29
  constructor(session: SessionHandle);
30
+ /**
31
+ * Waiters race the tail against their own generation's retirement, so a
32
+ * single operation that never settles cannot own the boundary forever. A
33
+ * woken waiter from a retired generation never runs its operation: the
34
+ * ordering it was promised no longer exists, and admitting it silently would
35
+ * be the one thing worse than telling the caller it was not admitted.
36
+ */
26
37
  private exclusive;
27
38
  private track;
39
+ /**
40
+ * Release the admission boundary after a cancellation that never settled.
41
+ * Nothing can make a hung ACP call return, so the stuck operation keeps its
42
+ * own promise — it just stops owning `tail` and stops holding a claim.
43
+ *
44
+ * Without this, one unsettled `interrupt` left every later producer queued
45
+ * behind it forever: a scheduled poll never returned, so the loop was never
46
+ * rescheduled and reported neither `started` nor `skipped_busy`, and an owner
47
+ * prompt stayed pending with nothing to resolve it.
48
+ *
49
+ * One-active-run safety is unchanged. The arbiter only orders admission;
50
+ * `tryScheduled` still asks the session itself whether it is idle, and the
51
+ * session still refuses to run two turns at once.
52
+ */
53
+ retireStalledAdmission(): void;
28
54
  queuePrompt(text: string, options?: SubmitPromptOptions): Promise<QueuedPrompt>;
29
55
  submitPrompt(text: string, options?: SubmitPromptOptions): Promise<TurnResult>;
30
56
  /**
@@ -45,7 +71,7 @@ export declare class RoleTurnArbiter implements SessionHandle {
45
71
  conversationSnapshot(): ConversationSnapshot;
46
72
  subscribeConversation(listener: (event: ConversationEventV1) => void): () => void;
47
73
  submitPromptBrowser(command: SubmitPromptCommand): Promise<PromptReceipt>;
48
- interrupt(source?: TurnCancellationSource): Promise<void>;
74
+ interrupt(source?: TurnCancellationSource): Promise<InterruptOutcome>;
49
75
  respondPermission(permissionId: string, optionId: string): boolean;
50
76
  respondPermissionV2(permissionId: string, optionId: string, sessionGeneration: string): 'accepted' | 'stale';
51
77
  eventsSince(seq: number): SessionEvent[];
@@ -1,3 +1,20 @@
1
+ import { ACP_CANCEL_DEADLINE_EXCEEDED, SessionControlError, interruptOutcome, } from './types.js';
2
+ /**
3
+ * Raised for admissions that were still waiting on a generation which had to be
4
+ * retired. It carries `ACP_CANCEL_DEADLINE_EXCEEDED` because that is the only
5
+ * thing that ever retires a generation, and because durable owner ingress
6
+ * already treats that reason as "never delivered — replay it".
7
+ */
8
+ class AdmissionRetiredError extends SessionControlError {
9
+ constructor() {
10
+ super('control-unavailable', 'admission was retired after a cancellation that never settled', ACP_CANCEL_DEADLINE_EXCEEDED);
11
+ }
12
+ }
13
+ function retirementSignal() {
14
+ let retire;
15
+ const promise = new Promise(resolve => { retire = resolve; });
16
+ return { promise, retire };
17
+ }
1
18
  /**
2
19
  * One in-process admission boundary for every producer targeting a role.
3
20
  * Scheduled callers get an atomic idle recheck plus submission; ordinary
@@ -11,21 +28,64 @@ export class RoleTurnArbiter {
11
28
  tail = Promise.resolve();
12
29
  unsettled = 0;
13
30
  stopping = false;
31
+ /** Bumped by `retireStalledAdmission`; every claim carries the one it was made under. */
32
+ generation = 0;
33
+ /** Resolves the instant the CURRENT generation is retired. */
34
+ retirement = retirementSignal();
14
35
  constructor(session) {
15
36
  this.session = session;
16
37
  this.backend = session.backend;
17
38
  this.pid = session.pid;
18
39
  }
40
+ /**
41
+ * Waiters race the tail against their own generation's retirement, so a
42
+ * single operation that never settles cannot own the boundary forever. A
43
+ * woken waiter from a retired generation never runs its operation: the
44
+ * ordering it was promised no longer exists, and admitting it silently would
45
+ * be the one thing worse than telling the caller it was not admitted.
46
+ */
19
47
  exclusive(operation) {
20
- const run = this.tail.then(operation);
48
+ const generation = this.generation;
49
+ const run = Promise.race([this.tail, this.retirement.promise]).then(() => {
50
+ if (generation !== this.generation)
51
+ throw new AdmissionRetiredError();
52
+ return operation();
53
+ });
21
54
  this.tail = run.then(() => undefined, () => undefined);
22
55
  return run;
23
56
  }
24
57
  track(queued) {
58
+ const generation = this.generation;
25
59
  this.unsettled++;
26
- const completion = queued.completion.finally(() => { this.unsettled = Math.max(0, this.unsettled - 1); });
60
+ const completion = queued.completion.finally(() => {
61
+ // A retired generation's claims were released in one step already.
62
+ if (generation === this.generation)
63
+ this.unsettled = Math.max(0, this.unsettled - 1);
64
+ });
27
65
  return { ...queued, completion };
28
66
  }
67
+ /**
68
+ * Release the admission boundary after a cancellation that never settled.
69
+ * Nothing can make a hung ACP call return, so the stuck operation keeps its
70
+ * own promise — it just stops owning `tail` and stops holding a claim.
71
+ *
72
+ * Without this, one unsettled `interrupt` left every later producer queued
73
+ * behind it forever: a scheduled poll never returned, so the loop was never
74
+ * rescheduled and reported neither `started` nor `skipped_busy`, and an owner
75
+ * prompt stayed pending with nothing to resolve it.
76
+ *
77
+ * One-active-run safety is unchanged. The arbiter only orders admission;
78
+ * `tryScheduled` still asks the session itself whether it is idle, and the
79
+ * session still refuses to run two turns at once.
80
+ */
81
+ retireStalledAdmission() {
82
+ const retired = this.retirement;
83
+ this.generation++;
84
+ this.retirement = retirementSignal();
85
+ this.tail = Promise.resolve();
86
+ this.unsettled = 0;
87
+ retired.retire();
88
+ }
29
89
  queuePrompt(text, options = {}) {
30
90
  return this.exclusive(async () => this.track(await this.session.queuePrompt(text, options)));
31
91
  }
@@ -54,13 +114,24 @@ export class RoleTurnArbiter {
54
114
  try {
55
115
  await beforeQueue?.();
56
116
  const queued = await this.session.queuePrompt(text, { interrupt: false, origin });
57
- if (queued.queuedBehind > 0)
117
+ // The race is lost, but the prompt was admitted and WILL run. Keep the
118
+ // claim so idle accounting matches reality instead of under-counting.
119
+ if (queued.queuedBehind > 0) {
120
+ this.track(queued);
58
121
  return { state: 'unavailable', error: 'scheduled admission race' };
122
+ }
59
123
  return { state: 'started', queued: this.track(queued) };
60
124
  }
61
125
  catch (error) {
62
126
  return { state: 'unavailable', error: error?.message ?? String(error) };
63
127
  }
128
+ }).catch(error => {
129
+ // A retired attempt was never submitted. Reporting that as an outcome
130
+ // instead of a rejection is what keeps the poll — and with it the loop's
131
+ // own rescheduling — from being abandoned mid-tick.
132
+ if (error instanceof AdmissionRetiredError)
133
+ return { state: 'unavailable', error: error.message };
134
+ throw error;
64
135
  });
65
136
  }
66
137
  stopScheduledAdmission() { this.stopping = true; }
@@ -87,7 +158,7 @@ export class RoleTurnArbiter {
87
158
  return this.exclusive(() => this.session.submitPromptBrowser(command));
88
159
  }
89
160
  interrupt(source = 'local-console') {
90
- return this.exclusive(() => this.session.interrupt(source));
161
+ return this.exclusive(async () => interruptOutcome(await this.session.interrupt(source)));
91
162
  }
92
163
  respondPermission(permissionId, optionId) {
93
164
  return this.session.respondPermission(permissionId, optionId);
@@ -2,7 +2,7 @@ import { randomBytes, randomUUID, timingSafeEqual } from 'node:crypto';
2
2
  import { chmodSync, existsSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
3
3
  import { createConnection, createServer } from 'node:net';
4
4
  import { join } from 'node:path';
5
- import { SessionControlError } from './types.js';
5
+ import { SessionControlError, interruptOutcome } from './types.js';
6
6
  const MAX_LINE_BYTES = 64 * 1024;
7
7
  /** Commands that require protocol version 3. */
8
8
  const V3_COMMANDS = new Set([
@@ -253,10 +253,13 @@ export class RoleControlServer {
253
253
  });
254
254
  return;
255
255
  }
256
- case 'interrupt':
257
- await this.session.interrupt('local-console');
258
- this.write(socket, { version: 1, id: request.id, ok: true });
256
+ case 'interrupt': {
257
+ // Forced recovery cancelled the turn just as surely as a cooperative
258
+ // stop did. Report HOW, never as a failed operation.
259
+ const outcome = interruptOutcome(await this.session.interrupt('local-console'));
260
+ this.write(socket, { version: 1, id: request.id, ok: true, result: outcome });
259
261
  return;
262
+ }
260
263
  case 'loop_status': {
261
264
  if (!this.loopManager)
262
265
  throw new SessionControlError('rejected', 'scheduled loops are unavailable for this role');
@@ -396,8 +399,11 @@ export class RoleControlServer {
396
399
  this.write(socket, { version: 1, id: request.id, ok: true, result: existing });
397
400
  return;
398
401
  }
399
- await this.session.interrupt('local-console');
400
- const receipt = { accepted: true, commandId: request.commandId, at: new Date().toISOString() };
402
+ const outcome = interruptOutcome(await this.session.interrupt('local-console'));
403
+ const receipt = {
404
+ accepted: true, commandId: request.commandId, at: new Date().toISOString(),
405
+ ...outcome,
406
+ };
401
407
  this.interruptCommands.set(request.commandId, receipt);
402
408
  if (this.interruptCommands.size > 200) {
403
409
  const oldest = this.interruptCommands.keys().next().value;
@@ -0,0 +1,109 @@
1
+ /**
2
+ * Crash-safe JSONL primitives for the session event stream.
3
+ *
4
+ * The 06:32 disk-full incident showed what an unguarded `appendFileSync` does
5
+ * under ENOSPC: the kernel short-writes, the already-written prefix stays on
6
+ * disk, and the next successful append lands directly on it — fusing a partial
7
+ * record with a valid later one into a single unparseable line. These helpers
8
+ * make an append all-or-nothing, keep a damaged byte range from ever swallowing
9
+ * a later record, and describe damage rather than quietly dropping it.
10
+ *
11
+ * Single-writer assumption: rollback truncates back to the size observed at the
12
+ * start of the append, so exactly one process may append to a given path.
13
+ */
14
+ /** The syscall surface an append needs, injectable so faults can be forced deterministically. */
15
+ export interface LogIo {
16
+ openSync(path: string, flags: string, mode?: number): number;
17
+ fstatSync(fd: number): {
18
+ size: number;
19
+ };
20
+ readSync(fd: number, buffer: Buffer, offset: number, length: number, position: number): number;
21
+ writeSync(fd: number, buffer: Buffer, offset: number, length: number): number;
22
+ ftruncateSync(fd: number, length: number): void;
23
+ closeSync(fd: number): void;
24
+ }
25
+ export declare const nodeLogIo: LogIo;
26
+ export declare class AtomicAppendError extends Error {
27
+ readonly rollbackFailed: boolean;
28
+ readonly cause?: unknown | undefined;
29
+ constructor(message: string, rollbackFailed: boolean, cause?: unknown | undefined);
30
+ }
31
+ export interface AppendResult {
32
+ /** True when a dangling partial record was closed off before this line was written. */
33
+ repairedBoundary: boolean;
34
+ }
35
+ export interface AppendOptions {
36
+ io?: LogIo;
37
+ /** Total attempts, including the first. Bounded so a full disk cannot spin. */
38
+ maxAttempts?: number;
39
+ mode?: number;
40
+ }
41
+ /**
42
+ * Append one line, all or nothing. A failed write is rolled back to the byte
43
+ * length observed before it started, so a partial record never persists; if the
44
+ * file already ends mid-record (damage from before this fix), the line is put on
45
+ * a fresh line so it cannot fuse with the damaged bytes.
46
+ */
47
+ export declare function appendLineAtomic(path: string, line: string, options?: AppendOptions): AppendResult;
48
+ export type DamageReason = 'truncated_tail' | 'interior_corruption';
49
+ export interface DamagedLine {
50
+ /** 1-based line number in the file as read. */
51
+ lineNumber: number;
52
+ /** True byte length of the damaged line, even when `raw` is capped. */
53
+ bytes: number;
54
+ reason: DamageReason;
55
+ /** The damaged bytes, capped for storage; `truncatedEvidence` says when. */
56
+ raw: string;
57
+ truncatedEvidence: boolean;
58
+ }
59
+ /** A run of sequence numbers that is simply gone — never recoverable, only reportable. */
60
+ export interface SequenceGap {
61
+ afterSeq: number;
62
+ beforeSeq: number;
63
+ missing: number;
64
+ }
65
+ export interface LogRecord {
66
+ version: 1;
67
+ seq: number;
68
+ [key: string]: unknown;
69
+ }
70
+ export interface ReadLogResult {
71
+ /** Records this version understands and can replay. */
72
+ records: LogRecord[];
73
+ /**
74
+ * Sequence numbers claimed by every readable entry in file order, whatever
75
+ * its version. A record written by a newer version is not replayable here,
76
+ * but it did occupy its number: counting it is what keeps a forward-compatible
77
+ * file from looking like it has a hole, and keeps continuation past it.
78
+ */
79
+ sequence: number[];
80
+ /** Highest sequence still readable anywhere in the file, for monotonic continuation. */
81
+ maxSeq: number;
82
+ damaged: DamagedLine[];
83
+ gaps: SequenceGap[];
84
+ }
85
+ /**
86
+ * Read the log line by line. One bad line costs exactly that line: everything
87
+ * before and after it is kept, and the damage is described rather than dropped.
88
+ */
89
+ export declare function readLog(path: string): ReadLogResult;
90
+ /**
91
+ * Missing sequence numbers between consecutive records. Records that parse
92
+ * perfectly still leave a hole when the writes between them never landed, so
93
+ * this is computed over whatever ordered run the caller cares about — including
94
+ * a rotated stream followed by its live successor, where the hole falls on the
95
+ * boundary and neither file can see it alone.
96
+ */
97
+ export declare function findSequenceGaps(sequence: number[]): SequenceGap[];
98
+ export interface QuarantineResult {
99
+ quarantined: boolean;
100
+ /** Present when the sidecar could not be written; the source file is untouched either way. */
101
+ error?: string;
102
+ sidecar: string;
103
+ }
104
+ /**
105
+ * Copy damaged bytes to a sidecar for later forensics. The damaged file itself
106
+ * is never read-modify-written, and a sidecar that cannot be created is reported
107
+ * rather than retried into the event writer.
108
+ */
109
+ export declare function quarantineDamage(path: string, damaged: DamagedLine[], source?: string): QuarantineResult;