@intx/inference 0.2.2 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/reactor.js CHANGED
@@ -13,6 +13,9 @@
13
13
  //
14
14
  // (INFERENCE.md § Agent Reactor)
15
15
  import { getLogger } from "@intx/log";
16
+ import { ApprovalDecision, signalKindToGateType } from "@intx/types";
17
+ import { canonicalJsonStringify } from "@intx/types/wire-definition-hash";
18
+ import { type } from "arktype";
16
19
  import { runInference } from "./harness.js";
17
20
  import { createCapabilities } from "./director.js";
18
21
  import { createGateManager } from "./gates.js";
@@ -21,21 +24,64 @@ import { createStateManager } from "./state.js";
21
24
  import { validateActions } from "./actions.js";
22
25
  import { createToolResultTurn, createInboundTurn, assertWellFormedToolSequence, } from "./turns.js";
23
26
  const logger = getLogger(["interchange", "reactor"]);
24
- function buildHarnessOpts(turns, source, options, signal, nextSeq, deps) {
25
- if (options !== undefined) {
26
- return {
27
- turns,
28
- source,
29
- inferenceOptions: options,
30
- signal,
31
- nextSeq,
32
- deps,
33
- };
34
- }
35
- return { turns, source, signal, nextSeq, deps };
27
+ // Sentinel returned by a per-call tool run when a before-tool extension parked
28
+ // the call on a gate. Distinct from every ToolResult so a suspended call is
29
+ // excluded from the tool-result history append and from tool.done continuation.
30
+ const SUSPENDED = Symbol("suspended");
31
+ // Exhaustiveness guard for the resume-dispatch switch. A newly added
32
+ // SignalKind or approval outcome that is not classified fails to type-check
33
+ // here, so the switch cannot silently drop an unhandled case.
34
+ function assertNever(x) {
35
+ throw new Error(`Unhandled resume case: ${JSON.stringify(x)}`);
36
+ }
37
+ function buildHarnessOpts(turns, source, options, signal, nextSeq, readMaterial, deps) {
38
+ // exactOptionalPropertyTypes is on: only set the optional keys when defined.
39
+ return {
40
+ turns,
41
+ source,
42
+ ...(options !== undefined ? { inferenceOptions: options } : {}),
43
+ signal,
44
+ nextSeq,
45
+ ...(readMaterial !== undefined ? { readMaterial } : {}),
46
+ deps,
47
+ };
36
48
  }
37
49
  const DEFAULT_GATE_TIMEOUT_MS = 3_600_000;
38
50
  const DEFAULT_SHUTDOWN_TIMEOUT_MS = 30_000;
51
+ const DEFAULT_DOOM_LOOP_THRESHOLD = 3;
52
+ /**
53
+ * Resolve the caller-facing `doomLoopThreshold` into the reactor's internal
54
+ * form: a positive integer when detection is active, or `null` when it is
55
+ * disabled. `undefined` (omitted) takes the default; `false` disables; a
56
+ * number is validated here — the construction edge is the one place that owns
57
+ * the default and rejects a malformed value loudly, so a stray `0`, negative,
58
+ * or non-integer throws rather than silently disarming the guard. Downstream
59
+ * code compares against the returned `number | null` and never sees the raw
60
+ * `false`, whose numeric coercion would otherwise trip the loop immediately.
61
+ */
62
+ function resolveDoomLoopThreshold(raw) {
63
+ if (raw === false)
64
+ return null;
65
+ if (raw === undefined)
66
+ return DEFAULT_DOOM_LOOP_THRESHOLD;
67
+ if (!Number.isInteger(raw) || raw < 1) {
68
+ throw new Error(`doomLoopThreshold must be a positive integer or false, got ${String(raw)}`);
69
+ }
70
+ return raw;
71
+ }
72
+ /**
73
+ * Order-independent identity of a batch of executed tool calls. Each call
74
+ * canonicalizes to its name and arguments (the call `id` is excluded, since it
75
+ * differs on every request); sorting makes a parallel batch match regardless
76
+ * of the order the model emitted its calls. Two turns share a signature when
77
+ * they run the same multiset of `(name, arguments)` pairs.
78
+ */
79
+ function toolBatchSignature(calls) {
80
+ return calls
81
+ .map((call) => canonicalJsonStringify({ name: call.name, arguments: call.arguments }))
82
+ .sort()
83
+ .join("\n");
84
+ }
39
85
  /**
40
86
  * Creates a reactor instance bound to the given configuration.
41
87
  * Call `start()` to begin the event loop.
@@ -49,6 +95,10 @@ export function createReactor(config) {
49
95
  failOverToNextSource = () => false, resetToPreferredSource = () => {
50
96
  /* single-source: nothing to reset */
51
97
  }, } = config;
98
+ // Resolved once at the construction edge: a positive integer while detection
99
+ // is active, or `null` when the caller disabled it with `false`. Every
100
+ // downstream comparison reads this binding, never the raw config value.
101
+ const doomLoopThreshold = resolveDoomLoopThreshold(config.doomLoopThreshold);
52
102
  // Monotonic sequence counter, scoped to this session.
53
103
  let seq = 0;
54
104
  function nextSeq() {
@@ -132,6 +182,10 @@ export function createReactor(config) {
132
182
  let running = false;
133
183
  let done = false;
134
184
  let shutdownStarted = false;
185
+ // Correlation state is empty until context loading and gate rehydration
186
+ // finish. Hold early deliveries so a resumed approval cannot be mistaken
187
+ // for a new conversation message during that startup window.
188
+ let startupDeliveries = [];
135
189
  // Per-message run-bracket state. Set when the loop dequeues a
136
190
  // message.received and begins per-message work; cleared at the
137
191
  // terminal point (wait/reply/done) or at a reactor-fatal abandon.
@@ -140,9 +194,22 @@ export function createReactor(config) {
140
194
  // produces unambiguous start/end pairs downstream.
141
195
  let currentMessageRunId = null;
142
196
  let currentMessageId = null;
197
+ // Doom-loop detection state, scoped to the current message run. Each executed
198
+ // tool-call turn is reduced to a batch signature; consecutive identical
199
+ // signatures accumulate here, and the run is broken when the count reaches
200
+ // `doomLoopThreshold`. This is run-scoped, not cycle-scoped: it resets only in
201
+ // `openMessageRun`, never in `resetCycleAccumulators`. It is also deliberately
202
+ // ephemeral (closure state, not persisted) -- a mid-run restart resets it to
203
+ // zero and the loop simply re-accumulates and trips a few turns later.
204
+ let lastToolBatchSignature = null;
205
+ let toolBatchRepeatCount = 0;
206
+ let lastToolBatchNames = [];
143
207
  function openMessageRun(messageId) {
144
208
  currentMessageRunId = crypto.randomUUID();
145
209
  currentMessageId = messageId;
210
+ lastToolBatchSignature = null;
211
+ toolBatchRepeatCount = 0;
212
+ lastToolBatchNames = [];
146
213
  emit({
147
214
  type: "message.run.started",
148
215
  seq: nextSeq(),
@@ -175,6 +242,10 @@ export function createReactor(config) {
175
242
  let cycleInferred = false;
176
243
  let cycleToolCallsExecuted = 0;
177
244
  let cycleCompactorName = null;
245
+ // A suspension registers a gate and may persist a pending operation. That is
246
+ // a durable state change even when the cycle ran no inference and completed
247
+ // no tool call, so it must force the cycle commit.
248
+ let cycleSuspended = false;
178
249
  // Director-supplied checkpoint message override; consumed exactly once.
179
250
  let pendingMessage = null;
180
251
  // AbortController for in-flight inference/tool operations.
@@ -197,6 +268,67 @@ export function createReactor(config) {
197
268
  // deliver() is fire-and-forget async, so two rapid delivers can interleave
198
269
  // across an await boundary in the validator, causing double-correlation.
199
270
  const correlatingIds = new Set();
271
+ // Decide how a correlated approval-kind pending operation resumes, granting
272
+ // any one-shot bypass synchronously so no delivery can interleave between the
273
+ // grant and the re-dispatch enqueued by the caller. An operation that carries
274
+ // a `suspendedCall` is an ask-flow suspension: the approver's decision routes
275
+ // it down the re-dispatch rail. An operation without one is an async-tool
276
+ // pending marker, which resumes on the normal gate-cleared rail.
277
+ //
278
+ // The nested switch is total: the outer `assertNever(op.kind)` rejects a
279
+ // future SignalKind at compile time, and the inner `assertNever` rejects a
280
+ // future decision outcome. A malformed decision body fails loud at the parse
281
+ // boundary before the switch.
282
+ function resumePendingOperation(op, message) {
283
+ if (op.suspendedCall === undefined) {
284
+ return { mode: "gate-cleared" };
285
+ }
286
+ const suspendedCall = op.suspendedCall;
287
+ if (message.content === undefined) {
288
+ throw new Error(`Correlated approval decision for ${op.correlationId} has no body to parse`);
289
+ }
290
+ let raw;
291
+ try {
292
+ raw = JSON.parse(message.content);
293
+ }
294
+ catch (cause) {
295
+ throw new Error(`Correlated approval decision for ${op.correlationId} is not valid JSON`, { cause });
296
+ }
297
+ const decision = ApprovalDecision(raw);
298
+ if (decision instanceof type.errors) {
299
+ throw new Error(`Correlated approval decision for ${op.correlationId} is malformed: ${decision.summary}`);
300
+ }
301
+ switch (op.kind) {
302
+ case "approval":
303
+ switch (decision.outcome) {
304
+ case "approved":
305
+ // Authorize the exact parked call to run once, then re-dispatch it.
306
+ // Grant on every before-tool extension: only the authz extension
307
+ // responds, but referencing it directly would re-couple the reactor
308
+ // to authz and break a deployment that runs without it.
309
+ for (const ext of beforeToolExtensions) {
310
+ ext.grantOneShot?.(suspendedCall.id);
311
+ }
312
+ return { mode: "redispatch", calls: [suspendedCall] };
313
+ case "rejected": {
314
+ // The approver denied the call. Answer the parked call with a
315
+ // synthetic error result rather than re-running it — no one-shot
316
+ // bypass is granted, so the tool never executes. The approver's
317
+ // reason, when present, is surfaced to the model verbatim.
318
+ const content = "denied by approver" +
319
+ (decision.message !== undefined ? `: ${decision.message}` : "");
320
+ return {
321
+ mode: "error_result",
322
+ result: { callId: suspendedCall.id, content, isError: true },
323
+ };
324
+ }
325
+ default:
326
+ return assertNever(decision.outcome);
327
+ }
328
+ default:
329
+ return assertNever(op.kind);
330
+ }
331
+ }
200
332
  async function tryCorrelate(message) {
201
333
  const correlationId = message.headers.interchangeCorrelationId;
202
334
  if (correlationId === undefined)
@@ -222,19 +354,77 @@ export function createReactor(config) {
222
354
  return false;
223
355
  }
224
356
  }
225
- // Clear the gate associated with this correlation, if any.
226
- const gate = gates.findByCorrelationId(correlationId);
227
- if (gate !== undefined) {
228
- gates.clear(gate.gateId);
357
+ // Capture the operation before removal so the resume dispatch can read its
358
+ // kind and suspended call. Removal happens only after the dispatch is
359
+ // decided, all inside this correlatingIds-guarded critical section so a
360
+ // double-deliver early-returns rather than double-dispatching.
361
+ const op = pending;
362
+ let dispatch;
363
+ try {
364
+ dispatch = resumePendingOperation(op, message);
229
365
  }
230
- correlations.remove(correlationId);
231
- if (stateManager !== null) {
232
- stateManager.removePendingOperation(correlationId);
233
- // Append the correlated message to conversation history so the model
234
- // sees the response content when it re-infers after the gate clears.
235
- const msg = createInboundTurn(message);
236
- if (msg !== null) {
237
- stateManager.appendTurn(msg);
366
+ catch (cause) {
367
+ correlatingIds.delete(correlationId);
368
+ throw cause;
369
+ }
370
+ const gate = gates.findByCorrelationId(correlationId);
371
+ switch (dispatch.mode) {
372
+ case "redispatch": {
373
+ // Clear the gate WITHOUT enqueuing gate.cleared: the re-dispatched call
374
+ // is the resumption, so a gate.cleared-driven re-infer would double the
375
+ // continuation. The re-dispatch's own tool.done drives the re-infer.
376
+ if (gate !== undefined) {
377
+ gates.clearSilently(gate.gateId);
378
+ if (stateManager !== null) {
379
+ stateManager.setGatesSnapshot(gates.snapshot());
380
+ }
381
+ }
382
+ correlations.remove(correlationId);
383
+ if (stateManager !== null) {
384
+ stateManager.removePendingOperation(correlationId);
385
+ }
386
+ // The grant is already recorded (synchronously, in
387
+ // resumePendingOperation) with no await since; enqueue the re-dispatch
388
+ // so it runs on the loop with normal event ordering. The director seeds
389
+ // its outstanding-result count off this event before the call's
390
+ // tool.done arrives.
391
+ enqueue({ type: "resume.execute_tools", calls: dispatch.calls });
392
+ break;
393
+ }
394
+ case "error_result": {
395
+ // The approver denied the call. Clear the gate SILENTLY (like the
396
+ // approved redispatch) so it cannot also trip onGateCleared and enqueue
397
+ // a second continuation. The synthetic error result answers the parked
398
+ // call; the director appends it and re-infers once.
399
+ if (gate !== undefined) {
400
+ gates.clearSilently(gate.gateId);
401
+ if (stateManager !== null) {
402
+ stateManager.setGatesSnapshot(gates.snapshot());
403
+ }
404
+ }
405
+ correlations.remove(correlationId);
406
+ if (stateManager !== null) {
407
+ stateManager.removePendingOperation(correlationId);
408
+ }
409
+ enqueue({ type: "resume.tool_result", result: dispatch.result });
410
+ break;
411
+ }
412
+ case "gate-cleared": {
413
+ // Async-tool resumption: clear the gate normally so the director
414
+ // re-infers, and append the correlated response to history so the model
415
+ // sees the content it was waiting on.
416
+ if (gate !== undefined) {
417
+ gates.clear(gate.gateId);
418
+ }
419
+ correlations.remove(correlationId);
420
+ if (stateManager !== null) {
421
+ stateManager.removePendingOperation(correlationId);
422
+ const msg = createInboundTurn(message);
423
+ if (msg !== null) {
424
+ stateManager.appendTurn(msg);
425
+ }
426
+ }
427
+ break;
238
428
  }
239
429
  }
240
430
  emit({
@@ -305,19 +495,11 @@ export function createReactor(config) {
305
495
  emitError(`writePrompt failed: ${cause instanceof Error ? cause.message : String(cause)}`, false);
306
496
  }
307
497
  const p = (async () => {
308
- // Per-source attempt budget for transient errors (quota/retryable/
309
- // timeout). Kept small because failover, not flogging one source, is
310
- // the recovery path: the harness already does its own mechanical
311
- // retry under each attempt, so this caps reactor-level same-source
312
- // retries at one before moving to the next source.
313
- const sameSourceAttempts = 2;
314
- const defaultRetryMs = 60_000;
315
498
  // Each cycle starts at the most-preferred source; a failover in a
316
499
  // prior cycle must not leave the agent permanently demoted.
317
500
  resetToPreferredSource();
318
- let attempt = 0;
319
501
  for (;;) {
320
- const harnessOpts = buildHarnessOpts(prompt, config.source, options, signal, nextSeq, deps);
502
+ const harnessOpts = buildHarnessOpts(prompt, config.source, options, signal, nextSeq, config.readMaterial, deps);
321
503
  let lastDone;
322
504
  let lastError;
323
505
  for await (const event of inferenceRunner(harnessOpts)) {
@@ -377,47 +559,17 @@ export function createReactor(config) {
377
559
  enqueue({ type: "inference.error", error: err, partial });
378
560
  return;
379
561
  }
380
- // A rate limit is the one category worth waiting out on the same
381
- // source: it clears with time, and the reactor's backoff is longer
382
- // than the harness's own per-call retry. The harness has already
383
- // exhausted its internal mechanical retries for retryable/timeout
384
- // by the time the reactor sees them, so those fail over rather than
385
- // re-running the same source (which would just retry-compound).
386
- if (err.category === "quota_exhausted") {
387
- attempt += 1;
388
- if (attempt < sameSourceAttempts && !signal.aborted) {
389
- const delayMs = err.retryAfterMs ?? defaultRetryMs;
390
- logger.warn `Rate limited, retrying same source after ${String(delayMs)}ms`;
391
- await new Promise((resolve) => {
392
- const timer = setTimeout(resolve, delayMs);
393
- const onAbort = () => {
394
- clearTimeout(timer);
395
- resolve();
396
- };
397
- signal.addEventListener("abort", onAbort, { once: true });
398
- });
399
- if (signal.aborted) {
400
- enqueue({
401
- type: "inference.error",
402
- error: {
403
- category: "aborted",
404
- message: "inference aborted during rate limit backoff",
405
- },
406
- partial,
407
- });
408
- return;
409
- }
410
- continue;
411
- }
412
- }
413
- // Same-source rate-limit budget exhausted, or a source-specific
414
- // failure (credential, protocol mismatch, retryable, timeout): fail
415
- // over to the next source. A pacing delay the leaving source asked
416
- // for must not gate the next source.
562
+ // Any remaining error (quota, credential, protocol mismatch,
563
+ // retryable, timeout) is source-specific. The harness wrapper owns
564
+ // mechanical retry and has already exhausted it against this source
565
+ // by the time the reactor sees the error, including honoring a
566
+ // provider Retry-After for quota, so re-running the same source
567
+ // would only retry-compound. Fail over to the next source instead.
568
+ // A pacing delay the leaving source asked for must not gate the
569
+ // next source.
417
570
  pendingPacingDelayMs = 0;
418
571
  if (failOverToNextSource()) {
419
572
  logger.warn `Failing over to next inference source after ${err.category}`;
420
- attempt = 0;
421
573
  continue;
422
574
  }
423
575
  // No further source to fail over to: surface the last error.
@@ -434,21 +586,37 @@ export function createReactor(config) {
434
586
  const state = stateManager;
435
587
  const signal = operationController.signal;
436
588
  const runOne = async (call) => {
437
- // Run before-tool extensions. First block or throw terminates the chain.
589
+ // Run before-tool extensions. The first non-allow decision terminates
590
+ // the chain: `block` answers the call with an error result, `suspend`
591
+ // parks it (no result, no tool.done).
438
592
  for (const ext of beforeToolExtensions) {
439
- let blockReason;
593
+ let decision;
440
594
  try {
441
- blockReason = await ext.beforeTool(call, state.snapshot(), signal);
595
+ decision = await ext.beforeTool(call, state.snapshot(), signal);
442
596
  }
443
597
  catch (cause) {
444
598
  const msg = cause instanceof Error ? cause.message : String(cause);
445
599
  emitError(`BeforeToolExtension threw for ${call.name}: ${msg}`, false);
446
- blockReason = msg;
600
+ decision = { type: "block", reason: msg };
447
601
  }
448
- if (blockReason !== undefined) {
602
+ if (decision.type === "suspend") {
603
+ // Park the call: register the gate, persist the pending operation,
604
+ // snapshot, and commit. The call is neither run nor answered — no
605
+ // tool.start, no tool.done, no tool-result turn. The gate clears
606
+ // when the correlated external decision is delivered.
607
+ await suspendOnGate({
608
+ gateType: decision.gate.type,
609
+ gateId: decision.gate.gateId,
610
+ timeoutMs: Math.max(1, decision.gate.timeoutAt - Date.now()),
611
+ correlationId: decision.gate.correlationId,
612
+ pendingOp: decision.pendingOp,
613
+ });
614
+ return SUSPENDED;
615
+ }
616
+ if (decision.type === "block") {
449
617
  const blocked = {
450
618
  callId: call.id,
451
- content: blockReason,
619
+ content: decision.reason,
452
620
  isError: true,
453
621
  };
454
622
  emit({
@@ -467,6 +635,10 @@ export function createReactor(config) {
467
635
  const gateId = `pending-${marker.correlationId}`;
468
636
  const op = {
469
637
  correlationId: marker.correlationId,
638
+ // Placeholder: async markers should carry their own SignalKind. The
639
+ // resume switch keys on suspendedCall presence (absent here) as the
640
+ // interim discriminator instead of on kind.
641
+ kind: "approval",
470
642
  registeredAt: Date.now(),
471
643
  gateId,
472
644
  ...(marker.expectedFrom !== undefined
@@ -489,22 +661,44 @@ export function createReactor(config) {
489
661
  }
490
662
  return current;
491
663
  };
492
- let results;
664
+ let outcomes;
493
665
  if (parallel) {
494
666
  const p = Promise.all(calls.map((c) => runOne(c)));
495
667
  void track(p);
496
- results = await p;
668
+ outcomes = await p;
497
669
  }
498
670
  else {
499
- results = [];
671
+ outcomes = [];
500
672
  for (const call of calls) {
501
673
  const p = runOne(call);
502
674
  void track(p);
503
- results.push(await p);
675
+ outcomes.push(await p);
504
676
  }
505
677
  }
678
+ // Suspended calls are parked, not answered: they contribute no tool
679
+ // result to history and no tool.done continuation event.
680
+ const results = outcomes.filter((o) => o !== SUSPENDED);
681
+ // Doom-loop accounting keys off the calls that actually ran, aligned to
682
+ // their outcome by index (both the parallel and serial paths above keep
683
+ // `outcomes` in `calls` order). A parked call contributes nothing, so a
684
+ // suspend-then-redispatch cycle counts its one real execution once. The
685
+ // loop reads `toolBatchRepeatCount` after this returns and breaks the run
686
+ // when it reaches the threshold. A `null` threshold means detection is
687
+ // disabled, so the accounting is skipped entirely.
688
+ const ranCalls = calls.filter((_call, i) => outcomes[i] !== SUSPENDED);
689
+ if (doomLoopThreshold !== null && ranCalls.length > 0) {
690
+ const signature = toolBatchSignature(ranCalls);
691
+ if (signature === lastToolBatchSignature) {
692
+ toolBatchRepeatCount += 1;
693
+ }
694
+ else {
695
+ lastToolBatchSignature = signature;
696
+ toolBatchRepeatCount = 1;
697
+ }
698
+ lastToolBatchNames = ranCalls.map((call) => call.name);
699
+ }
506
700
  cycleToolCallsExecuted += results.length;
507
- if (addToHistory && stateManager !== null) {
701
+ if (addToHistory && stateManager !== null && results.length > 0) {
508
702
  stateManager.appendTurn(createToolResultTurn(results));
509
703
  }
510
704
  for (const result of results) {
@@ -558,6 +752,7 @@ export function createReactor(config) {
558
752
  cycleInferred = false;
559
753
  cycleToolCallsExecuted = 0;
560
754
  cycleCompactorName = null;
755
+ cycleSuspended = false;
561
756
  }
562
757
  async function commitCycle() {
563
758
  if (stateManager === null)
@@ -567,7 +762,8 @@ export function createReactor(config) {
567
762
  // no override) commits nothing.
568
763
  const hasWork = cycleInferred ||
569
764
  cycleToolCallsExecuted > 0 ||
570
- cycleCompactorName !== null;
765
+ cycleCompactorName !== null ||
766
+ cycleSuspended;
571
767
  const hasOverride = pendingMessage !== null;
572
768
  if (!hasWork && !hasOverride) {
573
769
  resetCycleAccumulators();
@@ -606,6 +802,144 @@ export function createReactor(config) {
606
802
  tokenUsage: stateManager.getTokenUsage(),
607
803
  });
608
804
  }
805
+ let suspendingGate = null;
806
+ // Callback the gate manager invokes when a gate resolves, times out, or is
807
+ // shut down. Refreshes the snapshot and drives the loop's next step.
808
+ //
809
+ // A parked ask-flow approval that TIMES OUT ends without running its tool:
810
+ // it must be answered with a synthetic error result rather than left as a
811
+ // dangling tool_use. That path enqueues `resume.tool_result` INSTEAD OF
812
+ // `reactor.gate.cleared` — the two are mutually exclusive, because enqueuing
813
+ // both would drive two re-inferences for one timeout. Every other case (an
814
+ // async-marker pending op with no suspendedCall, no pending op at all, a
815
+ // `resolved`/`shutdown` reason, or a shutting-down reactor) keeps today's
816
+ // behavior: enqueue `reactor.gate.cleared` and let the director re-infer.
817
+ //
818
+ // A delivered `resolved` never reaches here on the ask rail — the redispatch
819
+ // and reject paths clear the gate silently (no onCleared) — so the timeout
820
+ // branch is gated on `reason === "timeout"` and shutdown stays on the plain
821
+ // path: a shutting-down reactor must not manufacture tool results.
822
+ function onGateCleared(gateId, reason) {
823
+ // A clear that fires while this gate's suspend is still committing must not
824
+ // take effect before `reactor.gate.blocked` is emitted. Record it and let
825
+ // suspendOnGate replay the full handler once the block is announced.
826
+ if (suspendingGate !== null &&
827
+ suspendingGate.gateId === gateId &&
828
+ suspendingGate.deferredClear === null) {
829
+ suspendingGate.deferredClear = { reason };
830
+ return;
831
+ }
832
+ if (stateManager !== null) {
833
+ stateManager.setGatesSnapshot(gates.snapshot());
834
+ }
835
+ if (reason === "timeout") {
836
+ const op = correlations.findByGateId(gateId);
837
+ if (op !== undefined && op.suspendedCall !== undefined) {
838
+ correlations.remove(op.correlationId);
839
+ if (stateManager !== null) {
840
+ stateManager.removePendingOperation(op.correlationId);
841
+ }
842
+ enqueue({
843
+ type: "resume.tool_result",
844
+ result: {
845
+ callId: op.suspendedCall.id,
846
+ content: "approval timed out",
847
+ isError: true,
848
+ },
849
+ });
850
+ return;
851
+ }
852
+ }
853
+ emit({
854
+ type: "reactor.gate.cleared",
855
+ seq: nextSeq(),
856
+ data: { gateId, reason },
857
+ });
858
+ enqueue({ type: "reactor.gate.cleared", gateId, reason });
859
+ }
860
+ // Parks the reactor on a gate. Shared by the director's `suspend` action and
861
+ // the before-tool `suspend` decision so both paths register the gate,
862
+ // durably persist any pending operation, snapshot the active gates, and
863
+ // commit before returning to the loop — a suspended reactor's state must be
864
+ // durable across restart. When `pendingOp` is supplied its correlation is
865
+ // registered and it is persisted; the director path has already persisted
866
+ // its pending operation (via the tool's pending marker), so it passes none.
867
+ async function suspendOnGate(args) {
868
+ const { gateType, gateId, timeoutMs, correlationId, pendingOp } = args;
869
+ if (pendingOp !== undefined) {
870
+ correlations.register(pendingOp);
871
+ if (stateManager !== null) {
872
+ stateManager.addPendingOperation(pendingOp);
873
+ }
874
+ }
875
+ // Track this suspend as in flight so a clear racing the commit below is
876
+ // deferred until `reactor.gate.blocked` has been emitted.
877
+ const inFlightSuspend = { gateId, deferredClear: null };
878
+ suspendingGate = inFlightSuspend;
879
+ // Register the gate. onGateCleared enqueues the cleared event so the loop
880
+ // processes it normally without blocking here.
881
+ void gates.register(gateId, gateType, timeoutMs, correlationId, onGateCleared);
882
+ if (stateManager !== null) {
883
+ stateManager.setGatesSnapshot(gates.snapshot());
884
+ }
885
+ // Registering the gate (and any pending operation) is a durable state
886
+ // change that must be committed even if this cycle did no other work.
887
+ cycleSuspended = true;
888
+ // Commit before the loop continues so the suspended state is durable
889
+ // across restart.
890
+ await commitCycle();
891
+ // Emit `reactor.gate.blocked` only AFTER the commit. This event resolves
892
+ // the `send()` awaiter as "suspended", and a downstream consumer (the warm
893
+ // agent's run-boundary durability mirror) reads the pending operation back
894
+ // out of the just-committed context store the instant `send()` settles.
895
+ // Emitting before the commit would resolve `send()` first, letting that
896
+ // mirror read a store that has not yet persisted the pending op -- it would
897
+ // durably mirror an empty pending-operation set and lose the approval
898
+ // snapshot, so a parked correlation could not be re-registered after a hub
899
+ // reconnect. This upholds persist-before-settle: the durable commit the
900
+ // header promises before returning to the loop lands before the suspension
901
+ // settles.
902
+ emit({
903
+ type: "reactor.gate.blocked",
904
+ seq: nextSeq(),
905
+ data: {
906
+ reason: gateType,
907
+ gateId,
908
+ ...(correlationId !== undefined ? { correlationId } : {}),
909
+ ...(pendingOp?.approvalSnapshot !== undefined
910
+ ? { approvalSnapshot: pendingOp.approvalSnapshot }
911
+ : {}),
912
+ },
913
+ });
914
+ // The suspension is announced. If the gate cleared while the commit was in
915
+ // flight, its handler was deferred to keep it after `blocked`; replay it
916
+ // now, in order.
917
+ suspendingGate = null;
918
+ if (inFlightSuspend.deferredClear !== null) {
919
+ onGateCleared(gateId, inFlightSuspend.deferredClear.reason);
920
+ }
921
+ }
922
+ // Re-registers a live gate and correlation for each pending operation loaded
923
+ // from the context store on restart. The remaining timeout is computed from
924
+ // the persisted absolute deadline (`timeoutAt`) against the current clock, so
925
+ // the deadline is preserved across the restart rather than restarted; a
926
+ // deadline already in the past clamps to 1ms so the gate fires on the next
927
+ // tick. An operation persisted without a `timeoutAt` (hold-indefinitely) has
928
+ // no deadline to preserve; the gate manager cannot express an indefinite
929
+ // hold, so it is armed with the session-level `gateTimeout` — the same
930
+ // effective timeout the director-suspend fallback uses — rather than a
931
+ // silent zero. This does not run through `suspendOnGate`: rehydration must
932
+ // not re-emit `reactor.gate.blocked` (the suspension already happened before
933
+ // the restart) and must not commit (nothing changed).
934
+ function rehydrateGates(ops) {
935
+ for (const op of ops) {
936
+ const timeoutMs = op.timeoutAt !== undefined
937
+ ? Math.max(1, op.timeoutAt - Date.now())
938
+ : gateTimeout;
939
+ correlations.register(op);
940
+ void gates.register(op.gateId, signalKindToGateType(op.kind), timeoutMs, op.correlationId, onGateCleared);
941
+ }
942
+ }
609
943
  // -------------------------------------------------------------------------
610
944
  // Main loop
611
945
  // -------------------------------------------------------------------------
@@ -651,6 +985,17 @@ export function createReactor(config) {
651
985
  }
652
986
  openMessageRun(event.message.headers.messageId);
653
987
  }
988
+ // A parked approval that ended without running its tool (rejected or
989
+ // timed out) carries a synthetic error result answering the parked call.
990
+ // Land it in history before the director decides so the tool_result turn
991
+ // closes the dangling tool_use and the re-inference the director returns
992
+ // sees a well-formed sequence. No tool ran, so no tool.done and no
993
+ // counter change accompany it.
994
+ if (event.type === "resume.tool_result") {
995
+ if (stateManager !== null) {
996
+ stateManager.appendTurn(createToolResultTurn([event.result]));
997
+ }
998
+ }
654
999
  let actions;
655
1000
  try {
656
1001
  actions = await director.decide(event, stateManager.snapshot(), capabilities);
@@ -725,34 +1070,13 @@ export function createReactor(config) {
725
1070
  const suspendAction = normalized.find((a) => a.type === "suspend");
726
1071
  if (suspendAction !== undefined && suspendAction.type === "suspend") {
727
1072
  const { gate } = suspendAction;
728
- const effectiveTimeout = gate.timeoutMs > 0 ? gate.timeoutMs : gateTimeout;
729
- emit({
730
- type: "reactor.gate.blocked",
731
- seq: nextSeq(),
732
- data: { reason: gate.type, gateId: gate.gateId },
733
- });
734
- if (stateManager !== null) {
735
- stateManager.setGatesSnapshot(gates.snapshot());
736
- }
737
- // Register the gate. The onCleared callback enqueues the cleared event
738
- // so the loop processes it normally without blocking here.
739
- void gates.register(gate.gateId, gate.type, effectiveTimeout, gate.correlationId, (gateId, reason) => {
740
- if (stateManager !== null) {
741
- stateManager.setGatesSnapshot(gates.snapshot());
742
- }
743
- emit({
744
- type: "reactor.gate.cleared",
745
- seq: nextSeq(),
746
- data: { gateId, reason },
747
- });
748
- enqueue({ type: "reactor.gate.cleared", gateId, reason });
1073
+ await suspendOnGate({
1074
+ gateType: gate.type,
1075
+ gateId: gate.gateId,
1076
+ timeoutMs: gate.timeoutMs > 0 ? gate.timeoutMs : gateTimeout,
1077
+ correlationId: gate.correlationId,
1078
+ pendingOp: undefined,
749
1079
  });
750
- if (stateManager !== null) {
751
- stateManager.setGatesSnapshot(gates.snapshot());
752
- }
753
- // Commit before the loop continues so the suspended-state turns are
754
- // durable across restart.
755
- await commitCycle();
756
1080
  continue;
757
1081
  }
758
1082
  // Handle reply — emit the content for the harness/supervisor to send.
@@ -811,6 +1135,17 @@ export function createReactor(config) {
811
1135
  const parallel = toolsAction.parallel !== false;
812
1136
  const addToHistory = toolsAction.addToHistory !== false;
813
1137
  await executeTools(toolsAction.calls, parallel, addToHistory);
1138
+ if (doomLoopThreshold !== null &&
1139
+ toolBatchRepeatCount >= doomLoopThreshold) {
1140
+ const tools = lastToolBatchNames.join(", ");
1141
+ const message = `Doom loop detected: an identical tool batch (${tools}) executed ` +
1142
+ `${String(doomLoopThreshold)} times consecutively`;
1143
+ emitError(message, true);
1144
+ closeMessageRun("failed", { message, kind: "doom_loop" });
1145
+ done = true;
1146
+ await initiateShutdown();
1147
+ break;
1148
+ }
814
1149
  continue;
815
1150
  }
816
1151
  // No infer/tools/reply/suspend/wait/compact action — if a checkpoint
@@ -873,19 +1208,45 @@ export function createReactor(config) {
873
1208
  initialUsage = loaded.tokenUsage;
874
1209
  }
875
1210
  catch (cause) {
1211
+ done = true;
1212
+ startupDeliveries = null;
876
1213
  logger.error `Context store load failed: ${cause}`;
877
1214
  emitError(`Context store load failed: ${cause instanceof Error ? cause.message : String(cause)}`, true);
878
1215
  emit({ type: "reactor.done", seq: nextSeq(), data: {} });
879
1216
  return;
880
1217
  }
881
1218
  stateManager = createStateManager(sessionId, initialTurns, initialOps, initialUsage);
882
- stateManager.setGatesSnapshot(gates.snapshot());
883
- emit({ type: "reactor.start", seq: nextSeq(), data: {} });
884
1219
  try {
1220
+ // Re-arm gates for operations that were suspended before the restart.
1221
+ // The state manager holds the loaded pending operations, but a gate is
1222
+ // in-memory and does not survive a restart; without this a reloaded
1223
+ // suspended agent is wedged (no live gate to clear, no correlation to
1224
+ // match). Each op re-registers its correlation and a live gate keyed on
1225
+ // the op's own gateId and correlationId, so a delivered signal clears
1226
+ // it exactly as the original suspension would have.
1227
+ //
1228
+ // Rehydration runs inside this try/catch because the pending operations
1229
+ // come from the context store — an untrusted external boundary — and
1230
+ // correlation/gate registration throws synchronously on a duplicate
1231
+ // correlationId or gateId. A throw must surface as reactor.error plus
1232
+ // reactor.done (matching the load-failure path), not brick the reactor
1233
+ // as a silent unhandled rejection.
1234
+ rehydrateGates(initialOps);
1235
+ stateManager.setGatesSnapshot(gates.snapshot());
1236
+ emit({ type: "reactor.start", seq: nextSeq(), data: {} });
1237
+ const bufferedDeliveries = startupDeliveries;
1238
+ startupDeliveries = null;
1239
+ if (bufferedDeliveries !== null) {
1240
+ for (const message of bufferedDeliveries) {
1241
+ processDelivery(message);
1242
+ }
1243
+ }
885
1244
  await loop();
886
1245
  }
887
1246
  catch (cause) {
888
1247
  const msg = cause instanceof Error ? cause.message : String(cause);
1248
+ done = true;
1249
+ startupDeliveries = null;
889
1250
  logger.error `Reactor loop threw unexpectedly: ${cause}`;
890
1251
  emitError(`Internal reactor error: ${msg}`, true);
891
1252
  closeMessageRun("failed", {
@@ -898,11 +1259,30 @@ export function createReactor(config) {
898
1259
  }
899
1260
  })();
900
1261
  }
901
- function deliver(message) {
902
- if (done)
903
- return;
1262
+ function processDelivery(message) {
904
1263
  void (async () => {
905
- const correlated = await tryCorrelate(message);
1264
+ let correlated;
1265
+ try {
1266
+ correlated = await tryCorrelate(message);
1267
+ }
1268
+ catch (cause) {
1269
+ // A correlation-path invariant failed (e.g. a malformed approval
1270
+ // decision). Surface it as a fatal reactor error rather than a silent
1271
+ // unhandled rejection, and stop the run — the resume cannot proceed on
1272
+ // a decision the reactor cannot trust.
1273
+ const msg = cause instanceof Error ? cause.message : String(cause);
1274
+ logger.error `Correlation dispatch failed: ${cause}`;
1275
+ emitError(`Correlation dispatch failed: ${msg}`, true);
1276
+ closeMessageRun("failed", {
1277
+ message: `Correlation dispatch failed: ${msg}`,
1278
+ kind: "reactor_fatal",
1279
+ });
1280
+ done = true;
1281
+ if (!shutdownStarted) {
1282
+ await initiateShutdown();
1283
+ }
1284
+ return;
1285
+ }
906
1286
  if (!correlated) {
907
1287
  emit({
908
1288
  type: "message.received",
@@ -913,7 +1293,21 @@ export function createReactor(config) {
913
1293
  }
914
1294
  })();
915
1295
  }
1296
+ function deliver(message) {
1297
+ if (done)
1298
+ return;
1299
+ if (startupDeliveries !== null) {
1300
+ startupDeliveries.push(message);
1301
+ return;
1302
+ }
1303
+ processDelivery(message);
1304
+ }
916
1305
  function abort(reason) {
1306
+ // The loop cannot dequeue the abort event while it is awaiting an active
1307
+ // inference or tool batch. Signal that operation immediately so it can
1308
+ // settle and return control to the loop, where the queued abort retains
1309
+ // its priority over every other event.
1310
+ operationController.abort();
917
1311
  enqueue({ type: "abort", reason });
918
1312
  }
919
1313
  return { start, deliver, abort };