@intx/inference 0.2.2 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapter.d.ts +4 -2
- package/dist/adapter.js +2 -2
- package/dist/assembly.d.ts +8 -1
- package/dist/assembly.js +2 -1
- package/dist/authz-extension.d.ts +15 -1
- package/dist/authz-extension.js +93 -9
- package/dist/correlation.d.ts +1 -0
- package/dist/correlation.js +8 -1
- package/dist/default-director.d.ts +1 -1
- package/dist/default-director.js +37 -8
- package/dist/gates.d.ts +1 -0
- package/dist/gates.js +24 -1
- package/dist/harness.js +109 -21
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/providers/anthropic.d.ts +5 -1
- package/dist/providers/anthropic.js +264 -17
- package/dist/providers/google-genai.d.ts +3 -1
- package/dist/providers/google-genai.js +224 -215
- package/dist/providers/index.d.ts +3 -3
- package/dist/providers/index.js +3 -3
- package/dist/providers/openai.d.ts +7 -1
- package/dist/providers/openai.js +353 -59
- package/dist/reactor.js +416 -103
- package/dist/transform.d.ts +1 -1
- package/dist/transform.js +19 -4
- package/package.json +5 -4
package/dist/reactor.js
CHANGED
|
@@ -13,6 +13,8 @@
|
|
|
13
13
|
//
|
|
14
14
|
// (INFERENCE.md § Agent Reactor)
|
|
15
15
|
import { getLogger } from "@intx/log";
|
|
16
|
+
import { ApprovalDecision, signalKindToGateType } from "@intx/types";
|
|
17
|
+
import { type } from "arktype";
|
|
16
18
|
import { runInference } from "./harness.js";
|
|
17
19
|
import { createCapabilities } from "./director.js";
|
|
18
20
|
import { createGateManager } from "./gates.js";
|
|
@@ -21,6 +23,16 @@ import { createStateManager } from "./state.js";
|
|
|
21
23
|
import { validateActions } from "./actions.js";
|
|
22
24
|
import { createToolResultTurn, createInboundTurn, assertWellFormedToolSequence, } from "./turns.js";
|
|
23
25
|
const logger = getLogger(["interchange", "reactor"]);
|
|
26
|
+
// Sentinel returned by a per-call tool run when a before-tool extension parked
|
|
27
|
+
// the call on a gate. Distinct from every ToolResult so a suspended call is
|
|
28
|
+
// excluded from the tool-result history append and from tool.done continuation.
|
|
29
|
+
const SUSPENDED = Symbol("suspended");
|
|
30
|
+
// Exhaustiveness guard for the resume-dispatch switch. A newly added
|
|
31
|
+
// SignalKind or approval outcome that is not classified fails to type-check
|
|
32
|
+
// here, so the switch cannot silently drop an unhandled case.
|
|
33
|
+
function assertNever(x) {
|
|
34
|
+
throw new Error(`Unhandled resume case: ${JSON.stringify(x)}`);
|
|
35
|
+
}
|
|
24
36
|
function buildHarnessOpts(turns, source, options, signal, nextSeq, deps) {
|
|
25
37
|
if (options !== undefined) {
|
|
26
38
|
return {
|
|
@@ -132,6 +144,10 @@ export function createReactor(config) {
|
|
|
132
144
|
let running = false;
|
|
133
145
|
let done = false;
|
|
134
146
|
let shutdownStarted = false;
|
|
147
|
+
// Correlation state is empty until context loading and gate rehydration
|
|
148
|
+
// finish. Hold early deliveries so a resumed approval cannot be mistaken
|
|
149
|
+
// for a new conversation message during that startup window.
|
|
150
|
+
let startupDeliveries = [];
|
|
135
151
|
// Per-message run-bracket state. Set when the loop dequeues a
|
|
136
152
|
// message.received and begins per-message work; cleared at the
|
|
137
153
|
// terminal point (wait/reply/done) or at a reactor-fatal abandon.
|
|
@@ -175,6 +191,10 @@ export function createReactor(config) {
|
|
|
175
191
|
let cycleInferred = false;
|
|
176
192
|
let cycleToolCallsExecuted = 0;
|
|
177
193
|
let cycleCompactorName = null;
|
|
194
|
+
// A suspension registers a gate and may persist a pending operation. That is
|
|
195
|
+
// a durable state change even when the cycle ran no inference and completed
|
|
196
|
+
// no tool call, so it must force the cycle commit.
|
|
197
|
+
let cycleSuspended = false;
|
|
178
198
|
// Director-supplied checkpoint message override; consumed exactly once.
|
|
179
199
|
let pendingMessage = null;
|
|
180
200
|
// AbortController for in-flight inference/tool operations.
|
|
@@ -197,6 +217,67 @@ export function createReactor(config) {
|
|
|
197
217
|
// deliver() is fire-and-forget async, so two rapid delivers can interleave
|
|
198
218
|
// across an await boundary in the validator, causing double-correlation.
|
|
199
219
|
const correlatingIds = new Set();
|
|
220
|
+
// Decide how a correlated approval-kind pending operation resumes, granting
|
|
221
|
+
// any one-shot bypass synchronously so no delivery can interleave between the
|
|
222
|
+
// grant and the re-dispatch enqueued by the caller. An operation that carries
|
|
223
|
+
// a `suspendedCall` is an ask-flow suspension: the approver's decision routes
|
|
224
|
+
// it down the re-dispatch rail. An operation without one is an async-tool
|
|
225
|
+
// pending marker, which resumes on the normal gate-cleared rail.
|
|
226
|
+
//
|
|
227
|
+
// The nested switch is total: the outer `assertNever(op.kind)` rejects a
|
|
228
|
+
// future SignalKind at compile time, and the inner `assertNever` rejects a
|
|
229
|
+
// future decision outcome. A malformed decision body fails loud at the parse
|
|
230
|
+
// boundary before the switch.
|
|
231
|
+
function resumePendingOperation(op, message) {
|
|
232
|
+
if (op.suspendedCall === undefined) {
|
|
233
|
+
return { mode: "gate-cleared" };
|
|
234
|
+
}
|
|
235
|
+
const suspendedCall = op.suspendedCall;
|
|
236
|
+
if (message.content === undefined) {
|
|
237
|
+
throw new Error(`Correlated approval decision for ${op.correlationId} has no body to parse`);
|
|
238
|
+
}
|
|
239
|
+
let raw;
|
|
240
|
+
try {
|
|
241
|
+
raw = JSON.parse(message.content);
|
|
242
|
+
}
|
|
243
|
+
catch (cause) {
|
|
244
|
+
throw new Error(`Correlated approval decision for ${op.correlationId} is not valid JSON`, { cause });
|
|
245
|
+
}
|
|
246
|
+
const decision = ApprovalDecision(raw);
|
|
247
|
+
if (decision instanceof type.errors) {
|
|
248
|
+
throw new Error(`Correlated approval decision for ${op.correlationId} is malformed: ${decision.summary}`);
|
|
249
|
+
}
|
|
250
|
+
switch (op.kind) {
|
|
251
|
+
case "approval":
|
|
252
|
+
switch (decision.outcome) {
|
|
253
|
+
case "approved":
|
|
254
|
+
// Authorize the exact parked call to run once, then re-dispatch it.
|
|
255
|
+
// Grant on every before-tool extension: only the authz extension
|
|
256
|
+
// responds, but referencing it directly would re-couple the reactor
|
|
257
|
+
// to authz and break a deployment that runs without it.
|
|
258
|
+
for (const ext of beforeToolExtensions) {
|
|
259
|
+
ext.grantOneShot?.(suspendedCall.id);
|
|
260
|
+
}
|
|
261
|
+
return { mode: "redispatch", calls: [suspendedCall] };
|
|
262
|
+
case "rejected": {
|
|
263
|
+
// The approver denied the call. Answer the parked call with a
|
|
264
|
+
// synthetic error result rather than re-running it — no one-shot
|
|
265
|
+
// bypass is granted, so the tool never executes. The approver's
|
|
266
|
+
// reason, when present, is surfaced to the model verbatim.
|
|
267
|
+
const content = "denied by approver" +
|
|
268
|
+
(decision.message !== undefined ? `: ${decision.message}` : "");
|
|
269
|
+
return {
|
|
270
|
+
mode: "error_result",
|
|
271
|
+
result: { callId: suspendedCall.id, content, isError: true },
|
|
272
|
+
};
|
|
273
|
+
}
|
|
274
|
+
default:
|
|
275
|
+
return assertNever(decision.outcome);
|
|
276
|
+
}
|
|
277
|
+
default:
|
|
278
|
+
return assertNever(op.kind);
|
|
279
|
+
}
|
|
280
|
+
}
|
|
200
281
|
async function tryCorrelate(message) {
|
|
201
282
|
const correlationId = message.headers.interchangeCorrelationId;
|
|
202
283
|
if (correlationId === undefined)
|
|
@@ -222,19 +303,77 @@ export function createReactor(config) {
|
|
|
222
303
|
return false;
|
|
223
304
|
}
|
|
224
305
|
}
|
|
225
|
-
//
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
306
|
+
// Capture the operation before removal so the resume dispatch can read its
|
|
307
|
+
// kind and suspended call. Removal happens only after the dispatch is
|
|
308
|
+
// decided, all inside this correlatingIds-guarded critical section so a
|
|
309
|
+
// double-deliver early-returns rather than double-dispatching.
|
|
310
|
+
const op = pending;
|
|
311
|
+
let dispatch;
|
|
312
|
+
try {
|
|
313
|
+
dispatch = resumePendingOperation(op, message);
|
|
229
314
|
}
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
315
|
+
catch (cause) {
|
|
316
|
+
correlatingIds.delete(correlationId);
|
|
317
|
+
throw cause;
|
|
318
|
+
}
|
|
319
|
+
const gate = gates.findByCorrelationId(correlationId);
|
|
320
|
+
switch (dispatch.mode) {
|
|
321
|
+
case "redispatch": {
|
|
322
|
+
// Clear the gate WITHOUT enqueuing gate.cleared: the re-dispatched call
|
|
323
|
+
// is the resumption, so a gate.cleared-driven re-infer would double the
|
|
324
|
+
// continuation. The re-dispatch's own tool.done drives the re-infer.
|
|
325
|
+
if (gate !== undefined) {
|
|
326
|
+
gates.clearSilently(gate.gateId);
|
|
327
|
+
if (stateManager !== null) {
|
|
328
|
+
stateManager.setGatesSnapshot(gates.snapshot());
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
correlations.remove(correlationId);
|
|
332
|
+
if (stateManager !== null) {
|
|
333
|
+
stateManager.removePendingOperation(correlationId);
|
|
334
|
+
}
|
|
335
|
+
// The grant is already recorded (synchronously, in
|
|
336
|
+
// resumePendingOperation) with no await since; enqueue the re-dispatch
|
|
337
|
+
// so it runs on the loop with normal event ordering. The director seeds
|
|
338
|
+
// its outstanding-result count off this event before the call's
|
|
339
|
+
// tool.done arrives.
|
|
340
|
+
enqueue({ type: "resume.execute_tools", calls: dispatch.calls });
|
|
341
|
+
break;
|
|
342
|
+
}
|
|
343
|
+
case "error_result": {
|
|
344
|
+
// The approver denied the call. Clear the gate SILENTLY (like the
|
|
345
|
+
// approved redispatch) so it cannot also trip onGateCleared and enqueue
|
|
346
|
+
// a second continuation. The synthetic error result answers the parked
|
|
347
|
+
// call; the director appends it and re-infers once.
|
|
348
|
+
if (gate !== undefined) {
|
|
349
|
+
gates.clearSilently(gate.gateId);
|
|
350
|
+
if (stateManager !== null) {
|
|
351
|
+
stateManager.setGatesSnapshot(gates.snapshot());
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
correlations.remove(correlationId);
|
|
355
|
+
if (stateManager !== null) {
|
|
356
|
+
stateManager.removePendingOperation(correlationId);
|
|
357
|
+
}
|
|
358
|
+
enqueue({ type: "resume.tool_result", result: dispatch.result });
|
|
359
|
+
break;
|
|
360
|
+
}
|
|
361
|
+
case "gate-cleared": {
|
|
362
|
+
// Async-tool resumption: clear the gate normally so the director
|
|
363
|
+
// re-infers, and append the correlated response to history so the model
|
|
364
|
+
// sees the content it was waiting on.
|
|
365
|
+
if (gate !== undefined) {
|
|
366
|
+
gates.clear(gate.gateId);
|
|
367
|
+
}
|
|
368
|
+
correlations.remove(correlationId);
|
|
369
|
+
if (stateManager !== null) {
|
|
370
|
+
stateManager.removePendingOperation(correlationId);
|
|
371
|
+
const msg = createInboundTurn(message);
|
|
372
|
+
if (msg !== null) {
|
|
373
|
+
stateManager.appendTurn(msg);
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
break;
|
|
238
377
|
}
|
|
239
378
|
}
|
|
240
379
|
emit({
|
|
@@ -305,17 +444,9 @@ export function createReactor(config) {
|
|
|
305
444
|
emitError(`writePrompt failed: ${cause instanceof Error ? cause.message : String(cause)}`, false);
|
|
306
445
|
}
|
|
307
446
|
const p = (async () => {
|
|
308
|
-
// Per-source attempt budget for transient errors (quota/retryable/
|
|
309
|
-
// timeout). Kept small because failover, not flogging one source, is
|
|
310
|
-
// the recovery path: the harness already does its own mechanical
|
|
311
|
-
// retry under each attempt, so this caps reactor-level same-source
|
|
312
|
-
// retries at one before moving to the next source.
|
|
313
|
-
const sameSourceAttempts = 2;
|
|
314
|
-
const defaultRetryMs = 60_000;
|
|
315
447
|
// Each cycle starts at the most-preferred source; a failover in a
|
|
316
448
|
// prior cycle must not leave the agent permanently demoted.
|
|
317
449
|
resetToPreferredSource();
|
|
318
|
-
let attempt = 0;
|
|
319
450
|
for (;;) {
|
|
320
451
|
const harnessOpts = buildHarnessOpts(prompt, config.source, options, signal, nextSeq, deps);
|
|
321
452
|
let lastDone;
|
|
@@ -377,47 +508,17 @@ export function createReactor(config) {
|
|
|
377
508
|
enqueue({ type: "inference.error", error: err, partial });
|
|
378
509
|
return;
|
|
379
510
|
}
|
|
380
|
-
//
|
|
381
|
-
//
|
|
382
|
-
//
|
|
383
|
-
//
|
|
384
|
-
//
|
|
385
|
-
//
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
if (attempt < sameSourceAttempts && !signal.aborted) {
|
|
389
|
-
const delayMs = err.retryAfterMs ?? defaultRetryMs;
|
|
390
|
-
logger.warn `Rate limited, retrying same source after ${String(delayMs)}ms`;
|
|
391
|
-
await new Promise((resolve) => {
|
|
392
|
-
const timer = setTimeout(resolve, delayMs);
|
|
393
|
-
const onAbort = () => {
|
|
394
|
-
clearTimeout(timer);
|
|
395
|
-
resolve();
|
|
396
|
-
};
|
|
397
|
-
signal.addEventListener("abort", onAbort, { once: true });
|
|
398
|
-
});
|
|
399
|
-
if (signal.aborted) {
|
|
400
|
-
enqueue({
|
|
401
|
-
type: "inference.error",
|
|
402
|
-
error: {
|
|
403
|
-
category: "aborted",
|
|
404
|
-
message: "inference aborted during rate limit backoff",
|
|
405
|
-
},
|
|
406
|
-
partial,
|
|
407
|
-
});
|
|
408
|
-
return;
|
|
409
|
-
}
|
|
410
|
-
continue;
|
|
411
|
-
}
|
|
412
|
-
}
|
|
413
|
-
// Same-source rate-limit budget exhausted, or a source-specific
|
|
414
|
-
// failure (credential, protocol mismatch, retryable, timeout): fail
|
|
415
|
-
// over to the next source. A pacing delay the leaving source asked
|
|
416
|
-
// for must not gate the next source.
|
|
511
|
+
// Any remaining error (quota, credential, protocol mismatch,
|
|
512
|
+
// retryable, timeout) is source-specific. The harness wrapper owns
|
|
513
|
+
// mechanical retry and has already exhausted it against this source
|
|
514
|
+
// by the time the reactor sees the error, including honoring a
|
|
515
|
+
// provider Retry-After for quota, so re-running the same source
|
|
516
|
+
// would only retry-compound. Fail over to the next source instead.
|
|
517
|
+
// A pacing delay the leaving source asked for must not gate the
|
|
518
|
+
// next source.
|
|
417
519
|
pendingPacingDelayMs = 0;
|
|
418
520
|
if (failOverToNextSource()) {
|
|
419
521
|
logger.warn `Failing over to next inference source after ${err.category}`;
|
|
420
|
-
attempt = 0;
|
|
421
522
|
continue;
|
|
422
523
|
}
|
|
423
524
|
// No further source to fail over to: surface the last error.
|
|
@@ -434,21 +535,37 @@ export function createReactor(config) {
|
|
|
434
535
|
const state = stateManager;
|
|
435
536
|
const signal = operationController.signal;
|
|
436
537
|
const runOne = async (call) => {
|
|
437
|
-
// Run before-tool extensions.
|
|
538
|
+
// Run before-tool extensions. The first non-allow decision terminates
|
|
539
|
+
// the chain: `block` answers the call with an error result, `suspend`
|
|
540
|
+
// parks it (no result, no tool.done).
|
|
438
541
|
for (const ext of beforeToolExtensions) {
|
|
439
|
-
let
|
|
542
|
+
let decision;
|
|
440
543
|
try {
|
|
441
|
-
|
|
544
|
+
decision = await ext.beforeTool(call, state.snapshot(), signal);
|
|
442
545
|
}
|
|
443
546
|
catch (cause) {
|
|
444
547
|
const msg = cause instanceof Error ? cause.message : String(cause);
|
|
445
548
|
emitError(`BeforeToolExtension threw for ${call.name}: ${msg}`, false);
|
|
446
|
-
|
|
549
|
+
decision = { type: "block", reason: msg };
|
|
550
|
+
}
|
|
551
|
+
if (decision.type === "suspend") {
|
|
552
|
+
// Park the call: register the gate, persist the pending operation,
|
|
553
|
+
// snapshot, and commit. The call is neither run nor answered — no
|
|
554
|
+
// tool.start, no tool.done, no tool-result turn. The gate clears
|
|
555
|
+
// when the correlated external decision is delivered.
|
|
556
|
+
await suspendOnGate({
|
|
557
|
+
gateType: decision.gate.type,
|
|
558
|
+
gateId: decision.gate.gateId,
|
|
559
|
+
timeoutMs: Math.max(1, decision.gate.timeoutAt - Date.now()),
|
|
560
|
+
correlationId: decision.gate.correlationId,
|
|
561
|
+
pendingOp: decision.pendingOp,
|
|
562
|
+
});
|
|
563
|
+
return SUSPENDED;
|
|
447
564
|
}
|
|
448
|
-
if (
|
|
565
|
+
if (decision.type === "block") {
|
|
449
566
|
const blocked = {
|
|
450
567
|
callId: call.id,
|
|
451
|
-
content:
|
|
568
|
+
content: decision.reason,
|
|
452
569
|
isError: true,
|
|
453
570
|
};
|
|
454
571
|
emit({
|
|
@@ -467,6 +584,10 @@ export function createReactor(config) {
|
|
|
467
584
|
const gateId = `pending-${marker.correlationId}`;
|
|
468
585
|
const op = {
|
|
469
586
|
correlationId: marker.correlationId,
|
|
587
|
+
// Placeholder: async markers should carry their own SignalKind. The
|
|
588
|
+
// resume switch keys on suspendedCall presence (absent here) as the
|
|
589
|
+
// interim discriminator instead of on kind.
|
|
590
|
+
kind: "approval",
|
|
470
591
|
registeredAt: Date.now(),
|
|
471
592
|
gateId,
|
|
472
593
|
...(marker.expectedFrom !== undefined
|
|
@@ -489,22 +610,25 @@ export function createReactor(config) {
|
|
|
489
610
|
}
|
|
490
611
|
return current;
|
|
491
612
|
};
|
|
492
|
-
let
|
|
613
|
+
let outcomes;
|
|
493
614
|
if (parallel) {
|
|
494
615
|
const p = Promise.all(calls.map((c) => runOne(c)));
|
|
495
616
|
void track(p);
|
|
496
|
-
|
|
617
|
+
outcomes = await p;
|
|
497
618
|
}
|
|
498
619
|
else {
|
|
499
|
-
|
|
620
|
+
outcomes = [];
|
|
500
621
|
for (const call of calls) {
|
|
501
622
|
const p = runOne(call);
|
|
502
623
|
void track(p);
|
|
503
|
-
|
|
624
|
+
outcomes.push(await p);
|
|
504
625
|
}
|
|
505
626
|
}
|
|
627
|
+
// Suspended calls are parked, not answered: they contribute no tool
|
|
628
|
+
// result to history and no tool.done continuation event.
|
|
629
|
+
const results = outcomes.filter((o) => o !== SUSPENDED);
|
|
506
630
|
cycleToolCallsExecuted += results.length;
|
|
507
|
-
if (addToHistory && stateManager !== null) {
|
|
631
|
+
if (addToHistory && stateManager !== null && results.length > 0) {
|
|
508
632
|
stateManager.appendTurn(createToolResultTurn(results));
|
|
509
633
|
}
|
|
510
634
|
for (const result of results) {
|
|
@@ -558,6 +682,7 @@ export function createReactor(config) {
|
|
|
558
682
|
cycleInferred = false;
|
|
559
683
|
cycleToolCallsExecuted = 0;
|
|
560
684
|
cycleCompactorName = null;
|
|
685
|
+
cycleSuspended = false;
|
|
561
686
|
}
|
|
562
687
|
async function commitCycle() {
|
|
563
688
|
if (stateManager === null)
|
|
@@ -567,7 +692,8 @@ export function createReactor(config) {
|
|
|
567
692
|
// no override) commits nothing.
|
|
568
693
|
const hasWork = cycleInferred ||
|
|
569
694
|
cycleToolCallsExecuted > 0 ||
|
|
570
|
-
cycleCompactorName !== null
|
|
695
|
+
cycleCompactorName !== null ||
|
|
696
|
+
cycleSuspended;
|
|
571
697
|
const hasOverride = pendingMessage !== null;
|
|
572
698
|
if (!hasWork && !hasOverride) {
|
|
573
699
|
resetCycleAccumulators();
|
|
@@ -606,6 +732,144 @@ export function createReactor(config) {
|
|
|
606
732
|
tokenUsage: stateManager.getTokenUsage(),
|
|
607
733
|
});
|
|
608
734
|
}
|
|
735
|
+
let suspendingGate = null;
|
|
736
|
+
// Callback the gate manager invokes when a gate resolves, times out, or is
|
|
737
|
+
// shut down. Refreshes the snapshot and drives the loop's next step.
|
|
738
|
+
//
|
|
739
|
+
// A parked ask-flow approval that TIMES OUT ends without running its tool:
|
|
740
|
+
// it must be answered with a synthetic error result rather than left as a
|
|
741
|
+
// dangling tool_use. That path enqueues `resume.tool_result` INSTEAD OF
|
|
742
|
+
// `reactor.gate.cleared` — the two are mutually exclusive, because enqueuing
|
|
743
|
+
// both would drive two re-inferences for one timeout. Every other case (an
|
|
744
|
+
// async-marker pending op with no suspendedCall, no pending op at all, a
|
|
745
|
+
// `resolved`/`shutdown` reason, or a shutting-down reactor) keeps today's
|
|
746
|
+
// behavior: enqueue `reactor.gate.cleared` and let the director re-infer.
|
|
747
|
+
//
|
|
748
|
+
// A delivered `resolved` never reaches here on the ask rail — the redispatch
|
|
749
|
+
// and reject paths clear the gate silently (no onCleared) — so the timeout
|
|
750
|
+
// branch is gated on `reason === "timeout"` and shutdown stays on the plain
|
|
751
|
+
// path: a shutting-down reactor must not manufacture tool results.
|
|
752
|
+
function onGateCleared(gateId, reason) {
|
|
753
|
+
// A clear that fires while this gate's suspend is still committing must not
|
|
754
|
+
// take effect before `reactor.gate.blocked` is emitted. Record it and let
|
|
755
|
+
// suspendOnGate replay the full handler once the block is announced.
|
|
756
|
+
if (suspendingGate !== null &&
|
|
757
|
+
suspendingGate.gateId === gateId &&
|
|
758
|
+
suspendingGate.deferredClear === null) {
|
|
759
|
+
suspendingGate.deferredClear = { reason };
|
|
760
|
+
return;
|
|
761
|
+
}
|
|
762
|
+
if (stateManager !== null) {
|
|
763
|
+
stateManager.setGatesSnapshot(gates.snapshot());
|
|
764
|
+
}
|
|
765
|
+
if (reason === "timeout") {
|
|
766
|
+
const op = correlations.findByGateId(gateId);
|
|
767
|
+
if (op !== undefined && op.suspendedCall !== undefined) {
|
|
768
|
+
correlations.remove(op.correlationId);
|
|
769
|
+
if (stateManager !== null) {
|
|
770
|
+
stateManager.removePendingOperation(op.correlationId);
|
|
771
|
+
}
|
|
772
|
+
enqueue({
|
|
773
|
+
type: "resume.tool_result",
|
|
774
|
+
result: {
|
|
775
|
+
callId: op.suspendedCall.id,
|
|
776
|
+
content: "approval timed out",
|
|
777
|
+
isError: true,
|
|
778
|
+
},
|
|
779
|
+
});
|
|
780
|
+
return;
|
|
781
|
+
}
|
|
782
|
+
}
|
|
783
|
+
emit({
|
|
784
|
+
type: "reactor.gate.cleared",
|
|
785
|
+
seq: nextSeq(),
|
|
786
|
+
data: { gateId, reason },
|
|
787
|
+
});
|
|
788
|
+
enqueue({ type: "reactor.gate.cleared", gateId, reason });
|
|
789
|
+
}
|
|
790
|
+
// Parks the reactor on a gate. Shared by the director's `suspend` action and
|
|
791
|
+
// the before-tool `suspend` decision so both paths register the gate,
|
|
792
|
+
// durably persist any pending operation, snapshot the active gates, and
|
|
793
|
+
// commit before returning to the loop — a suspended reactor's state must be
|
|
794
|
+
// durable across restart. When `pendingOp` is supplied its correlation is
|
|
795
|
+
// registered and it is persisted; the director path has already persisted
|
|
796
|
+
// its pending operation (via the tool's pending marker), so it passes none.
|
|
797
|
+
async function suspendOnGate(args) {
|
|
798
|
+
const { gateType, gateId, timeoutMs, correlationId, pendingOp } = args;
|
|
799
|
+
if (pendingOp !== undefined) {
|
|
800
|
+
correlations.register(pendingOp);
|
|
801
|
+
if (stateManager !== null) {
|
|
802
|
+
stateManager.addPendingOperation(pendingOp);
|
|
803
|
+
}
|
|
804
|
+
}
|
|
805
|
+
// Track this suspend as in flight so a clear racing the commit below is
|
|
806
|
+
// deferred until `reactor.gate.blocked` has been emitted.
|
|
807
|
+
const inFlightSuspend = { gateId, deferredClear: null };
|
|
808
|
+
suspendingGate = inFlightSuspend;
|
|
809
|
+
// Register the gate. onGateCleared enqueues the cleared event so the loop
|
|
810
|
+
// processes it normally without blocking here.
|
|
811
|
+
void gates.register(gateId, gateType, timeoutMs, correlationId, onGateCleared);
|
|
812
|
+
if (stateManager !== null) {
|
|
813
|
+
stateManager.setGatesSnapshot(gates.snapshot());
|
|
814
|
+
}
|
|
815
|
+
// Registering the gate (and any pending operation) is a durable state
|
|
816
|
+
// change that must be committed even if this cycle did no other work.
|
|
817
|
+
cycleSuspended = true;
|
|
818
|
+
// Commit before the loop continues so the suspended state is durable
|
|
819
|
+
// across restart.
|
|
820
|
+
await commitCycle();
|
|
821
|
+
// Emit `reactor.gate.blocked` only AFTER the commit. This event resolves
|
|
822
|
+
// the `send()` awaiter as "suspended", and a downstream consumer (the warm
|
|
823
|
+
// agent's run-boundary durability mirror) reads the pending operation back
|
|
824
|
+
// out of the just-committed context store the instant `send()` settles.
|
|
825
|
+
// Emitting before the commit would resolve `send()` first, letting that
|
|
826
|
+
// mirror read a store that has not yet persisted the pending op -- it would
|
|
827
|
+
// durably mirror an empty pending-operation set and lose the approval
|
|
828
|
+
// snapshot, so a parked correlation could not be re-registered after a hub
|
|
829
|
+
// reconnect. This upholds persist-before-settle: the durable commit the
|
|
830
|
+
// header promises before returning to the loop lands before the suspension
|
|
831
|
+
// settles.
|
|
832
|
+
emit({
|
|
833
|
+
type: "reactor.gate.blocked",
|
|
834
|
+
seq: nextSeq(),
|
|
835
|
+
data: {
|
|
836
|
+
reason: gateType,
|
|
837
|
+
gateId,
|
|
838
|
+
...(correlationId !== undefined ? { correlationId } : {}),
|
|
839
|
+
...(pendingOp?.approvalSnapshot !== undefined
|
|
840
|
+
? { approvalSnapshot: pendingOp.approvalSnapshot }
|
|
841
|
+
: {}),
|
|
842
|
+
},
|
|
843
|
+
});
|
|
844
|
+
// The suspension is announced. If the gate cleared while the commit was in
|
|
845
|
+
// flight, its handler was deferred to keep it after `blocked`; replay it
|
|
846
|
+
// now, in order.
|
|
847
|
+
suspendingGate = null;
|
|
848
|
+
if (inFlightSuspend.deferredClear !== null) {
|
|
849
|
+
onGateCleared(gateId, inFlightSuspend.deferredClear.reason);
|
|
850
|
+
}
|
|
851
|
+
}
|
|
852
|
+
// Re-registers a live gate and correlation for each pending operation loaded
|
|
853
|
+
// from the context store on restart. The remaining timeout is computed from
|
|
854
|
+
// the persisted absolute deadline (`timeoutAt`) against the current clock, so
|
|
855
|
+
// the deadline is preserved across the restart rather than restarted; a
|
|
856
|
+
// deadline already in the past clamps to 1ms so the gate fires on the next
|
|
857
|
+
// tick. An operation persisted without a `timeoutAt` (hold-indefinitely) has
|
|
858
|
+
// no deadline to preserve; the gate manager cannot express an indefinite
|
|
859
|
+
// hold, so it is armed with the session-level `gateTimeout` — the same
|
|
860
|
+
// effective timeout the director-suspend fallback uses — rather than a
|
|
861
|
+
// silent zero. This does not run through `suspendOnGate`: rehydration must
|
|
862
|
+
// not re-emit `reactor.gate.blocked` (the suspension already happened before
|
|
863
|
+
// the restart) and must not commit (nothing changed).
|
|
864
|
+
function rehydrateGates(ops) {
|
|
865
|
+
for (const op of ops) {
|
|
866
|
+
const timeoutMs = op.timeoutAt !== undefined
|
|
867
|
+
? Math.max(1, op.timeoutAt - Date.now())
|
|
868
|
+
: gateTimeout;
|
|
869
|
+
correlations.register(op);
|
|
870
|
+
void gates.register(op.gateId, signalKindToGateType(op.kind), timeoutMs, op.correlationId, onGateCleared);
|
|
871
|
+
}
|
|
872
|
+
}
|
|
609
873
|
// -------------------------------------------------------------------------
|
|
610
874
|
// Main loop
|
|
611
875
|
// -------------------------------------------------------------------------
|
|
@@ -651,6 +915,17 @@ export function createReactor(config) {
|
|
|
651
915
|
}
|
|
652
916
|
openMessageRun(event.message.headers.messageId);
|
|
653
917
|
}
|
|
918
|
+
// A parked approval that ended without running its tool (rejected or
|
|
919
|
+
// timed out) carries a synthetic error result answering the parked call.
|
|
920
|
+
// Land it in history before the director decides so the tool_result turn
|
|
921
|
+
// closes the dangling tool_use and the re-inference the director returns
|
|
922
|
+
// sees a well-formed sequence. No tool ran, so no tool.done and no
|
|
923
|
+
// counter change accompany it.
|
|
924
|
+
if (event.type === "resume.tool_result") {
|
|
925
|
+
if (stateManager !== null) {
|
|
926
|
+
stateManager.appendTurn(createToolResultTurn([event.result]));
|
|
927
|
+
}
|
|
928
|
+
}
|
|
654
929
|
let actions;
|
|
655
930
|
try {
|
|
656
931
|
actions = await director.decide(event, stateManager.snapshot(), capabilities);
|
|
@@ -725,34 +1000,13 @@ export function createReactor(config) {
|
|
|
725
1000
|
const suspendAction = normalized.find((a) => a.type === "suspend");
|
|
726
1001
|
if (suspendAction !== undefined && suspendAction.type === "suspend") {
|
|
727
1002
|
const { gate } = suspendAction;
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
if (stateManager !== null) {
|
|
735
|
-
stateManager.setGatesSnapshot(gates.snapshot());
|
|
736
|
-
}
|
|
737
|
-
// Register the gate. The onCleared callback enqueues the cleared event
|
|
738
|
-
// so the loop processes it normally without blocking here.
|
|
739
|
-
void gates.register(gate.gateId, gate.type, effectiveTimeout, gate.correlationId, (gateId, reason) => {
|
|
740
|
-
if (stateManager !== null) {
|
|
741
|
-
stateManager.setGatesSnapshot(gates.snapshot());
|
|
742
|
-
}
|
|
743
|
-
emit({
|
|
744
|
-
type: "reactor.gate.cleared",
|
|
745
|
-
seq: nextSeq(),
|
|
746
|
-
data: { gateId, reason },
|
|
747
|
-
});
|
|
748
|
-
enqueue({ type: "reactor.gate.cleared", gateId, reason });
|
|
1003
|
+
await suspendOnGate({
|
|
1004
|
+
gateType: gate.type,
|
|
1005
|
+
gateId: gate.gateId,
|
|
1006
|
+
timeoutMs: gate.timeoutMs > 0 ? gate.timeoutMs : gateTimeout,
|
|
1007
|
+
correlationId: gate.correlationId,
|
|
1008
|
+
pendingOp: undefined,
|
|
749
1009
|
});
|
|
750
|
-
if (stateManager !== null) {
|
|
751
|
-
stateManager.setGatesSnapshot(gates.snapshot());
|
|
752
|
-
}
|
|
753
|
-
// Commit before the loop continues so the suspended-state turns are
|
|
754
|
-
// durable across restart.
|
|
755
|
-
await commitCycle();
|
|
756
1010
|
continue;
|
|
757
1011
|
}
|
|
758
1012
|
// Handle reply — emit the content for the harness/supervisor to send.
|
|
@@ -873,19 +1127,45 @@ export function createReactor(config) {
|
|
|
873
1127
|
initialUsage = loaded.tokenUsage;
|
|
874
1128
|
}
|
|
875
1129
|
catch (cause) {
|
|
1130
|
+
done = true;
|
|
1131
|
+
startupDeliveries = null;
|
|
876
1132
|
logger.error `Context store load failed: ${cause}`;
|
|
877
1133
|
emitError(`Context store load failed: ${cause instanceof Error ? cause.message : String(cause)}`, true);
|
|
878
1134
|
emit({ type: "reactor.done", seq: nextSeq(), data: {} });
|
|
879
1135
|
return;
|
|
880
1136
|
}
|
|
881
1137
|
stateManager = createStateManager(sessionId, initialTurns, initialOps, initialUsage);
|
|
882
|
-
stateManager.setGatesSnapshot(gates.snapshot());
|
|
883
|
-
emit({ type: "reactor.start", seq: nextSeq(), data: {} });
|
|
884
1138
|
try {
|
|
1139
|
+
// Re-arm gates for operations that were suspended before the restart.
|
|
1140
|
+
// The state manager holds the loaded pending operations, but a gate is
|
|
1141
|
+
// in-memory and does not survive a restart; without this a reloaded
|
|
1142
|
+
// suspended agent is wedged (no live gate to clear, no correlation to
|
|
1143
|
+
// match). Each op re-registers its correlation and a live gate keyed on
|
|
1144
|
+
// the op's own gateId and correlationId, so a delivered signal clears
|
|
1145
|
+
// it exactly as the original suspension would have.
|
|
1146
|
+
//
|
|
1147
|
+
// Rehydration runs inside this try/catch because the pending operations
|
|
1148
|
+
// come from the context store — an untrusted external boundary — and
|
|
1149
|
+
// correlation/gate registration throws synchronously on a duplicate
|
|
1150
|
+
// correlationId or gateId. A throw must surface as reactor.error plus
|
|
1151
|
+
// reactor.done (matching the load-failure path), not brick the reactor
|
|
1152
|
+
// as a silent unhandled rejection.
|
|
1153
|
+
rehydrateGates(initialOps);
|
|
1154
|
+
stateManager.setGatesSnapshot(gates.snapshot());
|
|
1155
|
+
emit({ type: "reactor.start", seq: nextSeq(), data: {} });
|
|
1156
|
+
const bufferedDeliveries = startupDeliveries;
|
|
1157
|
+
startupDeliveries = null;
|
|
1158
|
+
if (bufferedDeliveries !== null) {
|
|
1159
|
+
for (const message of bufferedDeliveries) {
|
|
1160
|
+
processDelivery(message);
|
|
1161
|
+
}
|
|
1162
|
+
}
|
|
885
1163
|
await loop();
|
|
886
1164
|
}
|
|
887
1165
|
catch (cause) {
|
|
888
1166
|
const msg = cause instanceof Error ? cause.message : String(cause);
|
|
1167
|
+
done = true;
|
|
1168
|
+
startupDeliveries = null;
|
|
889
1169
|
logger.error `Reactor loop threw unexpectedly: ${cause}`;
|
|
890
1170
|
emitError(`Internal reactor error: ${msg}`, true);
|
|
891
1171
|
closeMessageRun("failed", {
|
|
@@ -898,11 +1178,30 @@ export function createReactor(config) {
|
|
|
898
1178
|
}
|
|
899
1179
|
})();
|
|
900
1180
|
}
|
|
901
|
-
function
|
|
902
|
-
if (done)
|
|
903
|
-
return;
|
|
1181
|
+
function processDelivery(message) {
|
|
904
1182
|
void (async () => {
|
|
905
|
-
|
|
1183
|
+
let correlated;
|
|
1184
|
+
try {
|
|
1185
|
+
correlated = await tryCorrelate(message);
|
|
1186
|
+
}
|
|
1187
|
+
catch (cause) {
|
|
1188
|
+
// A correlation-path invariant failed (e.g. a malformed approval
|
|
1189
|
+
// decision). Surface it as a fatal reactor error rather than a silent
|
|
1190
|
+
// unhandled rejection, and stop the run — the resume cannot proceed on
|
|
1191
|
+
// a decision the reactor cannot trust.
|
|
1192
|
+
const msg = cause instanceof Error ? cause.message : String(cause);
|
|
1193
|
+
logger.error `Correlation dispatch failed: ${cause}`;
|
|
1194
|
+
emitError(`Correlation dispatch failed: ${msg}`, true);
|
|
1195
|
+
closeMessageRun("failed", {
|
|
1196
|
+
message: `Correlation dispatch failed: ${msg}`,
|
|
1197
|
+
kind: "reactor_fatal",
|
|
1198
|
+
});
|
|
1199
|
+
done = true;
|
|
1200
|
+
if (!shutdownStarted) {
|
|
1201
|
+
await initiateShutdown();
|
|
1202
|
+
}
|
|
1203
|
+
return;
|
|
1204
|
+
}
|
|
906
1205
|
if (!correlated) {
|
|
907
1206
|
emit({
|
|
908
1207
|
type: "message.received",
|
|
@@ -913,7 +1212,21 @@ export function createReactor(config) {
|
|
|
913
1212
|
}
|
|
914
1213
|
})();
|
|
915
1214
|
}
|
|
1215
|
+
function deliver(message) {
|
|
1216
|
+
if (done)
|
|
1217
|
+
return;
|
|
1218
|
+
if (startupDeliveries !== null) {
|
|
1219
|
+
startupDeliveries.push(message);
|
|
1220
|
+
return;
|
|
1221
|
+
}
|
|
1222
|
+
processDelivery(message);
|
|
1223
|
+
}
|
|
916
1224
|
function abort(reason) {
|
|
1225
|
+
// The loop cannot dequeue the abort event while it is awaiting an active
|
|
1226
|
+
// inference or tool batch. Signal that operation immediately so it can
|
|
1227
|
+
// settle and return control to the loop, where the queued abort retains
|
|
1228
|
+
// its priority over every other event.
|
|
1229
|
+
operationController.abort();
|
|
917
1230
|
enqueue({ type: "abort", reason });
|
|
918
1231
|
}
|
|
919
1232
|
return { start, deliver, abort };
|