@animalabs/agent-framework 0.22.0 → 0.22.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (147) hide show
  1. package/README.md +11 -4
  2. package/dist/src/agent.d.ts.map +1 -1
  3. package/dist/src/agent.js +3 -1
  4. package/dist/src/agent.js.map +1 -1
  5. package/dist/src/api/server.d.ts.map +1 -1
  6. package/dist/src/api/server.js +41 -4
  7. package/dist/src/api/server.js.map +1 -1
  8. package/dist/src/framework.d.ts +319 -20
  9. package/dist/src/framework.d.ts.map +1 -1
  10. package/dist/src/framework.js +1500 -182
  11. package/dist/src/framework.js.map +1 -1
  12. package/dist/src/index.d.ts +2 -2
  13. package/dist/src/index.d.ts.map +1 -1
  14. package/dist/src/index.js.map +1 -1
  15. package/dist/src/mcpl/index.d.ts +2 -0
  16. package/dist/src/mcpl/index.d.ts.map +1 -1
  17. package/dist/src/mcpl/index.js.map +1 -1
  18. package/dist/src/mcpl/reference-fetcher.d.ts +2 -1
  19. package/dist/src/mcpl/reference-fetcher.d.ts.map +1 -1
  20. package/dist/src/mcpl/reference-fetcher.js +20 -9
  21. package/dist/src/mcpl/reference-fetcher.js.map +1 -1
  22. package/dist/src/mcpl/references.d.ts +29 -6
  23. package/dist/src/mcpl/references.d.ts.map +1 -1
  24. package/dist/src/mcpl/references.js +57 -16
  25. package/dist/src/mcpl/references.js.map +1 -1
  26. package/dist/src/mcpl/server-connection.d.ts +28 -1
  27. package/dist/src/mcpl/server-connection.d.ts.map +1 -1
  28. package/dist/src/mcpl/server-connection.js +32 -4
  29. package/dist/src/mcpl/server-connection.js.map +1 -1
  30. package/dist/src/module-registry.d.ts +9 -3
  31. package/dist/src/module-registry.d.ts.map +1 -1
  32. package/dist/src/module-registry.js +3 -2
  33. package/dist/src/module-registry.js.map +1 -1
  34. package/dist/src/modules/workspace/index.d.ts.map +1 -1
  35. package/dist/src/modules/workspace/index.js +46 -4
  36. package/dist/src/modules/workspace/index.js.map +1 -1
  37. package/dist/src/modules/workspace/observe.d.ts +13 -3
  38. package/dist/src/modules/workspace/observe.d.ts.map +1 -1
  39. package/dist/src/modules/workspace/observe.js +24 -4
  40. package/dist/src/modules/workspace/observe.js.map +1 -1
  41. package/dist/src/modules/workspace/reconcile.d.ts +14 -0
  42. package/dist/src/modules/workspace/reconcile.d.ts.map +1 -1
  43. package/dist/src/modules/workspace/reconcile.js +10 -2
  44. package/dist/src/modules/workspace/reconcile.js.map +1 -1
  45. package/dist/src/operator-log.d.ts +41 -0
  46. package/dist/src/operator-log.d.ts.map +1 -1
  47. package/dist/src/operator-log.js.map +1 -1
  48. package/dist/src/provider-waits.d.ts +216 -0
  49. package/dist/src/provider-waits.d.ts.map +1 -0
  50. package/dist/src/provider-waits.js +482 -0
  51. package/dist/src/provider-waits.js.map +1 -0
  52. package/dist/src/refusal-reactions.d.ts +14 -12
  53. package/dist/src/refusal-reactions.d.ts.map +1 -1
  54. package/dist/src/refusal-reactions.js +14 -12
  55. package/dist/src/refusal-reactions.js.map +1 -1
  56. package/dist/src/tool-result-guard.d.ts +30 -13
  57. package/dist/src/tool-result-guard.d.ts.map +1 -1
  58. package/dist/src/tool-result-guard.js +71 -25
  59. package/dist/src/tool-result-guard.js.map +1 -1
  60. package/dist/src/types/events.d.ts +12 -0
  61. package/dist/src/types/events.d.ts.map +1 -1
  62. package/dist/src/types/module.d.ts +12 -4
  63. package/dist/src/types/module.d.ts.map +1 -1
  64. package/dist/src/types/trace.d.ts +14 -3
  65. package/dist/src/types/trace.d.ts.map +1 -1
  66. package/dist/test/discord-awareness-outbox.test.js +3 -0
  67. package/dist/test/discord-awareness-outbox.test.js.map +1 -1
  68. package/dist/test/explicit-prose-routing.test.js +0 -2
  69. package/dist/test/explicit-prose-routing.test.js.map +1 -1
  70. package/dist/test/framework.test.js +160 -1
  71. package/dist/test/framework.test.js.map +1 -1
  72. package/dist/test/helpers/coalescing-fixture.d.ts.map +1 -1
  73. package/dist/test/helpers/coalescing-fixture.js +3 -2
  74. package/dist/test/helpers/coalescing-fixture.js.map +1 -1
  75. package/dist/test/inference-exhaustion-rewind.test.js +142 -14
  76. package/dist/test/inference-exhaustion-rewind.test.js.map +1 -1
  77. package/dist/test/inference-failure-bounds.test.d.ts +2 -0
  78. package/dist/test/inference-failure-bounds.test.d.ts.map +1 -0
  79. package/dist/test/inference-failure-bounds.test.js +1201 -0
  80. package/dist/test/inference-failure-bounds.test.js.map +1 -0
  81. package/dist/test/late-events-after-stop.test.d.ts +2 -0
  82. package/dist/test/late-events-after-stop.test.d.ts.map +1 -0
  83. package/dist/test/late-events-after-stop.test.js +122 -0
  84. package/dist/test/late-events-after-stop.test.js.map +1 -0
  85. package/dist/test/mcpl-awareness-barrier.test.js +141 -0
  86. package/dist/test/mcpl-awareness-barrier.test.js.map +1 -1
  87. package/dist/test/mcpl-coalescing.test.js +78 -16
  88. package/dist/test/mcpl-coalescing.test.js.map +1 -1
  89. package/dist/test/mcpl-config-diagnostics.test.js +6 -3
  90. package/dist/test/mcpl-config-diagnostics.test.js.map +1 -1
  91. package/dist/test/mcpl-manifest-changed.test.js +2 -1
  92. package/dist/test/mcpl-manifest-changed.test.js.map +1 -1
  93. package/dist/test/mcpl-reference-fetcher.test.js +232 -4
  94. package/dist/test/mcpl-reference-fetcher.test.js.map +1 -1
  95. package/dist/test/mcpl-reference-fields.test.js +69 -2
  96. package/dist/test/mcpl-reference-fields.test.js.map +1 -1
  97. package/dist/test/mcpl-ws-transport.test.js +103 -0
  98. package/dist/test/mcpl-ws-transport.test.js.map +1 -1
  99. package/dist/test/ops-alert.test.js +2 -1
  100. package/dist/test/ops-alert.test.js.map +1 -1
  101. package/dist/test/placed-reaction-baseline.test.d.ts +2 -0
  102. package/dist/test/placed-reaction-baseline.test.d.ts.map +1 -0
  103. package/dist/test/placed-reaction-baseline.test.js +312 -0
  104. package/dist/test/placed-reaction-baseline.test.js.map +1 -0
  105. package/dist/test/provider-acceleration-cooldown.test.js +5 -1
  106. package/dist/test/provider-acceleration-cooldown.test.js.map +1 -1
  107. package/dist/test/provider-waits.test.d.ts +2 -0
  108. package/dist/test/provider-waits.test.d.ts.map +1 -0
  109. package/dist/test/provider-waits.test.js +2196 -0
  110. package/dist/test/provider-waits.test.js.map +1 -0
  111. package/dist/test/reaction-baseline-platform.test.d.ts +2 -0
  112. package/dist/test/reaction-baseline-platform.test.d.ts.map +1 -0
  113. package/dist/test/reaction-baseline-platform.test.js +124 -0
  114. package/dist/test/reaction-baseline-platform.test.js.map +1 -0
  115. package/dist/test/refusal-reaction-baseline.test.js +10 -3
  116. package/dist/test/refusal-reaction-baseline.test.js.map +1 -1
  117. package/dist/test/save-recent-image.test.js +138 -2
  118. package/dist/test/save-recent-image.test.js.map +1 -1
  119. package/dist/test/surgery-marks-receipt.test.d.ts +2 -0
  120. package/dist/test/surgery-marks-receipt.test.d.ts.map +1 -0
  121. package/dist/test/surgery-marks-receipt.test.js +540 -0
  122. package/dist/test/surgery-marks-receipt.test.js.map +1 -0
  123. package/dist/test/tool-result-guard.test.js +221 -7
  124. package/dist/test/tool-result-guard.test.js.map +1 -1
  125. package/dist/test/workspace-three-way.test.js +115 -0
  126. package/dist/test/workspace-three-way.test.js.map +1 -1
  127. package/dist/tsconfig.tsbuildinfo +1 -1
  128. package/package.json +1 -1
  129. package/src/agent.ts +4 -1
  130. package/src/api/server.ts +43 -4
  131. package/src/framework.ts +1538 -189
  132. package/src/index.ts +7 -2
  133. package/src/mcpl/index.ts +2 -0
  134. package/src/mcpl/reference-fetcher.ts +18 -8
  135. package/src/mcpl/references.ts +57 -13
  136. package/src/mcpl/server-connection.ts +55 -4
  137. package/src/module-registry.ts +17 -9
  138. package/src/modules/workspace/index.ts +44 -4
  139. package/src/modules/workspace/observe.ts +31 -5
  140. package/src/modules/workspace/reconcile.ts +23 -2
  141. package/src/operator-log.ts +30 -0
  142. package/src/provider-waits.ts +523 -0
  143. package/src/refusal-reactions.ts +14 -12
  144. package/src/tool-result-guard.ts +72 -24
  145. package/src/types/events.ts +12 -0
  146. package/src/types/module.ts +12 -4
  147. package/src/types/trace.ts +14 -3
@@ -1,6 +1,6 @@
1
1
  import { ToolPresentation, presentationTools, isPresentationTool, renderCatalogue } from "./tool-presentation.js";
2
2
  import { dirname, join } from 'node:path';
3
- import { INLINE_WITHHELD_TEXT, classifyBlock, isInlineContradiction, referenceRegistry, referenceStubOrNull } from './mcpl/references.js';
3
+ import { EXPIRY_REASON, INLINE_WITHHELD_TEXT, REFERENCE_LIMITS, classifyBlock, isInlineContradiction, isReferenceExpired, referenceExpiry, referenceRegistry, referenceStubOrNull, referenceTypeLabel, sanitizeLabel } from './mcpl/references.js';
4
4
  import { ReferenceFetcher, DEFAULT_FETCH_MAX_BYTES, EAGER_FETCH_TIMEOUT_MS } from './mcpl/reference-fetcher.js';
5
5
  import { appendFileSync, existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs';
6
6
  import { JsStore } from '@animalabs/chronicle';
@@ -10,7 +10,7 @@ import { SUBCONSCIOUS_TOOLS, SUBCONSCIOUS_TOOL_NAMES } from './tune-out/tools.js
10
10
  import { TuneOutCoordinator, TUNE_OUT_DEFAULTS } from './tune-out/coordinator.js';
11
11
  import { ProcessQueueImpl } from './queue.js';
12
12
  import { DEFAULT_SHUTDOWN_TIMEOUT_MS, validateShutdownTimeout, shutdownErrorMessage } from './shutdown.js';
13
- import { REFUSAL_REACTIONS, REFUSAL_REACTION_FALLBACK } from './refusal-reactions.js';
13
+ import { REFUSAL_REACTIONS, REFUSAL_REACTION_FALLBACK, REFUSAL_REACTION_BASELINE } from './refusal-reactions.js';
14
14
  import { Agent } from './agent.js';
15
15
  import { ModuleRegistry, checkedToolResult, isStateExistsError, storableToolResult, thrownResult } from './module-registry.js';
16
16
  import { McplServerRegistry } from './mcpl/server-registry.js';
@@ -31,6 +31,7 @@ import { InferenceRouter } from './mcpl/inference-router.js';
31
31
  import { ChannelRegistry } from './mcpl/channel-registry.js';
32
32
  import { ConversationRouter, CONVERSATION_ROUTING_DEPRECATION_NOTICE } from './mcpl/conversation-router.js';
33
33
  import { safeSlice } from './safe-slice.js';
34
+ import { ProviderWaits, EVERY_MODEL, isReleaseModel, withoutControls } from './provider-waits.js';
34
35
  import { nonErrorResultText, toolResultDataToHistoryString, truncateForHistory, DEFAULT_TOOL_RESULT_INLINE_MAX_CHARS, } from './tool-result-history.js';
35
36
  import { TOOL_IMAGE_REF_RE, ToolImageLedger, formatPreservedImageSlots, parseImagePlaceholders, sha256Hex, splitPreservingImageSlots, } from './tool-image-ledger.js';
36
37
  import { randomUUID, createHash } from 'node:crypto';
@@ -290,6 +291,7 @@ import { CheckpointManager } from './mcpl/checkpoint-manager.js';
290
291
  import { isToolAllowed } from './mcpl/tool-policy.js';
291
292
  import { EventGate, formatShadowWarning } from './gate/event-gate.js';
292
293
  import { UsageTracker } from './usage/usage-tracker.js';
294
+ import { isWebSocketTransport } from './mcpl/transport.js';
293
295
  import { formatZonedDateTime, resolveTimeZone } from './timezone.js';
294
296
  import { DEFAULT_DISCORD_AWARENESS_EMOJI, DiscordAwarenessOutbox, defaultDiscordAwarenessOutboxPath, extractDiscordAwarenessRefs, } from './recovery/discord-awareness-outbox.js';
295
297
  import { OperatorActionError, OperatorLog, capIds, defaultOperatorLogPath, } from './operator-log.js';
@@ -361,6 +363,20 @@ const MAX_TURN_CHECKPOINTS = 20;
361
363
  const DEFAULT_DISCORD_AWARENESS_DEADLINE_MS = 10_000;
362
364
  const MIN_DISCORD_AWARENESS_DEADLINE_MS = 50;
363
365
  const MAX_DISCORD_AWARENESS_DEADLINE_MS = 60_000;
366
+ /** Several drains' outcomes as one: unavailable only if every one was. */
367
+ function combineDrainOutcomes(outcomes) {
368
+ if (outcomes.every((outcome) => outcome.status === 'unavailable')) {
369
+ return {
370
+ status: 'unavailable',
371
+ accounted: outcomes.reduce((count, outcome) => count + (outcome.status === 'unavailable' ? outcome.accounted : 0), 0),
372
+ };
373
+ }
374
+ return {
375
+ status: 'delivered',
376
+ delivered: outcomes.reduce((count, outcome) => count + (outcome.status === 'delivered' ? outcome.delivered : 0), 0),
377
+ failed: outcomes.reduce((count, outcome) => count + (outcome.status === 'delivered' ? outcome.failed : 0), 0),
378
+ };
379
+ }
364
380
  function normalizeDiscordAwarenessDeadline(value) {
365
381
  if (value === undefined || !Number.isFinite(value) || value <= 0) {
366
382
  return DEFAULT_DISCORD_AWARENESS_DEADLINE_MS;
@@ -459,11 +475,35 @@ function bodyGroupRun(cm, index) {
459
475
  group: win.messages[0].bodyGroupId !== undefined,
460
476
  };
461
477
  }
478
+ function describeMarkers(m) {
479
+ if (m.status === 'queued')
480
+ return `queued:${m.queued}`;
481
+ if (m.status === 'not-scheduled')
482
+ return 'not-scheduled(batch retired)';
483
+ if (m.status === 'unresolved')
484
+ return `unresolved(batch ${m.batchId} may still be delivered)`;
485
+ return 'none';
486
+ }
462
487
  function describeRequester(r) {
463
488
  if (!r)
464
489
  return 'unknown';
465
490
  return `${r.name ?? r.id ?? 'unknown'} (${r.via})`;
466
491
  }
492
+ /** What a held call or turn says of the wait that holds it: no call was
493
+ * made. The refusal of an auxiliary call and the trace of a turn held once
494
+ * compiled say it the same way. */
495
+ function providerWaitText(agentName, model, wait) {
496
+ return `Provider wait: ${model} is held for ${agentName} ` +
497
+ `${wait.until === null ? 'until an operator releases it' : `until ${new Date(wait.until).toISOString()}`} ` +
498
+ `(recorded ${new Date(wait.setAt).toISOString()}: ${wait.reason}); no call was made`;
499
+ }
500
+ /** Who asked for an operator act, as a durable record and a log line carry
501
+ * it. The name comes from a caller, so a control character (a forged log
502
+ * line) becomes a space, and it is bounded. */
503
+ function operatorLabel(by) {
504
+ const label = safeSlice(withoutControls(by).trim(), 0, 120);
505
+ return label === '' ? 'operator' : label;
506
+ }
467
507
  function hostCommandRequester(serverId, params) {
468
508
  return {
469
509
  via: `host-command:${serverId}`,
@@ -490,11 +530,11 @@ function isPermanentDiscordReactionFailure(message) {
490
530
  class DefaultErrorPolicy {
491
531
  maxRetries = 3;
492
532
  onInferenceError(error, _agentName, attempt) {
493
- if (error instanceof MembraneError && !error.retryable) {
533
+ if (isMembraneError(error) && error.retryable === false) {
494
534
  return { retry: false };
495
535
  }
496
536
  if (attempt < this.maxRetries) {
497
- const delayMs = error instanceof MembraneError && error.retryAfterMs !== undefined
537
+ const delayMs = isMembraneError(error) && typeof error.retryAfterMs === 'number'
498
538
  ? error.retryAfterMs
499
539
  : Math.pow(2, attempt) * 1000;
500
540
  return { retry: true, delayMs };
@@ -513,10 +553,32 @@ const DEFAULT_MAINTENANCE_INTERVAL_MS = 5000;
513
553
  /** Bound one pass so a large backlog yields to inference and other agents. */
514
554
  const MAINTENANCE_TICKS_PER_PASS = 8;
515
555
  const PROVIDER_ACCELERATION_DEFAULT_COOLDOWN_MS = 65_000;
556
+ /** Cap on waits the framework chooses itself. A provider's stated wait is
557
+ * never capped: it is a lower bound (see ProviderWaits). It is also the
558
+ * longest stated wait an ephemeral run sleeps in its turn: a longer one ends
559
+ * the run instead (nonOwningProviderWait). */
516
560
  const PROVIDER_ACCELERATION_MAX_COOLDOWN_MS = 10 * 60_000;
517
561
  const PROVIDER_ACCELERATION_JITTER_MS = 5_000;
562
+ /** Longest delay one setTimeout can represent (2^31-1 ms, about 24.8 days);
563
+ * a longer timer fires almost at once, so longer waits re-arm in steps. */
564
+ const MAX_TIMER_DELAY_MS = 2_147_483_647;
565
+ /** A stated wait recorded to hold a model longer than this, or until
566
+ * released, raises an ops alert. It is honoured in full either way: a hold
567
+ * that long is a quota-scale event or a provider's mistake, and a person
568
+ * decides which (release-provider-wait). Shorter waits are pacing. */
569
+ const PROVIDER_WAIT_ALERT_MS = 60 * 60_000;
570
+ /**
571
+ * A MembraneError, from this framework's copy of membrane or from another: the
572
+ * host supplies the Membrane instance (config.membrane), so with two copies
573
+ * installed its errors are instances of a class this framework's `instanceof`
574
+ * doesn't match. Recognised by name as well, as context-manager's errors are;
575
+ * its fields are the same, so callers still check each one's type.
576
+ */
577
+ function isMembraneError(error) {
578
+ return error instanceof MembraneError || (error instanceof Error && error.name === 'MembraneError');
579
+ }
518
580
  function isOrganizationAccelerationRateLimit(error) {
519
- if (!(error instanceof MembraneError) || error.type !== 'rate_limit')
581
+ if (!isMembraneError(error) || error.type !== 'rate_limit')
520
582
  return false;
521
583
  return (/organization(?:'s)?[^\n]{0,120}maximum(?:\s+usage)?\s+increase\s+rate/i.test(error.message) ||
522
584
  /organization(?:'s)?[^\n]{0,120}acceleration(?:\s+limit)?/i.test(error.message));
@@ -552,11 +614,267 @@ function extractGateFields(event) {
552
614
  : undefined;
553
615
  return { content, metadata, mount, paths };
554
616
  }
555
- /** Cap an API error message for inline use in a rewind marker: keep enough to
556
- * identify the failure class without pasting a wall of provider JSON into the
557
- * agent's context. */
558
- function truncateReason(reason, max = 160) {
559
- return reason.length <= max ? reason : reason.slice(0, max) + '…';
617
+ /**
618
+ * Longest failure text the inference-failure pipeline writes to one sink: the
619
+ * reason in the exhaustion stderr line, a field of a logs/failures.log record,
620
+ * an ops alert's message, the inference log's `error`, and a trace event's
621
+ * `error` and `stack`. A failure's text is whatever its source put in
622
+ * `Error.message`, and a provider can echo the entire rejected request there
623
+ * (one OpenAI-compatible 400 echoed ~1.7 MB). The sinks with shorter bounds
624
+ * name them below, and every cut goes through boundFailureText. This bound
625
+ * doesn't depend on membrane bounding its own errors.
626
+ */
627
+ const FAILURE_TEXT_MAX_CHARS = 2_000;
628
+ /** Longest failure reason written into an agent's own context (the
629
+ * `[inference-failed]` marker): enough for a provider's explanation, not a
630
+ * self-dump the agent then carries in every later request. */
631
+ const FAILURE_MARKER_REASON_MAX_CHARS = 600;
632
+ /** Longest failure reason in the rewind marker, also the agent's own context:
633
+ * enough to identify the failure class without a wall of provider JSON. */
634
+ const REWIND_MARKER_REASON_MAX_CHARS = 160;
635
+ /** Longest failure reason in an agent's status (`lastInference.lastError`) and
636
+ * in the compile-refused ops alert. */
637
+ const FAILURE_SUMMARY_MAX_CHARS = 300;
638
+ /** Longest alert message posted to the ops webhook. */
639
+ const OPS_WEBHOOK_MESSAGE_MAX_CHARS = 500;
640
+ /** Longest provider error code recorded verbatim; real codes are short tokens.
641
+ * A failure's name and type are bounded the same way. */
642
+ const PROVIDER_ERROR_CODE_MAX_CHARS = 128;
643
+ const isHighSurrogate = (text, index) => {
644
+ const code = text.charCodeAt(index);
645
+ return code >= 0xd800 && code <= 0xdbff;
646
+ };
647
+ const isLowSurrogate = (text, index) => {
648
+ const code = text.charCodeAt(index);
649
+ return code >= 0xdc00 && code <= 0xdfff;
650
+ };
651
+ /** Whether a cut before `index` would separate the halves of a surrogate pair. */
652
+ const splitsPair = (text, index) => index > 0 && index < text.length && isHighSurrogate(text, index - 1) && isLowSurrogate(text, index);
653
+ /**
654
+ * `text` when it fits within `max` characters; otherwise its head and tail
655
+ * around a marker stating how many of how many characters were omitted, the
656
+ * whole within `max`. A cut that would separate a surrogate pair moves toward
657
+ * omission, so the result never holds half of a pair the text held whole, and
658
+ * bounding bounded text returns it as is. Each sink cuts the whole text, so
659
+ * its marker states the whole length.
660
+ */
661
+ function boundFailureText(text, max = FAILURE_TEXT_MAX_CHARS) {
662
+ if (text.length <= max)
663
+ return text;
664
+ const markerLength = ` …[${text.length} of ${text.length} characters omitted]… `.length;
665
+ const budget = Math.max(0, max - markerLength);
666
+ const tailLength = Math.floor(budget / 4);
667
+ let headEnd = budget - tailLength;
668
+ if (splitsPair(text, headEnd))
669
+ headEnd--;
670
+ let tailStart = text.length - tailLength;
671
+ if (splitsPair(text, tailStart))
672
+ tailStart++;
673
+ return `${text.slice(0, headEnd)} …[${tailStart - headEnd} of ${text.length} characters omitted]… ${text.slice(tailStart)}`;
674
+ }
675
+ /** A provider error code, verbatim when token-sized; otherwise bounded. */
676
+ function boundProviderErrorCode(code) {
677
+ if (typeof code !== 'string')
678
+ return undefined;
679
+ return boundFailureText(code, PROVIDER_ERROR_CODE_MAX_CHARS);
680
+ }
681
+ /**
682
+ * A value's JSON for a failure sink, with every Error inside it written as its
683
+ * whole projection (failureDescription): serialising an Error writes its own
684
+ * fields, a MembraneError's whole `rawRequest` among them. The projection is
685
+ * left uncut, as a nested string is, so the sink's one cut of the JSON states
686
+ * its whole length. An Error is recognised by instanceof, so one from another
687
+ * realm (a vm context) is written as its own JSON, as any object is. Throws,
688
+ * or returns undefined, as JSON.stringify does.
689
+ */
690
+ function failureJson(value) {
691
+ return JSON.stringify(value, (_key, nested) => (nested instanceof Error ? failureDescription(nested) : nested));
692
+ }
693
+ /**
694
+ * The whole text describeFailure cuts. For an Error, its head (name,
695
+ * classification, status, provider code, each bounded) and its whole
696
+ * message. For a primitive, its String. For another object, its String when
697
+ * that's text of its own (a Buffer, an Error from another realm, a class
698
+ * with its own toString). An array, or an object whose String is empty or
699
+ * only its tag, such as `[object Object]`, is its JSON instead
700
+ * (failureJson), so its fields and elements are kept; its tag stands when
701
+ * JSON can't write it or finds nothing in it.
702
+ */
703
+ function failureDescription(error) {
704
+ try {
705
+ if (!(error instanceof Error)) {
706
+ if (typeof error !== 'object' || error === null)
707
+ return String(error);
708
+ if (!Array.isArray(error)) {
709
+ let text;
710
+ try {
711
+ text = String(error);
712
+ }
713
+ catch { /* no String of its own: its JSON, below */ }
714
+ if (text && !/^\[object [^\]]*\]$/.test(text))
715
+ return text;
716
+ }
717
+ try {
718
+ const json = failureJson(error);
719
+ if (json !== undefined && json !== '{}')
720
+ return json;
721
+ }
722
+ catch { /* its tag, below */ }
723
+ return Object.prototype.toString.call(error);
724
+ }
725
+ const fields = error;
726
+ // Every component is remote or caller-supplied text, the name included.
727
+ const classification = [
728
+ typeof fields.type === 'string' ? boundFailureText(fields.type, PROVIDER_ERROR_CODE_MAX_CHARS) : undefined,
729
+ typeof fields.httpStatus === 'number' ? `HTTP ${fields.httpStatus}` : undefined,
730
+ boundProviderErrorCode(fields.providerErrorCode),
731
+ typeof fields.retryable === 'boolean' ? `retryable=${fields.retryable}` : undefined,
732
+ ].filter((part) => part !== undefined);
733
+ const name = boundFailureText(String(error.name), PROVIDER_ERROR_CODE_MAX_CHARS);
734
+ const head = classification.length > 0 ? `${name} (${classification.join(', ')})` : name;
735
+ return `${head}: ${String(error.message)}`;
736
+ }
737
+ catch {
738
+ return '[failure could not be described]';
739
+ }
740
+ }
741
+ /**
742
+ * A bounded string describing a failure, for logs that would otherwise print
743
+ * the error object; it is one line unless one of its parts has newlines, as a
744
+ * message can. Printing the object prints its message whole, in the stack,
745
+ * and every own field: a MembraneError's `rawRequest` is the whole request
746
+ * that failed, and Node's console prints each of its strings, a long system
747
+ * prompt among them, up to 10,000 characters. The projection keeps what a
748
+ * reader needs (name, classification, status, provider code, message) and
749
+ * nothing else. The head's parts are bounded each, and the whole
750
+ * (failureDescription) is cut once, so its marker states the length of the
751
+ * head and the whole message.
752
+ */
753
+ function describeFailure(error) {
754
+ return boundFailureText(failureDescription(error));
755
+ }
756
+ /** A stack frame's line: an indent of spaces or tabs, then `at `. The indent
757
+ * never crosses a line break, so a run of blank lines isn't rescanned from
758
+ * each line's start. */
759
+ const FRAME_LINE = /^[ \t]+at /m;
760
+ /**
761
+ * describeFailure, then the error's stack frames, bounded, on the lines
762
+ * below: for a log of a failure the framework's own code may have thrown,
763
+ * where the frames say where. The stack opens with the message, and a
764
+ * message's own lines can look like frames (an echoed request quoting a stack
765
+ * trace), so when the message is the stack's header, before its first frame
766
+ * line, the frames are the stack from its first frame line after the
767
+ * message. Otherwise (a stack without its header, a message that isn't in
768
+ * the stack or is empty) they are the stack from its first frame line. The
769
+ * leak the projection closes was the object's own fields, not its stack.
770
+ */
771
+ function describeFailureWithFrames(error) {
772
+ let frames;
773
+ try {
774
+ const stack = error instanceof Error ? error.stack : undefined;
775
+ if (typeof stack === 'string') {
776
+ const message = String(error.message);
777
+ // The message is the stack's header only before its first frame: a
778
+ // stack without its header (copied, or frames only) can hold the
779
+ // message's text inside a frame's name.
780
+ const firstFrame = stack.search(FRAME_LINE);
781
+ const at = message ? stack.indexOf(message) : -1;
782
+ const frameLine = new RegExp(FRAME_LINE.source, 'gm');
783
+ frameLine.lastIndex = at >= 0 && (firstFrame < 0 || at < firstFrame) ? at + message.length : 0;
784
+ const found = frameLine.exec(stack);
785
+ if (found)
786
+ frames = boundFailureText(stack.slice(found.index));
787
+ }
788
+ }
789
+ catch { /* the projection alone */ }
790
+ const described = describeFailure(error);
791
+ return frames === undefined ? described : `${described}\n${frames}`;
792
+ }
793
+ /**
794
+ * A failure record's fields, each bounded, for logs/failures.log and the
795
+ * ops:alert trace: a string to FAILURE_TEXT_MAX_CHARS; an Error to its
796
+ * projection, since serialising one writes its own fields, a MembraneError's
797
+ * whole `rawRequest` among them; any other value to its JSON, with each Error
798
+ * inside it as its projection (failureJson), as the plain data that JSON
799
+ * describes when it fits FAILURE_TEXT_MAX_CHARS and as the JSON text,
800
+ * bounded, when it doesn't. So the trace's data is what failures.log writes:
801
+ * a Map or an object with toJSON whose JSON is small never hands a listener
802
+ * the value itself.
803
+ */
804
+ function boundFailureFields(record) {
805
+ const bounded = {};
806
+ for (const [key, value] of Object.entries(record)) {
807
+ if (typeof value === 'string')
808
+ bounded[key] = boundFailureText(value);
809
+ else if (value instanceof Error)
810
+ bounded[key] = describeFailure(value);
811
+ else {
812
+ let json;
813
+ try {
814
+ json = failureJson(value);
815
+ }
816
+ catch {
817
+ bounded[key] = describeFailure(value);
818
+ continue;
819
+ }
820
+ // A value JSON writes nothing for (a function, undefined) is left out,
821
+ // as the log's JSON leaves it out.
822
+ if (json === undefined)
823
+ continue;
824
+ bounded[key] = json.length <= FAILURE_TEXT_MAX_CHARS ? JSON.parse(json) : boundFailureText(json);
825
+ }
826
+ }
827
+ return bounded;
828
+ }
829
+ /**
830
+ * Whether the poison-history breaker may shed the newest exchange for a
831
+ * failure: an `invalid_request` at HTTP 400, 413 or 422 or with no status, or
832
+ * a `context_length` at 413. The status is a heuristic for when a shed may
833
+ * help, not proof that the history is at fault.
834
+ *
835
+ * 400 and 422 are how a provider answers a request that fails its
836
+ * validation, as a history it can't accept does, but other refusals come
837
+ * back the same way, such as Anthropic's 400 for a credit balance that's too
838
+ * low. A 413 is the request's size. When the newest exchange made it too
839
+ * large, as an oversized attachment does, shedding that exchange may bring
840
+ * the request back under the limit, and nothing else in the framework
841
+ * recovers a 413; when the history as a whole is too large, the shed doesn't
842
+ * help, and the breaker stops at its rewind cap. The 413 arm takes both
843
+ * labels a 413 arrives under, invalid_request and context_length, because
844
+ * membrane's label for a 413 differs by version and adapter. Other statuses
845
+ * keep the history, since shed history can't be got back and the status
846
+ * gives no sign that a shed would help: a 404 is a model or an endpoint that
847
+ * doesn't exist, and 405, 409, 410 and 415 are about the HTTP request or the
848
+ * resource, as a rule, rather than the conversation.
849
+ *
850
+ * The table test in test/inference-exhaustion-rewind.test.ts is this
851
+ * predicate's contract. The notes below on which membrane version gives which
852
+ * label are its history, and will go stale as hosts move to newer releases.
853
+ *
854
+ * Which failures reach the gate with which label depends on the membrane:
855
+ * - As membrane 0.5.86 classifies them: Anthropic files a 400 as
856
+ * invalid_request (after its context check) and its other 4xx as unknown.
857
+ * The chat adapters never file one as invalid_request, since they classify
858
+ * by the error's message text, and a context_length they file that way
859
+ * carries status 400 whatever the response's was. The raw-fetch Responses
860
+ * adapter files every status it doesn't list (404, 405, 409, 410, 413,
861
+ * 415, 422 …) as invalid_request, so it alone produces an invalid_request
862
+ * at 413 or 422, and no 0.5.86 adapter produces a context_length at 413.
863
+ * - From membrane #56 (merged, not yet in a release), one table serves every
864
+ * adapter. Auth codes and context overflow are checked on a 400 or 422
865
+ * first, a 404 is invalid_request, a 413 is context_length, and other 4xx
866
+ * are invalid_request. So a 413 still sheds, an Anthropic one included,
867
+ * while a context_length at 400 or 422, a token overflow, keeps the
868
+ * history. The 422 arm then also covers an OpenAI-compatible server, such
869
+ * as a FastAPI one, that answers a validation failure with 422.
870
+ *
871
+ * An error with no status, from a membrane that reports none, keeps the rule
872
+ * the breaker had before: the type alone.
873
+ */
874
+ export function shedMayHelp(errorType, httpStatus) {
875
+ if (httpStatus === 413)
876
+ return errorType === 'invalid_request' || errorType === 'context_length';
877
+ return errorType === 'invalid_request' && (httpStatus === undefined || httpStatus === 400 || httpStatus === 422);
560
878
  }
561
879
  /** The most of a thrown value's or a tool error's text that reaches a log line or a server's error answer. */
562
880
  const THROWN_TEXT_LIMIT = 500;
@@ -688,6 +1006,10 @@ export class AgentFramework {
688
1006
  providerAccelerationDefaultCooldownMs = PROVIDER_ACCELERATION_DEFAULT_COOLDOWN_MS;
689
1007
  providerAccelerationJitterMs = PROVIDER_ACCELERATION_JITTER_MS;
690
1008
  providerHoldHook;
1009
+ /** Provider waits (stated retry-after) per (agent, model), kept across restarts. */
1010
+ providerWaits;
1011
+ /** Wakers for error-policy retry waits in progress: stop() ends them. */
1012
+ retryWaitWakers = new Set();
691
1013
  providerAdmissionClosed = false;
692
1014
  /** Invalidates framework turns still assembling when stop begins. */
693
1015
  lifecycleGeneration = 0;
@@ -1069,6 +1391,13 @@ export class AgentFramework {
1069
1391
  this.operatorLog = operatorLog;
1070
1392
  this.store = store;
1071
1393
  this.ownsStore = ownsStore;
1394
+ // Unreadable recorded waits hold every agent's provider calls, and a
1395
+ // persistent cause never reads again: a person has to hear of it.
1396
+ this.providerWaits = new ProviderWaits(store, {
1397
+ onUnreadable: (error) => this.opsAlert('provider-wait', 'framework', `recorded provider waits could not be read (${error}): no provider call is admitted for any agent ` +
1398
+ 'until they read again (re-read every 30 s) or an operator releases an agent\'s waits with ' +
1399
+ 'release-provider-wait; such a release lasts for this process only', { data: { model: EVERY_MODEL, until: null } }),
1400
+ });
1072
1401
  this.membrane = membrane;
1073
1402
  this.inferencePolicy = inferencePolicy;
1074
1403
  this.errorPolicy = errorPolicy;
@@ -1085,14 +1414,16 @@ export class AgentFramework {
1085
1414
  emitTrace: (e) => this.emitTrace({ ...e }),
1086
1415
  });
1087
1416
  // Initialize module registry with callbacks
1088
- this.moduleRegistry = new ModuleRegistry(store, this.queue, {
1417
+ // A module's events, through pushEvent() or its deprecated queue handle,
1418
+ // go through pushModuleEvent(), which names the module.
1419
+ this.moduleRegistry = new ModuleRegistry(store, { push: (event, moduleName) => this.pushModuleEvent(event, moduleName, false) }, {
1089
1420
  getAgents: () => Array.from(this.agents.values()),
1090
1421
  addMessage: (p, c, m) => this.addMessage(p, c, m),
1091
1422
  editMessage: (id, c) => this.editMessage(id, c),
1092
1423
  removeMessage: (id) => this.removeMessage(id),
1093
1424
  getMessage: (id) => this.getMessage(id),
1094
1425
  queryMessages: (filter) => this.queryMessages(filter),
1095
- pushEvent: (event) => this.pushEvent(event),
1426
+ pushEvent: (event, moduleName) => this.pushModuleEvent(event, moduleName, true),
1096
1427
  onTrace: (listener) => this.onTrace(listener),
1097
1428
  // The ONLY entry that confers module origin — a private closure the
1098
1429
  // framework hands to its own module registry. The public
@@ -1565,6 +1896,8 @@ export class AgentFramework {
1565
1896
  // store is still open, rather than leaving them to a reboot replay.
1566
1897
  this.ackDeferredWrites();
1567
1898
  this.providerAdmissionClosed = true;
1899
+ for (const wake of [...this.retryWaitWakers])
1900
+ wake();
1568
1901
  this.lifecycleGeneration++;
1569
1902
  // The generation this stop ends; withheld spills close for it alone, so a
1570
1903
  // start() made before this stop finishes reopens them for its own.
@@ -1716,7 +2049,13 @@ export class AgentFramework {
1716
2049
  providerGateBlocked(agentName) {
1717
2050
  const gate = this.providerGates.get(agentName);
1718
2051
  return (gate?.primaryDepth ?? 0) > 0 || (gate?.primaryPending ?? false) ||
1719
- this.providerAccelerationCooldowns.has(agentName);
2052
+ this.cooldownBlocksAuxiliary(agentName);
2053
+ }
2054
+ /** Acceleration and host holds park the whole agent; a provider wait holds
2055
+ * only its own model, which auxiliary admission checks per call. */
2056
+ cooldownBlocksAuxiliary(agentName) {
2057
+ const cooldown = this.providerAccelerationCooldowns.get(agentName);
2058
+ return cooldown !== undefined && cooldown.waitModel === undefined;
1720
2059
  }
1721
2060
  acquirePrimaryProviderGate(agentName) {
1722
2061
  const gate = this.providerGate(agentName);
@@ -1730,7 +2069,7 @@ export class AgentFramework {
1730
2069
  }
1731
2070
  flushAuxiliaryAdmission(agentName) {
1732
2071
  const gate = this.providerGate(agentName);
1733
- if (gate.primaryDepth > 0 || gate.primaryPending || this.providerAccelerationCooldowns.has(agentName))
2072
+ if (gate.primaryDepth > 0 || gate.primaryPending || this.cooldownBlocksAuxiliary(agentName))
1734
2073
  return;
1735
2074
  for (const resolve of gate.auxiliaryWaiters.splice(0))
1736
2075
  resolve();
@@ -1741,8 +2080,9 @@ export class AgentFramework {
1741
2080
  return;
1742
2081
  await new Promise((resolve) => gate.idleWaiters.push(resolve));
1743
2082
  }
1744
- async withAuxiliaryAdmission(agentName, run) {
2083
+ async withAuxiliaryAdmission(agentName, run, model) {
1745
2084
  const gate = this.providerGate(agentName);
2085
+ this.refuseHeldProviderModel(agentName, model);
1746
2086
  if (this.providerGateBlocked(agentName)) {
1747
2087
  gate.deferredAuxiliary++;
1748
2088
  try {
@@ -1755,17 +2095,23 @@ export class AgentFramework {
1755
2095
  if (this.providerAdmissionClosed) {
1756
2096
  throw new Error(`Provider admission closed while stopping (${agentName})`);
1757
2097
  }
2098
+ // A wait recorded while this call was parked binds it too.
2099
+ this.refuseHeldProviderModel(agentName, model);
1758
2100
  gate.auxiliaryInFlight++;
1759
2101
  try {
1760
2102
  return await run();
1761
2103
  }
1762
2104
  catch (error) {
1763
2105
  // Compression can be the first call to find a spent quota (idle agent,
1764
- // scheduled maintenance). Arm the host hold so later passes wait at the
1765
- // gate instead of retrying into it; this pass still fails to its caller.
2106
+ // scheduled maintenance). Record a stated wait for this call's model,
2107
+ // and arm the host hold so later passes wait at the gate instead of
2108
+ // retrying into it; this pass still fails to its caller.
1766
2109
  const agent = this.agents.get(agentName);
1767
- if (agent && error instanceof Error)
2110
+ if (agent && error instanceof Error) {
2111
+ if (model !== undefined)
2112
+ this.recordProviderWait(agentName, model, error);
1768
2113
  this.holdProviderAcceleration(agent, error, undefined, true);
2114
+ }
1769
2115
  throw error;
1770
2116
  }
1771
2117
  finally {
@@ -1776,22 +2122,62 @@ export class AgentFramework {
1776
2122
  }
1777
2123
  }
1778
2124
  auxiliaryMembraneFor(agentName) {
2125
+ return this.membraneAdmitting((run, model) => this.withAuxiliaryAdmission(agentName, run, model));
2126
+ }
2127
+ /**
2128
+ * A conversation agent's auxiliary calls (its context manager's
2129
+ * compression). A conversation agent takes no provider gate: admission
2130
+ * belongs to the persistent resident, and a generation-unique fork must not
2131
+ * leave a gate behind. But its stated waits are recorded for (agent, model)
2132
+ * as a resident's are, so they bind its auxiliary calls too: a held model
2133
+ * is refused without a call, and an auxiliary call's own stated wait is
2134
+ * recorded for it. (An ephemeral run records nothing, its name being
2135
+ * single-use, so its context manager keeps the plain membrane.)
2136
+ */
2137
+ conversationAuxiliaryMembraneFor(agentName) {
2138
+ return this.membraneAdmitting(async (run, model) => {
2139
+ this.refuseHeldProviderModel(agentName, model);
2140
+ try {
2141
+ return await run();
2142
+ }
2143
+ catch (error) {
2144
+ if (model !== undefined && error instanceof Error)
2145
+ this.recordProviderWait(agentName, model, error);
2146
+ throw error;
2147
+ }
2148
+ });
2149
+ }
2150
+ /** The framework's membrane, with each complete() call passed through
2151
+ * `admit` along with the model it names. */
2152
+ membraneAdmitting(admit) {
1779
2153
  const target = this.membrane;
1780
2154
  return new Proxy(target, { get: (obj, prop, receiver) => {
1781
2155
  const value = Reflect.get(obj, prop, receiver);
1782
2156
  if (prop === 'complete' && typeof value === 'function') {
1783
- return (...args) => this.withAuxiliaryAdmission(agentName, () => Reflect.apply(value, this.membrane, args));
2157
+ return (...args) => {
2158
+ const model = args[0]?.config?.model;
2159
+ return admit(() => Reflect.apply(value, this.membrane, args), typeof model === 'string' ? model : undefined);
2160
+ };
1784
2161
  }
1785
2162
  return typeof value === 'function' ? value.bind(this.membrane) : value;
1786
2163
  } });
1787
2164
  }
2165
+ /** An acceleration hold's length: AF's own pacing (`ownMs`: its default
2166
+ * plus jitter, capped), or the provider's stated wait in its place
2167
+ * (`stated`). The stated wait is a lower bound: jitter only adds to it,
2168
+ * and the cap applies only to AF's own default. A negative stated wait is
2169
+ * a stated 0, as in ProviderWaits: the 1 s floor, plus jitter. */
1788
2170
  accelerationCooldownMs(agentName, error) {
1789
- const base = error.retryAfterMs ?? this.providerAccelerationDefaultCooldownMs;
1790
2171
  let hash = 0;
1791
2172
  for (const ch of agentName)
1792
2173
  hash = ((hash * 31) + ch.charCodeAt(0)) >>> 0;
1793
2174
  const jitter = this.providerAccelerationJitterMs > 0 ? hash % (this.providerAccelerationJitterMs + 1) : 0;
1794
- return Math.min(PROVIDER_ACCELERATION_MAX_COOLDOWN_MS, Math.max(1_000, base) + jitter);
2175
+ const ownMs = Math.min(PROVIDER_ACCELERATION_MAX_COOLDOWN_MS, Math.max(1_000, this.providerAccelerationDefaultCooldownMs) + jitter);
2176
+ const stated = error.retryAfterMs;
2177
+ if (typeof stated === 'number' && Number.isFinite(stated)) {
2178
+ return { delayMs: Math.max(1_000, stated) + jitter, ownMs, stated: true };
2179
+ }
2180
+ return { delayMs: ownMs, ownMs, stated: false };
1795
2181
  }
1796
2182
  sameInferenceRequest(a, b) {
1797
2183
  return a.agentName === b.agentName && a.reason === b.reason && a.source === b.source &&
@@ -1804,11 +2190,15 @@ export class AgentFramework {
1804
2190
  const hold = this.providerHoldHook(error, agentName, { model: this.agents.get(agentName)?.model });
1805
2191
  if (!hold || !Number.isFinite(hold.holdMs) || hold.holdMs <= 0)
1806
2192
  return undefined;
2193
+ // The reason is printed, kept in the cooldown and shown in the status
2194
+ // snapshot (cooldownReason); without the host's own, it is the error's
2195
+ // message, which can echo the whole request.
1807
2196
  return { holdMs: Math.min(PROVIDER_ACCELERATION_MAX_COOLDOWN_MS, Math.max(1_000, hold.holdMs)),
1808
- reason: typeof hold.reason === 'string' && hold.reason ? hold.reason : error.message };
2197
+ reason: boundFailureText(typeof hold.reason === 'string' && hold.reason ? hold.reason : String(error.message)) };
1809
2198
  }
1810
2199
  catch (err) {
1811
- console.error(`[provider-cooldown] providerHold hook threw for ${agentName}; treating as no hold:`, err);
2200
+ // The hook is handed the model's error, and may throw it back.
2201
+ console.error(`[provider-cooldown] providerHold hook threw for ${agentName}; treating as no hold: ${describeFailureWithFrames(err)}`);
1812
2202
  return undefined;
1813
2203
  }
1814
2204
  }
@@ -1822,7 +2212,7 @@ export class AgentFramework {
1822
2212
  return false;
1823
2213
  clearTimeout(cooldown.timer);
1824
2214
  cooldown.until = Date.now() + hold.holdMs;
1825
- cooldown.timer = setTimeout(() => this.releaseProviderAccelerationCooldown(agentName), hold.holdMs);
2215
+ cooldown.timer = setTimeout(() => this.releaseProviderAccelerationCooldown(agentName), Math.min(hold.holdMs, MAX_TIMER_DELAY_MS));
1826
2216
  cooldown.timer.unref?.();
1827
2217
  if (hold.reason !== cooldown.reason) {
1828
2218
  cooldown.reason = hold.reason;
@@ -1833,26 +2223,42 @@ export class AgentFramework {
1833
2223
  holdProviderAcceleration(agent, error, trigger, auxiliary = false) {
1834
2224
  if (this.ephemeralRuns.has(agent.name) || this.conversationAgentHomes.has(agent.name))
1835
2225
  return false;
2226
+ // A provider's stated wait binds later calls to the primary model, across
2227
+ // restarts. (An auxiliary call records its own model's wait at the gate.)
2228
+ const wait = auxiliary ? undefined : this.recordProviderWait(agent.name, agent.model, error);
1836
2229
  // The host knows more than the message wording does: ask it first. An
1837
2230
  // auxiliary failure arms host holds only — the built-in acceleration
1838
2231
  // cooldown stays a primary-path mechanism.
1839
2232
  const hostHold = this.consultProviderHold(error, agent.name);
1840
- const acceleration = !hostHold && !auxiliary && isOrganizationAccelerationRateLimit(error);
1841
- if (!acceleration && !hostHold)
1842
- return false;
2233
+ // A failure the provider says may not be retried keeps its terminal
2234
+ // disposition, whatever its wording (a stated wait still binds after it).
2235
+ const acceleration = !hostHold && !auxiliary && error.retryable !== false
2236
+ && isOrganizationAccelerationRateLimit(error);
2237
+ // The stated wait binds later calls whatever the failure was. Retaining
2238
+ // and retrying THIS request is for a failure the provider says may be
2239
+ // retried: a non-retryable one keeps its terminal disposition (marker,
2240
+ // failure streak), and the wait applies to the calls that follow it.
2241
+ if (!acceleration && !hostHold) {
2242
+ return wait !== undefined && isMembraneError(error) && error.retryable === true
2243
+ && this.holdForProviderWait(agent, wait, trigger ? [trigger] : [], true);
2244
+ }
1843
2245
  const now = Date.now();
1844
- const delayMs = hostHold ? hostHold.holdMs : this.accelerationCooldownMs(agent.name, error);
2246
+ const pacing = hostHold ? undefined : this.accelerationCooldownMs(agent.name, error);
2247
+ const delayMs = hostHold ? hostHold.holdMs : pacing.delayMs;
1845
2248
  const existing = this.providerAccelerationCooldowns.get(agent.name);
1846
2249
  const held = existing?.heldRequests ?? [];
1847
2250
  if (trigger && !held.some((r) => this.sameInferenceRequest(r, trigger)))
1848
2251
  held.push(trigger);
1849
2252
  if (existing)
1850
2253
  clearTimeout(existing.timer);
1851
- const timer = setTimeout(() => this.releaseProviderAccelerationCooldown(agent.name), delayMs);
2254
+ const timer = setTimeout(() => this.releaseProviderAccelerationCooldown(agent.name), Math.min(delayMs, MAX_TIMER_DELAY_MS));
1852
2255
  timer.unref?.();
1853
2256
  this.providerAccelerationCooldowns.set(agent.name, { startedAt: existing?.startedAt ?? now,
1854
- until: now + delayMs, timer, heldRequests: held, reason: hostHold?.reason ?? error.message,
2257
+ until: now + delayMs, timer, heldRequests: held, reason: hostHold?.reason ?? boundFailureText(String(error.message)),
1855
2258
  failures: (existing?.failures ?? 0) + 1, ...(hostHold ? { hostHoldError: error } : {}),
2259
+ // The stated wait (recorded above for agent.model) set this hold's
2260
+ // length: an operator release of it returns the hold to AF's own pacing.
2261
+ ...(pacing?.stated ? { statedWait: { model: agent.model, ownUntil: now + pacing.ownMs } } : {}),
1856
2262
  ...(auxiliary && (!existing || existing.auxiliaryOrigin) ? { auxiliaryOrigin: true } : {}) });
1857
2263
  if (auxiliary && !existing) {
1858
2264
  console.error(`[provider-cooldown] agent=${agent.name} host hold (${hostHold.reason}) from an auxiliary call — ` +
@@ -1862,20 +2268,101 @@ export class AgentFramework {
1862
2268
  const gate = this.providerGate(agent.name);
1863
2269
  gate.primaryPending = true;
1864
2270
  this.providerAccelerationRecoveries.set(agent.name, { startedAt: existing?.startedAt ?? now,
1865
- failures: (existing?.failures ?? 0) + 1, heldRequests: held.length, reason: error.message });
2271
+ failures: (existing?.failures ?? 0) + 1, heldRequests: held.length, reason: boundFailureText(String(error.message)) });
1866
2272
  console.error(`[provider-cooldown] agent=${agent.name} ` +
1867
2273
  `${hostHold ? `host hold (${hostHold.reason})` : 'organization acceleration 429'} — ` +
1868
2274
  `holding primary/auxiliary for ${delayMs}ms; ${held.length} request(s) retained`);
1869
2275
  return true;
1870
2276
  }
2277
+ /**
2278
+ * A provider's stated wait on a turn that doesn't own provider admission:
2279
+ * a conversation agent's or an ephemeral run's. Left to the error policy,
2280
+ * which returns a stated wait as it is, a 429 stating two hours would hold
2281
+ * the turn asleep two hours, up to three times, with nothing recorded and
2282
+ * nothing raised (membrane #56 review, M3). So:
2283
+ * - A conversation agent's turns come through the scheduler as a resident's
2284
+ * do, so it treats the wait as a resident does: recorded for (agent,
2285
+ * model), alerted past PROVIDER_WAIT_ALERT_MS, and the turn held until the
2286
+ * wait passes or an operator releases it ('held'). A non-retryable failure
2287
+ * records the wait for the calls after it and keeps its own disposition.
2288
+ * - An ephemeral run's name is single-use, so a wait recorded for it would
2289
+ * bind nothing, and its caller is waiting on it. A retryable stated wait
2290
+ * up to PROVIDER_ACCELERATION_MAX_COOLDOWN_MS is left to the error policy,
2291
+ * slept and retried. A longer one, or one that can't be represented, ends
2292
+ * the run now ('fail'): it is never retried before the provider's time,
2293
+ * and the caller hears why at once.
2294
+ * Anything else is the error policy's.
2295
+ */
2296
+ nonOwningProviderWait(agent, error, trigger) {
2297
+ // From either copy of membrane (isMembraneError), a wait is stated only
2298
+ // as a number, as the default error policy reads it.
2299
+ if (!isMembraneError(error) || typeof error.retryAfterMs !== 'number')
2300
+ return undefined;
2301
+ if (this.conversationAgentHomes.has(agent.name)) {
2302
+ const wait = this.recordProviderWait(agent.name, agent.model, error);
2303
+ return error.retryable === true && wait !== undefined && this.holdForProviderWait(agent, wait, trigger ? [trigger] : [], true)
2304
+ ? 'held'
2305
+ : undefined;
2306
+ }
2307
+ if (this.ephemeralRuns.has(agent.name) && error.retryable === true
2308
+ && !(error.retryAfterMs <= PROVIDER_ACCELERATION_MAX_COOLDOWN_MS)) {
2309
+ return 'fail';
2310
+ }
2311
+ return undefined;
2312
+ }
2313
+ /** Why an ephemeral run ended instead of sleeping a provider's stated wait,
2314
+ * for its caller. */
2315
+ statedWaitNotSlept(error) {
2316
+ const stated = isMembraneError(error) ? error.retryAfterMs : undefined;
2317
+ const cap = `${PROVIDER_ACCELERATION_MAX_COOLDOWN_MS / 1000} s`;
2318
+ if (typeof stated !== 'number' || Number.isNaN(stated)) {
2319
+ return `${error.message} (the provider's stated wait is not a number, and an ephemeral run sleeps only a ` +
2320
+ `stated wait of ${cap} or less, so it ends now)`;
2321
+ }
2322
+ const asked = Number.isFinite(stated) ? `${Math.ceil(stated / 1000)} s` : 'longer than can be represented';
2323
+ return `${error.message} (the provider asked to wait ${asked}; an ephemeral run doesn't sleep a stated wait ` +
2324
+ `longer than ${cap}, so it ends now)`;
2325
+ }
2326
+ /** When a hold's timer next fires: at its deadline, stepped past what one
2327
+ * timer can hold. A hold with no deadline (a wait held until released, or
2328
+ * the stand-in for unreadable records) asks again as often as unreadable
2329
+ * records are read again: records that read again end it, and nothing
2330
+ * announces that. */
2331
+ providerHoldTimerMs(remaining) {
2332
+ return Number.isFinite(remaining)
2333
+ ? Math.min(remaining, MAX_TIMER_DELAY_MS)
2334
+ : this.providerWaits?.retryReadMs ?? MAX_TIMER_DELAY_MS;
2335
+ }
1871
2336
  releaseProviderAccelerationCooldown(agentName) {
1872
2337
  const cooldown = this.providerAccelerationCooldowns.get(agentName);
1873
2338
  if (!cooldown)
1874
2339
  return;
2340
+ // A wait hold lasts exactly as long as its wait binds: longer once the
2341
+ // wait is extended, while it is indefinite, and over once no wait binds
2342
+ // (passed, released, or the stand-in for unreadable records lifted because
2343
+ // they read again). A different primary model now is not bound by it.
2344
+ if (cooldown.waitModel !== undefined && this.agents.get(agentName)?.model === cooldown.waitModel) {
2345
+ const wait = this.providerWaits?.active(agentName, cooldown.waitModel);
2346
+ cooldown.until = wait ? (wait.until ?? Number.POSITIVE_INFINITY) : Date.now();
2347
+ }
2348
+ const remaining = cooldown.until - Date.now();
2349
+ if (remaining > 0 && (cooldown.waitModel === undefined || this.agents.get(agentName)?.model === cooldown.waitModel)) {
2350
+ clearTimeout(cooldown.timer);
2351
+ cooldown.timer = setTimeout(() => this.releaseProviderAccelerationCooldown(agentName), this.providerHoldTimerMs(remaining));
2352
+ cooldown.timer.unref?.();
2353
+ return;
2354
+ }
1875
2355
  if (this.extendHostProviderHold(agentName, cooldown))
1876
2356
  return;
1877
2357
  clearTimeout(cooldown.timer);
1878
2358
  this.providerAccelerationCooldowns.delete(agentName);
2359
+ if (cooldown.waitModel !== undefined) {
2360
+ const model = this.agents.get(agentName)?.model;
2361
+ console.error(`[provider-wait] agent=${agentName} model=${cooldown.waitModel} ` +
2362
+ (model !== undefined && model !== cooldown.waitModel
2363
+ ? `no longer holds primary turns: the primary model is now ${model} (the wait stays recorded)`
2364
+ : `primary hold ended after ${Date.now() - cooldown.startedAt}ms`));
2365
+ }
1879
2366
  if (cooldown.auxiliaryOrigin && cooldown.heldRequests.length === 0) {
1880
2367
  // Nobody asked for an inference while parked: just reopen the gate.
1881
2368
  console.error(`[provider-cooldown] agent=${agentName} auxiliary hold released after ${Date.now() - cooldown.startedAt}ms`);
@@ -1893,6 +2380,199 @@ export class AgentFramework {
1893
2380
  console.error(`[provider-cooldown] agent=${agentName} released after ${Date.now() - cooldown.startedAt}ms — ` +
1894
2381
  `fresh compile queued with ${requests.length} retained request(s)`);
1895
2382
  }
2383
+ /**
2384
+ * Wait an error policy's retry delay, all of it: the delay can be a
2385
+ * provider's retry-after, and a single timer past 2^31-1 ms fires almost at
2386
+ * once. Steps through longer waits; stop() wakes it so a shutting-down
2387
+ * framework is never left with a wait parked for days.
2388
+ */
2389
+ async waitForRetry(ms) {
2390
+ // An infinite delay waits until stop(), like any wait too long to end; a
2391
+ // delay that isn't a number, or is 0 or less, waits none.
2392
+ let remaining = ms > 0 ? ms : 0;
2393
+ while (remaining > 0 && !this.providerAdmissionClosed) {
2394
+ const step = Math.min(remaining, MAX_TIMER_DELAY_MS);
2395
+ remaining -= step;
2396
+ await new Promise((resolve) => {
2397
+ const wakers = this.retryWaitWakers;
2398
+ const done = () => { clearTimeout(timer); wakers.delete(done); resolve(); };
2399
+ const timer = setTimeout(done, step);
2400
+ wakers.add(done);
2401
+ });
2402
+ }
2403
+ }
2404
+ /**
2405
+ * Record a provider's stated wait (a classified error carrying
2406
+ * `retryAfterMs`) for (agent, model). Returns the wait that binds now,
2407
+ * which is the longer of this one and any outstanding one. A wait that
2408
+ * newly binds past PROVIDER_WAIT_ALERT_MS, or until released, raises an
2409
+ * ops alert.
2410
+ */
2411
+ recordProviderWait(agentName, model, error) {
2412
+ // From either copy of membrane: the host supplies the Membrane instance,
2413
+ // and a wait is stated only as a number, as the default policy reads it.
2414
+ if (!isMembraneError(error) || typeof error.retryAfterMs !== 'number' || !this.providerWaits)
2415
+ return undefined;
2416
+ // The provider's words, bounded; a line break in them must not start a
2417
+ // line that reads as the framework's own where the reason is logged.
2418
+ const reason = safeSlice(withoutControls(error.message), 0, 300);
2419
+ const { wait, changed } = this.providerWaits.set(agentName, model, error.retryAfterMs, reason);
2420
+ // A wait already over (0, or a negative "retry now") holds nothing to report.
2421
+ const remaining = wait.until === null ? Number.POSITIVE_INFINITY : wait.until - Date.now();
2422
+ if (changed && remaining > 0) {
2423
+ const until = wait.until === null
2424
+ ? 'until an operator releases it (the stated wait cannot be represented as an instant)'
2425
+ : `until ${new Date(wait.until).toISOString()}`;
2426
+ const stated = `${error.type}${error.httpStatus !== undefined ? ` HTTP ${error.httpStatus}` : ''}, retry-after ${String(error.retryAfterMs)}ms`;
2427
+ console.error(`[provider-wait] agent=${agentName} model=${model} holds ${until}: ${stated}`);
2428
+ if (remaining > PROVIDER_WAIT_ALERT_MS) {
2429
+ this.opsAlert('provider-wait', agentName, `provider wait on ${model} holds ${until} (${stated}: ${reason}); ` +
2430
+ 'if the provider did not mean that, release it with release-provider-wait', { data: { model, until: wait.until === null ? null : new Date(wait.until).toISOString() } });
2431
+ }
2432
+ }
2433
+ return wait;
2434
+ }
2435
+ /**
2436
+ * Hold this agent's primary turns for a provider wait on its model. The
2437
+ * requests are kept and released together, into one fresh compile, when
2438
+ * the wait ends: the same path an acceleration cooldown takes, without
2439
+ * parking auxiliary calls to other models.
2440
+ */
2441
+ holdForProviderWait(agent, wait, requests, failed = false) {
2442
+ const now = Date.now();
2443
+ const until = wait.until ?? Number.POSITIVE_INFINITY;
2444
+ if (until <= now)
2445
+ return false;
2446
+ const existing = this.providerAccelerationCooldowns.get(agent.name);
2447
+ const held = existing?.heldRequests ?? [];
2448
+ for (const request of requests) {
2449
+ if (!held.some((r) => this.sameInferenceRequest(r, request)))
2450
+ held.push(request);
2451
+ }
2452
+ // An acceleration or host hold already parks the whole agent; its release
2453
+ // re-checks this wait at admission.
2454
+ if (existing && existing.waitModel === undefined)
2455
+ return true;
2456
+ if (existing)
2457
+ clearTimeout(existing.timer);
2458
+ const timer = setTimeout(() => this.releaseProviderAccelerationCooldown(agent.name), this.providerHoldTimerMs(until - now));
2459
+ timer.unref?.();
2460
+ this.providerAccelerationCooldowns.set(agent.name, {
2461
+ startedAt: existing?.startedAt ?? now, until, timer, heldRequests: held,
2462
+ // A failed call counts; a turn arriving at a wait recorded earlier doesn't.
2463
+ reason: wait.reason, failures: (existing?.failures ?? 0) + (failed ? 1 : 0), waitModel: wait.model,
2464
+ });
2465
+ if (!existing) {
2466
+ console.error(`[provider-wait] agent=${agent.name} model=${wait.model} holding primary turns ` +
2467
+ `${wait.until === null ? 'until released' : `until ${new Date(wait.until).toISOString()}`}; ${held.length} request(s) retained`);
2468
+ }
2469
+ return true;
2470
+ }
2471
+ /**
2472
+ * Refuse an auxiliary call to a model the provider asked this agent to wait
2473
+ * on, without calling it. The refusal is a retryable rate limit carrying
2474
+ * the remaining wait, so a caller that paces (context-manager's compression
2475
+ * lane, from its corrective round #138 on) waits it out instead of spending
2476
+ * a call. It is marked
2477
+ * `providerAdmission: 'deferred'`: no call was made, so a caller can keep
2478
+ * the wait without counting a failed call (room-225 #47927).
2479
+ */
2480
+ refuseHeldProviderModel(agentName, model) {
2481
+ // A call that names no model can't be matched to a model's wait, but the
2482
+ // hold that stands for unreadable records binds every model, so it binds
2483
+ // that call too (EVERY_MODEL: only that hold, or a release of it, says so).
2484
+ const wait = this.providerWaits?.active(agentName, model ?? EVERY_MODEL);
2485
+ if (!wait)
2486
+ return;
2487
+ const now = Date.now();
2488
+ throw Object.assign(new MembraneError({
2489
+ type: 'rate_limit',
2490
+ retryable: true,
2491
+ ...(wait.until !== null ? { retryAfterMs: Math.max(0, wait.until - now) } : {}),
2492
+ message: providerWaitText(agentName, model ?? EVERY_MODEL, wait),
2493
+ rawError: undefined,
2494
+ }), { providerAdmission: 'deferred' });
2495
+ }
2496
+ /**
2497
+ * Release provider waits: one model's for an agent, or every model's. The
2498
+ * explicit operator override for a wait that should not be honoured (the
2499
+ * passing of time needs no call). Recorded, so a restart does not bring the
2500
+ * wait back; held primary turns are released into one fresh compile. An
2501
+ * acceleration hold the released wait lengthened keeps only AF's own pacing
2502
+ * for its failure; a host hold stays the host's.
2503
+ */
2504
+ releaseProviderWait(agentName, model, by = 'operator') {
2505
+ // A release names one of this framework's agents, or the agent of a wait
2506
+ // the listing shows (a disposed conversation fork's wait stays recorded):
2507
+ // an operator can release anything the listing shows. Any other name (a
2508
+ // typo, or '*' and 'framework', which stand for the hold on unreadable
2509
+ // records in its listing and its alert) is refused before anything is
2510
+ // recorded: it would lift nothing and still read as a success. That hold
2511
+ // is lifted one agent at a time.
2512
+ if (!this.agents.has(agentName) && !this.providerWaits?.recorded(agentName)) {
2513
+ throw new Error(`Unknown agent: ${safeSlice(JSON.stringify(agentName), 0, 120)}. A release names one of this ` +
2514
+ "framework's agents or an agent with a recorded wait, and the hold on unreadable provider waits is lifted " +
2515
+ 'one agent at a time');
2516
+ }
2517
+ by = operatorLabel(String(by));
2518
+ // '*' (EVERY_MODEL, as providerWaits lists the unreadable-records hold)
2519
+ // names every model, the same as omitting it, everywhere below.
2520
+ if (model === EVERY_MODEL)
2521
+ model = undefined;
2522
+ const released = this.providerWaits?.release(agentName, model, by) ?? [];
2523
+ for (const { wait, release } of released) {
2524
+ console.error(`[provider-wait] agent=${agentName} model=${wait.model} released by ${by} (${release})`);
2525
+ }
2526
+ // The agent's context strategy may pace its own provider calls by the
2527
+ // same stated wait (context-manager's compression lane, from its
2528
+ // corrective round #138 on; 0.14.0 has no releaseCompressionPause, so this
2529
+ // calls nothing there). The release is the operator's word on that wait
2530
+ // too, so it is offered to the lane with its scope even when no wait
2531
+ // binds here: the lane can hold a pause after
2532
+ // this framework's own wait is gone (the records read again, the wait
2533
+ // passed, or it was released in another process). The strategy's own
2534
+ // backoff stays, and the receipt still lists only waits released here.
2535
+ // Read inside the try: the release is already recorded, so a strategy
2536
+ // that throws must not keep the hold below from being released.
2537
+ try {
2538
+ const strategy = this.agents.get(agentName)?.getContextManager().getStrategy();
2539
+ if (strategy?.releaseCompressionPause?.(model)) {
2540
+ console.error(`[provider-wait] agent=${agentName} model=${model ?? '*'} compression lane's provider wait released`);
2541
+ }
2542
+ }
2543
+ catch (error) {
2544
+ console.error(`[provider-wait] agent=${agentName} could not offer the release to the compression lane: ${safeSlice(error instanceof Error ? error.message : String(error), 0, 300)}`);
2545
+ }
2546
+ const cooldown = this.providerAccelerationCooldowns.get(agentName);
2547
+ const covers = (held) => held !== undefined && (model === undefined || held === model);
2548
+ if (cooldown && covers(cooldown.waitModel)) {
2549
+ // A wait hold asks its wait again: released, the hold ends.
2550
+ this.releaseProviderAccelerationCooldown(agentName);
2551
+ }
2552
+ else if (cooldown?.statedWait && covers(cooldown.statedWait.model)) {
2553
+ // An acceleration hold the stated wait lengthened keeps only AF's own
2554
+ // pacing for that failure, and is never lengthened by a release. A host
2555
+ // hold carries no stated wait: its length stays the host's.
2556
+ cooldown.until = Math.min(cooldown.until, cooldown.statedWait.ownUntil);
2557
+ console.error(`[provider-wait] agent=${agentName} model=${cooldown.statedWait.model} acceleration hold keeps AF's own pacing only, ` +
2558
+ (cooldown.until > Date.now() ? `until ${new Date(cooldown.until).toISOString()}` : 'which has passed'));
2559
+ delete cooldown.statedWait;
2560
+ this.releaseProviderAccelerationCooldown(agentName);
2561
+ }
2562
+ return released.map(({ wait, release }) => ({
2563
+ model: wait.model, until: wait.until === null ? null : new Date(wait.until).toISOString(), release,
2564
+ }));
2565
+ }
2566
+ /** Provider waits binding now, for health and doctor tooling. */
2567
+ providerWaitSnapshot(agentName) {
2568
+ return (this.providerWaits?.list(agentName) ?? []).map((wait) => ({
2569
+ agent: wait.agent,
2570
+ model: wait.model,
2571
+ until: wait.until === null ? null : new Date(wait.until).toISOString(),
2572
+ reason: wait.reason,
2573
+ setAt: new Date(wait.setAt).toISOString(),
2574
+ }));
2575
+ }
1896
2576
  recordProviderAccelerationRecovery(agent, request, stopReason) {
1897
2577
  const recovery = this.providerAccelerationRecoveries.get(agent.name);
1898
2578
  if (!recovery?.releasedAt)
@@ -1934,9 +2614,12 @@ export class AgentFramework {
1934
2614
  if (this.maintenancePass || !this.running)
1935
2615
  return;
1936
2616
  const pass = this.runQueuedMaintenance().catch((error) => {
1937
- const reason = error instanceof Error ? error.message : String(error);
2617
+ // The alert cuts its own copies from the whole message, so the
2618
+ // webhook's states the whole length.
2619
+ const wholeReason = error instanceof Error ? String(error.message) : failureDescription(error);
2620
+ const reason = boundFailureText(wholeReason);
1938
2621
  console.error('[context-maintenance] pass failed:', reason);
1939
- this.opsAlert('context-maintenance-failed', 'framework', reason, { data: { scope: 'pass' } });
2622
+ this.opsAlert('context-maintenance-failed', 'framework', wholeReason, { data: { scope: 'pass' } });
1940
2623
  });
1941
2624
  this.maintenancePass = pass;
1942
2625
  void pass.finally(() => {
@@ -2005,9 +2688,10 @@ export class AgentFramework {
2005
2688
  }
2006
2689
  }
2007
2690
  catch (error) {
2008
- record.error = error instanceof Error ? error.message : String(error);
2691
+ const wholeError = error instanceof Error ? String(error.message) : failureDescription(error);
2692
+ record.error = boundFailureText(wholeError);
2009
2693
  console.error(`[context-maintenance] agent=${agent.name} failed:`, record.error);
2010
- this.opsAlert('context-maintenance-failed', agent.name, record.error, {
2694
+ this.opsAlert('context-maintenance-failed', agent.name, wholeError, {
2011
2695
  data: {
2012
2696
  scope: 'agent',
2013
2697
  ...(record.pendingBefore ? { pending: record.pendingBefore } : {}),
@@ -2111,19 +2795,41 @@ export class AgentFramework {
2111
2795
  this.queue.push(event);
2112
2796
  this.emitTrace({ type: 'process:received', processEvent: event });
2113
2797
  }
2798
+ /**
2799
+ * A module's event. stop() closes the queue before it stops modules, so a
2800
+ * module's event can arrive after that: from a listener that fires while
2801
+ * stop() runs, or from the module's own stop(). Nothing will process it,
2802
+ * and a throw there can go unhandled or fail stop(), so it's dropped with a
2803
+ * log line naming the module. Otherwise it's queued as pushEvent() queues
2804
+ * it, or, through the deprecated queue handle, without the trace.
2805
+ */
2806
+ pushModuleEvent(event, moduleName, traced) {
2807
+ if (this.queue.isClosed) {
2808
+ // A tool result takes pushEvent()'s own drop, whose line names the
2809
+ // call, the tool and its outcome.
2810
+ if (event.type === 'tool-result') {
2811
+ this.pushEvent(event);
2812
+ return;
2813
+ }
2814
+ console.error(`[framework] a '${event.type}' event from module ${moduleName ?? '(unnamed)'} arrived after stop() began — dropped`);
2815
+ return;
2816
+ }
2817
+ if (traced)
2818
+ this.pushEvent(event);
2819
+ else
2820
+ this.queue.push(event);
2821
+ }
2114
2822
  /**
2115
2823
  * Add a trace event listener for observability.
2116
2824
  */
2117
2825
  onTrace(listener) {
2118
- this.traceListeners.push(listener);
2826
+ const registration = { listener, active: true };
2827
+ this.traceListeners.push(registration);
2119
2828
  // Return an unsubscribe so callers with a bounded lifetime (e.g. modules
2120
2829
  // that get torn down and recreated on session switch) don't leak
2121
2830
  // listeners. Existing callers that ignore the return value are unaffected.
2122
- return () => {
2123
- const idx = this.traceListeners.indexOf(listener);
2124
- if (idx >= 0)
2125
- this.traceListeners.splice(idx, 1);
2126
- };
2831
+ // It removes this subscription, whatever else the same function holds.
2832
+ return () => this.removeTraceRegistration(registration);
2127
2833
  }
2128
2834
  /**
2129
2835
  * Public accessor for the MCPL channel registry.
@@ -2135,13 +2841,19 @@ export class AgentFramework {
2135
2841
  return this.channelRegistry;
2136
2842
  }
2137
2843
  /**
2138
- * Remove a trace event listener.
2844
+ * Remove a trace event listener: its first subscription, if it holds more
2845
+ * than one.
2139
2846
  */
2140
2847
  offTrace(listener) {
2141
- const index = this.traceListeners.indexOf(listener);
2142
- if (index >= 0) {
2848
+ const registration = this.traceListeners.find((r) => r.listener === listener);
2849
+ if (registration)
2850
+ this.removeTraceRegistration(registration);
2851
+ }
2852
+ removeTraceRegistration(registration) {
2853
+ registration.active = false;
2854
+ const index = this.traceListeners.indexOf(registration);
2855
+ if (index >= 0)
2143
2856
  this.traceListeners.splice(index, 1);
2144
- }
2145
2857
  }
2146
2858
  /**
2147
2859
  * Add a module at runtime.
@@ -2280,16 +2992,29 @@ export class AgentFramework {
2280
2992
  async autoFetchReferences(content, serverId, eager) {
2281
2993
  if (!content)
2282
2994
  return;
2283
- const fetcher = this.getReferenceFetcher();
2284
- const eligible = [];
2995
+ // Register every block first, then judge each uri once, by its latest
2996
+ // record. The registry keys records by uri and evicts the one registered
2997
+ // least recently, so a uri this result mentions keeps its record through
2998
+ // the result's later blocks, short of more distinct references in one
2999
+ // result than the registry holds; past that, a uri can get a second
3000
+ // record here, hence the key by uri. A uri that any of this result's
3001
+ // blocks calls expired takes no slot, whichever block registered last. A
3002
+ // later result's fresh expiry still supersedes, through register().
3003
+ const byUri = new Map();
2285
3004
  for (const block of content) {
2286
3005
  const c = classifyBlock(block);
2287
3006
  if (c.kind !== 'reference')
2288
3007
  continue;
2289
3008
  const record = referenceRegistry.register(c.testimony, serverId);
2290
- if (eager && fetcher.isEagerEligible(record))
2291
- eligible.push(record);
3009
+ const expiredHere = byUri.get(c.testimony.uri)?.expiredHere === true || isReferenceExpired(c.testimony);
3010
+ byUri.set(c.testimony.uri, { record, expiredHere });
2292
3011
  }
3012
+ if (!eager)
3013
+ return;
3014
+ const fetcher = this.getReferenceFetcher();
3015
+ const eligible = [...byUri.values()]
3016
+ .filter(({ record, expiredHere }) => !expiredHere && fetcher.isEagerEligible(record))
3017
+ .map(({ record }) => record);
2293
3018
  // Parallel with a count cap: N small fetches cost one round-trip of
2294
3019
  // wall-clock, and anything past the cap stays lazy (stub says so).
2295
3020
  const MAX_EAGER_PER_RESULT = 8;
@@ -3885,7 +4610,8 @@ export class AgentFramework {
3885
4610
  return;
3886
4611
  run.lastActivity = Date.now();
3887
4612
  if (outcome.stopReason === 'exhausted') {
3888
- run.settle.reject(new Error(outcome.error ?? 'Unknown error'));
4613
+ // The caller may log or relay the message, which can echo the whole request.
4614
+ run.settle.reject(new Error(boundFailureText(outcome.error ?? 'Unknown error')));
3889
4615
  return;
3890
4616
  }
3891
4617
  run.settle.resolve({
@@ -4415,7 +5141,8 @@ export class AgentFramework {
4415
5141
  if (parts[0] !== 'discord')
4416
5142
  return;
4417
5143
  const channelId = parts[parts.length - 1];
4418
- const emoji = REFUSAL_REACTIONS[category] ?? REFUSAL_REACTION_FALLBACK;
5144
+ // Own names only: an inherited one such as 'constructor' takes the fallback.
5145
+ const emoji = Object.hasOwn(REFUSAL_REACTIONS, category) ? REFUSAL_REACTIONS[category] : REFUSAL_REACTION_FALLBACK;
4419
5146
  // Resolve the MCPL server that owns the locus channel and call
4420
5147
  // tools/call directly on its connection (bare tool name — no prefix
4421
5148
  // games), bypassing the agent event queue so no synthetic tool-result
@@ -4528,9 +5255,10 @@ export class AgentFramework {
4528
5255
  ? `the model refused on them (content filter: ${category}). Their content is ` +
4529
5256
  `not reproduced here (so this note can't re-trigger the filter); the ` +
4530
5257
  `originals remain in the raw record`
4531
- : `the model API kept rejecting the conversation on them (${category}). ` +
4532
- `Their content is not reproduced here (so this note can't re-trigger the ` +
4533
- `rejection); the originals remain in the raw record`;
5258
+ : `the model API kept refusing the requests that carried them (${category}), ` +
5259
+ `so that the conversation could go on. Their content is not reproduced ` +
5260
+ `here (so this note can't trigger the same refusal); the originals remain ` +
5261
+ `in the raw record`;
4534
5262
  const text = `[refusal-rewind] The ${count} most recent turn(s) were set aside because ` +
4535
5263
  `${why}. You are clear to continue — if you ` +
4536
5264
  `were mid-task, take a different approach; otherwise carry on with whatever ` +
@@ -4675,6 +5403,10 @@ export class AgentFramework {
4675
5403
  *
4676
5404
  * host-status — report the quiesce state (drained / parked wakes /
4677
5405
  * running background scripts).
5406
+ *
5407
+ * release-provider-wait — end an agent's provider wait (a provider's
5408
+ * stated retry-after, kept across restarts) for one `model` or all, as an
5409
+ * explicit, recorded operator override.
4678
5410
  */
4679
5411
  async handleHostCommand(serverId, params) {
4680
5412
  if (params.command !== 'undo' &&
@@ -4684,7 +5416,8 @@ export class AgentFramework {
4684
5416
  params.command !== 'quiesce' &&
4685
5417
  params.command !== 'resume' &&
4686
5418
  params.command !== 'maintain' &&
4687
- params.command !== 'host-status') {
5419
+ params.command !== 'host-status' &&
5420
+ params.command !== 'release-provider-wait') {
4688
5421
  return { ok: false, error: `Unknown host command: ${String(params.command)}` };
4689
5422
  }
4690
5423
  // Host-scoped verbs (issue #122) — no agent resolution.
@@ -4720,6 +5453,31 @@ export class AgentFramework {
4720
5453
  if (params.command === 'host-status') {
4721
5454
  return { ok: true, hostMode: this.getHostModeStatus() };
4722
5455
  }
5456
+ // A release names its agent, as host.releaseProviderWait does: the first
5457
+ // agent by default could lift the wrong agent's waits. The name may be an
5458
+ // agent the framework no longer has whose wait the listing still shows
5459
+ // (a disposed conversation fork); releaseProviderWait decides that.
5460
+ if (params.command === 'release-provider-wait') {
5461
+ if (typeof params.agentName !== 'string' || params.agentName === '') {
5462
+ return { ok: false, error: 'release-provider-wait: agentName must name the agent whose waits to release' };
5463
+ }
5464
+ // Only an omitted model, or '*', releases every model: a malformed one
5465
+ // is refused here, before anything changes, never widened.
5466
+ if (!isReleaseModel(params.model)) {
5467
+ return { ok: false, error: "release-provider-wait: model must be a model name, or '*' (or omitted) for every model" };
5468
+ }
5469
+ const by = operatorLabel(String(requester));
5470
+ let released;
5471
+ try {
5472
+ released = this.releaseProviderWait(params.agentName, params.model, by);
5473
+ }
5474
+ catch (error) {
5475
+ return { ok: false, error: error instanceof Error ? error.message : String(error) };
5476
+ }
5477
+ console.error(`[host-command] release-provider-wait agent=${params.agentName} model=${params.model ?? '*'} by=${by} ` +
5478
+ `(server=${serverId}): ${released.length} wait(s) lifted`);
5479
+ return { ok: true, released };
5480
+ }
4723
5481
  const agentName = params.agentName ?? [...this.agents.keys()][0];
4724
5482
  if (!agentName || !this.agents.has(agentName)) {
4725
5483
  return { ok: false, error: `Unknown agent: ${String(agentName)}` };
@@ -4884,6 +5642,7 @@ export class AgentFramework {
4884
5642
  ok: true,
4885
5643
  messagesRemoved: r.messagesRemoved,
4886
5644
  removedRefs: r.removedRefs,
5645
+ markers: r.markers,
4887
5646
  lastVisible: r.lastVisible,
4888
5647
  };
4889
5648
  }
@@ -4918,7 +5677,9 @@ export class AgentFramework {
4918
5677
  if (undone === 0) {
4919
5678
  return { ok: true, undone: 0, requested, lastVisible: null };
4920
5679
  }
4921
- await this.syncDiscordAwarenessMarkers();
5680
+ // The branch has moved: delivery of the marks it changes starts behind
5681
+ // the data-plane gate, and the command doesn't wait for Discord.
5682
+ this.startDiscordAwarenessDelivery('turn undo', agentName);
4922
5683
  return { ok: true, undone, requested, lastVisible: await this.lastVisiblePreview(agentName) };
4923
5684
  }
4924
5685
  /**
@@ -5434,6 +6195,12 @@ export class AgentFramework {
5434
6195
  * Discord messages that left the live context get awareness markers via the
5435
6196
  * durable outbox, exactly as message-granular `undo` does.
5436
6197
  *
6198
+ * Returns once the switch has landed: the body is applied, with a
6199
+ * marker-scheduling receipt (`markers`). It does not wait for Discord to
6200
+ * accept the reactions: delivery is started, behind the MCPL data-plane
6201
+ * gate, and continues without the caller or the store reservation waiting
6202
+ * on it; its per-message outcomes stay in the outbox ledger.
6203
+ *
5437
6204
  * Throws `OperatorActionError` (`agent-busy`, `unknown-message`, …) — the
5438
6205
  * agent must be idle; nothing is queued.
5439
6206
  */
@@ -5473,9 +6240,12 @@ export class AgentFramework {
5473
6240
  throw new OperatorActionError('invalid', 'Rollback branch name must differ from the active branch');
5474
6241
  }
5475
6242
  // Gate + reserve the whole store (see reserveStoreForSurgery); held
5476
- // until the switch has landed or been rolled back.
6243
+ // until the switch has landed and its marker scheduling has been
6244
+ // recorded or reported, or the switch has been rolled back. Remote
6245
+ // delivery is not waited on.
5477
6246
  const release = this.reserveStoreForSurgery('roll back', agentName);
5478
6247
  let branchName;
6248
+ let markers;
5479
6249
  try {
5480
6250
  // Prepare the external side effect before switching Chronicle. If the
5481
6251
  // process dies after the switch but before activate(), startup
@@ -5503,9 +6273,14 @@ export class AgentFramework {
5503
6273
  const restored = await this.restoreSourceBranch(cm, sourceBranch, targetBranch, agentName);
5504
6274
  throw new OperatorActionError('failed', `Rollback failed; ${restored}: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
5505
6275
  }
5506
- if (markerBatch)
5507
- this.discordAwarenessOutbox.activate(markerBatch.id);
5508
- await this.syncDiscordAwarenessMarkers();
6276
+ // The body change has landed. Nothing after this point turns the
6277
+ // rollback into a failure: marks are an outbox outcome, reported
6278
+ // apart from the body (`markers`).
6279
+ markers = this.activateSurgeryMarkers(markerBatch, 'rollback', agentName);
6280
+ // Installs the MCPL data-plane gate synchronously, before the store is
6281
+ // released, and starts delivery without awaiting it (the first
6282
+ // reaction request may already be on the wire when this returns).
6283
+ this.startDiscordAwarenessDelivery('rollback', agentName);
5509
6284
  this.materializeConfigMountAfterBranchSwitch();
5510
6285
  }
5511
6286
  finally {
@@ -5513,7 +6288,8 @@ export class AgentFramework {
5513
6288
  }
5514
6289
  console.error(`[operator] rollback agent=${agentName} to=${tailMessageId}` +
5515
6290
  `${tailMessageId !== opts.messageId ? ` (requested ${opts.messageId}, snapped to end of body group)` : ''}` +
5516
- ` removed=${messagesRemoved} branch=${branchName} by=${describeRequester(opts.requester)}`);
6291
+ ` removed=${messagesRemoved} branch=${branchName} marks=${describeMarkers(markers)}` +
6292
+ ` by=${describeRequester(opts.requester)}`);
5517
6293
  this.recordOperatorAction({
5518
6294
  ...logBase,
5519
6295
  result: {
@@ -5522,6 +6298,7 @@ export class AgentFramework {
5522
6298
  tailMessageId,
5523
6299
  messagesRemoved,
5524
6300
  discordRefs: removedRefs.length,
6301
+ markers,
5525
6302
  },
5526
6303
  });
5527
6304
  return {
@@ -5531,6 +6308,7 @@ export class AgentFramework {
5531
6308
  messagesRemoved,
5532
6309
  tailMessageId,
5533
6310
  removedRefs,
6311
+ markers,
5534
6312
  lastVisible: await this.lastVisiblePreview(agentName),
5535
6313
  };
5536
6314
  }
@@ -5546,6 +6324,9 @@ export class AgentFramework {
5546
6324
  * removed together. Discord originals get awareness markers via the outbox;
5547
6325
  * the batch is `explicit`-activated only after every removal succeeded, and
5548
6326
  * a crash mid-way is finished at next boot (resumePreparedDiscordSuppressions).
6327
+ * Like rollbackToMessage, it returns once the body change has landed, with
6328
+ * a marker-scheduling receipt (`markers`); delivery continues without the
6329
+ * caller waiting on it.
5549
6330
  *
5550
6331
  * Not retroactive over derived state: a message already folded into an
5551
6332
  * autobiographical summary stays in that summary — roll back to before it
@@ -5596,9 +6377,12 @@ export class AgentFramework {
5596
6377
  throw new OperatorActionError('invalid', 'Suppression branch name must differ from the active branch');
5597
6378
  }
5598
6379
  // Gate + reserve the whole store (see reserveStoreForSurgery); held
5599
- // until the fork is fully redacted or the source is restored.
6380
+ // until the fork is fully redacted and its marker scheduling has been
6381
+ // recorded or reported, or the source is restored. Remote delivery is
6382
+ // not waited on.
5600
6383
  const release = this.reserveStoreForSurgery('suppress', agentName);
5601
6384
  let createdBranch;
6385
+ let markers;
5602
6386
  try {
5603
6387
  const markerBatch = this.discordAwarenessOutbox?.prepare({
5604
6388
  agentName,
@@ -5638,8 +6422,6 @@ export class AgentFramework {
5638
6422
  else
5639
6423
  cm.removeMessage(iv.fromId);
5640
6424
  }
5641
- if (markerBatch)
5642
- this.discordAwarenessOutbox.activate(markerBatch.id);
5643
6425
  }
5644
6426
  catch (error) {
5645
6427
  // A partially suppressed branch is not safe to serve from. Keep it
@@ -5649,14 +6431,26 @@ export class AgentFramework {
5649
6431
  const restored = await this.restoreSourceBranch(cm, sourceBranch, createdBranch, agentName);
5650
6432
  throw new OperatorActionError('failed', `Suppression failed on ${createdBranch}; ${restored}: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
5651
6433
  }
5652
- await this.syncDiscordAwarenessMarkers();
6434
+ // Every redaction committed: the body change has landed and the fork
6435
+ // is safe to serve from. Activating the marker batch is bookkeeping
6436
+ // about a world-side effect; if it fails, the suppression still
6437
+ // stands, and the receipt says what became of the marks: not
6438
+ // scheduled once the batch is retired, or unresolved if it can't be.
6439
+ // (The batch's resume journal has nothing left to do, since every
6440
+ // interval has committed, so retiring the batch gives up only the
6441
+ // marks.)
6442
+ markers = this.activateSurgeryMarkers(markerBatch, 'suppress', agentName);
6443
+ // Installs the MCPL data-plane gate synchronously, before the store is
6444
+ // released, and starts delivery without awaiting it (the first
6445
+ // reaction request may already be on the wire when this returns).
6446
+ this.startDiscordAwarenessDelivery('suppress', agentName);
5653
6447
  this.materializeConfigMountAfterBranchSwitch();
5654
6448
  }
5655
6449
  finally {
5656
6450
  release();
5657
6451
  }
5658
6452
  console.error(`[operator] suppress agent=${agentName} removed=${messagesRemoved} branch=${createdBranch} ` +
5659
- `by=${describeRequester(opts.requester)}`);
6453
+ `marks=${describeMarkers(markers)} by=${describeRequester(opts.requester)}`);
5660
6454
  const loggedIds = capIds(removedIds);
5661
6455
  this.recordOperatorAction({
5662
6456
  ...logBase,
@@ -5667,6 +6461,7 @@ export class AgentFramework {
5667
6461
  removedIds: loggedIds.ids,
5668
6462
  ...(loggedIds.truncated ? { removedIdsTruncated: true } : {}),
5669
6463
  discordRefs: removedRefs.length,
6464
+ markers,
5670
6465
  },
5671
6466
  });
5672
6467
  return {
@@ -5676,6 +6471,7 @@ export class AgentFramework {
5676
6471
  messagesRemoved,
5677
6472
  removedIds,
5678
6473
  removedRefs,
6474
+ markers,
5679
6475
  lastVisible: await this.lastVisiblePreview(agentName),
5680
6476
  };
5681
6477
  }
@@ -5684,6 +6480,97 @@ export class AgentFramework {
5684
6480
  throw error;
5685
6481
  }
5686
6482
  }
6483
+ /**
6484
+ * Record a surgery's prepared marker batch as active once its body change
6485
+ * has landed, and say what happened. Never throws: the body is already
6486
+ * applied, so a ledger failure here is reported in the receipt rather than
6487
+ * as a failed surgery an operator might retry. A batch that could not be
6488
+ * activated is retired, so no later reconciliation can deliver it behind a
6489
+ * receipt saying it was not scheduled (`not-scheduled`). If retiring fails
6490
+ * too, the receipt says only that the outcome is `unresolved`: the batch is
6491
+ * retained, and it may still be delivered. A rollback's batch
6492
+ * (activated by target branch) may be promoted by the very next
6493
+ * reconciliation, the one the delivery it starts runs; a suppression's
6494
+ * (activated explicitly) waits for the next start.
6495
+ *
6496
+ * A batch with no refs (a suppression journal with nothing addressable) is
6497
+ * still activated, which closes its resume journal. It schedules no marks,
6498
+ * so it reports `none` whatever the ledger does. If it can be neither
6499
+ * activated nor retired, its journal is left prepared and closes at the
6500
+ * next start: every interval it names has already committed, so the resume
6501
+ * skips them all and activates it.
6502
+ */
6503
+ activateSurgeryMarkers(batch, verb, agentName) {
6504
+ if (!batch)
6505
+ return { status: 'none', queued: 0 };
6506
+ try {
6507
+ this.discordAwarenessOutbox.activate(batch.id);
6508
+ }
6509
+ catch (error) {
6510
+ const detail = error instanceof Error ? error.message : String(error);
6511
+ let retired = false;
6512
+ let retireDetail = '';
6513
+ try {
6514
+ retired = this.discordAwarenessOutbox.discard(batch.id);
6515
+ }
6516
+ catch (discardError) {
6517
+ retireDetail = discardError instanceof Error ? discardError.message : String(discardError);
6518
+ }
6519
+ console.error(`[discord-awareness] ${verb} agent=${agentName}: body applied, but marker batch ${batch.id} ` +
6520
+ `could not be activated (${detail}); ` +
6521
+ (retired ? 'batch retired' : `batch retained${retireDetail ? ` (retire failed: ${retireDetail})` : ''}`));
6522
+ if (batch.refs.length === 0)
6523
+ return { status: 'none', queued: 0 };
6524
+ this.opsAlert(retired ? 'discord-awareness-not-scheduled' : 'discord-awareness-unresolved', agentName, retired
6525
+ ? `${verb} applied, but its ${batch.refs.length} awareness mark(s) were not scheduled ` +
6526
+ `(batch ${batch.id} retired): ${detail}`
6527
+ : `${verb} applied, but awareness batch ${batch.id} (${batch.refs.length} mark(s)) could be ` +
6528
+ `neither activated nor retired; it may still be delivered: ${detail}`, { data: { batchId: batch.id, retired } });
6529
+ return retired
6530
+ ? { status: 'not-scheduled', queued: 0, error: detail }
6531
+ : { status: 'unresolved', queued: 0, batchId: batch.id, error: detail };
6532
+ }
6533
+ if (batch.refs.length === 0)
6534
+ return { status: 'none', queued: 0 };
6535
+ return { status: 'queued', queued: batch.refs.length, batchId: batch.id };
6536
+ }
6537
+ /**
6538
+ * Start awareness delivery after a branch move (a surgery, a switch, an
6539
+ * undo or redo) without the caller awaiting it. No operator response waits
6540
+ * for Discord to accept marks: the MCPL data-plane gate alone holds channel
6541
+ * traffic that arrives after it until they are attempted.
6542
+ * `syncDiscordAwarenessMarkers` installs that gate synchronously, before
6543
+ * its first await, so calling this while a surgery still holds the store
6544
+ * puts the gate up before the store is released. With no drain running, the
6545
+ * drain starts in the same synchronous run (the first reaction request can
6546
+ * be written before this returns); with one running, this generation's pass
6547
+ * follows it (drainForGeneration). Either way delivery continues
6548
+ * independently: the caller's response and any store reservation stop
6549
+ * waiting on remote reactions. A turn may run during delivery when its wake
6550
+ * carries no channel traffic (a scheduler timer, a provider admission), and
6551
+ * also when its channel traffic was admitted before the gate went up
6552
+ * (queued when a surgery began, or arriving during its switch or fork): the
6553
+ * surgery's store reservation, which used to hold such a turn through the
6554
+ * drain, now ends once the gate is up. A failure is accounted by the gate
6555
+ * itself: the current generation's failure recycles the connections, and a
6556
+ * superseded one leaves that to the newer generation, whose own pass meets
6557
+ * the same ledger. Here it is logged and raised as an ops alert, and the
6558
+ * next sync, reconnect or start delivers what is still pending.
6559
+ *
6560
+ * @param after What moved the branch, for the log and the alert
6561
+ * ("rollback", "branch switch", "undo" …).
6562
+ * @param agentName The agent the move was for, or "framework" for a
6563
+ * branch switch that names none.
6564
+ */
6565
+ startDiscordAwarenessDelivery(after, agentName = 'framework') {
6566
+ if (!this.discordAwarenessOutbox)
6567
+ return;
6568
+ this.syncDiscordAwarenessMarkers().catch((error) => {
6569
+ const detail = error instanceof Error ? error.message : String(error);
6570
+ console.error(`[discord-awareness] delivery after ${after} agent=${agentName} failed: ${detail}`);
6571
+ this.opsAlert('discord-awareness-delivery', agentName, `Awareness mark delivery after ${after} failed: ${detail}`);
6572
+ });
6573
+ }
5687
6574
  /** Re-materialize config files from the (new) active branch — fire and
5688
6575
  * forget; the gate picks changes up via mtime. Mirrors undoLastTurn. */
5689
6576
  materializeConfigMountAfterBranchSwitch() {
@@ -6068,7 +6955,7 @@ export class AgentFramework {
6068
6955
  // divergence breaks the compile prefix).
6069
6956
  const { blocks: toolResultContent, spilled } = await this.buildStoredToolResultContent(agent.name, currentState.toolResults, maxChars);
6070
6957
  const membraneResults = currentState.toolResults.map(tc => this.toMembraneToolResult(tc.id, tc.result, maxChars, spilled.get(tc.id)));
6071
- agent.toolResultGuard.storeResults(toolResultContent, membraneResults, currentState.toolResults, turnRowMetadata);
6958
+ agent.toolResultGuard.storeResults(toolResultContent, membraneResults, currentState.toolResults, turnRowMetadata, maxChars);
6072
6959
  // Flush any messages that were deferred while this turn was in
6073
6960
  // flight. Route to the PRIMARY agent — deferred messages are
6074
6961
  // framework-level (module messages, push events, subagent return
@@ -6675,10 +7562,10 @@ export class AgentFramework {
6675
7562
  isolate: true,
6676
7563
  // Strategy instances are stateful — never share the template's.
6677
7564
  strategy: router.strategyFactory?.() ?? new PassthroughStrategy(),
6678
- // Dynamic conversation forks retain the established provider policy in
6679
- // this bounded first slice. Provider cooldown ownership belongs to the
6680
- // persistent resident only; generation-unique forks must not leak gates.
6681
- membrane: this.membrane,
7565
+ // Provider cooldown ownership belongs to the persistent resident only;
7566
+ // generation-unique forks must not leak gates. A fork's own recorded
7567
+ // provider waits still bind its compression calls.
7568
+ membrane: this.conversationAuxiliaryMembraneFor(name),
6682
7569
  debugLogContext: !!process.env.DEBUG_CONTEXT,
6683
7570
  });
6684
7571
  // Seed with the template's compiled context, renaming the template
@@ -6745,6 +7632,18 @@ export class AgentFramework {
6745
7632
  this.evictTurnCheckpoints(agentName);
6746
7633
  if (agent)
6747
7634
  this.logicalTurnToolCalls.delete(agent);
7635
+ // A provider-wait hold (nonOwningProviderWait) goes with the agent: left,
7636
+ // its timer would requeue held wakes for an agent that is gone, and a wait
7637
+ // held until released would re-arm for the name indefinitely. The recorded
7638
+ // wait stays, and binds the name if its channel binds again.
7639
+ const cooldown = this.providerAccelerationCooldowns.get(agentName);
7640
+ if (cooldown) {
7641
+ clearTimeout(cooldown.timer);
7642
+ this.providerAccelerationCooldowns.delete(agentName);
7643
+ console.error(`[provider-wait] agent=${agentName} disposed while held` +
7644
+ `${cooldown.waitModel !== undefined ? ` on ${cooldown.waitModel}` : ''}: ` +
7645
+ `${cooldown.heldRequests.length} held request(s) dropped with it (the recorded wait stays)`);
7646
+ }
6748
7647
  this.emitTrace({
6749
7648
  type: 'mcpl:conversation-disposed',
6750
7649
  agentName,
@@ -7587,7 +8486,10 @@ export class AgentFramework {
7587
8486
  }
7588
8487
  const providerCooldown = this.providerAccelerationCooldowns?.get(agentName);
7589
8488
  if (providerCooldown) {
7590
- if (now < providerCooldown.until || this.extendHostProviderHold(agentName, providerCooldown)) {
8489
+ // A provider wait binds its own model; with a different primary model
8490
+ // selected now, the turn proceeds (the wait stays recorded).
8491
+ const otherModel = providerCooldown.waitModel !== undefined && providerCooldown.waitModel !== agent.model;
8492
+ if (!otherModel && (now < providerCooldown.until || this.extendHostProviderHold(agentName, providerCooldown))) {
7591
8493
  for (const req of requests) {
7592
8494
  if (!providerCooldown.heldRequests.some((r) => this.sameInferenceRequest(r, req))) {
7593
8495
  providerCooldown.heldRequests.push(req);
@@ -7608,6 +8510,13 @@ export class AgentFramework {
7608
8510
  recovery.releasedAt = now;
7609
8511
  requests = [...providerCooldown.heldRequests, ...requests];
7610
8512
  }
8513
+ // A provider's stated wait on this agent's primary model, including one
8514
+ // recorded before a restart, holds the turn until it passes. An
8515
+ // ephemeral run records nothing, its name being single-use, so no wait,
8516
+ // known or unreadable, binds it (as at the last boundary).
8517
+ const providerWait = this.providerWaits?.active(agentName, agent.model);
8518
+ if (providerWait && !this.ephemeralRuns.has(agentName) && this.holdForProviderWait(agent, providerWait, requests))
8519
+ continue;
7611
8520
  // Skip if agent is busy (inferring, streaming, or waiting for tools) —
7612
8521
  // or if a turn is alive at all (activeTurnTokens): the state machine
7613
8522
  // goes idle at endTurn's reset while driveStream teardown is still
@@ -8325,6 +9234,13 @@ export class AgentFramework {
8325
9234
  }
8326
9235
  async startAgentStream(agent, trigger, attempt = 0, providerGateAlreadyHeld = false) {
8327
9236
  const ownsProviderGate = !this.ephemeralRuns.has(agent.name) && !this.conversationAgentHomes.has(agent.name);
9237
+ // A stopping framework starts no turn, whoever owns provider admission
9238
+ // (conversation agents and ephemeral runs do not take the gate).
9239
+ if (this.providerAdmissionClosed) {
9240
+ if (ownsProviderGate && providerGateAlreadyHeld)
9241
+ this.releasePrimaryProviderGate(agent.name);
9242
+ return;
9243
+ }
8328
9244
  if (ownsProviderGate && !providerGateAlreadyHeld) {
8329
9245
  this.acquirePrimaryProviderGate(agent.name);
8330
9246
  const gate = this.providerGate(agent.name);
@@ -8371,7 +9287,7 @@ export class AgentFramework {
8371
9287
  await this.startAgentStream(agent, trigger, attempt, true);
8372
9288
  }).catch((error) => {
8373
9289
  this.releasePrimaryProviderGate(agent.name);
8374
- console.error(`[provider-cooldown] failed to resume primary for ${agent.name}:`, error);
9290
+ console.error(`[provider-cooldown] failed to resume primary for ${agent.name}: ${describeFailureWithFrames(error)}`);
8375
9291
  });
8376
9292
  return;
8377
9293
  }
@@ -8380,6 +9296,36 @@ export class AgentFramework {
8380
9296
  return;
8381
9297
  }
8382
9298
  }
9299
+ // The last admission boundary before this primary call. With the gate
9300
+ // held and no auxiliary call in flight, no new provider wait can be
9301
+ // recorded for this agent before the call (auxiliary admission waits on
9302
+ // the gate). But one may have been recorded since the scheduler's check:
9303
+ // an auxiliary call this wake parked behind can fail with a stated wait
9304
+ // on the primary model. That wait binds this wake too: hold it, don't call.
9305
+ // A conversation agent's waits are recorded and held as a resident's are
9306
+ // (nonOwningProviderWait), and its retries and in-turn restarts reach this
9307
+ // boundary without the scheduler: a wait recorded meanwhile, by its own
9308
+ // auxiliary call or a non-retryable failure, binds them here too. An
9309
+ // ephemeral run records nothing, so nothing binds it here. A frame whose
9310
+ // agent was replaced or removed (a fork disposed while its retry slept)
9311
+ // installs no hold: its turn is dropped after the compile, as main drops
9312
+ // any such turn.
9313
+ if (!this.ephemeralRuns.has(agent.name) && this.agents.get(agent.name) === agent) {
9314
+ const wait = this.providerWaits?.active(agent.name, agent.model);
9315
+ if (wait && this.holdForProviderWait(agent, wait, trigger ? [trigger] : [])) {
9316
+ if (ownsProviderGate)
9317
+ this.releasePrimaryProviderGate(agent.name);
9318
+ // A retry or an in-turn restart arrives with its turn still open at
9319
+ // the event gate: the failed attempt, or the predecessor that kept
9320
+ // liveness for it, leaves closing it to this frame. Held here, no
9321
+ // frame would close it, and while it is open every gate event
9322
+ // buffers, for every agent, until the wait ends.
9323
+ this.eventGate?.onInferenceEnded(agent.name);
9324
+ console.error(`[provider-wait] agent=${agent.name} model=${agent.model} binds at admission ` +
9325
+ `(recorded after the wake was scheduled); wake held, not started`);
9326
+ return;
9327
+ }
9328
+ }
8383
9329
  // Mark the turn alive before ANYTHING awaits (hooks, compile, stream
8384
9330
  // setup): from here until this turn's teardown, cross-turn writers defer
8385
9331
  // (addMessage guard) instead of appending — nothing may enter the window
@@ -8728,6 +9674,28 @@ export class AgentFramework {
8728
9674
  this.eventGate?.onInferenceEnded(agent.name);
8729
9675
  return false;
8730
9676
  }
9677
+ // A conversation agent takes no provider gate, so its own compression
9678
+ // can record a wait while this turn compiles, after the last boundary
9679
+ // checked. Check its waits again here, where nothing has been sent, and
9680
+ // hold the turn as the boundary would rather than send it into the wait.
9681
+ if (this.conversationAgentHomes.has(agent.name)) {
9682
+ const wait = this.providerWaits?.active(agent.name, agent.model);
9683
+ if (wait && this.holdForProviderWait(agent, wait, trigger ? [trigger] : [])) {
9684
+ console.error(`[provider-wait] agent=${agent.name} model=${agent.model} binds after the compile ` +
9685
+ `(recorded while the turn compiled); turn held, nothing sent`);
9686
+ // The turn emitted inference:started before its compile; every turn
9687
+ // that does gets one terminal event, which listeners (a typing loop)
9688
+ // rely on, as the failure sites' holds emit inference:failed.
9689
+ this.emitTrace({ type: 'inference:failed', agentName: agent.name, error: providerWaitText(agent.name, agent.model, wait) });
9690
+ stream.cancel();
9691
+ agent.cancelStream();
9692
+ this.activeStreams.delete(agent.name);
9693
+ this.pendingAssistantBlocks.delete(agent.name);
9694
+ this.channelRegistry?.stopTyping();
9695
+ this.eventGate?.onInferenceEnded(agent.name);
9696
+ return false;
9697
+ }
9698
+ }
8731
9699
  const handle = this.driveStream(agent, stream, requestSnapshot, trigger, attempt, compiledRequest, ownsProviderGate, takeKvSubmission, drainKvSubmissionIds, new Set(tools.map((tool) => tool.name)));
8732
9700
  this.activeStreams.set(agent.name, handle);
8733
9701
  // Handoff: driveStream captured the token in its synchronous prefix;
@@ -8753,9 +9721,27 @@ export class AgentFramework {
8753
9721
  this.eventGate?.onInferenceEnded(agent.name);
8754
9722
  return false;
8755
9723
  }
8756
- const action = this.errorPolicy.onInferenceError(err, agent.name, attempt);
9724
+ const statedWait = ownsProviderGate ? undefined : this.nonOwningProviderWait(agent, err, trigger);
9725
+ if (statedWait === 'held') {
9726
+ // A conversation agent held by the provider's stated wait, as a
9727
+ // resident is: the hold's release owns the later compile.
9728
+ this.eventGate?.onInferenceEnded(agent.name);
9729
+ return false;
9730
+ }
9731
+ const action = statedWait === 'fail' ? { retry: false } : this.errorPolicy.onInferenceError(err, agent.name, attempt);
8757
9732
  if (action.retry) {
8758
- await new Promise((resolve) => setTimeout(resolve, action.delayMs));
9733
+ // A policy's delay can be a provider's retry-after: wait all of it.
9734
+ await this.waitForRetry(action.delayMs);
9735
+ if (this.providerAdmissionClosed) {
9736
+ // stop() ended the wait: that cancels the retry, it does not hasten
9737
+ // it. Nothing is written (the store is closing); this frame still
9738
+ // owns its token, since no retry frame replaced it.
9739
+ if (this.activeTurnTokens.get(agent.name) === turnToken) {
9740
+ this.activeTurnTokens.delete(agent.name);
9741
+ this.activeTurnTriggers.delete(agent.name);
9742
+ }
9743
+ return false;
9744
+ }
8759
9745
  // The retry re-enters startAgentStream, which replaces this frame's
8760
9746
  // turn token with its own; cleanup belongs to the innermost frame.
8761
9747
  await this.startAgentStream(agent, trigger, attempt + 1);
@@ -8773,16 +9759,18 @@ export class AgentFramework {
8773
9759
  this.settleAgent(agent.name, {
8774
9760
  stopReason: 'exhausted',
8775
9761
  speech: '',
8776
- error: err.message,
9762
+ error: statedWait === 'fail' ? this.statedWaitNotSlept(err) : err.message,
8777
9763
  }, startLogical);
8778
9764
  this.emitTrace({
8779
9765
  type: 'inference:exhausted',
8780
9766
  agentName: agent.name,
8781
9767
  error: err.message,
8782
- // Drives the poison-history breaker (`invalid_request`: a 400-class
8783
- // rejection of the history itself — retrying the same context can
8784
- // never succeed; auth/abort/context_length are also non-retryable
8785
- // but must NOT cost history) and the OverBudget drain breaker
9768
+ // Drives the poison-history breaker (shedMayHelp: an
9769
+ // `invalid_request` at HTTP 400, 413 or 422 or with no status, or a
9770
+ // `context_length` at 413, suggests the provider refused the
9771
+ // request's content or size, which a shed of the newest exchange may
9772
+ // fix; auth/abort and other context_length failures are also
9773
+ // non-retryable but must NOT cost history) and the OverBudget drain breaker
8786
9774
  // (`over_budget`: compile refused to fit the hard budget — this is
8787
9775
  // the site that sees it, since compile runs before the stream
8788
9776
  // exists). `retryable` is kept for observability.
@@ -9374,7 +10362,7 @@ export class AgentFramework {
9374
10362
  if (readyState.status === 'ready') {
9375
10363
  const cap = this.resolveToolResultInlineCap(agent).cap;
9376
10364
  const { blocks: toolResultContent, spilled } = await this.buildStoredToolResultContent(agent.name, readyState.toolResults, cap);
9377
- agent.toolResultGuard.storeResults(toolResultContent, readyState.toolResults.map((tc) => this.toMembraneToolResult(tc.id, tc.result, cap, spilled.get(tc.id))), readyState.toolResults, this.silentTurnRowMetadata(agent.name));
10365
+ agent.toolResultGuard.storeResults(toolResultContent, readyState.toolResults.map((tc) => this.toMembraneToolResult(tc.id, tc.result, cap, spilled.get(tc.id))), readyState.toolResults, this.silentTurnRowMetadata(agent.name), cap);
9378
10366
  // The response is already complete: this batch is never
9379
10367
  // submitted in this turn, so it cannot be refused. Admit it.
9380
10368
  agent.toolResultGuard.settleTurnEnded();
@@ -9597,7 +10585,7 @@ export class AgentFramework {
9597
10585
  // border is close, rather than noise on every near-threshold
9598
10586
  // flip. (It must not re-enter any agent's context — the emoji
9599
10587
  // is suppressed at the surface via
9600
- // DISCORD_SUPPRESS_REACTION_EMOJIS; without that, marking a
10588
+ // DISCORD_SUPPRESSED_REACTIONS_BASELINE; without that, marking a
9601
10589
  // refusal feeds the next one.)
9602
10590
  if (!handledByRewind) {
9603
10591
  void this.reactToRefusal(agent.name, category);
@@ -9869,25 +10857,39 @@ export class AgentFramework {
9869
10857
  this.eventGate?.onInferenceEnded(agent.name);
9870
10858
  break;
9871
10859
  }
9872
- const action = this.errorPolicy.onInferenceError(err, agent.name, attempt);
10860
+ const statedWait = ownsProviderGate ? undefined : this.nonOwningProviderWait(agent, err, trigger);
10861
+ if (statedWait === 'held') {
10862
+ lifecyclePhase = 'failed';
10863
+ if (this.agents.get(agent.name) === agent && agent.streamId === myStreamId)
10864
+ this.eventGate?.onInferenceEnded(agent.name);
10865
+ break;
10866
+ }
10867
+ const action = statedWait === 'fail' ? { retry: false } : this.errorPolicy.onInferenceError(err, agent.name, attempt);
9873
10868
  if (action.retry) {
9874
- await new Promise((resolve) => setTimeout(resolve, action.delayMs));
9875
- await this.startAgentStream(agent, trigger, attempt + 1);
10869
+ // A policy's delay can be a provider's retry-after: wait all of it.
10870
+ await this.waitForRetry(action.delayMs);
10871
+ // stop() ending the wait cancels the retry (this frame's finally
10872
+ // clears the token); it is never permission to retry at once.
10873
+ if (!this.providerAdmissionClosed)
10874
+ await this.startAgentStream(agent, trigger, attempt + 1);
9876
10875
  }
9877
10876
  else {
9878
10877
  this.settleAgent(agent.name, {
9879
10878
  stopReason: 'exhausted',
9880
10879
  speech: '',
9881
- error: err.message,
10880
+ error: statedWait === 'fail' ? this.statedWaitNotSlept(err) : err.message,
9882
10881
  }, this.agents.get(agent.name) === agent && agent.streamId === myStreamId ? streamRouting.logical : undefined);
9883
10882
  this.emitTrace({
9884
10883
  type: 'inference:exhausted',
9885
10884
  agentName: agent.name,
9886
10885
  error: err.message,
9887
10886
  // Drives the poison-history breaker. `retryable` is kept for
9888
- // observability, but the breakers gate on `errorType` — only an
9889
- // `invalid_request` (a 400-class rejection of the history
9890
- // itself) means retrying the same context can never succeed.
10887
+ // observability, but the breaker gates on
10888
+ // shedMayHelp(errorType, httpStatus): only an `invalid_request`
10889
+ // at HTTP 400, 413 or 422 or with no status, or a
10890
+ // `context_length` at 413, suggests the provider refused the
10891
+ // request's content or size, which a shed of the newest
10892
+ // exchange may fix.
9891
10893
  ...this.classifyInferenceError(err),
9892
10894
  });
9893
10895
  if (this.agents.get(agent.name) === agent && agent.streamId === myStreamId)
@@ -10102,6 +11104,23 @@ export class AgentFramework {
10102
11104
  this.eventGate?.onInferenceEnded(agent.name);
10103
11105
  return;
10104
11106
  }
11107
+ // The same rule for a turn that doesn't own provider admission as at
11108
+ // the other two failure sites: a conversation agent's stated wait is
11109
+ // recorded and its turn held; an ephemeral run whose wait it won't sleep
11110
+ // says why it ends.
11111
+ const statedWait = ownsProviderGate ? undefined : this.nonOwningProviderWait(agent, err, trigger);
11112
+ if (statedWait === 'held') {
11113
+ this.emitTrace({ type: 'inference:failed', agentName: agent.name, error: err.message, stack: err.stack });
11114
+ this.logInference({ timestamp: startTime, agentName: agent.name, requestId, success: false,
11115
+ error: `Provider wait: ${err.message}`,
11116
+ request: compiledRequest ?? { note: 'streaming request rate-limited' }, durationMs });
11117
+ this.abortAgentScript(agent.name, 'provider wait');
11118
+ lifecyclePhase = 'failed';
11119
+ agent.reset();
11120
+ if (this.agents.get(agent.name) === agent && agent.streamId === myStreamId)
11121
+ this.eventGate?.onInferenceEnded(agent.name);
11122
+ return;
11123
+ }
10105
11124
  this.emitTrace({
10106
11125
  type: 'inference:failed',
10107
11126
  agentName: agent.name,
@@ -10111,7 +11130,7 @@ export class AgentFramework {
10111
11130
  this.settleAgent(agent.name, {
10112
11131
  stopReason: 'exhausted',
10113
11132
  speech: '',
10114
- error: err.message,
11133
+ error: statedWait === 'fail' ? this.statedWaitNotSlept(err) : err.message,
10115
11134
  }, streamRouting.logical);
10116
11135
  this.emitTrace({
10117
11136
  type: 'inference:exhausted',
@@ -11428,6 +12447,10 @@ export class AgentFramework {
11428
12447
  logInference(entry) {
11429
12448
  // Store large request/response as blobs
11430
12449
  const entryToStore = { ...entry };
12450
+ // The failure text is stored inline in every record; the request that
12451
+ // produced it is already stored separately as a blob.
12452
+ if (typeof entryToStore.error === 'string')
12453
+ entryToStore.error = boundFailureText(entryToStore.error);
11431
12454
  // Blob threshold: 10KB - typical context-heavy requests exceed this
11432
12455
  const BLOB_THRESHOLD = 10000;
11433
12456
  // Both branches persist the JSON view, never the live object: a compiled
@@ -11931,7 +12954,10 @@ export class AgentFramework {
11931
12954
  // without this the only durable record of a failed inference is a field in
11932
12955
  // llm-calls.jsonl — invisible to operator, agent, and monitoring.
11933
12956
  if (event.type === 'inference:exhausted') {
11934
- this.noteInferenceExhausted(event.agentName ?? 'unknown', event.error ?? 'unknown error', event.retryable, event.errorType);
12957
+ this.noteInferenceExhausted(event.agentName ?? 'unknown', event.error ?? 'unknown error', event.retryable, event.errorType, {
12958
+ httpStatus: event.httpStatus,
12959
+ providerErrorCode: event.providerErrorCode,
12960
+ });
11935
12961
  }
11936
12962
  else if (event.type === 'inference:completed') {
11937
12963
  // A successful response — even mid-turn between tool calls — proves the
@@ -11949,7 +12975,28 @@ export class AgentFramework {
11949
12975
  ...event,
11950
12976
  timestamp: Date.now(),
11951
12977
  };
11952
- for (const listener of this.traceListeners) {
12978
+ // Listeners (web UI, subagent observers, host loggers) receive bounded
12979
+ // failure text. An event's `error` and `stack` strings are whatever its
12980
+ // source's Error carried, which for a provider failure can be an echo of
12981
+ // the whole request.
12982
+ const bounded = traceEvent;
12983
+ if (typeof bounded.error === 'string')
12984
+ bounded.error = boundFailureText(bounded.error);
12985
+ if (typeof bounded.stack === 'string')
12986
+ bounded.stack = boundFailureText(bounded.stack);
12987
+ // An event goes to the subscriptions present when its delivery starts,
12988
+ // each one only if it is still active when its turn comes, as the DOM's
12989
+ // EventTarget delivers: a subscription made during delivery waits for the
12990
+ // next event, and one removed during delivery is skipped if its turn
12991
+ // hasn't come. Each onTrace call is a subscription of its own, so a
12992
+ // function subscribed twice is called twice. Iterating the live array let
12993
+ // an unsubscribe during delivery make the listener after the one being
12994
+ // called miss the event, since splice shifts the rest under the iterator.
12995
+ for (const registration of [...this.traceListeners]) {
12996
+ if (!registration.active)
12997
+ continue;
12998
+ // Called as a plain function, as before, so the registration isn't its `this`.
12999
+ const { listener } = registration;
11953
13000
  try {
11954
13001
  listener(traceEvent);
11955
13002
  }
@@ -11972,15 +13019,18 @@ export class AgentFramework {
11972
13019
  * inference, so this never causes a retry/wake loop.
11973
13020
  * 3. escalation — after N consecutive failures the agent is hard-down;
11974
13021
  * log that loudly (a repeated identical failure is the textbook signal).
11975
- * 4. breaker — when the failures are an `invalid_request` (a 400-class
11976
- * rejection of the history itself, e.g. corrupted tool_use/tool_result
11977
- * pairing or an oversized attachment), retrying the same context can
11978
- * never succeed: every new push event wakes the agent onto the same
11979
- * poisoned history forever. At the hard-down threshold, automatically
11980
- * quarantine: shed the newest complete exchange (the same forced-rewind
11981
- * primitive `/unstick` uses — shedNewestTurn never orphans a
11982
- * tool_use/thinking block) and retry, bounded by the same rewind cap so
11983
- * the breaker can never eat the whole history.
13022
+ * 4. breaker — when a failure at or past the hard-down threshold suggests
13023
+ * the provider refused the request's content or size (shedMayHelp:
13024
+ * an `invalid_request` at HTTP 400, 413 or 422 or with no status, or a
13025
+ * `context_length` at 413, such as a provider's answer to a corrupted
13026
+ * tool_use/tool_result pairing or an oversized attachment), retrying
13027
+ * the same context is likely to fail the same way, and every new push
13028
+ * event would wake the agent onto that same history. At the hard-down
13029
+ * threshold, automatically quarantine: shed the newest complete
13030
+ * exchange (the same forced-rewind primitive `/unstick` uses —
13031
+ * shedNewestTurn never orphans a tool_use/thinking block) and retry,
13032
+ * bounded by the same rewind cap so the breaker can never eat the
13033
+ * whole history.
11984
13034
  */
11985
13035
  /**
11986
13036
  * Live health snapshot for /healthz and doctor tooling: gate state, queued
@@ -12005,11 +13055,14 @@ export class AgentFramework {
12005
13055
  primaryPending: gate?.primaryPending ?? false,
12006
13056
  auxiliaryInFlight: gate?.auxiliaryInFlight ?? 0,
12007
13057
  auxiliaryDeferred: gate?.deferredAuxiliary ?? 0,
12008
- cooldownUntil: cooldown?.until ?? null,
13058
+ // Null while none, and while a provider wait holds until released.
13059
+ cooldownUntil: cooldown !== undefined && Number.isFinite(cooldown.until) ? cooldown.until : null,
12009
13060
  cooldownReason: cooldown?.reason ?? null,
12010
13061
  hostHold: cooldown?.hostHoldError !== undefined,
12011
13062
  heldRequests: cooldown?.heldRequests.length ?? 0,
12012
13063
  lastRecovery: this.providerAccelerationLastRecovery?.get(name) ?? null,
13064
+ /** Provider waits (stated retry-after) binding this agent now, per model. */
13065
+ providerWaits: this.providerWaitSnapshot(name).map(({ agent: _agent, ...wait }) => wait),
12013
13066
  };
12014
13067
  })(),
12015
13068
  lastInference: this.lastInferenceAt.get(name) ?? null,
@@ -12024,8 +13077,12 @@ export class AgentFramework {
12024
13077
  * and kind-specific fields (additive only — doctor parses by regex). */
12025
13078
  logFailure(record) {
12026
13079
  try {
13080
+ // Every field is bounded: `reason` is conventionally free text, and any
13081
+ // caller's field may carry an error message, or an Error whose own
13082
+ // fields include the whole request.
13083
+ const bounded = boundFailureFields(record);
12027
13084
  mkdirSync('logs', { recursive: true });
12028
- appendFileSync('logs/failures.log', JSON.stringify({ at: new Date().toISOString(), ...record }) + '\n');
13085
+ appendFileSync('logs/failures.log', JSON.stringify({ at: new Date().toISOString(), ...bounded }) + '\n');
12029
13086
  }
12030
13087
  catch { /* best-effort */ }
12031
13088
  }
@@ -12048,11 +13105,25 @@ export class AgentFramework {
12048
13105
  notifyOps(kind, agentName, message, data) {
12049
13106
  this.opsAlert(kind, agentName, message, { data });
12050
13107
  }
12051
- opsAlert(kind, agentName, message, opts) {
13108
+ opsAlert(kind, agentName, rawMessage, opts) {
13109
+ // A JavaScript caller of notifyOps or notifyOpsAlert can pass a message
13110
+ // that isn't a string; it's described as a thrown value is, rather than
13111
+ // thrown on (a string is its own description).
13112
+ const wholeMessage = failureDescription(rawMessage);
13113
+ const message = boundFailureText(wholeMessage);
13114
+ // The caller's data goes to failures.log and to trace listeners, bounded
13115
+ // the same way for both. A JavaScript caller's null is no data, and data
13116
+ // that isn't a plain object (a string, an array, a Buffer) is kept as one
13117
+ // field, `data`, rather than spread into a field per character or element.
13118
+ const rawData = opts?.data;
13119
+ const plain = typeof rawData === 'object' && rawData !== null &&
13120
+ [Object.prototype, null].includes(Object.getPrototypeOf(rawData));
13121
+ const data = rawData == null ? undefined
13122
+ : boundFailureFields(plain ? rawData : { data: rawData });
12052
13123
  if (!opts?.skipLog) {
12053
- this.logFailure({ agent: agentName, kind, reason: message, ...opts?.data });
13124
+ this.logFailure({ agent: agentName, kind, reason: message, ...data });
12054
13125
  }
12055
- this.emitTrace({ type: 'ops:alert', kind, agentName, message, data: opts?.data });
13126
+ this.emitTrace({ type: 'ops:alert', kind, agentName, message, data });
12056
13127
  const hook = process.env.CONNECTOME_OPS_WEBHOOK;
12057
13128
  if (!hook)
12058
13129
  return;
@@ -12066,7 +13137,7 @@ export class AgentFramework {
12066
13137
  method: 'POST',
12067
13138
  headers: { 'content-type': 'application/json' },
12068
13139
  body: JSON.stringify({
12069
- content: `\u{1F6A8} **${agentName}** ${kind}: ${message.slice(0, 500)}`,
13140
+ content: `\u{1F6A8} **${agentName}** ${kind}: ${boundFailureText(wholeMessage, OPS_WEBHOOK_MESSAGE_MAX_CHARS)}`,
12070
13141
  }),
12071
13142
  }).catch((err) => console.error(`[ops-alert] webhook post failed (${key}):`, err instanceof Error ? err.message : err));
12072
13143
  }
@@ -12103,9 +13174,12 @@ export class AgentFramework {
12103
13174
  }
12104
13175
  /**
12105
13176
  * Classify a terminal inference error for the `inference:exhausted` trace.
12106
- * Single-sourced for every emit site: the downstream breakers (poison-history
12107
- * quarantine, OverBudget drain kick) gate on `errorType`, so classification
12108
- * drift between sites would silently disable a safety net.
13177
+ * Single-sourced for every emit site that has an error to classify (a
13178
+ * stream the framework aborted has none): the downstream breakers gate on
13179
+ * what it returns (the poison-history quarantine on `errorType` and
13180
+ * `httpStatus`, through shedMayHelp; the OverBudget drain kick on
13181
+ * `errorType`), so classification drift between sites would silently
13182
+ * change when a safety net acts.
12109
13183
  *
12110
13184
  * context-manager's refusal errors are matched by a real `instanceof` since
12111
13185
  * CM exports them from its package root (context-manager#41 / #71; floor
@@ -12117,8 +13191,18 @@ export class AgentFramework {
12117
13191
  * can be reworded without warning.
12118
13192
  */
12119
13193
  classifyInferenceError(err) {
12120
- if (err instanceof MembraneError) {
12121
- return { retryable: err.retryable, errorType: err.type };
13194
+ // From either copy of membrane (isMembraneError). The fields are read as
13195
+ // data, and the type is bounded like describeFailure's, for every sink the
13196
+ // classification reaches (the trace, the marker, the logs).
13197
+ if (isMembraneError(err)) {
13198
+ const fields = err;
13199
+ const providerErrorCode = boundProviderErrorCode(fields.providerErrorCode);
13200
+ return {
13201
+ ...(typeof fields.retryable === 'boolean' ? { retryable: fields.retryable } : {}),
13202
+ ...(typeof fields.type === 'string' ? { errorType: boundFailureText(fields.type, PROVIDER_ERROR_CODE_MAX_CHARS) } : {}),
13203
+ ...(typeof fields.httpStatus === 'number' ? { httpStatus: fields.httpStatus } : {}),
13204
+ ...(providerErrorCode !== undefined ? { providerErrorCode } : {}),
13205
+ };
12122
13206
  }
12123
13207
  if (err instanceof OverBudgetError || err.name === 'OverBudgetError') {
12124
13208
  return { errorType: 'over_budget' };
@@ -12142,31 +13226,72 @@ export class AgentFramework {
12142
13226
  notifyOpsAlert(kind, agentName, message, data) {
12143
13227
  this.opsAlert(kind, agentName, message, { data });
12144
13228
  }
12145
- noteInferenceExhausted(agentName, reason, retryable, errorType) {
13229
+ noteInferenceExhausted(agentName, rawReason, retryable, rawErrorType, detail) {
13230
+ // The reason is the failure's own message, and a provider can echo the
13231
+ // whole rejected request in it: one household's [inference-failed]
13232
+ // marker carried ~1.7 MB of its own request back into its context. Every
13233
+ // sink below gets an excerpt cut from the whole reason at its own bound
13234
+ // (boundFailureText, so its marker states the whole length): the stderr
13235
+ // line, failures.log and the hard-down alert FAILURE_TEXT_MAX_CHARS, the
13236
+ // [inference-failed] marker FAILURE_MARKER_REASON_MAX_CHARS, the status's
13237
+ // lastError and the compile-refused alert FAILURE_SUMMARY_MAX_CHARS, and
13238
+ // the rewind marker REWIND_MARKER_REASON_MAX_CHARS. The classification
13239
+ // travels as structured fields beside the text.
13240
+ //
13241
+ // A reason that isn't a string (the stream paths always pass a message)
13242
+ // is described rather than allowed to throw before anything is recorded.
13243
+ // Its whole description is the whole reason, cut once by each sink.
13244
+ const wholeReason = typeof rawReason === 'string' ? rawReason : failureDescription(rawReason);
13245
+ // The type is bounded wherever it's written, as classifyInferenceError
13246
+ // bounds the types it reads, since a caller can pass one it didn't classify.
13247
+ const errorType = typeof rawErrorType === 'string'
13248
+ ? boundFailureText(rawErrorType, PROVIDER_ERROR_CODE_MAX_CHARS) : rawErrorType;
13249
+ const reason = boundFailureText(wholeReason);
13250
+ const markerReason = boundFailureText(wholeReason, FAILURE_MARKER_REASON_MAX_CHARS);
13251
+ const httpStatus = typeof detail?.httpStatus === 'number' ? detail.httpStatus : undefined;
13252
+ const providerErrorCode = boundProviderErrorCode(detail?.providerErrorCode);
13253
+ const classification = {
13254
+ ...(errorType !== undefined ? { errorType } : {}),
13255
+ ...(retryable !== undefined ? { retryable } : {}),
13256
+ ...(httpStatus !== undefined ? { httpStatus } : {}),
13257
+ ...(providerErrorCode !== undefined ? { providerErrorCode } : {}),
13258
+ };
13259
+ // Each excerpt says how long the whole was when it was cut.
13260
+ const cutAt = (excerpt) => excerpt === wholeReason ? {} : { reasonChars: wholeReason.length };
12146
13261
  const streak = (this.consecutiveInferenceFailures.get(agentName) ?? 0) + 1;
12147
13262
  this.consecutiveInferenceFailures.set(agentName, streak);
12148
- this.lastInferenceAt.set(agentName, { ...this.lastInferenceAt.get(agentName), failedAt: Date.now(), lastError: reason.slice(0, 300) });
13263
+ this.lastInferenceAt.set(agentName, { ...this.lastInferenceAt.get(agentName), failedAt: Date.now(), lastError: boundFailureText(wholeReason, FAILURE_SUMMARY_MAX_CHARS) });
12149
13264
  // (1) Durable stderr line — works in headless/daemon mode with no client.
12150
- console.error(`[inference-failed] agent=${agentName} consecutive=${streak}: ${reason}`);
13265
+ const labels = [
13266
+ errorType !== undefined ? `type=${errorType}` : undefined,
13267
+ httpStatus !== undefined ? `status=${httpStatus}` : undefined,
13268
+ providerErrorCode !== undefined ? `code=${providerErrorCode}` : undefined,
13269
+ ].filter((label) => label !== undefined);
13270
+ console.error(`[inference-failed] agent=${agentName} consecutive=${streak}` +
13271
+ `${labels.length > 0 ? ` ${labels.join(' ')}` : ''}: ${reason}`);
12151
13272
  // (1b) Machine-greppable durable record, independent of journald/unit log
12152
13273
  // redirects: logs/failures.log under the host's working directory. This is
12153
- // what connectome-doctor reads. Legacy fields kept; `kind` is additive.
12154
- this.logFailure({ agent: agentName, consecutive: streak, reason, kind: 'inference-exhausted' });
13274
+ // what connectome-doctor reads. Legacy fields kept; `kind` and the
13275
+ // classification fields are additive.
13276
+ this.logFailure({ agent: agentName, consecutive: streak, reason, kind: 'inference-exhausted', ...classification, ...cutAt(reason) });
12155
13277
  // (2) Agent-facing chronicle marker (no inference triggered → no loop).
12156
13278
  const agent = this.agents.get(agentName);
12157
13279
  if (agent && process.env.SUPPRESS_INFERENCE_FAILED_MARKER !== '1') {
13280
+ const cause = [errorType, httpStatus !== undefined ? `HTTP ${httpStatus}` : undefined]
13281
+ .filter((part) => part !== undefined)
13282
+ .join(', ');
12158
13283
  try {
12159
13284
  agent.getContextManager().addMessage('user', [{
12160
13285
  type: 'text',
12161
13286
  text: `[inference-failed] Your previous turn did not complete: the model ` +
12162
- `call failed and produced no response, so nothing was sent. Reason: ` +
12163
- `${reason}. If this recurs with the same cause, change approach ` +
13287
+ `call failed${cause ? ` (${cause})` : ''} and produced no response, so nothing was sent. Reason: ` +
13288
+ `${markerReason}. If this recurs with the same cause, change approach ` +
12164
13289
  `rather than retrying identically (e.g. drop an oversized attachment ` +
12165
13290
  `or an unsupported setting).`,
12166
- }], { system: true, kind: 'inference-failed', reason, consecutive: streak });
13291
+ }], { system: true, kind: 'inference-failed', reason: markerReason, consecutive: streak, ...classification, ...cutAt(markerReason) });
12167
13292
  }
12168
13293
  catch (err) {
12169
- console.error(`[inference-failed] could not record chronicle marker for ${agentName}:`, err);
13294
+ console.error(`[inference-failed] could not record chronicle marker for ${agentName}: ${describeFailureWithFrames(err)}`);
12170
13295
  }
12171
13296
  }
12172
13297
  // (2b) OverBudget deadlock breaker. When compile fails with
@@ -12184,16 +13309,22 @@ export class AgentFramework {
12184
13309
  // (e.g. a reason string that crossed a serialization boundary). It matches
12185
13310
  // CM's current OverBudgetError wording and MAY rot if CM rewords it — the
12186
13311
  // errorType gate is the one that's load-bearing.
13312
+ //
13313
+ // The fallback applies only to an unclassified reason. A classified
13314
+ // failure (a provider's 400, say) already says what it is, and its
13315
+ // message is the provider's text, which can quote anything, including
13316
+ // these phrases from the agent's own context.
12187
13317
  const overBudget = errorType === 'over_budget' ||
12188
13318
  errorType === 'context_refusal' ||
12189
- /exceed hard budget|no summary covers/i.test(reason);
13319
+ // Decided on the whole reason: presentation bounds never change it.
13320
+ (errorType === undefined && /exceed hard budget|no summary covers/i.test(wholeReason));
12190
13321
  // Observability: a refused compile means the agent cannot think AT ALL
12191
13322
  // this turn — surface it on the ops-alert pipeline (ops:alert trace +
12192
13323
  // webhook → fleet-watch) immediately, not only at the hard-down streak.
12193
13324
  // The per-(agent,kind) cooldown throttles repeats; failures.log already
12194
13325
  // has the per-exhaustion record from logFailure above.
12195
13326
  if (overBudget) {
12196
- this.opsAlert('context-refusal', agentName, `compile refused (${errorType ?? 'unclassified'}): ${reason.slice(0, 300)} — ` +
13327
+ this.opsAlert('context-refusal', agentName, `compile refused (${errorType ?? 'unclassified'}): ${boundFailureText(wholeReason, FAILURE_SUMMARY_MAX_CHARS)} — ` +
12197
13328
  `drain breaker ${agent ? 'kicking' : 'unavailable (no agent handle)'}; ` +
12198
13329
  `if this persists, raise contextBudgetTokens or overBudgetGraceRatio.`, { skipLog: true });
12199
13330
  }
@@ -12210,7 +13341,7 @@ export class AgentFramework {
12210
13341
  console.error(`[inference-failed] drain kicked for ${agentName} (OverBudget breaker, ${ticks} ticks)`);
12211
13342
  }
12212
13343
  catch (err) {
12213
- console.error(`[inference-failed] drain kick failed for ${agentName} after ${ticks} ticks:`, err);
13344
+ console.error(`[inference-failed] drain kick failed for ${agentName} after ${ticks} ticks: ${describeFailureWithFrames(err)}`);
12214
13345
  }
12215
13346
  finally {
12216
13347
  this.overBudgetDrainInFlight.delete(agentName);
@@ -12224,18 +13355,26 @@ export class AgentFramework {
12224
13355
  // Ops alert (webhook + ops:alert trace), throttled per (agent, kind) —
12225
13356
  // a hard-down agent re-posts every ~15 min, not on every failed retry.
12226
13357
  // failures.log already got the per-exhaustion record above.
12227
- this.opsAlert('hard-down', agentName, `${streak} consecutive inference failures. Last reason: ${reason}`, { skipLog: true });
12228
- // (4) Poison-history breaker: ONLY for `invalid_request` — a 400-class
12229
- // rejection of the history itself. Deliberately NOT keyed on
12230
- // `retryable === false`, which membrane also returns for auth (expired
12231
- // key), abort (deliberate cancel), context_length (compression's job,
12232
- // and shedding newest is the wrong direction), safety and unsupported —
12233
- // none of which mean the history is poisoned, and all of which would
12234
- // otherwise shed good exchanges and stamp a false "the API kept
12235
- // rejecting your history" marker. `retryable` is kept only for the trace.
13358
+ // The excerpt leaves room for the prefix, so the alert's own bound
13359
+ // never cuts it a second time.
13360
+ const hardDown = `${streak} consecutive inference failures. Last reason: `;
13361
+ this.opsAlert('hard-down', agentName, hardDown + boundFailureText(wholeReason, FAILURE_TEXT_MAX_CHARS - hardDown.length), { skipLog: true });
13362
+ // (4) Poison-history breaker: ONLY when shedMayHelp(errorType,
13363
+ // httpStatus) holds, for an `invalid_request` at HTTP 400, 413 or 422 or
13364
+ // with no status, or a `context_length` at 413, which suggests the
13365
+ // provider refused the request's content or size. The current failure
13366
+ // decides, whatever failed before it in the streak. Deliberately NOT
13367
+ // keyed on `retryable === false`, which membrane also returns for auth
13368
+ // (expired key), abort (deliberate cancel), a context_length at 400 or
13369
+ // 422 (a token overflow, which the context budget exists to prevent;
13370
+ // shedding the newest exchange is the wrong direction for it), safety
13371
+ // and unsupported. For none of them is shedding the newest exchange
13372
+ // likely to help, and each would otherwise shed good exchanges and
13373
+ // stamp a rewind marker saying the API kept refusing them.
13374
+ // `retryable` is kept only for the trace.
12236
13375
  void retryable;
12237
- if (errorType === 'invalid_request' && agent) {
12238
- this.quarantinePoisonedHistory(agent, reason);
13376
+ if (agent && shedMayHelp(errorType, httpStatus)) {
13377
+ this.quarantinePoisonedHistory(agent, wholeReason);
12239
13378
  }
12240
13379
  }
12241
13380
  }
@@ -12245,14 +13384,14 @@ export class AgentFramework {
12245
13384
  * primitives as the refusal auto-rewind / `/unstick` (shedNewestTurn +
12246
13385
  * the single consolidated episode marker) — and queues a retry so the
12247
13386
  * rewound history is verified immediately instead of waiting for the next
12248
- * push event to wake the agent onto the same poisoned context.
13387
+ * push event to wake the agent onto the same context.
12249
13388
  *
12250
13389
  * Bounded: at most `refusalHandling.maxRewinds` (default 3, hard cap 10)
12251
13390
  * sheds per failure episode; the budget resets on any successful inference.
12252
13391
  * At the cap (or with nothing left to shed) it stops — the agent stays up
12253
13392
  * and hard-down logging continues, but no further history is consumed.
12254
13393
  */
12255
- quarantinePoisonedHistory(agent, reason) {
13394
+ quarantinePoisonedHistory(agent, wholeReason) {
12256
13395
  const cap = Math.max(1, Math.min(10, agent.refusalHandling?.maxRewinds ?? 3));
12257
13396
  const used = this.exhaustionRewinds.get(agent.name) ?? 0;
12258
13397
  if (used >= cap) {
@@ -12267,7 +13406,7 @@ export class AgentFramework {
12267
13406
  return;
12268
13407
  }
12269
13408
  this.exhaustionRewinds.set(agent.name, used + 1);
12270
- const count = this.updateRewindMarker(agent, truncateReason(reason), 'inference-failure');
13409
+ const count = this.updateRewindMarker(agent, boundFailureText(wholeReason, REWIND_MARKER_REASON_MAX_CHARS), 'inference-failure');
12271
13410
  console.error(`[inference-rewind] agent=${agent.name} auto-quarantined ${rec.descriptor} ` +
12272
13411
  `(${count} turn(s) shed this episode, cap ${cap}) after non-retryable ` +
12273
13412
  `inference failures — retrying on the rewound history.`);
@@ -12461,7 +13600,11 @@ export class AgentFramework {
12461
13600
  // Discover tools from all connected servers
12462
13601
  await this.refreshMcplTools();
12463
13602
  }
12464
- /** Reconcile the durable ledger with Chronicle, then deliver every server's work. */
13603
+ /**
13604
+ * Reconcile the durable ledger with Chronicle, then deliver every server's
13605
+ * work. A drain still running is waited for and then followed by a fresh
13606
+ * pass (drainForGeneration).
13607
+ */
12465
13608
  async syncDiscordAwarenessMarkers(_onlyServerId) {
12466
13609
  if (!this.discordAwarenessOutbox)
12467
13610
  return;
@@ -12629,34 +13772,66 @@ export class AgentFramework {
12629
13772
  throw new DiscordAwarenessAccountingError('branch reconciliation', error);
12630
13773
  }
12631
13774
  const pending = this.readDiscordAwarenessPending();
12632
- if (pending.length === 0) {
13775
+ // A drain that is still running holds operations it read before this
13776
+ // reconciliation, and it sends them whatever the ledger now says. A branch
13777
+ // move can hide them from `pending`: restoring a rollback's source makes
13778
+ // its marks undesired, and an add whose reply has not arrived is not yet
13779
+ // present, so the ledger shows nothing to do while the drain goes on to
13780
+ // send its saved adds and then the removals they need. So a generation
13781
+ // accounts for every running drain as well as for every pending
13782
+ // operation, and waits for those drains instead of releasing the data
13783
+ // planes beside them (drainForGeneration).
13784
+ const serverIds = [...new Set([
13785
+ ...pending.map((operation) => operation.ref.serverId),
13786
+ ...this.discordAwarenessDrains.keys(),
13787
+ ])];
13788
+ if (serverIds.length === 0) {
12633
13789
  return {
12634
13790
  requiresBarrier: false,
12635
13791
  promise: Promise.resolve({ status: 'delivered', delivered: 0, failed: 0 }),
12636
13792
  };
12637
13793
  }
12638
- const serverIds = [...new Set(pending.map((operation) => operation.ref.serverId))];
12639
- const promise = Promise.all(serverIds.map((serverId) => this.drainDiscordAwarenessOutbox(serverId))).then((outcomes) => {
12640
- if (outcomes.every((outcome) => outcome.status === 'unavailable')) {
12641
- return {
12642
- status: 'unavailable',
12643
- accounted: outcomes.reduce((count, outcome) => count + (outcome.status === 'unavailable' ? outcome.accounted : 0), 0),
12644
- };
12645
- }
12646
- return {
12647
- status: 'delivered',
12648
- delivered: outcomes.reduce((count, outcome) => count + (outcome.status === 'delivered' ? outcome.delivered : 0), 0),
12649
- failed: outcomes.reduce((count, outcome) => count + (outcome.status === 'delivered' ? outcome.failed : 0), 0),
12650
- };
12651
- });
13794
+ const promise = Promise.all(serverIds.map((serverId) => this.drainForGeneration(serverId))).then(combineDrainOutcomes);
12652
13795
  return { requiresBarrier: true, promise };
12653
13796
  }
13797
+ /**
13798
+ * One server's drain for a new generation: a gate completes only after
13799
+ * every operation pending when it was armed has been attempted. A drain
13800
+ * already running can't promise that. It never re-sends a key it has
13801
+ * attempted, so an operation re-pended while it runs (a rollback, its undo,
13802
+ * then a redo, all mid-drain) stays pending, and it may already have made
13803
+ * its last read of the ledger, missing a batch activated since. So a
13804
+ * generation that finds a drain running waits for it, then drains again
13805
+ * with a fresh pass that reads the ledger anew. Two generations waiting on
13806
+ * the same drain share the fresh pass that follows it.
13807
+ */
13808
+ drainForGeneration(serverId) {
13809
+ const running = this.discordAwarenessDrains.get(serverId);
13810
+ if (!running)
13811
+ return this.drainDiscordAwarenessOutbox(serverId);
13812
+ return running.then((first) => this.drainDiscordAwarenessOutbox(serverId)
13813
+ .then((fresh) => combineDrainOutcomes([first, fresh])));
13814
+ }
13815
+ /**
13816
+ * Wait out the awareness gate for an inbound event: the generation current
13817
+ * now, and any generation installed while waiting, until none is current
13818
+ * or the current one is one already waited for. A handler that waited only
13819
+ * for the generation it first saw would go on behind a newer one armed
13820
+ * meanwhile. Rejects as the generation it waited for rejects.
13821
+ */
13822
+ async awaitDiscordAwarenessBarriers() {
13823
+ let waited = null;
13824
+ for (let barrier = this.discordAwarenessBarrier; barrier && barrier !== waited; barrier = this.discordAwarenessBarrier) {
13825
+ waited = barrier;
13826
+ await barrier.promise;
13827
+ }
13828
+ }
12654
13829
  /**
12655
13830
  * Install one framework-global Discord-awareness generation before releasing
12656
- * any inference-bearing event. With pending work, every MCPL data plane is
12657
- * paused while all control planes remain live. With no pending work, ready()
12658
- * runs synchronously so request responders preserve their historical
12659
- * same-stack behavior.
13831
+ * any inference-bearing event. With pending work or a drain still running,
13832
+ * every MCPL data plane is paused while all control planes remain live.
13833
+ * With neither, ready() runs synchronously so request responders preserve
13834
+ * their historical same-stack behavior.
12660
13835
  */
12661
13836
  installMcplDataPlaneGate() {
12662
13837
  for (const connection of this.mcplServerRegistry?.getAllServers() ?? []) {
@@ -12744,6 +13919,13 @@ export class AgentFramework {
12744
13919
  if (!this.mcplServerRegistry || !this.mcplHostCapabilities) {
12745
13920
  throw new Error('MCPL subsystem is not initialized');
12746
13921
  }
13922
+ // Every connect needs a readable awareness ledger: a runtime connect's
13923
+ // awareness barrier reconciles it, and a stdio child taking the
13924
+ // protective default derives that from it. Read it first, so an
13925
+ // unreadable ledger refuses the connect before anything is registered
13926
+ // under its id or spawned (a reconnecting server would otherwise be left
13927
+ // registered as a closed stub). With no outbox configured nothing is read.
13928
+ this.getPlacedReactionBaseline();
12747
13929
  // Register prefix + config for tool dispatch routing (idempotent with
12748
13930
  // the pre-registration pass in initializeMcpl).
12749
13931
  const prefix = config.toolPrefix ?? `mcpl--${config.id}`;
@@ -12754,7 +13936,31 @@ export class AgentFramework {
12754
13936
  if (this.channelRegistry) {
12755
13937
  this.channelRegistry.setSubscriptionPolicy(config.id, config.channelSubscription ?? 'manual');
12756
13938
  }
12757
- const connection = await this.mcplServerRegistry.addServer(config, this.mcplHostCapabilities);
13939
+ // A stdio child's protective reaction default, DISCORD_SUPPRESSED_REACTIONS_BASELINE
13940
+ // (WebSocket servers own their environment outside this process). The
13941
+ // caller's config stays unchanged, so a runtime restart derives afresh.
13942
+ // The default is derived at each spawn: the env below is a getter, which
13943
+ // the transport reads at first connect, at every automatic reconnect and
13944
+ // at a stub's retry, so retained markers added since the last spawn are
13945
+ // covered.
13946
+ let connectionConfig = config;
13947
+ if (!isWebSocketTransport(config)) {
13948
+ const declared = this.declaredReactionBaseline(config);
13949
+ if (declared) {
13950
+ console.error(`[mcpl] ${config.id}: DISCORD_SUPPRESSED_REACTIONS_BASELINE ${declared.source === 'env' ? 'from its env' : 'inherited from the host'} ` +
13951
+ `replaces the framework's protective default`);
13952
+ }
13953
+ connectionConfig = { ...config };
13954
+ Object.defineProperty(connectionConfig, 'env', {
13955
+ enumerable: true,
13956
+ configurable: true,
13957
+ get: () => ({
13958
+ ...config.env,
13959
+ DISCORD_SUPPRESSED_REACTIONS_BASELINE: declared?.value ?? this.getPlacedReactionBaseline().join(','),
13960
+ }),
13961
+ });
13962
+ }
13963
+ const connection = await this.mcplServerRegistry.addServer(connectionConfig, this.mcplHostCapabilities);
12758
13964
  // Wire listeners before either startup staging or the runtime global gate
12759
13965
  // releases control traffic needed for registration and marker service.
12760
13966
  this.wireMcplEvents(connection);
@@ -13076,6 +14282,91 @@ export class AgentFramework {
13076
14282
  // ==========================================================================
13077
14283
  // Runtime MCPL server lifecycle (agent-facing hot deploy/restart/unload)
13078
14284
  // ==========================================================================
14285
+ /**
14286
+ * Snapshot the protective Discord reaction default for this deployment, as
14287
+ * suppression tokens: the forms the Discord adapter's suppression matcher
14288
+ * reads, not marker payloads for an add, a remove or a replay. It covers
14289
+ * the reactions the framework places: refusal markers, the adapter's own
14290
+ * default awareness marker, the configured awareness marker, and retained
14291
+ * outbox markers (including offline recovery choices and marks that can
14292
+ * later be removed or replayed). A custom emoji appears as its bare ID,
14293
+ * which suppression matches even after a rename; stored markers and the
14294
+ * actual add/remove payloads stay unchanged. Explicit adapter or operator
14295
+ * suppression settings still take precedence. Throws on an unreadable
14296
+ * ledger rather than reporting a partial baseline.
14297
+ */
14298
+ getPlacedReactionBaseline() {
14299
+ let retained;
14300
+ try {
14301
+ retained = this.discordAwarenessOutbox?.batches().map((batch) => batch.emoji) ?? [];
14302
+ }
14303
+ catch (error) {
14304
+ throw new DiscordAwarenessAccountingError('reaction-baseline ledger read', error);
14305
+ }
14306
+ // Discord events carry the custom emoji's name and ID separately. Full
14307
+ // marker forms belong in add/remove calls, but suppression matches their
14308
+ // stable ID (including after a rename), rather than the composite token.
14309
+ const suppressionToken = (emoji) => {
14310
+ const custom = /^(?:<a?:\w+:(\d{17,20})>|(?:a:)?\w+:(\d{17,20}))$/.exec(emoji.trim());
14311
+ return custom?.[1] ?? custom?.[2] ?? emoji;
14312
+ };
14313
+ return [...new Set([
14314
+ ...REFUSAL_REACTION_BASELINE,
14315
+ DEFAULT_DISCORD_AWARENESS_EMOJI,
14316
+ this.discordAwarenessEmoji,
14317
+ ...retained,
14318
+ ].map(suppressionToken))].filter((emoji) => emoji.length > 0);
14319
+ }
14320
+ /**
14321
+ * The reaction baseline an operator chose for a stdio server, if any: a
14322
+ * value declared in its env, or one inherited from the host with
14323
+ * `inheritEnv`, the declared one first. A value with no tokens, as
14324
+ * discord-mcpl's parser reads them, counts as unset: the adapter splits on
14325
+ * commas and keeps a token only when its normalizer leaves something of it
14326
+ * (blank, `:`, `::` and a lone U+FE0F leave nothing), so such a value (an
14327
+ * empty systemd `Environment=`, say, or `${A},${B}` or `:${NAME}:` with the
14328
+ * variables empty) would otherwise turn the protective default off without
14329
+ * anyone choosing that.
14330
+ *
14331
+ * Windows names are matched in both sources: Workers expose a case-
14332
+ * sensitive `process.env` copy. Within one source, Node's spawn keeps the
14333
+ * first spelling in sorted order, so each source is resolved that way.
14334
+ *
14335
+ * A value is read as the child will see it. A config loaded from JSON can
14336
+ * carry a non-string there despite the type, and spawn writes every
14337
+ * defined value as its string form, so a number is a declared value, not a
14338
+ * failed connect. A null counts as unset, like a blank: written in JSON it
14339
+ * means no value, and read as spawn's "null" it would replace the
14340
+ * protective default with a list that suppresses nothing.
14341
+ */
14342
+ declaredReactionBaseline(config) {
14343
+ // A token as discord-mcpl's parser keeps one: `parseSuppressionEnvTokens`
14344
+ // trims each comma-separated token and keeps it when
14345
+ // `normalizeReactionEmoji` (discord-mcpl `src/filters.ts`) leaves
14346
+ // something, after it strips U+FE0F, trims, and strips one leading and one
14347
+ // trailing colon. The same steps in the same order, so the two can be
14348
+ // compared when either changes.
14349
+ const isToken = (token) => token.trim().replace(/\uFE0F/g, '').trim().replace(/^:|:$/g, '') !== '';
14350
+ const valueIn = (env) => {
14351
+ let raw;
14352
+ if (process.platform !== 'win32')
14353
+ raw = env?.DISCORD_SUPPRESSED_REACTIONS_BASELINE;
14354
+ else {
14355
+ const key = Object.keys(env ?? {}).sort()
14356
+ .find((name) => name.toUpperCase() === 'DISCORD_SUPPRESSED_REACTIONS_BASELINE');
14357
+ raw = key === undefined ? undefined : env?.[key];
14358
+ }
14359
+ if (raw === undefined || raw === null)
14360
+ return undefined;
14361
+ const value = String(raw);
14362
+ return value.split(',').some(isToken) ? value : undefined;
14363
+ };
14364
+ const declared = valueIn(config.env);
14365
+ if (declared !== undefined)
14366
+ return { value: declared, source: 'env' };
14367
+ const inherited = config.inheritEnv ? valueIn(process.env) : undefined;
14368
+ return inherited !== undefined ? { value: inherited, source: 'inherited' } : undefined;
14369
+ }
13079
14370
  /**
13080
14371
  * Connect a new MCPL server at runtime. Refreshes the tool list and
13081
14372
  * notifies the agent of newly available tools. Throws if the MCPL
@@ -13141,6 +14432,12 @@ export class AgentFramework {
13141
14432
  if (!config) {
13142
14433
  throw new Error(`MCPL server "${id}" is not configured`);
13143
14434
  }
14435
+ // Every connect reads the awareness ledger: its awareness barrier
14436
+ // reconciles it, and a stdio child taking the protective default derives
14437
+ // that from it. Read it before disconnecting, so an unreadable ledger
14438
+ // refuses the restart and leaves the running child up, whatever the
14439
+ // server's baseline or transport. (No outbox configured: nothing to read.)
14440
+ this.getPlacedReactionBaseline();
13144
14441
  await this.disconnectMcplServer(id);
13145
14442
  await this.connectMcplServer(config);
13146
14443
  }
@@ -13207,9 +14504,11 @@ export class AgentFramework {
13207
14504
  connection.on('push-event', async (params, responder) => {
13208
14505
  const answer = answerOnce(responder, 'push/event', connection.id);
13209
14506
  try {
13210
- const barrier = this.discordAwarenessBarrier;
13211
- if (barrier)
13212
- await barrier.promise;
14507
+ // Only when a generation is current: with none, the event is handled in
14508
+ // the turn it arrives, as it always was (the push coalescer's ordering of
14509
+ // a push against its retraction depends on that).
14510
+ if (this.discordAwarenessBarrier)
14511
+ await this.awaitDiscordAwarenessBarriers();
13213
14512
  // Checked here, after the barrier and before anything records the
13214
14513
  // push as accepted: a stopped host refuses it.
13215
14514
  if (this.queue?.isClosed) {
@@ -13237,9 +14536,8 @@ export class AgentFramework {
13237
14536
  if (this.inferenceRouter && responder) {
13238
14537
  const answer = answerOnce(responder, 'inference/request', connection.id);
13239
14538
  try {
13240
- const barrier = this.discordAwarenessBarrier;
13241
- if (barrier)
13242
- await barrier.promise;
14539
+ if (this.discordAwarenessBarrier)
14540
+ await this.awaitDiscordAwarenessBarriers();
13243
14541
  // A stopping host starts no inference for a server: refused, like a
13244
14542
  // push, before the router runs.
13245
14543
  if (this.queue?.isClosed) {
@@ -13331,9 +14629,8 @@ export class AgentFramework {
13331
14629
  connection.on('channels-incoming', async (params, responder) => {
13332
14630
  const answer = answerOnce(responder, 'channels/incoming', connection.id);
13333
14631
  try {
13334
- const barrier = this.discordAwarenessBarrier;
13335
- if (barrier)
13336
- await barrier.promise;
14632
+ if (this.discordAwarenessBarrier)
14633
+ await this.awaitDiscordAwarenessBarriers();
13337
14634
  if (this.queue?.isClosed) {
13338
14635
  this.refuseServerItem('channels/incoming', connection.id, answer, { kind: 'refused' });
13339
14636
  return;
@@ -13350,9 +14647,8 @@ export class AgentFramework {
13350
14647
  return;
13351
14648
  const answer = answerOnce(responder, 'host/command', connection.id);
13352
14649
  try {
13353
- const barrier = this.discordAwarenessBarrier;
13354
- if (barrier)
13355
- await barrier.promise;
14650
+ if (this.discordAwarenessBarrier)
14651
+ await this.awaitDiscordAwarenessBarriers();
13356
14652
  // A stopping host runs no host command: refused like the requests
13357
14653
  // above, before its handler runs.
13358
14654
  if (this.queue?.isClosed) {
@@ -13394,7 +14690,7 @@ export class AgentFramework {
13394
14690
  // grant) before the data-plane barrier releases — otherwise the first
13395
14691
  // post-reconnect events land while the grant is still empty and are
13396
14692
  // rejected fail-closed instead of delivered.
13397
- void (async () => {
14693
+ const reregistered = (async () => {
13398
14694
  try {
13399
14695
  const config = this.mcplServerConfigs.get(connection.id);
13400
14696
  if (config) {
@@ -13407,7 +14703,11 @@ export class AgentFramework {
13407
14703
  this.releaseMcplDataPlaneGate(awarenessBarrier);
13408
14704
  this.handleToolsListChanged(connection.id);
13409
14705
  })();
13410
- void awarenessBarrier.promise.then(() => {
14706
+ // The gate completes, and the data planes go ready, only once both the
14707
+ // awareness drain and the re-registration have finished. A drain can
14708
+ // settle first (one already running, or little to send), and the grant
14709
+ // must exist before the first data-plane event flows.
14710
+ void Promise.all([awarenessBarrier.promise, reregistered]).then(() => {
13411
14711
  if (awarenessBarrier.requiresBarrier
13412
14712
  && !this.completeMcplDataPlaneGate(awarenessBarrier))
13413
14713
  return;
@@ -14466,9 +15766,21 @@ export class AgentFramework {
14466
15766
  };
14467
15767
  }
14468
15768
  if (candidate.kind === 'reference') {
15769
+ // Point at fetch_reference only where it can deliver: not for a
15770
+ // reference this process holds no record of (fetch_reference
15771
+ // looks the id up in the same registry), nor, as the stub does,
15772
+ // for an unfetched one the fetcher would refuse as expired.
15773
+ const record = referenceRegistry.get(candidate.refId);
15774
+ const expiry = record && !record.fetchedPath ? referenceExpiry(record.testimony) : null;
14469
15775
  throw new Error(`save_recent_image: image at index ${position} is a reference (${candidate.refId}, ${candidate.mediaType}, ` +
14470
15776
  `from ${describeTool(candidate)}) — its bytes are not held here. Nothing written. ` +
14471
- 'Fetch it with fetch_reference and save the materialized file instead.');
15777
+ (!record
15778
+ ? 'This process holds no record of it (a restart or an eviction drops records, and a stub quoted ' +
15779
+ "from elsewhere never had one), so fetch_reference can't fetch it either. If its tool can produce " +
15780
+ 'it again, re-run that tool for a fresh reference.'
15781
+ : expiry
15782
+ ? `It can't be fetched either — ${EXPIRY_REASON[expiry]}.`
15783
+ : 'Fetch it with fetch_reference and save the materialized file instead.'));
14472
15784
  }
14473
15785
  if (candidate.ref === null) {
14474
15786
  throw new Error(`save_recent_image: image at index ${position} is a tool-result image from ${describeTool(candidate)} ` +
@@ -14568,14 +15880,16 @@ export class AgentFramework {
14568
15880
  consider({ kind: 'tool', ref: ph.ref, toolCallId, toolName, mediaType: ph.mediaType, messagesBack });
14569
15881
  continue;
14570
15882
  }
14571
- // Reference stub: the registry's testimony (when this process
14572
- // still has the record) says whether it was an image; the
14573
- // sniffed stub text is the fallback. Non-image references
14574
- // are not slots.
14575
- const testimony = referenceRegistry.get(ph.refId)?.testimony.mimeType?.toLowerCase() ?? ph.mediaType;
14576
- if (!testimony || !testimony.startsWith('image/'))
15883
+ // Reference stub: the registry's claimed type (when this process
15884
+ // still has the record), else the first image/ token in the
15885
+ // stub's text, which can come from its name, says whether it was
15886
+ // an image. Either is read as the stub shows a type, sanitized
15887
+ // and bounded. Non-image references are not slots.
15888
+ const claimed = referenceRegistry.get(ph.refId)?.testimony.mimeType ?? ph.mediaType ?? undefined;
15889
+ const mediaType = referenceTypeLabel(claimed).toLowerCase();
15890
+ if (!mediaType.startsWith('image/'))
14577
15891
  continue;
14578
- consider({ kind: 'reference', refId: ph.refId, mediaType: testimony, toolCallId, toolName, messagesBack });
15892
+ consider({ kind: 'reference', refId: ph.refId, mediaType, toolCallId, toolName, messagesBack });
14579
15893
  }
14580
15894
  };
14581
15895
  // Same-batch results (e.g. a snapshot dispatched alongside the save)
@@ -14622,10 +15936,14 @@ export class AgentFramework {
14622
15936
  const block = content[b];
14623
15937
  if (block?.type === 'image') {
14624
15938
  const inline = block.source?.type === 'base64' && typeof block.source.data === 'string';
15939
+ // The sender's word, shown in the receipt and stored with the
15940
+ // file, so it's taken as a label; with nothing left of it, it
15941
+ // counts as absent.
15942
+ const claimed = block.source?.mediaType;
14625
15943
  consider({
14626
15944
  kind: 'attachment',
14627
15945
  data: inline ? block.source.data : null,
14628
- mediaType: block.source?.mediaType ?? 'image/png',
15946
+ mediaType: (typeof claimed === 'string' ? sanitizeLabel(claimed, REFERENCE_LIMITS.mimeType) : '') || 'image/png',
14629
15947
  messagesBack,
14630
15948
  });
14631
15949
  }