@animalabs/agent-framework 0.22.0 → 0.22.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (147) hide show
  1. package/README.md +11 -4
  2. package/dist/src/agent.d.ts.map +1 -1
  3. package/dist/src/agent.js +3 -1
  4. package/dist/src/agent.js.map +1 -1
  5. package/dist/src/api/server.d.ts.map +1 -1
  6. package/dist/src/api/server.js +41 -4
  7. package/dist/src/api/server.js.map +1 -1
  8. package/dist/src/framework.d.ts +319 -20
  9. package/dist/src/framework.d.ts.map +1 -1
  10. package/dist/src/framework.js +1500 -182
  11. package/dist/src/framework.js.map +1 -1
  12. package/dist/src/index.d.ts +2 -2
  13. package/dist/src/index.d.ts.map +1 -1
  14. package/dist/src/index.js.map +1 -1
  15. package/dist/src/mcpl/index.d.ts +2 -0
  16. package/dist/src/mcpl/index.d.ts.map +1 -1
  17. package/dist/src/mcpl/index.js.map +1 -1
  18. package/dist/src/mcpl/reference-fetcher.d.ts +2 -1
  19. package/dist/src/mcpl/reference-fetcher.d.ts.map +1 -1
  20. package/dist/src/mcpl/reference-fetcher.js +20 -9
  21. package/dist/src/mcpl/reference-fetcher.js.map +1 -1
  22. package/dist/src/mcpl/references.d.ts +29 -6
  23. package/dist/src/mcpl/references.d.ts.map +1 -1
  24. package/dist/src/mcpl/references.js +57 -16
  25. package/dist/src/mcpl/references.js.map +1 -1
  26. package/dist/src/mcpl/server-connection.d.ts +28 -1
  27. package/dist/src/mcpl/server-connection.d.ts.map +1 -1
  28. package/dist/src/mcpl/server-connection.js +32 -4
  29. package/dist/src/mcpl/server-connection.js.map +1 -1
  30. package/dist/src/module-registry.d.ts +9 -3
  31. package/dist/src/module-registry.d.ts.map +1 -1
  32. package/dist/src/module-registry.js +3 -2
  33. package/dist/src/module-registry.js.map +1 -1
  34. package/dist/src/modules/workspace/index.d.ts.map +1 -1
  35. package/dist/src/modules/workspace/index.js +46 -4
  36. package/dist/src/modules/workspace/index.js.map +1 -1
  37. package/dist/src/modules/workspace/observe.d.ts +13 -3
  38. package/dist/src/modules/workspace/observe.d.ts.map +1 -1
  39. package/dist/src/modules/workspace/observe.js +24 -4
  40. package/dist/src/modules/workspace/observe.js.map +1 -1
  41. package/dist/src/modules/workspace/reconcile.d.ts +14 -0
  42. package/dist/src/modules/workspace/reconcile.d.ts.map +1 -1
  43. package/dist/src/modules/workspace/reconcile.js +10 -2
  44. package/dist/src/modules/workspace/reconcile.js.map +1 -1
  45. package/dist/src/operator-log.d.ts +41 -0
  46. package/dist/src/operator-log.d.ts.map +1 -1
  47. package/dist/src/operator-log.js.map +1 -1
  48. package/dist/src/provider-waits.d.ts +216 -0
  49. package/dist/src/provider-waits.d.ts.map +1 -0
  50. package/dist/src/provider-waits.js +482 -0
  51. package/dist/src/provider-waits.js.map +1 -0
  52. package/dist/src/refusal-reactions.d.ts +14 -12
  53. package/dist/src/refusal-reactions.d.ts.map +1 -1
  54. package/dist/src/refusal-reactions.js +14 -12
  55. package/dist/src/refusal-reactions.js.map +1 -1
  56. package/dist/src/tool-result-guard.d.ts +30 -13
  57. package/dist/src/tool-result-guard.d.ts.map +1 -1
  58. package/dist/src/tool-result-guard.js +71 -25
  59. package/dist/src/tool-result-guard.js.map +1 -1
  60. package/dist/src/types/events.d.ts +12 -0
  61. package/dist/src/types/events.d.ts.map +1 -1
  62. package/dist/src/types/module.d.ts +12 -4
  63. package/dist/src/types/module.d.ts.map +1 -1
  64. package/dist/src/types/trace.d.ts +14 -3
  65. package/dist/src/types/trace.d.ts.map +1 -1
  66. package/dist/test/discord-awareness-outbox.test.js +3 -0
  67. package/dist/test/discord-awareness-outbox.test.js.map +1 -1
  68. package/dist/test/explicit-prose-routing.test.js +0 -2
  69. package/dist/test/explicit-prose-routing.test.js.map +1 -1
  70. package/dist/test/framework.test.js +160 -1
  71. package/dist/test/framework.test.js.map +1 -1
  72. package/dist/test/helpers/coalescing-fixture.d.ts.map +1 -1
  73. package/dist/test/helpers/coalescing-fixture.js +3 -2
  74. package/dist/test/helpers/coalescing-fixture.js.map +1 -1
  75. package/dist/test/inference-exhaustion-rewind.test.js +142 -14
  76. package/dist/test/inference-exhaustion-rewind.test.js.map +1 -1
  77. package/dist/test/inference-failure-bounds.test.d.ts +2 -0
  78. package/dist/test/inference-failure-bounds.test.d.ts.map +1 -0
  79. package/dist/test/inference-failure-bounds.test.js +1201 -0
  80. package/dist/test/inference-failure-bounds.test.js.map +1 -0
  81. package/dist/test/late-events-after-stop.test.d.ts +2 -0
  82. package/dist/test/late-events-after-stop.test.d.ts.map +1 -0
  83. package/dist/test/late-events-after-stop.test.js +122 -0
  84. package/dist/test/late-events-after-stop.test.js.map +1 -0
  85. package/dist/test/mcpl-awareness-barrier.test.js +141 -0
  86. package/dist/test/mcpl-awareness-barrier.test.js.map +1 -1
  87. package/dist/test/mcpl-coalescing.test.js +78 -16
  88. package/dist/test/mcpl-coalescing.test.js.map +1 -1
  89. package/dist/test/mcpl-config-diagnostics.test.js +6 -3
  90. package/dist/test/mcpl-config-diagnostics.test.js.map +1 -1
  91. package/dist/test/mcpl-manifest-changed.test.js +2 -1
  92. package/dist/test/mcpl-manifest-changed.test.js.map +1 -1
  93. package/dist/test/mcpl-reference-fetcher.test.js +232 -4
  94. package/dist/test/mcpl-reference-fetcher.test.js.map +1 -1
  95. package/dist/test/mcpl-reference-fields.test.js +69 -2
  96. package/dist/test/mcpl-reference-fields.test.js.map +1 -1
  97. package/dist/test/mcpl-ws-transport.test.js +103 -0
  98. package/dist/test/mcpl-ws-transport.test.js.map +1 -1
  99. package/dist/test/ops-alert.test.js +2 -1
  100. package/dist/test/ops-alert.test.js.map +1 -1
  101. package/dist/test/placed-reaction-baseline.test.d.ts +2 -0
  102. package/dist/test/placed-reaction-baseline.test.d.ts.map +1 -0
  103. package/dist/test/placed-reaction-baseline.test.js +312 -0
  104. package/dist/test/placed-reaction-baseline.test.js.map +1 -0
  105. package/dist/test/provider-acceleration-cooldown.test.js +5 -1
  106. package/dist/test/provider-acceleration-cooldown.test.js.map +1 -1
  107. package/dist/test/provider-waits.test.d.ts +2 -0
  108. package/dist/test/provider-waits.test.d.ts.map +1 -0
  109. package/dist/test/provider-waits.test.js +2196 -0
  110. package/dist/test/provider-waits.test.js.map +1 -0
  111. package/dist/test/reaction-baseline-platform.test.d.ts +2 -0
  112. package/dist/test/reaction-baseline-platform.test.d.ts.map +1 -0
  113. package/dist/test/reaction-baseline-platform.test.js +124 -0
  114. package/dist/test/reaction-baseline-platform.test.js.map +1 -0
  115. package/dist/test/refusal-reaction-baseline.test.js +10 -3
  116. package/dist/test/refusal-reaction-baseline.test.js.map +1 -1
  117. package/dist/test/save-recent-image.test.js +138 -2
  118. package/dist/test/save-recent-image.test.js.map +1 -1
  119. package/dist/test/surgery-marks-receipt.test.d.ts +2 -0
  120. package/dist/test/surgery-marks-receipt.test.d.ts.map +1 -0
  121. package/dist/test/surgery-marks-receipt.test.js +540 -0
  122. package/dist/test/surgery-marks-receipt.test.js.map +1 -0
  123. package/dist/test/tool-result-guard.test.js +221 -7
  124. package/dist/test/tool-result-guard.test.js.map +1 -1
  125. package/dist/test/workspace-three-way.test.js +115 -0
  126. package/dist/test/workspace-three-way.test.js.map +1 -1
  127. package/dist/tsconfig.tsbuildinfo +1 -1
  128. package/package.json +1 -1
  129. package/src/agent.ts +4 -1
  130. package/src/api/server.ts +43 -4
  131. package/src/framework.ts +1538 -189
  132. package/src/index.ts +7 -2
  133. package/src/mcpl/index.ts +2 -0
  134. package/src/mcpl/reference-fetcher.ts +18 -8
  135. package/src/mcpl/references.ts +57 -13
  136. package/src/mcpl/server-connection.ts +55 -4
  137. package/src/module-registry.ts +17 -9
  138. package/src/modules/workspace/index.ts +44 -4
  139. package/src/modules/workspace/observe.ts +31 -5
  140. package/src/modules/workspace/reconcile.ts +23 -2
  141. package/src/operator-log.ts +30 -0
  142. package/src/provider-waits.ts +523 -0
  143. package/src/refusal-reactions.ts +14 -12
  144. package/src/tool-result-guard.ts +72 -24
  145. package/src/types/events.ts +12 -0
  146. package/src/types/module.ts +12 -4
  147. package/src/types/trace.ts +14 -3
package/src/framework.ts CHANGED
@@ -1,6 +1,6 @@
1
1
  import { ToolPresentation, presentationTools, isPresentationTool, renderCatalogue, type PresentationSnapshot } from "./tool-presentation.js";
2
2
  import { dirname, join } from 'node:path';
3
- import { INLINE_WITHHELD_TEXT, classifyBlock, isInlineContradiction, referenceRegistry, referenceStubOrNull } from './mcpl/references.js';
3
+ import { EXPIRY_REASON, INLINE_WITHHELD_TEXT, REFERENCE_LIMITS, classifyBlock, isInlineContradiction, isReferenceExpired, referenceExpiry, referenceRegistry, referenceStubOrNull, referenceTypeLabel, sanitizeLabel } from './mcpl/references.js';
4
4
  import { ReferenceFetcher, DEFAULT_FETCH_MAX_BYTES, EAGER_FETCH_TIMEOUT_MS } from './mcpl/reference-fetcher.js';
5
5
  import { appendFileSync, existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs';
6
6
  import { JsStore } from '@animalabs/chronicle';
@@ -62,7 +62,7 @@ import type {
62
62
  } from './types/index.js';
63
63
  import { ProcessQueueImpl } from './queue.js';
64
64
  import { DEFAULT_SHUTDOWN_TIMEOUT_MS, validateShutdownTimeout, shutdownErrorMessage } from './shutdown.js';
65
- import { REFUSAL_REACTIONS, REFUSAL_REACTION_FALLBACK } from './refusal-reactions.js';
65
+ import { REFUSAL_REACTIONS, REFUSAL_REACTION_FALLBACK, REFUSAL_REACTION_BASELINE } from './refusal-reactions.js';
66
66
  import { Agent } from './agent.js';
67
67
  import { ModuleRegistry, checkedToolResult, isStateExistsError, storableToolResult, thrownResult } from './module-registry.js';
68
68
  import { McplServerRegistry } from './mcpl/server-registry.js';
@@ -88,6 +88,7 @@ import { InferenceRouter } from './mcpl/inference-router.js';
88
88
  import { ChannelRegistry, type ChannelToolOrigin } from './mcpl/channel-registry.js';
89
89
  import { ConversationRouter, CONVERSATION_ROUTING_DEPRECATION_NOTICE } from './mcpl/conversation-router.js';
90
90
  import { safeSlice } from './safe-slice.js';
91
+ import { ProviderWaits, EVERY_MODEL, isReleaseModel, withoutControls, type ProviderWait } from './provider-waits.js';
91
92
  import type { WorkspaceModule } from './modules/workspace/index.js';
92
93
  import type { ToolResultGuard, WithheldSpill } from './tool-result-guard.js';
93
94
  import {
@@ -394,6 +395,7 @@ import { EventGate, formatShadowWarning } from './gate/event-gate.js';
394
395
  import { UsageTracker, type PersistedUsageState } from './usage/usage-tracker.js';
395
396
  import type { SessionUsageSnapshot, UsageUpdatedEvent } from './usage/types.js';
396
397
  import type { McplServerConnection } from './mcpl/server-connection.js';
398
+ import { isWebSocketTransport } from './mcpl/transport.js';
397
399
  import type {
398
400
  McplServerConfig,
399
401
  McplHostCapabilities,
@@ -412,12 +414,14 @@ import {
412
414
  DiscordAwarenessOutbox,
413
415
  defaultDiscordAwarenessOutboxPath,
414
416
  extractDiscordAwarenessRefs,
417
+ type DiscordAwarenessBatch,
415
418
  } from './recovery/discord-awareness-outbox.js';
416
419
  import {
417
420
  OperatorActionError,
418
421
  OperatorLog,
419
422
  capIds,
420
423
  defaultOperatorLogPath,
424
+ type SurgeryMarkerReceipt,
421
425
  type OperatorLogEntry,
422
426
  type OperatorLogInput,
423
427
  type OperatorRequester,
@@ -608,6 +612,21 @@ interface DiscordAwarenessBarrier {
608
612
  promise: Promise<DiscordAwarenessDrainOutcome>;
609
613
  }
610
614
 
615
+ /** Several drains' outcomes as one: unavailable only if every one was. */
616
+ function combineDrainOutcomes(outcomes: DiscordAwarenessDrainOutcome[]): DiscordAwarenessDrainOutcome {
617
+ if (outcomes.every((outcome) => outcome.status === 'unavailable')) {
618
+ return {
619
+ status: 'unavailable',
620
+ accounted: outcomes.reduce((count, outcome) => count + (outcome.status === 'unavailable' ? outcome.accounted : 0), 0),
621
+ };
622
+ }
623
+ return {
624
+ status: 'delivered',
625
+ delivered: outcomes.reduce((count, outcome) => count + (outcome.status === 'delivered' ? outcome.delivered : 0), 0),
626
+ failed: outcomes.reduce((count, outcome) => count + (outcome.status === 'delivered' ? outcome.failed : 0), 0),
627
+ };
628
+ }
629
+
611
630
  function normalizeDiscordAwarenessDeadline(value: number | undefined): number {
612
631
  if (value === undefined || !Number.isFinite(value) || value <= 0) {
613
632
  return DEFAULT_DISCORD_AWARENESS_DEADLINE_MS;
@@ -814,11 +833,35 @@ function bodyGroupRun(
814
833
  };
815
834
  }
816
835
 
836
+ function describeMarkers(m: SurgeryMarkerReceipt): string {
837
+ if (m.status === 'queued') return `queued:${m.queued}`;
838
+ if (m.status === 'not-scheduled') return 'not-scheduled(batch retired)';
839
+ if (m.status === 'unresolved') return `unresolved(batch ${m.batchId} may still be delivered)`;
840
+ return 'none';
841
+ }
842
+
817
843
  function describeRequester(r: OperatorRequester | undefined): string {
818
844
  if (!r) return 'unknown';
819
845
  return `${r.name ?? r.id ?? 'unknown'} (${r.via})`;
820
846
  }
821
847
 
848
+ /** What a held call or turn says of the wait that holds it: no call was
849
+ * made. The refusal of an auxiliary call and the trace of a turn held once
850
+ * compiled say it the same way. */
851
+ function providerWaitText(agentName: string, model: string, wait: ProviderWait): string {
852
+ return `Provider wait: ${model} is held for ${agentName} ` +
853
+ `${wait.until === null ? 'until an operator releases it' : `until ${new Date(wait.until).toISOString()}`} ` +
854
+ `(recorded ${new Date(wait.setAt).toISOString()}: ${wait.reason}); no call was made`;
855
+ }
856
+
857
+ /** Who asked for an operator act, as a durable record and a log line carry
858
+ * it. The name comes from a caller, so a control character (a forged log
859
+ * line) becomes a space, and it is bounded. */
860
+ function operatorLabel(by: string): string {
861
+ const label = safeSlice(withoutControls(by).trim(), 0, 120);
862
+ return label === '' ? 'operator' : label;
863
+ }
864
+
822
865
  function hostCommandRequester(serverId: string, params: HostCommandParams): OperatorRequester {
823
866
  return {
824
867
  via: `host-command:${serverId}`,
@@ -848,11 +891,11 @@ class DefaultErrorPolicy implements ErrorPolicy {
848
891
  maxRetries = 3;
849
892
 
850
893
  onInferenceError(error: Error, _agentName: string, attempt: number): ErrorAction {
851
- if (error instanceof MembraneError && !error.retryable) {
894
+ if (isMembraneError(error) && error.retryable === false) {
852
895
  return { retry: false };
853
896
  }
854
897
  if (attempt < this.maxRetries) {
855
- const delayMs = error instanceof MembraneError && error.retryAfterMs !== undefined
898
+ const delayMs = isMembraneError(error) && typeof error.retryAfterMs === 'number'
856
899
  ? error.retryAfterMs
857
900
  : Math.pow(2, attempt) * 1000;
858
901
  return { retry: true, delayMs };
@@ -894,6 +937,17 @@ interface ProviderAccelerationCooldown {
894
937
  /** Armed by an auxiliary (compression) failure with no primary involved:
895
938
  * release reopens auxiliary admission without synthesising an inference. */
896
939
  auxiliaryOrigin?: boolean;
940
+ /** Set when this hold is a provider wait (a stated retry-after) for the
941
+ * agent's primary model: it holds primary turns only. Auxiliary calls are
942
+ * admitted per model by the provider waits themselves, so a compression
943
+ * model the provider did not limit keeps working. */
944
+ waitModel?: string;
945
+ /** Set on an acceleration hold whose length is the provider's stated wait
946
+ * (recorded for `model`), which replaced AF's own pacing for the failure.
947
+ * `ownUntil` is when that pacing alone would have ended the hold: a
948
+ * release of the stated wait shortens the hold to it, never past it. A
949
+ * host hold never carries this: its length is the host's. */
950
+ statedWait?: { model: string; ownUntil: number };
897
951
  }
898
952
  interface ProviderAccelerationRecovery {
899
953
  startedAt: number;
@@ -915,10 +969,33 @@ interface ProviderAccelerationReceipt {
915
969
  stopReason: string;
916
970
  }
917
971
  const PROVIDER_ACCELERATION_DEFAULT_COOLDOWN_MS = 65_000;
972
+ /** Cap on waits the framework chooses itself. A provider's stated wait is
973
+ * never capped: it is a lower bound (see ProviderWaits). It is also the
974
+ * longest stated wait an ephemeral run sleeps in its turn: a longer one ends
975
+ * the run instead (nonOwningProviderWait). */
918
976
  const PROVIDER_ACCELERATION_MAX_COOLDOWN_MS = 10 * 60_000;
919
977
  const PROVIDER_ACCELERATION_JITTER_MS = 5_000;
978
+ /** Longest delay one setTimeout can represent (2^31-1 ms, about 24.8 days);
979
+ * a longer timer fires almost at once, so longer waits re-arm in steps. */
980
+ const MAX_TIMER_DELAY_MS = 2_147_483_647;
981
+ /** A stated wait recorded to hold a model longer than this, or until
982
+ * released, raises an ops alert. It is honoured in full either way: a hold
983
+ * that long is a quota-scale event or a provider's mistake, and a person
984
+ * decides which (release-provider-wait). Shorter waits are pacing. */
985
+ const PROVIDER_WAIT_ALERT_MS = 60 * 60_000;
986
+ /**
987
+ * A MembraneError, from this framework's copy of membrane or from another: the
988
+ * host supplies the Membrane instance (config.membrane), so with two copies
989
+ * installed its errors are instances of a class this framework's `instanceof`
990
+ * doesn't match. Recognised by name as well, as context-manager's errors are;
991
+ * its fields are the same, so callers still check each one's type.
992
+ */
993
+ function isMembraneError(error: unknown): error is MembraneError {
994
+ return error instanceof MembraneError || (error instanceof Error && error.name === 'MembraneError');
995
+ }
996
+
920
997
  function isOrganizationAccelerationRateLimit(error: Error): error is MembraneError {
921
- if (!(error instanceof MembraneError) || error.type !== 'rate_limit') return false;
998
+ if (!isMembraneError(error) || error.type !== 'rate_limit') return false;
922
999
  return (
923
1000
  /organization(?:'s)?[^\n]{0,120}maximum(?:\s+usage)?\s+increase\s+rate/i.test(error.message) ||
924
1001
  /organization(?:'s)?[^\n]{0,120}acceleration(?:\s+limit)?/i.test(error.message)
@@ -993,6 +1070,9 @@ interface HostCommandParams {
993
1070
  abandon?: boolean;
994
1071
  /** For the `resume` command: override a failing feasibility verdict. */
995
1072
  force?: boolean;
1073
+ /** For the `release-provider-wait` command: the model whose wait to release
1074
+ * (every model's when omitted). */
1075
+ model?: string;
996
1076
  requesterId?: string;
997
1077
  requesterName?: string;
998
1078
  }
@@ -1015,6 +1095,14 @@ interface Deferred<T> {
1015
1095
  reject: (reason?: unknown) => void;
1016
1096
  }
1017
1097
 
1098
+ /** One onTrace subscription. Removing it, by its unsubscribe or by offTrace,
1099
+ * clears `active` as it splices it out, so a delivery already under way
1100
+ * skips it (see emitTrace). */
1101
+ interface TraceRegistration {
1102
+ listener: TraceEventListener;
1103
+ active: boolean;
1104
+ }
1105
+
1018
1106
  /** Per-run state for a framework-driven ephemeral agent: the settle signal
1019
1107
  * the caller awaits plus the liveness/progress bookkeeping the watchdogs and
1020
1108
  * the stream driver share. One map entry per run — created and torn down in
@@ -1043,11 +1131,256 @@ interface EphemeralRun {
1043
1131
  toolCallsCount: number;
1044
1132
  }
1045
1133
 
1046
- /** Cap an API error message for inline use in a rewind marker: keep enough to
1047
- * identify the failure class without pasting a wall of provider JSON into the
1048
- * agent's context. */
1049
- function truncateReason(reason: string, max = 160): string {
1050
- return reason.length <= max ? reason : reason.slice(0, max) + '…';
1134
+ /**
1135
+ * Longest failure text the inference-failure pipeline writes to one sink: the
1136
+ * reason in the exhaustion stderr line, a field of a logs/failures.log record,
1137
+ * an ops alert's message, the inference log's `error`, and a trace event's
1138
+ * `error` and `stack`. A failure's text is whatever its source put in
1139
+ * `Error.message`, and a provider can echo the entire rejected request there
1140
+ * (one OpenAI-compatible 400 echoed ~1.7 MB). The sinks with shorter bounds
1141
+ * name them below, and every cut goes through boundFailureText. This bound
1142
+ * doesn't depend on membrane bounding its own errors.
1143
+ */
1144
+ const FAILURE_TEXT_MAX_CHARS = 2_000;
1145
+ /** Longest failure reason written into an agent's own context (the
1146
+ * `[inference-failed]` marker): enough for a provider's explanation, not a
1147
+ * self-dump the agent then carries in every later request. */
1148
+ const FAILURE_MARKER_REASON_MAX_CHARS = 600;
1149
+ /** Longest failure reason in the rewind marker, also the agent's own context:
1150
+ * enough to identify the failure class without a wall of provider JSON. */
1151
+ const REWIND_MARKER_REASON_MAX_CHARS = 160;
1152
+ /** Longest failure reason in an agent's status (`lastInference.lastError`) and
1153
+ * in the compile-refused ops alert. */
1154
+ const FAILURE_SUMMARY_MAX_CHARS = 300;
1155
+ /** Longest alert message posted to the ops webhook. */
1156
+ const OPS_WEBHOOK_MESSAGE_MAX_CHARS = 500;
1157
+ /** Longest provider error code recorded verbatim; real codes are short tokens.
1158
+ * A failure's name and type are bounded the same way. */
1159
+ const PROVIDER_ERROR_CODE_MAX_CHARS = 128;
1160
+
1161
+ const isHighSurrogate = (text: string, index: number): boolean => {
1162
+ const code = text.charCodeAt(index);
1163
+ return code >= 0xd800 && code <= 0xdbff;
1164
+ };
1165
+ const isLowSurrogate = (text: string, index: number): boolean => {
1166
+ const code = text.charCodeAt(index);
1167
+ return code >= 0xdc00 && code <= 0xdfff;
1168
+ };
1169
+ /** Whether a cut before `index` would separate the halves of a surrogate pair. */
1170
+ const splitsPair = (text: string, index: number): boolean =>
1171
+ index > 0 && index < text.length && isHighSurrogate(text, index - 1) && isLowSurrogate(text, index);
1172
+
1173
+ /**
1174
+ * `text` when it fits within `max` characters; otherwise its head and tail
1175
+ * around a marker stating how many of how many characters were omitted, the
1176
+ * whole within `max`. A cut that would separate a surrogate pair moves toward
1177
+ * omission, so the result never holds half of a pair the text held whole, and
1178
+ * bounding bounded text returns it as is. Each sink cuts the whole text, so
1179
+ * its marker states the whole length.
1180
+ */
1181
+ function boundFailureText(text: string, max: number = FAILURE_TEXT_MAX_CHARS): string {
1182
+ if (text.length <= max) return text;
1183
+ const markerLength = ` …[${text.length} of ${text.length} characters omitted]… `.length;
1184
+ const budget = Math.max(0, max - markerLength);
1185
+ const tailLength = Math.floor(budget / 4);
1186
+ let headEnd = budget - tailLength;
1187
+ if (splitsPair(text, headEnd)) headEnd--;
1188
+ let tailStart = text.length - tailLength;
1189
+ if (splitsPair(text, tailStart)) tailStart++;
1190
+ return `${text.slice(0, headEnd)} …[${tailStart - headEnd} of ${text.length} characters omitted]… ${text.slice(tailStart)}`;
1191
+ }
1192
+
1193
+ /** A provider error code, verbatim when token-sized; otherwise bounded. */
1194
+ function boundProviderErrorCode(code: unknown): string | undefined {
1195
+ if (typeof code !== 'string') return undefined;
1196
+ return boundFailureText(code, PROVIDER_ERROR_CODE_MAX_CHARS);
1197
+ }
1198
+
1199
+ /**
1200
+ * A value's JSON for a failure sink, with every Error inside it written as its
1201
+ * whole projection (failureDescription): serialising an Error writes its own
1202
+ * fields, a MembraneError's whole `rawRequest` among them. The projection is
1203
+ * left uncut, as a nested string is, so the sink's one cut of the JSON states
1204
+ * its whole length. An Error is recognised by instanceof, so one from another
1205
+ * realm (a vm context) is written as its own JSON, as any object is. Throws,
1206
+ * or returns undefined, as JSON.stringify does.
1207
+ */
1208
+ function failureJson(value: unknown): string | undefined {
1209
+ return JSON.stringify(value, (_key, nested: unknown) => (nested instanceof Error ? failureDescription(nested) : nested));
1210
+ }
1211
+
1212
+ /**
1213
+ * The whole text describeFailure cuts. For an Error, its head (name,
1214
+ * classification, status, provider code, each bounded) and its whole
1215
+ * message. For a primitive, its String. For another object, its String when
1216
+ * that's text of its own (a Buffer, an Error from another realm, a class
1217
+ * with its own toString). An array, or an object whose String is empty or
1218
+ * only its tag, such as `[object Object]`, is its JSON instead
1219
+ * (failureJson), so its fields and elements are kept; its tag stands when
1220
+ * JSON can't write it or finds nothing in it.
1221
+ */
1222
+ function failureDescription(error: unknown): string {
1223
+ try {
1224
+ if (!(error instanceof Error)) {
1225
+ if (typeof error !== 'object' || error === null) return String(error);
1226
+ if (!Array.isArray(error)) {
1227
+ let text: string | undefined;
1228
+ try { text = String(error); } catch { /* no String of its own: its JSON, below */ }
1229
+ if (text && !/^\[object [^\]]*\]$/.test(text)) return text;
1230
+ }
1231
+ try {
1232
+ const json = failureJson(error);
1233
+ if (json !== undefined && json !== '{}') return json;
1234
+ } catch { /* its tag, below */ }
1235
+ return Object.prototype.toString.call(error);
1236
+ }
1237
+ const fields = error as Error & {
1238
+ type?: unknown; httpStatus?: unknown; providerErrorCode?: unknown; retryable?: unknown;
1239
+ };
1240
+ // Every component is remote or caller-supplied text, the name included.
1241
+ const classification = [
1242
+ typeof fields.type === 'string' ? boundFailureText(fields.type, PROVIDER_ERROR_CODE_MAX_CHARS) : undefined,
1243
+ typeof fields.httpStatus === 'number' ? `HTTP ${fields.httpStatus}` : undefined,
1244
+ boundProviderErrorCode(fields.providerErrorCode),
1245
+ typeof fields.retryable === 'boolean' ? `retryable=${fields.retryable}` : undefined,
1246
+ ].filter((part): part is string => part !== undefined);
1247
+ const name = boundFailureText(String(error.name), PROVIDER_ERROR_CODE_MAX_CHARS);
1248
+ const head = classification.length > 0 ? `${name} (${classification.join(', ')})` : name;
1249
+ return `${head}: ${String(error.message)}`;
1250
+ } catch {
1251
+ return '[failure could not be described]';
1252
+ }
1253
+ }
1254
+
1255
+ /**
1256
+ * A bounded string describing a failure, for logs that would otherwise print
1257
+ * the error object; it is one line unless one of its parts has newlines, as a
1258
+ * message can. Printing the object prints its message whole, in the stack,
1259
+ * and every own field: a MembraneError's `rawRequest` is the whole request
1260
+ * that failed, and Node's console prints each of its strings, a long system
1261
+ * prompt among them, up to 10,000 characters. The projection keeps what a
1262
+ * reader needs (name, classification, status, provider code, message) and
1263
+ * nothing else. The head's parts are bounded each, and the whole
1264
+ * (failureDescription) is cut once, so its marker states the length of the
1265
+ * head and the whole message.
1266
+ */
1267
+ function describeFailure(error: unknown): string {
1268
+ return boundFailureText(failureDescription(error));
1269
+ }
1270
+
1271
+ /** A stack frame's line: an indent of spaces or tabs, then `at `. The indent
1272
+ * never crosses a line break, so a run of blank lines isn't rescanned from
1273
+ * each line's start. */
1274
+ const FRAME_LINE = /^[ \t]+at /m;
1275
+
1276
+ /**
1277
+ * describeFailure, then the error's stack frames, bounded, on the lines
1278
+ * below: for a log of a failure the framework's own code may have thrown,
1279
+ * where the frames say where. The stack opens with the message, and a
1280
+ * message's own lines can look like frames (an echoed request quoting a stack
1281
+ * trace), so when the message is the stack's header, before its first frame
1282
+ * line, the frames are the stack from its first frame line after the
1283
+ * message. Otherwise (a stack without its header, a message that isn't in
1284
+ * the stack or is empty) they are the stack from its first frame line. The
1285
+ * leak the projection closes was the object's own fields, not its stack.
1286
+ */
1287
+ function describeFailureWithFrames(error: unknown): string {
1288
+ let frames: string | undefined;
1289
+ try {
1290
+ const stack = error instanceof Error ? error.stack : undefined;
1291
+ if (typeof stack === 'string') {
1292
+ const message = String((error as Error).message);
1293
+ // The message is the stack's header only before its first frame: a
1294
+ // stack without its header (copied, or frames only) can hold the
1295
+ // message's text inside a frame's name.
1296
+ const firstFrame = stack.search(FRAME_LINE);
1297
+ const at = message ? stack.indexOf(message) : -1;
1298
+ const frameLine = new RegExp(FRAME_LINE.source, 'gm');
1299
+ frameLine.lastIndex = at >= 0 && (firstFrame < 0 || at < firstFrame) ? at + message.length : 0;
1300
+ const found = frameLine.exec(stack);
1301
+ if (found) frames = boundFailureText(stack.slice(found.index));
1302
+ }
1303
+ } catch { /* the projection alone */ }
1304
+ const described = describeFailure(error);
1305
+ return frames === undefined ? described : `${described}\n${frames}`;
1306
+ }
1307
+
1308
+ /**
1309
+ * A failure record's fields, each bounded, for logs/failures.log and the
1310
+ * ops:alert trace: a string to FAILURE_TEXT_MAX_CHARS; an Error to its
1311
+ * projection, since serialising one writes its own fields, a MembraneError's
1312
+ * whole `rawRequest` among them; any other value to its JSON, with each Error
1313
+ * inside it as its projection (failureJson), as the plain data that JSON
1314
+ * describes when it fits FAILURE_TEXT_MAX_CHARS and as the JSON text,
1315
+ * bounded, when it doesn't. So the trace's data is what failures.log writes:
1316
+ * a Map or an object with toJSON whose JSON is small never hands a listener
1317
+ * the value itself.
1318
+ */
1319
+ function boundFailureFields(record: Record<string, unknown>): Record<string, unknown> {
1320
+ const bounded: Record<string, unknown> = {};
1321
+ for (const [key, value] of Object.entries(record)) {
1322
+ if (typeof value === 'string') bounded[key] = boundFailureText(value);
1323
+ else if (value instanceof Error) bounded[key] = describeFailure(value);
1324
+ else {
1325
+ let json: string | undefined;
1326
+ try { json = failureJson(value); } catch { bounded[key] = describeFailure(value); continue; }
1327
+ // A value JSON writes nothing for (a function, undefined) is left out,
1328
+ // as the log's JSON leaves it out.
1329
+ if (json === undefined) continue;
1330
+ bounded[key] = json.length <= FAILURE_TEXT_MAX_CHARS ? JSON.parse(json) : boundFailureText(json);
1331
+ }
1332
+ }
1333
+ return bounded;
1334
+ }
1335
+
1336
+ /**
1337
+ * Whether the poison-history breaker may shed the newest exchange for a
1338
+ * failure: an `invalid_request` at HTTP 400, 413 or 422 or with no status, or
1339
+ * a `context_length` at 413. The status is a heuristic for when a shed may
1340
+ * help, not proof that the history is at fault.
1341
+ *
1342
+ * 400 and 422 are how a provider answers a request that fails its
1343
+ * validation, as a history it can't accept does, but other refusals come
1344
+ * back the same way, such as Anthropic's 400 for a credit balance that's too
1345
+ * low. A 413 is the request's size. When the newest exchange made it too
1346
+ * large, as an oversized attachment does, shedding that exchange may bring
1347
+ * the request back under the limit, and nothing else in the framework
1348
+ * recovers a 413; when the history as a whole is too large, the shed doesn't
1349
+ * help, and the breaker stops at its rewind cap. The 413 arm takes both
1350
+ * labels a 413 arrives under, invalid_request and context_length, because
1351
+ * membrane's label for a 413 differs by version and adapter. Other statuses
1352
+ * keep the history, since shed history can't be got back and the status
1353
+ * gives no sign that a shed would help: a 404 is a model or an endpoint that
1354
+ * doesn't exist, and 405, 409, 410 and 415 are about the HTTP request or the
1355
+ * resource, as a rule, rather than the conversation.
1356
+ *
1357
+ * The table test in test/inference-exhaustion-rewind.test.ts is this
1358
+ * predicate's contract. The notes below on which membrane version gives which
1359
+ * label are its history, and will go stale as hosts move to newer releases.
1360
+ *
1361
+ * Which failures reach the gate with which label depends on the membrane:
1362
+ * - As membrane 0.5.86 classifies them: Anthropic files a 400 as
1363
+ * invalid_request (after its context check) and its other 4xx as unknown.
1364
+ * The chat adapters never file one as invalid_request, since they classify
1365
+ * by the error's message text, and a context_length they file that way
1366
+ * carries status 400 whatever the response's was. The raw-fetch Responses
1367
+ * adapter files every status it doesn't list (404, 405, 409, 410, 413,
1368
+ * 415, 422 …) as invalid_request, so it alone produces an invalid_request
1369
+ * at 413 or 422, and no 0.5.86 adapter produces a context_length at 413.
1370
+ * - From membrane #56 (merged, not yet in a release), one table serves every
1371
+ * adapter. Auth codes and context overflow are checked on a 400 or 422
1372
+ * first, a 404 is invalid_request, a 413 is context_length, and other 4xx
1373
+ * are invalid_request. So a 413 still sheds, an Anthropic one included,
1374
+ * while a context_length at 400 or 422, a token overflow, keeps the
1375
+ * history. The 422 arm then also covers an OpenAI-compatible server, such
1376
+ * as a FastAPI one, that answers a validation failure with 422.
1377
+ *
1378
+ * An error with no status, from a membrane that reports none, keeps the rule
1379
+ * the breaker had before: the type alone.
1380
+ */
1381
+ export function shedMayHelp(errorType: string | undefined, httpStatus: number | undefined): boolean {
1382
+ if (httpStatus === 413) return errorType === 'invalid_request' || errorType === 'context_length';
1383
+ return errorType === 'invalid_request' && (httpStatus === undefined || httpStatus === 400 || httpStatus === 422);
1051
1384
  }
1052
1385
 
1053
1386
  /** The most of a thrown value's or a tool error's text that reaches a log line or a server's error answer. */
@@ -1174,7 +1507,7 @@ export class AgentFramework {
1174
1507
  private quiesced = false;
1175
1508
  private quiesceReason?: string;
1176
1509
  private quiescedAt?: number;
1177
- private traceListeners: TraceEventListener[] = [];
1510
+ private traceListeners: TraceRegistration[] = [];
1178
1511
  private syncIntervalMs: number;
1179
1512
  private syncTimer: ReturnType<typeof setInterval> | null = null;
1180
1513
  private maintenanceIntervalMs: number;
@@ -1190,6 +1523,10 @@ export class AgentFramework {
1190
1523
  private providerAccelerationDefaultCooldownMs = PROVIDER_ACCELERATION_DEFAULT_COOLDOWN_MS;
1191
1524
  private providerAccelerationJitterMs = PROVIDER_ACCELERATION_JITTER_MS;
1192
1525
  private providerHoldHook: ProviderHoldHook | undefined;
1526
+ /** Provider waits (stated retry-after) per (agent, model), kept across restarts. */
1527
+ private providerWaits: ProviderWaits | undefined;
1528
+ /** Wakers for error-policy retry waits in progress: stop() ends them. */
1529
+ private retryWaitWakers = new Set<() => void>();
1193
1530
  private providerAdmissionClosed = false;
1194
1531
  /** Invalidates framework turns still assembling when stop begins. */
1195
1532
  private lifecycleGeneration = 0;
@@ -1621,6 +1958,15 @@ export class AgentFramework {
1621
1958
  this.operatorLog = operatorLog;
1622
1959
  this.store = store;
1623
1960
  this.ownsStore = ownsStore;
1961
+ // Unreadable recorded waits hold every agent's provider calls, and a
1962
+ // persistent cause never reads again: a person has to hear of it.
1963
+ this.providerWaits = new ProviderWaits(store, {
1964
+ onUnreadable: (error) => this.opsAlert('provider-wait', 'framework',
1965
+ `recorded provider waits could not be read (${error}): no provider call is admitted for any agent ` +
1966
+ 'until they read again (re-read every 30 s) or an operator releases an agent\'s waits with ' +
1967
+ 'release-provider-wait; such a release lasts for this process only',
1968
+ { data: { model: EVERY_MODEL, until: null } }),
1969
+ });
1624
1970
  this.membrane = membrane;
1625
1971
  this.inferencePolicy = inferencePolicy;
1626
1972
  this.errorPolicy = errorPolicy;
@@ -1638,14 +1984,16 @@ export class AgentFramework {
1638
1984
  });
1639
1985
 
1640
1986
  // Initialize module registry with callbacks
1641
- this.moduleRegistry = new ModuleRegistry(store, this.queue, {
1987
+ // A module's events, through pushEvent() or its deprecated queue handle,
1988
+ // go through pushModuleEvent(), which names the module.
1989
+ this.moduleRegistry = new ModuleRegistry(store, { push: (event, moduleName) => this.pushModuleEvent(event, moduleName, false) }, {
1642
1990
  getAgents: () => Array.from(this.agents.values()),
1643
1991
  addMessage: (p, c, m) => this.addMessage(p, c, m),
1644
1992
  editMessage: (id, c) => this.editMessage(id, c),
1645
1993
  removeMessage: (id) => this.removeMessage(id),
1646
1994
  getMessage: (id) => this.getMessage(id),
1647
1995
  queryMessages: (filter) => this.queryMessages(filter),
1648
- pushEvent: (event) => this.pushEvent(event),
1996
+ pushEvent: (event, moduleName) => this.pushModuleEvent(event, moduleName, true),
1649
1997
  onTrace: (listener) => this.onTrace(listener),
1650
1998
  // The ONLY entry that confers module origin — a private closure the
1651
1999
  // framework hands to its own module registry. The public
@@ -2179,6 +2527,7 @@ export class AgentFramework {
2179
2527
  // store is still open, rather than leaving them to a reboot replay.
2180
2528
  this.ackDeferredWrites();
2181
2529
  this.providerAdmissionClosed = true;
2530
+ for (const wake of [...this.retryWaitWakers]) wake();
2182
2531
  this.lifecycleGeneration++;
2183
2532
  // The generation this stop ends; withheld spills close for it alone, so a
2184
2533
  // start() made before this stop finishes reopens them for its own.
@@ -2340,7 +2689,13 @@ export class AgentFramework {
2340
2689
  private providerGateBlocked(agentName: string): boolean {
2341
2690
  const gate = this.providerGates.get(agentName);
2342
2691
  return (gate?.primaryDepth ?? 0) > 0 || (gate?.primaryPending ?? false) ||
2343
- this.providerAccelerationCooldowns.has(agentName);
2692
+ this.cooldownBlocksAuxiliary(agentName);
2693
+ }
2694
+ /** Acceleration and host holds park the whole agent; a provider wait holds
2695
+ * only its own model, which auxiliary admission checks per call. */
2696
+ private cooldownBlocksAuxiliary(agentName: string): boolean {
2697
+ const cooldown = this.providerAccelerationCooldowns.get(agentName);
2698
+ return cooldown !== undefined && cooldown.waitModel === undefined;
2344
2699
  }
2345
2700
  private acquirePrimaryProviderGate(agentName: string): void {
2346
2701
  const gate = this.providerGate(agentName); gate.primaryPending = false; gate.primaryDepth++;
@@ -2351,7 +2706,7 @@ export class AgentFramework {
2351
2706
  }
2352
2707
  private flushAuxiliaryAdmission(agentName: string): void {
2353
2708
  const gate = this.providerGate(agentName);
2354
- if (gate.primaryDepth > 0 || gate.primaryPending || this.providerAccelerationCooldowns.has(agentName)) return;
2709
+ if (gate.primaryDepth > 0 || gate.primaryPending || this.cooldownBlocksAuxiliary(agentName)) return;
2355
2710
  for (const resolve of gate.auxiliaryWaiters.splice(0)) resolve();
2356
2711
  }
2357
2712
  private async waitForAuxiliaryIdle(agentName: string): Promise<void> {
@@ -2359,8 +2714,9 @@ export class AgentFramework {
2359
2714
  if (gate.auxiliaryInFlight === 0) return;
2360
2715
  await new Promise<void>((resolve) => gate.idleWaiters.push(resolve));
2361
2716
  }
2362
- private async withAuxiliaryAdmission<T>(agentName: string, run: () => Promise<T>): Promise<T> {
2717
+ private async withAuxiliaryAdmission<T>(agentName: string, run: () => Promise<T>, model?: string): Promise<T> {
2363
2718
  const gate = this.providerGate(agentName);
2719
+ this.refuseHeldProviderModel(agentName, model);
2364
2720
  if (this.providerGateBlocked(agentName)) {
2365
2721
  gate.deferredAuxiliary++;
2366
2722
  try {
@@ -2372,14 +2728,20 @@ export class AgentFramework {
2372
2728
  if (this.providerAdmissionClosed) {
2373
2729
  throw new Error(`Provider admission closed while stopping (${agentName})`);
2374
2730
  }
2731
+ // A wait recorded while this call was parked binds it too.
2732
+ this.refuseHeldProviderModel(agentName, model);
2375
2733
  gate.auxiliaryInFlight++;
2376
2734
  try { return await run(); }
2377
2735
  catch (error) {
2378
2736
  // Compression can be the first call to find a spent quota (idle agent,
2379
- // scheduled maintenance). Arm the host hold so later passes wait at the
2380
- // gate instead of retrying into it; this pass still fails to its caller.
2737
+ // scheduled maintenance). Record a stated wait for this call's model,
2738
+ // and arm the host hold so later passes wait at the gate instead of
2739
+ // retrying into it; this pass still fails to its caller.
2381
2740
  const agent = this.agents.get(agentName);
2382
- if (agent && error instanceof Error) this.holdProviderAcceleration(agent, error, undefined, true);
2741
+ if (agent && error instanceof Error) {
2742
+ if (model !== undefined) this.recordProviderWait(agentName, model, error);
2743
+ this.holdProviderAcceleration(agent, error, undefined, true);
2744
+ }
2383
2745
  throw error;
2384
2746
  }
2385
2747
  finally {
@@ -2388,21 +2750,58 @@ export class AgentFramework {
2388
2750
  }
2389
2751
  }
2390
2752
  private auxiliaryMembraneFor(agentName: string): Membrane {
2753
+ return this.membraneAdmitting((run, model) => this.withAuxiliaryAdmission(agentName, run, model));
2754
+ }
2755
+ /**
2756
+ * A conversation agent's auxiliary calls (its context manager's
2757
+ * compression). A conversation agent takes no provider gate: admission
2758
+ * belongs to the persistent resident, and a generation-unique fork must not
2759
+ * leave a gate behind. But its stated waits are recorded for (agent, model)
2760
+ * as a resident's are, so they bind its auxiliary calls too: a held model
2761
+ * is refused without a call, and an auxiliary call's own stated wait is
2762
+ * recorded for it. (An ephemeral run records nothing, its name being
2763
+ * single-use, so its context manager keeps the plain membrane.)
2764
+ */
2765
+ private conversationAuxiliaryMembraneFor(agentName: string): Membrane {
2766
+ return this.membraneAdmitting(async (run, model) => {
2767
+ this.refuseHeldProviderModel(agentName, model);
2768
+ try { return await run(); }
2769
+ catch (error) {
2770
+ if (model !== undefined && error instanceof Error) this.recordProviderWait(agentName, model, error);
2771
+ throw error;
2772
+ }
2773
+ });
2774
+ }
2775
+ /** The framework's membrane, with each complete() call passed through
2776
+ * `admit` along with the model it names. */
2777
+ private membraneAdmitting(admit: (run: () => Promise<unknown>, model: string | undefined) => Promise<unknown>): Membrane {
2391
2778
  const target = this.membrane as unknown as Record<PropertyKey, unknown>;
2392
2779
  return new Proxy(target, { get: (obj, prop, receiver) => {
2393
2780
  const value = Reflect.get(obj, prop, receiver);
2394
2781
  if (prop === 'complete' && typeof value === 'function') {
2395
- return (...args: unknown[]) => this.withAuxiliaryAdmission(
2396
- agentName, () => Reflect.apply(value, this.membrane, args) as Promise<unknown>);
2782
+ return (...args: unknown[]) => {
2783
+ const model = (args[0] as { config?: { model?: unknown } } | undefined)?.config?.model;
2784
+ return admit(() => Reflect.apply(value, this.membrane, args) as Promise<unknown>,
2785
+ typeof model === 'string' ? model : undefined);
2786
+ };
2397
2787
  }
2398
2788
  return typeof value === 'function' ? value.bind(this.membrane) : value;
2399
2789
  }}) as unknown as Membrane;
2400
2790
  }
2401
- private accelerationCooldownMs(agentName: string, error: MembraneError): number {
2402
- const base = error.retryAfterMs ?? this.providerAccelerationDefaultCooldownMs;
2791
+ /** An acceleration hold's length: AF's own pacing (`ownMs`: its default
2792
+ * plus jitter, capped), or the provider's stated wait in its place
2793
+ * (`stated`). The stated wait is a lower bound: jitter only adds to it,
2794
+ * and the cap applies only to AF's own default. A negative stated wait is
2795
+ * a stated 0, as in ProviderWaits: the 1 s floor, plus jitter. */
2796
+ private accelerationCooldownMs(agentName: string, error: MembraneError): { delayMs: number; ownMs: number; stated: boolean } {
2403
2797
  let hash = 0; for (const ch of agentName) hash = ((hash * 31) + ch.charCodeAt(0)) >>> 0;
2404
2798
  const jitter = this.providerAccelerationJitterMs > 0 ? hash % (this.providerAccelerationJitterMs + 1) : 0;
2405
- return Math.min(PROVIDER_ACCELERATION_MAX_COOLDOWN_MS, Math.max(1_000, base) + jitter);
2799
+ const ownMs = Math.min(PROVIDER_ACCELERATION_MAX_COOLDOWN_MS, Math.max(1_000, this.providerAccelerationDefaultCooldownMs) + jitter);
2800
+ const stated = error.retryAfterMs;
2801
+ if (typeof stated === 'number' && Number.isFinite(stated)) {
2802
+ return { delayMs: Math.max(1_000, stated) + jitter, ownMs, stated: true };
2803
+ }
2804
+ return { delayMs: ownMs, ownMs, stated: false };
2406
2805
  }
2407
2806
  private sameInferenceRequest(a: InferenceRequest, b: InferenceRequest): boolean {
2408
2807
  return a.agentName === b.agentName && a.reason === b.reason && a.source === b.source &&
@@ -2413,10 +2812,14 @@ export class AgentFramework {
2413
2812
  try {
2414
2813
  const hold = this.providerHoldHook(error, agentName, { model: this.agents.get(agentName)?.model });
2415
2814
  if (!hold || !Number.isFinite(hold.holdMs) || hold.holdMs <= 0) return undefined;
2815
+ // The reason is printed, kept in the cooldown and shown in the status
2816
+ // snapshot (cooldownReason); without the host's own, it is the error's
2817
+ // message, which can echo the whole request.
2416
2818
  return { holdMs: Math.min(PROVIDER_ACCELERATION_MAX_COOLDOWN_MS, Math.max(1_000, hold.holdMs)),
2417
- reason: typeof hold.reason === 'string' && hold.reason ? hold.reason : error.message };
2819
+ reason: boundFailureText(typeof hold.reason === 'string' && hold.reason ? hold.reason : String(error.message)) };
2418
2820
  } catch (err) {
2419
- console.error(`[provider-cooldown] providerHold hook threw for ${agentName}; treating as no hold:`, err);
2821
+ // The hook is handed the model's error, and may throw it back.
2822
+ console.error(`[provider-cooldown] providerHold hook threw for ${agentName}; treating as no hold: ${describeFailureWithFrames(err)}`);
2420
2823
  return undefined;
2421
2824
  }
2422
2825
  }
@@ -2428,7 +2831,7 @@ export class AgentFramework {
2428
2831
  if (!hold) return false;
2429
2832
  clearTimeout(cooldown.timer);
2430
2833
  cooldown.until = Date.now() + hold.holdMs;
2431
- cooldown.timer = setTimeout(() => this.releaseProviderAccelerationCooldown(agentName), hold.holdMs);
2834
+ cooldown.timer = setTimeout(() => this.releaseProviderAccelerationCooldown(agentName), Math.min(hold.holdMs, MAX_TIMER_DELAY_MS));
2432
2835
  cooldown.timer.unref?.();
2433
2836
  if (hold.reason !== cooldown.reason) {
2434
2837
  cooldown.reason = hold.reason;
@@ -2438,21 +2841,39 @@ export class AgentFramework {
2438
2841
  }
2439
2842
  private holdProviderAcceleration(agent: Agent, error: Error, trigger?: InferenceRequest, auxiliary = false): boolean {
2440
2843
  if (this.ephemeralRuns.has(agent.name) || this.conversationAgentHomes.has(agent.name)) return false;
2844
+ // A provider's stated wait binds later calls to the primary model, across
2845
+ // restarts. (An auxiliary call records its own model's wait at the gate.)
2846
+ const wait = auxiliary ? undefined : this.recordProviderWait(agent.name, agent.model, error);
2441
2847
  // The host knows more than the message wording does: ask it first. An
2442
2848
  // auxiliary failure arms host holds only — the built-in acceleration
2443
2849
  // cooldown stays a primary-path mechanism.
2444
2850
  const hostHold = this.consultProviderHold(error, agent.name);
2445
- const acceleration = !hostHold && !auxiliary && isOrganizationAccelerationRateLimit(error);
2446
- if (!acceleration && !hostHold) return false;
2447
- const now = Date.now(); const delayMs = hostHold ? hostHold.holdMs : this.accelerationCooldownMs(agent.name, error as MembraneError);
2851
+ // A failure the provider says may not be retried keeps its terminal
2852
+ // disposition, whatever its wording (a stated wait still binds after it).
2853
+ const acceleration = !hostHold && !auxiliary && (error as MembraneError).retryable !== false
2854
+ && isOrganizationAccelerationRateLimit(error);
2855
+ // The stated wait binds later calls whatever the failure was. Retaining
2856
+ // and retrying THIS request is for a failure the provider says may be
2857
+ // retried: a non-retryable one keeps its terminal disposition (marker,
2858
+ // failure streak), and the wait applies to the calls that follow it.
2859
+ if (!acceleration && !hostHold) {
2860
+ return wait !== undefined && isMembraneError(error) && error.retryable === true
2861
+ && this.holdForProviderWait(agent, wait, trigger ? [trigger] : [], true);
2862
+ }
2863
+ const now = Date.now();
2864
+ const pacing = hostHold ? undefined : this.accelerationCooldownMs(agent.name, error as MembraneError);
2865
+ const delayMs = hostHold ? hostHold.holdMs : pacing!.delayMs;
2448
2866
  const existing = this.providerAccelerationCooldowns.get(agent.name);
2449
2867
  const held = existing?.heldRequests ?? [];
2450
2868
  if (trigger && !held.some((r) => this.sameInferenceRequest(r, trigger))) held.push(trigger);
2451
2869
  if (existing) clearTimeout(existing.timer);
2452
- const timer = setTimeout(() => this.releaseProviderAccelerationCooldown(agent.name), delayMs); timer.unref?.();
2870
+ const timer = setTimeout(() => this.releaseProviderAccelerationCooldown(agent.name), Math.min(delayMs, MAX_TIMER_DELAY_MS)); timer.unref?.();
2453
2871
  this.providerAccelerationCooldowns.set(agent.name, { startedAt: existing?.startedAt ?? now,
2454
- until: now + delayMs, timer, heldRequests: held, reason: hostHold?.reason ?? error.message,
2872
+ until: now + delayMs, timer, heldRequests: held, reason: hostHold?.reason ?? boundFailureText(String(error.message)),
2455
2873
  failures: (existing?.failures ?? 0) + 1, ...(hostHold ? { hostHoldError: error } : {}),
2874
+ // The stated wait (recorded above for agent.model) set this hold's
2875
+ // length: an operator release of it returns the hold to AF's own pacing.
2876
+ ...(pacing?.stated ? { statedWait: { model: agent.model, ownUntil: now + pacing.ownMs } } : {}),
2456
2877
  ...(auxiliary && (!existing || existing.auxiliaryOrigin) ? { auxiliaryOrigin: true } : {}) });
2457
2878
  if (auxiliary && !existing) {
2458
2879
  console.error(`[provider-cooldown] agent=${agent.name} host hold (${hostHold!.reason}) from an auxiliary call — ` +
@@ -2461,16 +2882,96 @@ export class AgentFramework {
2461
2882
  }
2462
2883
  const gate = this.providerGate(agent.name); gate.primaryPending = true;
2463
2884
  this.providerAccelerationRecoveries.set(agent.name, { startedAt: existing?.startedAt ?? now,
2464
- failures: (existing?.failures ?? 0) + 1, heldRequests: held.length, reason: error.message });
2885
+ failures: (existing?.failures ?? 0) + 1, heldRequests: held.length, reason: boundFailureText(String(error.message)) });
2465
2886
  console.error(`[provider-cooldown] agent=${agent.name} ` +
2466
2887
  `${hostHold ? `host hold (${hostHold.reason})` : 'organization acceleration 429'} — ` +
2467
2888
  `holding primary/auxiliary for ${delayMs}ms; ${held.length} request(s) retained`);
2468
2889
  return true;
2469
2890
  }
2891
+ /**
2892
+ * A provider's stated wait on a turn that doesn't own provider admission:
2893
+ * a conversation agent's or an ephemeral run's. Left to the error policy,
2894
+ * which returns a stated wait as it is, a 429 stating two hours would hold
2895
+ * the turn asleep two hours, up to three times, with nothing recorded and
2896
+ * nothing raised (membrane #56 review, M3). So:
2897
+ * - A conversation agent's turns come through the scheduler as a resident's
2898
+ * do, so it treats the wait as a resident does: recorded for (agent,
2899
+ * model), alerted past PROVIDER_WAIT_ALERT_MS, and the turn held until the
2900
+ * wait passes or an operator releases it ('held'). A non-retryable failure
2901
+ * records the wait for the calls after it and keeps its own disposition.
2902
+ * - An ephemeral run's name is single-use, so a wait recorded for it would
2903
+ * bind nothing, and its caller is waiting on it. A retryable stated wait
2904
+ * up to PROVIDER_ACCELERATION_MAX_COOLDOWN_MS is left to the error policy,
2905
+ * slept and retried. A longer one, or one that can't be represented, ends
2906
+ * the run now ('fail'): it is never retried before the provider's time,
2907
+ * and the caller hears why at once.
2908
+ * Anything else is the error policy's.
2909
+ */
2910
+ private nonOwningProviderWait(agent: Agent, error: Error, trigger?: InferenceRequest): 'held' | 'fail' | undefined {
2911
+ // From either copy of membrane (isMembraneError), a wait is stated only
2912
+ // as a number, as the default error policy reads it.
2913
+ if (!isMembraneError(error) || typeof error.retryAfterMs !== 'number') return undefined;
2914
+ if (this.conversationAgentHomes.has(agent.name)) {
2915
+ const wait = this.recordProviderWait(agent.name, agent.model, error);
2916
+ return error.retryable === true && wait !== undefined && this.holdForProviderWait(agent, wait, trigger ? [trigger] : [], true)
2917
+ ? 'held'
2918
+ : undefined;
2919
+ }
2920
+ if (this.ephemeralRuns.has(agent.name) && error.retryable === true
2921
+ && !(error.retryAfterMs <= PROVIDER_ACCELERATION_MAX_COOLDOWN_MS)) {
2922
+ return 'fail';
2923
+ }
2924
+ return undefined;
2925
+ }
2926
+ /** Why an ephemeral run ended instead of sleeping a provider's stated wait,
2927
+ * for its caller. */
2928
+ private statedWaitNotSlept(error: Error): string {
2929
+ const stated = isMembraneError(error) ? error.retryAfterMs : undefined;
2930
+ const cap = `${PROVIDER_ACCELERATION_MAX_COOLDOWN_MS / 1000} s`;
2931
+ if (typeof stated !== 'number' || Number.isNaN(stated)) {
2932
+ return `${error.message} (the provider's stated wait is not a number, and an ephemeral run sleeps only a ` +
2933
+ `stated wait of ${cap} or less, so it ends now)`;
2934
+ }
2935
+ const asked = Number.isFinite(stated) ? `${Math.ceil(stated / 1000)} s` : 'longer than can be represented';
2936
+ return `${error.message} (the provider asked to wait ${asked}; an ephemeral run doesn't sleep a stated wait ` +
2937
+ `longer than ${cap}, so it ends now)`;
2938
+ }
2939
+ /** When a hold's timer next fires: at its deadline, stepped past what one
2940
+ * timer can hold. A hold with no deadline (a wait held until released, or
2941
+ * the stand-in for unreadable records) asks again as often as unreadable
2942
+ * records are read again: records that read again end it, and nothing
2943
+ * announces that. */
2944
+ private providerHoldTimerMs(remaining: number): number {
2945
+ return Number.isFinite(remaining)
2946
+ ? Math.min(remaining, MAX_TIMER_DELAY_MS)
2947
+ : this.providerWaits?.retryReadMs ?? MAX_TIMER_DELAY_MS;
2948
+ }
2470
2949
  private releaseProviderAccelerationCooldown(agentName: string): void {
2471
2950
  const cooldown = this.providerAccelerationCooldowns.get(agentName); if (!cooldown) return;
2951
+ // A wait hold lasts exactly as long as its wait binds: longer once the
2952
+ // wait is extended, while it is indefinite, and over once no wait binds
2953
+ // (passed, released, or the stand-in for unreadable records lifted because
2954
+ // they read again). A different primary model now is not bound by it.
2955
+ if (cooldown.waitModel !== undefined && this.agents.get(agentName)?.model === cooldown.waitModel) {
2956
+ const wait = this.providerWaits?.active(agentName, cooldown.waitModel);
2957
+ cooldown.until = wait ? (wait.until ?? Number.POSITIVE_INFINITY) : Date.now();
2958
+ }
2959
+ const remaining = cooldown.until - Date.now();
2960
+ if (remaining > 0 && (cooldown.waitModel === undefined || this.agents.get(agentName)?.model === cooldown.waitModel)) {
2961
+ clearTimeout(cooldown.timer);
2962
+ cooldown.timer = setTimeout(() => this.releaseProviderAccelerationCooldown(agentName), this.providerHoldTimerMs(remaining));
2963
+ cooldown.timer.unref?.();
2964
+ return;
2965
+ }
2472
2966
  if (this.extendHostProviderHold(agentName, cooldown)) return;
2473
2967
  clearTimeout(cooldown.timer); this.providerAccelerationCooldowns.delete(agentName);
2968
+ if (cooldown.waitModel !== undefined) {
2969
+ const model = this.agents.get(agentName)?.model;
2970
+ console.error(`[provider-wait] agent=${agentName} model=${cooldown.waitModel} ` +
2971
+ (model !== undefined && model !== cooldown.waitModel
2972
+ ? `no longer holds primary turns: the primary model is now ${model} (the wait stays recorded)`
2973
+ : `primary hold ended after ${Date.now() - cooldown.startedAt}ms`));
2974
+ }
2474
2975
  if (cooldown.auxiliaryOrigin && cooldown.heldRequests.length === 0) {
2475
2976
  // Nobody asked for an inference while parked: just reopen the gate.
2476
2977
  console.error(`[provider-cooldown] agent=${agentName} auxiliary hold released after ${Date.now() - cooldown.startedAt}ms`);
@@ -2485,6 +2986,201 @@ export class AgentFramework {
2485
2986
  console.error(`[provider-cooldown] agent=${agentName} released after ${Date.now() - cooldown.startedAt}ms — ` +
2486
2987
  `fresh compile queued with ${requests.length} retained request(s)`);
2487
2988
  }
2989
+ /**
2990
+ * Wait an error policy's retry delay, all of it: the delay can be a
2991
+ * provider's retry-after, and a single timer past 2^31-1 ms fires almost at
2992
+ * once. Steps through longer waits; stop() wakes it so a shutting-down
2993
+ * framework is never left with a wait parked for days.
2994
+ */
2995
+ private async waitForRetry(ms: number): Promise<void> {
2996
+ // An infinite delay waits until stop(), like any wait too long to end; a
2997
+ // delay that isn't a number, or is 0 or less, waits none.
2998
+ let remaining = ms > 0 ? ms : 0;
2999
+ while (remaining > 0 && !this.providerAdmissionClosed) {
3000
+ const step = Math.min(remaining, MAX_TIMER_DELAY_MS);
3001
+ remaining -= step;
3002
+ await new Promise<void>((resolve) => {
3003
+ const wakers = this.retryWaitWakers;
3004
+ const done = (): void => { clearTimeout(timer); wakers.delete(done); resolve(); };
3005
+ const timer = setTimeout(done, step);
3006
+ wakers.add(done);
3007
+ });
3008
+ }
3009
+ }
3010
+
3011
+ /**
3012
+ * Record a provider's stated wait (a classified error carrying
3013
+ * `retryAfterMs`) for (agent, model). Returns the wait that binds now,
3014
+ * which is the longer of this one and any outstanding one. A wait that
3015
+ * newly binds past PROVIDER_WAIT_ALERT_MS, or until released, raises an
3016
+ * ops alert.
3017
+ */
3018
+ private recordProviderWait(agentName: string, model: string, error: Error): ProviderWait | undefined {
3019
+ // From either copy of membrane: the host supplies the Membrane instance,
3020
+ // and a wait is stated only as a number, as the default policy reads it.
3021
+ if (!isMembraneError(error) || typeof error.retryAfterMs !== 'number' || !this.providerWaits) return undefined;
3022
+ // The provider's words, bounded; a line break in them must not start a
3023
+ // line that reads as the framework's own where the reason is logged.
3024
+ const reason = safeSlice(withoutControls(error.message), 0, 300);
3025
+ const { wait, changed } = this.providerWaits.set(agentName, model, error.retryAfterMs, reason);
3026
+ // A wait already over (0, or a negative "retry now") holds nothing to report.
3027
+ const remaining = wait.until === null ? Number.POSITIVE_INFINITY : wait.until - Date.now();
3028
+ if (changed && remaining > 0) {
3029
+ const until = wait.until === null
3030
+ ? 'until an operator releases it (the stated wait cannot be represented as an instant)'
3031
+ : `until ${new Date(wait.until).toISOString()}`;
3032
+ const stated = `${error.type}${error.httpStatus !== undefined ? ` HTTP ${error.httpStatus}` : ''}, retry-after ${String(error.retryAfterMs)}ms`;
3033
+ console.error(`[provider-wait] agent=${agentName} model=${model} holds ${until}: ${stated}`);
3034
+ if (remaining > PROVIDER_WAIT_ALERT_MS) {
3035
+ this.opsAlert('provider-wait', agentName,
3036
+ `provider wait on ${model} holds ${until} (${stated}: ${reason}); ` +
3037
+ 'if the provider did not mean that, release it with release-provider-wait',
3038
+ { data: { model, until: wait.until === null ? null : new Date(wait.until).toISOString() } });
3039
+ }
3040
+ }
3041
+ return wait;
3042
+ }
3043
+
3044
+ /**
3045
+ * Hold this agent's primary turns for a provider wait on its model. The
3046
+ * requests are kept and released together, into one fresh compile, when
3047
+ * the wait ends: the same path an acceleration cooldown takes, without
3048
+ * parking auxiliary calls to other models.
3049
+ */
3050
+ private holdForProviderWait(agent: Agent, wait: ProviderWait, requests: InferenceRequest[], failed = false): boolean {
3051
+ const now = Date.now();
3052
+ const until = wait.until ?? Number.POSITIVE_INFINITY;
3053
+ if (until <= now) return false;
3054
+ const existing = this.providerAccelerationCooldowns.get(agent.name);
3055
+ const held = existing?.heldRequests ?? [];
3056
+ for (const request of requests) {
3057
+ if (!held.some((r) => this.sameInferenceRequest(r, request))) held.push(request);
3058
+ }
3059
+ // An acceleration or host hold already parks the whole agent; its release
3060
+ // re-checks this wait at admission.
3061
+ if (existing && existing.waitModel === undefined) return true;
3062
+ if (existing) clearTimeout(existing.timer);
3063
+ const timer = setTimeout(() => this.releaseProviderAccelerationCooldown(agent.name), this.providerHoldTimerMs(until - now));
3064
+ timer.unref?.();
3065
+ this.providerAccelerationCooldowns.set(agent.name, {
3066
+ startedAt: existing?.startedAt ?? now, until, timer, heldRequests: held,
3067
+ // A failed call counts; a turn arriving at a wait recorded earlier doesn't.
3068
+ reason: wait.reason, failures: (existing?.failures ?? 0) + (failed ? 1 : 0), waitModel: wait.model,
3069
+ });
3070
+ if (!existing) {
3071
+ console.error(`[provider-wait] agent=${agent.name} model=${wait.model} holding primary turns ` +
3072
+ `${wait.until === null ? 'until released' : `until ${new Date(wait.until).toISOString()}`}; ${held.length} request(s) retained`);
3073
+ }
3074
+ return true;
3075
+ }
3076
+
3077
+ /**
3078
+ * Refuse an auxiliary call to a model the provider asked this agent to wait
3079
+ * on, without calling it. The refusal is a retryable rate limit carrying
3080
+ * the remaining wait, so a caller that paces (context-manager's compression
3081
+ * lane, from its corrective round #138 on) waits it out instead of spending
3082
+ * a call. It is marked
3083
+ * `providerAdmission: 'deferred'`: no call was made, so a caller can keep
3084
+ * the wait without counting a failed call (room-225 #47927).
3085
+ */
3086
+ private refuseHeldProviderModel(agentName: string, model: string | undefined): void {
3087
+ // A call that names no model can't be matched to a model's wait, but the
3088
+ // hold that stands for unreadable records binds every model, so it binds
3089
+ // that call too (EVERY_MODEL: only that hold, or a release of it, says so).
3090
+ const wait = this.providerWaits?.active(agentName, model ?? EVERY_MODEL);
3091
+ if (!wait) return;
3092
+ const now = Date.now();
3093
+ throw Object.assign(new MembraneError({
3094
+ type: 'rate_limit',
3095
+ retryable: true,
3096
+ ...(wait.until !== null ? { retryAfterMs: Math.max(0, wait.until - now) } : {}),
3097
+ message: providerWaitText(agentName, model ?? EVERY_MODEL, wait),
3098
+ rawError: undefined,
3099
+ }), { providerAdmission: 'deferred' as const });
3100
+ }
3101
+
3102
+ /**
3103
+ * Release provider waits: one model's for an agent, or every model's. The
3104
+ * explicit operator override for a wait that should not be honoured (the
3105
+ * passing of time needs no call). Recorded, so a restart does not bring the
3106
+ * wait back; held primary turns are released into one fresh compile. An
3107
+ * acceleration hold the released wait lengthened keeps only AF's own pacing
3108
+ * for its failure; a host hold stays the host's.
3109
+ */
3110
+ releaseProviderWait(agentName: string, model?: string, by = 'operator'): Array<{
3111
+ model: string; until: string | null; release: 'recorded' | 'pending' | 'in-process override';
3112
+ }> {
3113
+ // A release names one of this framework's agents, or the agent of a wait
3114
+ // the listing shows (a disposed conversation fork's wait stays recorded):
3115
+ // an operator can release anything the listing shows. Any other name (a
3116
+ // typo, or '*' and 'framework', which stand for the hold on unreadable
3117
+ // records in its listing and its alert) is refused before anything is
3118
+ // recorded: it would lift nothing and still read as a success. That hold
3119
+ // is lifted one agent at a time.
3120
+ if (!this.agents.has(agentName) && !this.providerWaits?.recorded(agentName)) {
3121
+ throw new Error(`Unknown agent: ${safeSlice(JSON.stringify(agentName), 0, 120)}. A release names one of this ` +
3122
+ "framework's agents or an agent with a recorded wait, and the hold on unreadable provider waits is lifted " +
3123
+ 'one agent at a time');
3124
+ }
3125
+ by = operatorLabel(String(by));
3126
+ // '*' (EVERY_MODEL, as providerWaits lists the unreadable-records hold)
3127
+ // names every model, the same as omitting it, everywhere below.
3128
+ if (model === EVERY_MODEL) model = undefined;
3129
+ const released = this.providerWaits?.release(agentName, model, by) ?? [];
3130
+ for (const { wait, release } of released) {
3131
+ console.error(`[provider-wait] agent=${agentName} model=${wait.model} released by ${by} (${release})`);
3132
+ }
3133
+ // The agent's context strategy may pace its own provider calls by the
3134
+ // same stated wait (context-manager's compression lane, from its
3135
+ // corrective round #138 on; 0.14.0 has no releaseCompressionPause, so this
3136
+ // calls nothing there). The release is the operator's word on that wait
3137
+ // too, so it is offered to the lane with its scope even when no wait
3138
+ // binds here: the lane can hold a pause after
3139
+ // this framework's own wait is gone (the records read again, the wait
3140
+ // passed, or it was released in another process). The strategy's own
3141
+ // backoff stays, and the receipt still lists only waits released here.
3142
+ // Read inside the try: the release is already recorded, so a strategy
3143
+ // that throws must not keep the hold below from being released.
3144
+ try {
3145
+ const strategy = this.agents.get(agentName)?.getContextManager().getStrategy() as
3146
+ { releaseCompressionPause?: (model?: string) => boolean } | undefined;
3147
+ if (strategy?.releaseCompressionPause?.(model)) {
3148
+ console.error(`[provider-wait] agent=${agentName} model=${model ?? '*'} compression lane's provider wait released`);
3149
+ }
3150
+ } catch (error) {
3151
+ console.error(`[provider-wait] agent=${agentName} could not offer the release to the compression lane: ${safeSlice(error instanceof Error ? error.message : String(error), 0, 300)}`);
3152
+ }
3153
+ const cooldown = this.providerAccelerationCooldowns.get(agentName);
3154
+ const covers = (held: string | undefined): boolean => held !== undefined && (model === undefined || held === model);
3155
+ if (cooldown && covers(cooldown.waitModel)) {
3156
+ // A wait hold asks its wait again: released, the hold ends.
3157
+ this.releaseProviderAccelerationCooldown(agentName);
3158
+ } else if (cooldown?.statedWait && covers(cooldown.statedWait.model)) {
3159
+ // An acceleration hold the stated wait lengthened keeps only AF's own
3160
+ // pacing for that failure, and is never lengthened by a release. A host
3161
+ // hold carries no stated wait: its length stays the host's.
3162
+ cooldown.until = Math.min(cooldown.until, cooldown.statedWait.ownUntil);
3163
+ console.error(`[provider-wait] agent=${agentName} model=${cooldown.statedWait.model} acceleration hold keeps AF's own pacing only, ` +
3164
+ (cooldown.until > Date.now() ? `until ${new Date(cooldown.until).toISOString()}` : 'which has passed'));
3165
+ delete cooldown.statedWait;
3166
+ this.releaseProviderAccelerationCooldown(agentName);
3167
+ }
3168
+ return released.map(({ wait, release }) => ({
3169
+ model: wait.model, until: wait.until === null ? null : new Date(wait.until).toISOString(), release,
3170
+ }));
3171
+ }
3172
+
3173
+ /** Provider waits binding now, for health and doctor tooling. */
3174
+ providerWaitSnapshot(agentName?: string): Array<{ agent: string; model: string; until: string | null; reason: string; setAt: string }> {
3175
+ return (this.providerWaits?.list(agentName) ?? []).map((wait) => ({
3176
+ agent: wait.agent,
3177
+ model: wait.model,
3178
+ until: wait.until === null ? null : new Date(wait.until).toISOString(),
3179
+ reason: wait.reason,
3180
+ setAt: new Date(wait.setAt).toISOString(),
3181
+ }));
3182
+ }
3183
+
2488
3184
  private recordProviderAccelerationRecovery(agent: Agent, request: NormalizedRequest | undefined, stopReason: string): void {
2489
3185
  const recovery = this.providerAccelerationRecoveries.get(agent.name); if (!recovery?.releasedAt) return;
2490
3186
  this.providerAccelerationRecoveries.delete(agent.name);
@@ -2519,7 +3215,10 @@ export class AgentFramework {
2519
3215
  private startQueuedMaintenance(): void {
2520
3216
  if (this.maintenancePass || !this.running) return;
2521
3217
  const pass = this.runQueuedMaintenance().catch((error) => {
2522
- const reason = error instanceof Error ? error.message : String(error);
3218
+ // The alert cuts its own copies from the whole message, so the
3219
+ // webhook's states the whole length.
3220
+ const wholeReason = error instanceof Error ? String(error.message) : failureDescription(error);
3221
+ const reason = boundFailureText(wholeReason);
2523
3222
  console.error(
2524
3223
  '[context-maintenance] pass failed:',
2525
3224
  reason,
@@ -2527,7 +3226,7 @@ export class AgentFramework {
2527
3226
  this.opsAlert(
2528
3227
  'context-maintenance-failed',
2529
3228
  'framework',
2530
- reason,
3229
+ wholeReason,
2531
3230
  { data: { scope: 'pass' } },
2532
3231
  );
2533
3232
  });
@@ -2597,12 +3296,13 @@ export class AgentFramework {
2597
3296
  record.ticks++;
2598
3297
  }
2599
3298
  } catch (error) {
2600
- record.error = error instanceof Error ? error.message : String(error);
3299
+ const wholeError = error instanceof Error ? String(error.message) : failureDescription(error);
3300
+ record.error = boundFailureText(wholeError);
2601
3301
  console.error(`[context-maintenance] agent=${agent.name} failed:`, record.error);
2602
3302
  this.opsAlert(
2603
3303
  'context-maintenance-failed',
2604
3304
  agent.name,
2605
- record.error,
3305
+ wholeError,
2606
3306
  {
2607
3307
  data: {
2608
3308
  scope: 'agent',
@@ -2712,18 +3412,40 @@ export class AgentFramework {
2712
3412
  this.emitTrace({ type: 'process:received', processEvent: event });
2713
3413
  }
2714
3414
 
3415
+ /**
3416
+ * A module's event. stop() closes the queue before it stops modules, so a
3417
+ * module's event can arrive after that: from a listener that fires while
3418
+ * stop() runs, or from the module's own stop(). Nothing will process it,
3419
+ * and a throw there can go unhandled or fail stop(), so it's dropped with a
3420
+ * log line naming the module. Otherwise it's queued as pushEvent() queues
3421
+ * it, or, through the deprecated queue handle, without the trace.
3422
+ */
3423
+ private pushModuleEvent(event: ProcessEvent, moduleName: string | undefined, traced: boolean): void {
3424
+ if (this.queue.isClosed) {
3425
+ // A tool result takes pushEvent()'s own drop, whose line names the
3426
+ // call, the tool and its outcome.
3427
+ if (event.type === 'tool-result') {
3428
+ this.pushEvent(event);
3429
+ return;
3430
+ }
3431
+ console.error(`[framework] a '${event.type}' event from module ${moduleName ?? '(unnamed)'} arrived after stop() began — dropped`);
3432
+ return;
3433
+ }
3434
+ if (traced) this.pushEvent(event);
3435
+ else this.queue.push(event);
3436
+ }
3437
+
2715
3438
  /**
2716
3439
  * Add a trace event listener for observability.
2717
3440
  */
2718
3441
  onTrace(listener: TraceEventListener): () => void {
2719
- this.traceListeners.push(listener);
3442
+ const registration: TraceRegistration = { listener, active: true };
3443
+ this.traceListeners.push(registration);
2720
3444
  // Return an unsubscribe so callers with a bounded lifetime (e.g. modules
2721
3445
  // that get torn down and recreated on session switch) don't leak
2722
3446
  // listeners. Existing callers that ignore the return value are unaffected.
2723
- return () => {
2724
- const idx = this.traceListeners.indexOf(listener);
2725
- if (idx >= 0) this.traceListeners.splice(idx, 1);
2726
- };
3447
+ // It removes this subscription, whatever else the same function holds.
3448
+ return () => this.removeTraceRegistration(registration);
2727
3449
  }
2728
3450
 
2729
3451
  /**
@@ -2737,13 +3459,18 @@ export class AgentFramework {
2737
3459
  }
2738
3460
 
2739
3461
  /**
2740
- * Remove a trace event listener.
3462
+ * Remove a trace event listener: its first subscription, if it holds more
3463
+ * than one.
2741
3464
  */
2742
3465
  offTrace(listener: TraceEventListener): void {
2743
- const index = this.traceListeners.indexOf(listener);
2744
- if (index >= 0) {
2745
- this.traceListeners.splice(index, 1);
2746
- }
3466
+ const registration = this.traceListeners.find((r) => r.listener === listener);
3467
+ if (registration) this.removeTraceRegistration(registration);
3468
+ }
3469
+
3470
+ private removeTraceRegistration(registration: TraceRegistration): void {
3471
+ registration.active = false;
3472
+ const index = this.traceListeners.indexOf(registration);
3473
+ if (index >= 0) this.traceListeners.splice(index, 1);
2747
3474
  }
2748
3475
 
2749
3476
  /**
@@ -2891,14 +3618,27 @@ export class AgentFramework {
2891
3618
  * says fetch_reference instead of a path. */
2892
3619
  private async autoFetchReferences(content: unknown[] | undefined, serverId: string, eager: boolean): Promise<void> {
2893
3620
  if (!content) return;
2894
- const fetcher = this.getReferenceFetcher();
2895
- const eligible: import('./mcpl/references.js').ReferenceRecord[] = [];
3621
+ // Register every block first, then judge each uri once, by its latest
3622
+ // record. The registry keys records by uri and evicts the one registered
3623
+ // least recently, so a uri this result mentions keeps its record through
3624
+ // the result's later blocks, short of more distinct references in one
3625
+ // result than the registry holds; past that, a uri can get a second
3626
+ // record here, hence the key by uri. A uri that any of this result's
3627
+ // blocks calls expired takes no slot, whichever block registered last. A
3628
+ // later result's fresh expiry still supersedes, through register().
3629
+ const byUri = new Map<string, { record: import('./mcpl/references.js').ReferenceRecord; expiredHere: boolean }>();
2896
3630
  for (const block of content) {
2897
3631
  const c = classifyBlock(block);
2898
3632
  if (c.kind !== 'reference') continue;
2899
3633
  const record = referenceRegistry.register(c.testimony, serverId);
2900
- if (eager && fetcher.isEagerEligible(record)) eligible.push(record);
3634
+ const expiredHere = byUri.get(c.testimony.uri)?.expiredHere === true || isReferenceExpired(c.testimony);
3635
+ byUri.set(c.testimony.uri, { record, expiredHere });
2901
3636
  }
3637
+ if (!eager) return;
3638
+ const fetcher = this.getReferenceFetcher();
3639
+ const eligible = [...byUri.values()]
3640
+ .filter(({ record, expiredHere }) => !expiredHere && fetcher.isEagerEligible(record))
3641
+ .map(({ record }) => record);
2902
3642
  // Parallel with a count cap: N small fetches cost one round-trip of
2903
3643
  // wall-clock, and anything past the cap stays lazy (stub says so).
2904
3644
  const MAX_EAGER_PER_RESULT = 8;
@@ -4634,7 +5374,8 @@ export class AgentFramework {
4634
5374
  if (!run) return;
4635
5375
  run.lastActivity = Date.now();
4636
5376
  if (outcome.stopReason === 'exhausted') {
4637
- run.settle.reject(new Error(outcome.error ?? 'Unknown error'));
5377
+ // The caller may log or relay the message, which can echo the whole request.
5378
+ run.settle.reject(new Error(boundFailureText(outcome.error ?? 'Unknown error')));
4638
5379
  return;
4639
5380
  }
4640
5381
  run.settle.resolve({
@@ -5205,7 +5946,8 @@ export class AgentFramework {
5205
5946
  const parts = incoming.channelId.split(':');
5206
5947
  if (parts[0] !== 'discord') return;
5207
5948
  const channelId = parts[parts.length - 1];
5208
- const emoji = REFUSAL_REACTIONS[category] ?? REFUSAL_REACTION_FALLBACK;
5949
+ // Own names only: an inherited one such as 'constructor' takes the fallback.
5950
+ const emoji = Object.hasOwn(REFUSAL_REACTIONS, category) ? REFUSAL_REACTIONS[category] : REFUSAL_REACTION_FALLBACK;
5209
5951
  // Resolve the MCPL server that owns the locus channel and call
5210
5952
  // tools/call directly on its connection (bare tool name — no prefix
5211
5953
  // games), bypassing the agent event queue so no synthetic tool-result
@@ -5330,9 +6072,10 @@ export class AgentFramework {
5330
6072
  ? `the model refused on them (content filter: ${category}). Their content is ` +
5331
6073
  `not reproduced here (so this note can't re-trigger the filter); the ` +
5332
6074
  `originals remain in the raw record`
5333
- : `the model API kept rejecting the conversation on them (${category}). ` +
5334
- `Their content is not reproduced here (so this note can't re-trigger the ` +
5335
- `rejection); the originals remain in the raw record`;
6075
+ : `the model API kept refusing the requests that carried them (${category}), ` +
6076
+ `so that the conversation could go on. Their content is not reproduced ` +
6077
+ `here (so this note can't trigger the same refusal); the originals remain ` +
6078
+ `in the raw record`;
5336
6079
  const text =
5337
6080
  `[refusal-rewind] The ${count} most recent turn(s) were set aside because ` +
5338
6081
  `${why}. You are clear to continue — if you ` +
@@ -5495,6 +6238,10 @@ export class AgentFramework {
5495
6238
  *
5496
6239
  * host-status — report the quiesce state (drained / parked wakes /
5497
6240
  * running background scripts).
6241
+ *
6242
+ * release-provider-wait — end an agent's provider wait (a provider's
6243
+ * stated retry-after, kept across restarts) for one `model` or all, as an
6244
+ * explicit, recorded operator override.
5498
6245
  */
5499
6246
  private async handleHostCommand(
5500
6247
  serverId: string,
@@ -5514,6 +6261,9 @@ export class AgentFramework {
5514
6261
  /** Discord addresses removed by message-granular undo. The durable outbox
5515
6262
  * owns eventual delivery; this is also returned for immediate surfaces. */
5516
6263
  removedRefs?: Array<{ serverId: string; channelId: string; messageId: string }>;
6264
+ /** For message-granular undo: whether awareness marks were scheduled
6265
+ * (not whether Discord has accepted them). */
6266
+ markers?: SurgeryMarkerReceipt;
5517
6267
  hidden?: number;
5518
6268
  /** For `hide`: the Discord (channelId, messageId) of each removed message
5519
6269
  * that carried one — so the surface can mark them with a reaction. */
@@ -5529,6 +6279,8 @@ export class AgentFramework {
5529
6279
  /** For `nudge`: agent state at queue time ('idle' = runs immediately,
5530
6280
  * else it runs once the current turn settles). */
5531
6281
  agentStatus?: string;
6282
+ /** For `release-provider-wait`: the waits released. */
6283
+ released?: Array<{ model: string; until: string | null }>;
5532
6284
  }> {
5533
6285
  if (
5534
6286
  params.command !== 'undo' &&
@@ -5538,7 +6290,8 @@ export class AgentFramework {
5538
6290
  params.command !== 'quiesce' &&
5539
6291
  params.command !== 'resume' &&
5540
6292
  params.command !== 'maintain' &&
5541
- params.command !== 'host-status'
6293
+ params.command !== 'host-status' &&
6294
+ params.command !== 'release-provider-wait'
5542
6295
  ) {
5543
6296
  return { ok: false, error: `Unknown host command: ${String(params.command)}` };
5544
6297
  }
@@ -5575,6 +6328,30 @@ export class AgentFramework {
5575
6328
  if (params.command === 'host-status') {
5576
6329
  return { ok: true, hostMode: this.getHostModeStatus() };
5577
6330
  }
6331
+ // A release names its agent, as host.releaseProviderWait does: the first
6332
+ // agent by default could lift the wrong agent's waits. The name may be an
6333
+ // agent the framework no longer has whose wait the listing still shows
6334
+ // (a disposed conversation fork); releaseProviderWait decides that.
6335
+ if (params.command === 'release-provider-wait') {
6336
+ if (typeof params.agentName !== 'string' || params.agentName === '') {
6337
+ return { ok: false, error: 'release-provider-wait: agentName must name the agent whose waits to release' };
6338
+ }
6339
+ // Only an omitted model, or '*', releases every model: a malformed one
6340
+ // is refused here, before anything changes, never widened.
6341
+ if (!isReleaseModel(params.model)) {
6342
+ return { ok: false, error: "release-provider-wait: model must be a model name, or '*' (or omitted) for every model" };
6343
+ }
6344
+ const by = operatorLabel(String(requester));
6345
+ let released: ReturnType<AgentFramework['releaseProviderWait']>;
6346
+ try {
6347
+ released = this.releaseProviderWait(params.agentName, params.model, by);
6348
+ } catch (error) {
6349
+ return { ok: false, error: error instanceof Error ? error.message : String(error) };
6350
+ }
6351
+ console.error(`[host-command] release-provider-wait agent=${params.agentName} model=${params.model ?? '*'} by=${by} ` +
6352
+ `(server=${serverId}): ${released.length} wait(s) lifted`);
6353
+ return { ok: true, released };
6354
+ }
5578
6355
 
5579
6356
  const agentName = params.agentName ?? [...this.agents.keys()][0];
5580
6357
  if (!agentName || !this.agents.has(agentName)) {
@@ -5757,6 +6534,7 @@ export class AgentFramework {
5757
6534
  ok: true,
5758
6535
  messagesRemoved: r.messagesRemoved,
5759
6536
  removedRefs: r.removedRefs,
6537
+ markers: r.markers,
5760
6538
  lastVisible: r.lastVisible,
5761
6539
  };
5762
6540
  } catch (error) {
@@ -5793,7 +6571,9 @@ export class AgentFramework {
5793
6571
  return { ok: true, undone: 0, requested, lastVisible: null };
5794
6572
  }
5795
6573
 
5796
- await this.syncDiscordAwarenessMarkers();
6574
+ // The branch has moved: delivery of the marks it changes starts behind
6575
+ // the data-plane gate, and the command doesn't wait for Discord.
6576
+ this.startDiscordAwarenessDelivery('turn undo', agentName);
5797
6577
 
5798
6578
  return { ok: true, undone, requested, lastVisible: await this.lastVisiblePreview(agentName) };
5799
6579
  }
@@ -6388,6 +7168,12 @@ export class AgentFramework {
6388
7168
  * Discord messages that left the live context get awareness markers via the
6389
7169
  * durable outbox, exactly as message-granular `undo` does.
6390
7170
  *
7171
+ * Returns once the switch has landed: the body is applied, with a
7172
+ * marker-scheduling receipt (`markers`). It does not wait for Discord to
7173
+ * accept the reactions: delivery is started, behind the MCPL data-plane
7174
+ * gate, and continues without the caller or the store reservation waiting
7175
+ * on it; its per-message outcomes stay in the outbox ledger.
7176
+ *
6391
7177
  * Throws `OperatorActionError` (`agent-busy`, `unknown-message`, …) — the
6392
7178
  * agent must be idle; nothing is queued.
6393
7179
  */
@@ -6410,6 +7196,8 @@ export class AgentFramework {
6410
7196
  * the group's last shard so the sharded message stays whole. */
6411
7197
  tailMessageId: string;
6412
7198
  removedRefs: Array<{ serverId: string; channelId: string; messageId: string }>;
7199
+ /** Whether awareness marks were scheduled; never a delivery claim. */
7200
+ markers: SurgeryMarkerReceipt;
6413
7201
  lastVisible: { participant?: string; role?: string; preview?: string } | null;
6414
7202
  }> {
6415
7203
  const logBase: OperatorLogInput = {
@@ -6447,9 +7235,12 @@ export class AgentFramework {
6447
7235
  }
6448
7236
 
6449
7237
  // Gate + reserve the whole store (see reserveStoreForSurgery); held
6450
- // until the switch has landed or been rolled back.
7238
+ // until the switch has landed and its marker scheduling has been
7239
+ // recorded or reported, or the switch has been rolled back. Remote
7240
+ // delivery is not waited on.
6451
7241
  const release = this.reserveStoreForSurgery('roll back', agentName);
6452
7242
  let branchName: string;
7243
+ let markers: SurgeryMarkerReceipt;
6453
7244
  try {
6454
7245
  // Prepare the external side effect before switching Chronicle. If the
6455
7246
  // process dies after the switch but before activate(), startup
@@ -6480,8 +7271,14 @@ export class AgentFramework {
6480
7271
  { cause: error },
6481
7272
  );
6482
7273
  }
6483
- if (markerBatch) this.discordAwarenessOutbox!.activate(markerBatch.id);
6484
- await this.syncDiscordAwarenessMarkers();
7274
+ // The body change has landed. Nothing after this point turns the
7275
+ // rollback into a failure: marks are an outbox outcome, reported
7276
+ // apart from the body (`markers`).
7277
+ markers = this.activateSurgeryMarkers(markerBatch, 'rollback', agentName);
7278
+ // Installs the MCPL data-plane gate synchronously, before the store is
7279
+ // released, and starts delivery without awaiting it (the first
7280
+ // reaction request may already be on the wire when this returns).
7281
+ this.startDiscordAwarenessDelivery('rollback', agentName);
6485
7282
  this.materializeConfigMountAfterBranchSwitch();
6486
7283
  } finally {
6487
7284
  release();
@@ -6490,7 +7287,8 @@ export class AgentFramework {
6490
7287
  console.error(
6491
7288
  `[operator] rollback agent=${agentName} to=${tailMessageId}` +
6492
7289
  `${tailMessageId !== opts.messageId ? ` (requested ${opts.messageId}, snapped to end of body group)` : ''}` +
6493
- ` removed=${messagesRemoved} branch=${branchName} by=${describeRequester(opts.requester)}`,
7290
+ ` removed=${messagesRemoved} branch=${branchName} marks=${describeMarkers(markers)}` +
7291
+ ` by=${describeRequester(opts.requester)}`,
6494
7292
  );
6495
7293
  this.recordOperatorAction({
6496
7294
  ...logBase,
@@ -6500,6 +7298,7 @@ export class AgentFramework {
6500
7298
  tailMessageId,
6501
7299
  messagesRemoved,
6502
7300
  discordRefs: removedRefs.length,
7301
+ markers,
6503
7302
  },
6504
7303
  });
6505
7304
  return {
@@ -6509,6 +7308,7 @@ export class AgentFramework {
6509
7308
  messagesRemoved,
6510
7309
  tailMessageId,
6511
7310
  removedRefs,
7311
+ markers,
6512
7312
  lastVisible: await this.lastVisiblePreview(agentName),
6513
7313
  };
6514
7314
  } catch (error) {
@@ -6524,6 +7324,9 @@ export class AgentFramework {
6524
7324
  * removed together. Discord originals get awareness markers via the outbox;
6525
7325
  * the batch is `explicit`-activated only after every removal succeeded, and
6526
7326
  * a crash mid-way is finished at next boot (resumePreparedDiscordSuppressions).
7327
+ * Like rollbackToMessage, it returns once the body change has landed, with
7328
+ * a marker-scheduling receipt (`markers`); delivery continues without the
7329
+ * caller waiting on it.
6527
7330
  *
6528
7331
  * Not retroactive over derived state: a message already folded into an
6529
7332
  * autobiographical summary stays in that summary — roll back to before it
@@ -6545,6 +7348,8 @@ export class AgentFramework {
6545
7348
  messagesRemoved: number;
6546
7349
  removedIds: string[];
6547
7350
  removedRefs: Array<{ serverId: string; channelId: string; messageId: string }>;
7351
+ /** Whether awareness marks were scheduled; never a delivery claim. */
7352
+ markers: SurgeryMarkerReceipt;
6548
7353
  lastVisible: { participant?: string; role?: string; preview?: string } | null;
6549
7354
  }> {
6550
7355
  const requestedIds = [...new Set(opts.messageIds.map(String))];
@@ -6597,9 +7402,12 @@ export class AgentFramework {
6597
7402
  throw new OperatorActionError('invalid', 'Suppression branch name must differ from the active branch');
6598
7403
  }
6599
7404
  // Gate + reserve the whole store (see reserveStoreForSurgery); held
6600
- // until the fork is fully redacted or the source is restored.
7405
+ // until the fork is fully redacted and its marker scheduling has been
7406
+ // recorded or reported, or the source is restored. Remote delivery is
7407
+ // not waited on.
6601
7408
  const release = this.reserveStoreForSurgery('suppress', agentName);
6602
7409
  let createdBranch: string;
7410
+ let markers: SurgeryMarkerReceipt;
6603
7411
  try {
6604
7412
  const markerBatch = this.discordAwarenessOutbox?.prepare({
6605
7413
  agentName,
@@ -6640,7 +7448,6 @@ export class AgentFramework {
6640
7448
  if (iv.group || iv.fromId !== iv.toId) cm.removeMessages(iv.fromId as MessageId, iv.toId as MessageId);
6641
7449
  else cm.removeMessage(iv.fromId as MessageId);
6642
7450
  }
6643
- if (markerBatch) this.discordAwarenessOutbox!.activate(markerBatch.id);
6644
7451
  } catch (error) {
6645
7452
  // A partially suppressed branch is not safe to serve from. Keep it
6646
7453
  // for diagnosis, put the agent back on the untouched source — and
@@ -6653,7 +7460,19 @@ export class AgentFramework {
6653
7460
  { cause: error },
6654
7461
  );
6655
7462
  }
6656
- await this.syncDiscordAwarenessMarkers();
7463
+ // Every redaction committed: the body change has landed and the fork
7464
+ // is safe to serve from. Activating the marker batch is bookkeeping
7465
+ // about a world-side effect; if it fails, the suppression still
7466
+ // stands, and the receipt says what became of the marks: not
7467
+ // scheduled once the batch is retired, or unresolved if it can't be.
7468
+ // (The batch's resume journal has nothing left to do, since every
7469
+ // interval has committed, so retiring the batch gives up only the
7470
+ // marks.)
7471
+ markers = this.activateSurgeryMarkers(markerBatch, 'suppress', agentName);
7472
+ // Installs the MCPL data-plane gate synchronously, before the store is
7473
+ // released, and starts delivery without awaiting it (the first
7474
+ // reaction request may already be on the wire when this returns).
7475
+ this.startDiscordAwarenessDelivery('suppress', agentName);
6657
7476
  this.materializeConfigMountAfterBranchSwitch();
6658
7477
  } finally {
6659
7478
  release();
@@ -6661,7 +7480,7 @@ export class AgentFramework {
6661
7480
 
6662
7481
  console.error(
6663
7482
  `[operator] suppress agent=${agentName} removed=${messagesRemoved} branch=${createdBranch} ` +
6664
- `by=${describeRequester(opts.requester)}`,
7483
+ `marks=${describeMarkers(markers)} by=${describeRequester(opts.requester)}`,
6665
7484
  );
6666
7485
  const loggedIds = capIds(removedIds);
6667
7486
  this.recordOperatorAction({
@@ -6673,6 +7492,7 @@ export class AgentFramework {
6673
7492
  removedIds: loggedIds.ids,
6674
7493
  ...(loggedIds.truncated ? { removedIdsTruncated: true } : {}),
6675
7494
  discordRefs: removedRefs.length,
7495
+ markers,
6676
7496
  },
6677
7497
  });
6678
7498
  return {
@@ -6682,6 +7502,7 @@ export class AgentFramework {
6682
7502
  messagesRemoved,
6683
7503
  removedIds,
6684
7504
  removedRefs,
7505
+ markers,
6685
7506
  lastVisible: await this.lastVisiblePreview(agentName),
6686
7507
  };
6687
7508
  } catch (error) {
@@ -6690,6 +7511,108 @@ export class AgentFramework {
6690
7511
  }
6691
7512
  }
6692
7513
 
7514
+ /**
7515
+ * Record a surgery's prepared marker batch as active once its body change
7516
+ * has landed, and say what happened. Never throws: the body is already
7517
+ * applied, so a ledger failure here is reported in the receipt rather than
7518
+ * as a failed surgery an operator might retry. A batch that could not be
7519
+ * activated is retired, so no later reconciliation can deliver it behind a
7520
+ * receipt saying it was not scheduled (`not-scheduled`). If retiring fails
7521
+ * too, the receipt says only that the outcome is `unresolved`: the batch is
7522
+ * retained, and it may still be delivered. A rollback's batch
7523
+ * (activated by target branch) may be promoted by the very next
7524
+ * reconciliation, the one the delivery it starts runs; a suppression's
7525
+ * (activated explicitly) waits for the next start.
7526
+ *
7527
+ * A batch with no refs (a suppression journal with nothing addressable) is
7528
+ * still activated, which closes its resume journal. It schedules no marks,
7529
+ * so it reports `none` whatever the ledger does. If it can be neither
7530
+ * activated nor retired, its journal is left prepared and closes at the
7531
+ * next start: every interval it names has already committed, so the resume
7532
+ * skips them all and activates it.
7533
+ */
7534
+ private activateSurgeryMarkers(
7535
+ batch: DiscordAwarenessBatch | null,
7536
+ verb: 'rollback' | 'suppress',
7537
+ agentName: string,
7538
+ ): SurgeryMarkerReceipt {
7539
+ if (!batch) return { status: 'none', queued: 0 };
7540
+ try {
7541
+ this.discordAwarenessOutbox!.activate(batch.id);
7542
+ } catch (error) {
7543
+ const detail = error instanceof Error ? error.message : String(error);
7544
+ let retired = false;
7545
+ let retireDetail = '';
7546
+ try {
7547
+ retired = this.discordAwarenessOutbox!.discard(batch.id);
7548
+ } catch (discardError) {
7549
+ retireDetail = discardError instanceof Error ? discardError.message : String(discardError);
7550
+ }
7551
+ console.error(
7552
+ `[discord-awareness] ${verb} agent=${agentName}: body applied, but marker batch ${batch.id} ` +
7553
+ `could not be activated (${detail}); ` +
7554
+ (retired ? 'batch retired' : `batch retained${retireDetail ? ` (retire failed: ${retireDetail})` : ''}`),
7555
+ );
7556
+ if (batch.refs.length === 0) return { status: 'none', queued: 0 };
7557
+ this.opsAlert(
7558
+ retired ? 'discord-awareness-not-scheduled' : 'discord-awareness-unresolved',
7559
+ agentName,
7560
+ retired
7561
+ ? `${verb} applied, but its ${batch.refs.length} awareness mark(s) were not scheduled ` +
7562
+ `(batch ${batch.id} retired): ${detail}`
7563
+ : `${verb} applied, but awareness batch ${batch.id} (${batch.refs.length} mark(s)) could be ` +
7564
+ `neither activated nor retired; it may still be delivered: ${detail}`,
7565
+ { data: { batchId: batch.id, retired } },
7566
+ );
7567
+ return retired
7568
+ ? { status: 'not-scheduled', queued: 0, error: detail }
7569
+ : { status: 'unresolved', queued: 0, batchId: batch.id, error: detail };
7570
+ }
7571
+ if (batch.refs.length === 0) return { status: 'none', queued: 0 };
7572
+ return { status: 'queued', queued: batch.refs.length, batchId: batch.id };
7573
+ }
7574
+
7575
+ /**
7576
+ * Start awareness delivery after a branch move (a surgery, a switch, an
7577
+ * undo or redo) without the caller awaiting it. No operator response waits
7578
+ * for Discord to accept marks: the MCPL data-plane gate alone holds channel
7579
+ * traffic that arrives after it until they are attempted.
7580
+ * `syncDiscordAwarenessMarkers` installs that gate synchronously, before
7581
+ * its first await, so calling this while a surgery still holds the store
7582
+ * puts the gate up before the store is released. With no drain running, the
7583
+ * drain starts in the same synchronous run (the first reaction request can
7584
+ * be written before this returns); with one running, this generation's pass
7585
+ * follows it (drainForGeneration). Either way delivery continues
7586
+ * independently: the caller's response and any store reservation stop
7587
+ * waiting on remote reactions. A turn may run during delivery when its wake
7588
+ * carries no channel traffic (a scheduler timer, a provider admission), and
7589
+ * also when its channel traffic was admitted before the gate went up
7590
+ * (queued when a surgery began, or arriving during its switch or fork): the
7591
+ * surgery's store reservation, which used to hold such a turn through the
7592
+ * drain, now ends once the gate is up. A failure is accounted by the gate
7593
+ * itself: the current generation's failure recycles the connections, and a
7594
+ * superseded one leaves that to the newer generation, whose own pass meets
7595
+ * the same ledger. Here it is logged and raised as an ops alert, and the
7596
+ * next sync, reconnect or start delivers what is still pending.
7597
+ *
7598
+ * @param after What moved the branch, for the log and the alert
7599
+ * ("rollback", "branch switch", "undo" …).
7600
+ * @param agentName The agent the move was for, or "framework" for a
7601
+ * branch switch that names none.
7602
+ */
7603
+ startDiscordAwarenessDelivery(after: string, agentName = 'framework'): void {
7604
+ if (!this.discordAwarenessOutbox) return;
7605
+ this.syncDiscordAwarenessMarkers().catch((error) => {
7606
+ const detail = error instanceof Error ? error.message : String(error);
7607
+ console.error(`[discord-awareness] delivery after ${after} agent=${agentName} failed: ${detail}`);
7608
+ this.opsAlert(
7609
+ 'discord-awareness-delivery',
7610
+ agentName,
7611
+ `Awareness mark delivery after ${after} failed: ${detail}`,
7612
+ );
7613
+ });
7614
+ }
7615
+
6693
7616
  /** Re-materialize config files from the (new) active branch — fire and
6694
7617
  * forget; the gate picks changes up via mtime. Mirrors undoLastTurn. */
6695
7618
  private materializeConfigMountAfterBranchSwitch(): void {
@@ -7129,7 +8052,7 @@ export class AgentFramework {
7129
8052
  const membraneResults = currentState.toolResults.map(tc =>
7130
8053
  this.toMembraneToolResult(tc.id, tc.result, maxChars, spilled.get(tc.id))
7131
8054
  );
7132
- agent.toolResultGuard.storeResults(toolResultContent, membraneResults, currentState.toolResults, turnRowMetadata);
8055
+ agent.toolResultGuard.storeResults(toolResultContent, membraneResults, currentState.toolResults, turnRowMetadata, maxChars);
7133
8056
 
7134
8057
  // Flush any messages that were deferred while this turn was in
7135
8058
  // flight. Route to the PRIMARY agent — deferred messages are
@@ -7822,10 +8745,10 @@ export class AgentFramework {
7822
8745
  isolate: true,
7823
8746
  // Strategy instances are stateful — never share the template's.
7824
8747
  strategy: router.strategyFactory?.() ?? new PassthroughStrategy(),
7825
- // Dynamic conversation forks retain the established provider policy in
7826
- // this bounded first slice. Provider cooldown ownership belongs to the
7827
- // persistent resident only; generation-unique forks must not leak gates.
7828
- membrane: this.membrane,
8748
+ // Provider cooldown ownership belongs to the persistent resident only;
8749
+ // generation-unique forks must not leak gates. A fork's own recorded
8750
+ // provider waits still bind its compression calls.
8751
+ membrane: this.conversationAuxiliaryMembraneFor(name),
7829
8752
  debugLogContext: !!process.env.DEBUG_CONTEXT,
7830
8753
  });
7831
8754
 
@@ -7893,6 +8816,18 @@ export class AgentFramework {
7893
8816
  this.conversationAgentHomes.delete(agentName);
7894
8817
  this.evictTurnCheckpoints(agentName);
7895
8818
  if (agent) this.logicalTurnToolCalls.delete(agent);
8819
+ // A provider-wait hold (nonOwningProviderWait) goes with the agent: left,
8820
+ // its timer would requeue held wakes for an agent that is gone, and a wait
8821
+ // held until released would re-arm for the name indefinitely. The recorded
8822
+ // wait stays, and binds the name if its channel binds again.
8823
+ const cooldown = this.providerAccelerationCooldowns.get(agentName);
8824
+ if (cooldown) {
8825
+ clearTimeout(cooldown.timer);
8826
+ this.providerAccelerationCooldowns.delete(agentName);
8827
+ console.error(`[provider-wait] agent=${agentName} disposed while held` +
8828
+ `${cooldown.waitModel !== undefined ? ` on ${cooldown.waitModel}` : ''}: ` +
8829
+ `${cooldown.heldRequests.length} held request(s) dropped with it (the recorded wait stays)`);
8830
+ }
7896
8831
  this.emitTrace({
7897
8832
  type: 'mcpl:conversation-disposed',
7898
8833
  agentName,
@@ -8751,7 +9686,10 @@ export class AgentFramework {
8751
9686
 
8752
9687
  const providerCooldown = this.providerAccelerationCooldowns?.get(agentName);
8753
9688
  if (providerCooldown) {
8754
- if (now < providerCooldown.until || this.extendHostProviderHold(agentName, providerCooldown)) {
9689
+ // A provider wait binds its own model; with a different primary model
9690
+ // selected now, the turn proceeds (the wait stays recorded).
9691
+ const otherModel = providerCooldown.waitModel !== undefined && providerCooldown.waitModel !== agent.model;
9692
+ if (!otherModel && (now < providerCooldown.until || this.extendHostProviderHold(agentName, providerCooldown))) {
8755
9693
  for (const req of requests) {
8756
9694
  if (!providerCooldown.heldRequests.some((r) => this.sameInferenceRequest(r, req))) {
8757
9695
  providerCooldown.heldRequests.push(req);
@@ -8770,6 +9708,12 @@ export class AgentFramework {
8770
9708
  if (recovery) recovery.releasedAt = now;
8771
9709
  requests = [...providerCooldown.heldRequests, ...requests];
8772
9710
  }
9711
+ // A provider's stated wait on this agent's primary model, including one
9712
+ // recorded before a restart, holds the turn until it passes. An
9713
+ // ephemeral run records nothing, its name being single-use, so no wait,
9714
+ // known or unreadable, binds it (as at the last boundary).
9715
+ const providerWait = this.providerWaits?.active(agentName, agent.model);
9716
+ if (providerWait && !this.ephemeralRuns.has(agentName) && this.holdForProviderWait(agent, providerWait, requests)) continue;
8773
9717
 
8774
9718
  // Skip if agent is busy (inferring, streaming, or waiting for tools) —
8775
9719
  // or if a turn is alive at all (activeTurnTokens): the state machine
@@ -9537,6 +10481,12 @@ export class AgentFramework {
9537
10481
  ): Promise<void> {
9538
10482
  const ownsProviderGate =
9539
10483
  !this.ephemeralRuns.has(agent.name) && !this.conversationAgentHomes.has(agent.name);
10484
+ // A stopping framework starts no turn, whoever owns provider admission
10485
+ // (conversation agents and ephemeral runs do not take the gate).
10486
+ if (this.providerAdmissionClosed) {
10487
+ if (ownsProviderGate && providerGateAlreadyHeld) this.releasePrimaryProviderGate(agent.name);
10488
+ return;
10489
+ }
9540
10490
  if (ownsProviderGate && !providerGateAlreadyHeld) {
9541
10491
  this.acquirePrimaryProviderGate(agent.name);
9542
10492
  const gate = this.providerGate(agent.name);
@@ -9583,7 +10533,7 @@ export class AgentFramework {
9583
10533
  await this.startAgentStream(agent, trigger, attempt, true);
9584
10534
  }).catch((error) => {
9585
10535
  this.releasePrimaryProviderGate(agent.name);
9586
- console.error(`[provider-cooldown] failed to resume primary for ${agent.name}:`, error);
10536
+ console.error(`[provider-cooldown] failed to resume primary for ${agent.name}: ${describeFailureWithFrames(error)}`);
9587
10537
  });
9588
10538
  return;
9589
10539
  }
@@ -9592,6 +10542,35 @@ export class AgentFramework {
9592
10542
  return;
9593
10543
  }
9594
10544
  }
10545
+ // The last admission boundary before this primary call. With the gate
10546
+ // held and no auxiliary call in flight, no new provider wait can be
10547
+ // recorded for this agent before the call (auxiliary admission waits on
10548
+ // the gate). But one may have been recorded since the scheduler's check:
10549
+ // an auxiliary call this wake parked behind can fail with a stated wait
10550
+ // on the primary model. That wait binds this wake too: hold it, don't call.
10551
+ // A conversation agent's waits are recorded and held as a resident's are
10552
+ // (nonOwningProviderWait), and its retries and in-turn restarts reach this
10553
+ // boundary without the scheduler: a wait recorded meanwhile, by its own
10554
+ // auxiliary call or a non-retryable failure, binds them here too. An
10555
+ // ephemeral run records nothing, so nothing binds it here. A frame whose
10556
+ // agent was replaced or removed (a fork disposed while its retry slept)
10557
+ // installs no hold: its turn is dropped after the compile, as main drops
10558
+ // any such turn.
10559
+ if (!this.ephemeralRuns.has(agent.name) && this.agents.get(agent.name) === agent) {
10560
+ const wait = this.providerWaits?.active(agent.name, agent.model);
10561
+ if (wait && this.holdForProviderWait(agent, wait, trigger ? [trigger] : [])) {
10562
+ if (ownsProviderGate) this.releasePrimaryProviderGate(agent.name);
10563
+ // A retry or an in-turn restart arrives with its turn still open at
10564
+ // the event gate: the failed attempt, or the predecessor that kept
10565
+ // liveness for it, leaves closing it to this frame. Held here, no
10566
+ // frame would close it, and while it is open every gate event
10567
+ // buffers, for every agent, until the wait ends.
10568
+ this.eventGate?.onInferenceEnded(agent.name);
10569
+ console.error(`[provider-wait] agent=${agent.name} model=${agent.model} binds at admission ` +
10570
+ `(recorded after the wake was scheduled); wake held, not started`);
10571
+ return;
10572
+ }
10573
+ }
9595
10574
  // Mark the turn alive before ANYTHING awaits (hooks, compile, stream
9596
10575
  // setup): from here until this turn's teardown, cross-turn writers defer
9597
10576
  // (addMessage guard) instead of appending — nothing may enter the window
@@ -9967,6 +10946,28 @@ export class AgentFramework {
9967
10946
  this.eventGate?.onInferenceEnded(agent.name);
9968
10947
  return false;
9969
10948
  }
10949
+ // A conversation agent takes no provider gate, so its own compression
10950
+ // can record a wait while this turn compiles, after the last boundary
10951
+ // checked. Check its waits again here, where nothing has been sent, and
10952
+ // hold the turn as the boundary would rather than send it into the wait.
10953
+ if (this.conversationAgentHomes.has(agent.name)) {
10954
+ const wait = this.providerWaits?.active(agent.name, agent.model);
10955
+ if (wait && this.holdForProviderWait(agent, wait, trigger ? [trigger] : [])) {
10956
+ console.error(`[provider-wait] agent=${agent.name} model=${agent.model} binds after the compile ` +
10957
+ `(recorded while the turn compiled); turn held, nothing sent`);
10958
+ // The turn emitted inference:started before its compile; every turn
10959
+ // that does gets one terminal event, which listeners (a typing loop)
10960
+ // rely on, as the failure sites' holds emit inference:failed.
10961
+ this.emitTrace({ type: 'inference:failed', agentName: agent.name, error: providerWaitText(agent.name, agent.model, wait) });
10962
+ stream.cancel();
10963
+ agent.cancelStream();
10964
+ this.activeStreams.delete(agent.name);
10965
+ this.pendingAssistantBlocks.delete(agent.name);
10966
+ this.channelRegistry?.stopTyping();
10967
+ this.eventGate?.onInferenceEnded(agent.name);
10968
+ return false;
10969
+ }
10970
+ }
9970
10971
 
9971
10972
  const handle = this.driveStream(
9972
10973
  agent,
@@ -10004,10 +11005,28 @@ export class AgentFramework {
10004
11005
  this.eventGate?.onInferenceEnded(agent.name);
10005
11006
  return false;
10006
11007
  }
11008
+ const statedWait = ownsProviderGate ? undefined : this.nonOwningProviderWait(agent, err, trigger);
11009
+ if (statedWait === 'held') {
11010
+ // A conversation agent held by the provider's stated wait, as a
11011
+ // resident is: the hold's release owns the later compile.
11012
+ this.eventGate?.onInferenceEnded(agent.name);
11013
+ return false;
11014
+ }
10007
11015
 
10008
- const action = this.errorPolicy.onInferenceError(err, agent.name, attempt);
11016
+ const action: ErrorAction = statedWait === 'fail' ? { retry: false } : this.errorPolicy.onInferenceError(err, agent.name, attempt);
10009
11017
  if (action.retry) {
10010
- await new Promise((resolve) => setTimeout(resolve, action.delayMs));
11018
+ // A policy's delay can be a provider's retry-after: wait all of it.
11019
+ await this.waitForRetry(action.delayMs);
11020
+ if (this.providerAdmissionClosed) {
11021
+ // stop() ended the wait: that cancels the retry, it does not hasten
11022
+ // it. Nothing is written (the store is closing); this frame still
11023
+ // owns its token, since no retry frame replaced it.
11024
+ if (this.activeTurnTokens.get(agent.name) === turnToken) {
11025
+ this.activeTurnTokens.delete(agent.name);
11026
+ this.activeTurnTriggers.delete(agent.name);
11027
+ }
11028
+ return false;
11029
+ }
10011
11030
  // The retry re-enters startAgentStream, which replaces this frame's
10012
11031
  // turn token with its own; cleanup belongs to the innermost frame.
10013
11032
  await this.startAgentStream(agent, trigger, attempt + 1);
@@ -10024,16 +11043,18 @@ export class AgentFramework {
10024
11043
  this.settleAgent(agent.name, {
10025
11044
  stopReason: 'exhausted',
10026
11045
  speech: '',
10027
- error: err.message,
11046
+ error: statedWait === 'fail' ? this.statedWaitNotSlept(err) : err.message,
10028
11047
  }, startLogical);
10029
11048
  this.emitTrace({
10030
11049
  type: 'inference:exhausted',
10031
11050
  agentName: agent.name,
10032
11051
  error: err.message,
10033
- // Drives the poison-history breaker (`invalid_request`: a 400-class
10034
- // rejection of the history itself — retrying the same context can
10035
- // never succeed; auth/abort/context_length are also non-retryable
10036
- // but must NOT cost history) and the OverBudget drain breaker
11052
+ // Drives the poison-history breaker (shedMayHelp: an
11053
+ // `invalid_request` at HTTP 400, 413 or 422 or with no status, or a
11054
+ // `context_length` at 413, suggests the provider refused the
11055
+ // request's content or size, which a shed of the newest exchange may
11056
+ // fix; auth/abort and other context_length failures are also
11057
+ // non-retryable but must NOT cost history) and the OverBudget drain breaker
10037
11058
  // (`over_budget`: compile refused to fit the hard budget — this is
10038
11059
  // the site that sees it, since compile runs before the stream
10039
11060
  // exists). `retryable` is kept for observability.
@@ -10669,7 +11690,7 @@ export class AgentFramework {
10669
11690
  );
10670
11691
  agent.toolResultGuard.storeResults(toolResultContent, readyState.toolResults.map((tc) =>
10671
11692
  this.toMembraneToolResult(tc.id, tc.result, cap, spilled.get(tc.id))), readyState.toolResults,
10672
- this.silentTurnRowMetadata(agent.name));
11693
+ this.silentTurnRowMetadata(agent.name), cap);
10673
11694
  // The response is already complete: this batch is never
10674
11695
  // submitted in this turn, so it cannot be refused. Admit it.
10675
11696
  agent.toolResultGuard.settleTurnEnded();
@@ -10931,7 +11952,7 @@ export class AgentFramework {
10931
11952
  // border is close, rather than noise on every near-threshold
10932
11953
  // flip. (It must not re-enter any agent's context — the emoji
10933
11954
  // is suppressed at the surface via
10934
- // DISCORD_SUPPRESS_REACTION_EMOJIS; without that, marking a
11955
+ // DISCORD_SUPPRESSED_REACTIONS_BASELINE; without that, marking a
10935
11956
  // refusal feeds the next one.)
10936
11957
  if (!handledByRewind) {
10937
11958
  void this.reactToRefusal(agent.name, category);
@@ -11212,25 +12233,37 @@ export class AgentFramework {
11212
12233
  if (this.agents.get(agent.name) === agent && agent.streamId === myStreamId) this.eventGate?.onInferenceEnded(agent.name);
11213
12234
  break;
11214
12235
  }
12236
+ const statedWait = ownsProviderGate ? undefined : this.nonOwningProviderWait(agent, err, trigger);
12237
+ if (statedWait === 'held') {
12238
+ lifecyclePhase = 'failed';
12239
+ if (this.agents.get(agent.name) === agent && agent.streamId === myStreamId) this.eventGate?.onInferenceEnded(agent.name);
12240
+ break;
12241
+ }
11215
12242
 
11216
- const action = this.errorPolicy.onInferenceError(err, agent.name, attempt);
12243
+ const action: ErrorAction = statedWait === 'fail' ? { retry: false } : this.errorPolicy.onInferenceError(err, agent.name, attempt);
11217
12244
  if (action.retry) {
11218
- await new Promise((resolve) => setTimeout(resolve, action.delayMs));
11219
- await this.startAgentStream(agent, trigger, attempt + 1);
12245
+ // A policy's delay can be a provider's retry-after: wait all of it.
12246
+ await this.waitForRetry(action.delayMs);
12247
+ // stop() ending the wait cancels the retry (this frame's finally
12248
+ // clears the token); it is never permission to retry at once.
12249
+ if (!this.providerAdmissionClosed) await this.startAgentStream(agent, trigger, attempt + 1);
11220
12250
  } else {
11221
12251
  this.settleAgent(agent.name, {
11222
12252
  stopReason: 'exhausted',
11223
12253
  speech: '',
11224
- error: err.message,
12254
+ error: statedWait === 'fail' ? this.statedWaitNotSlept(err) : err.message,
11225
12255
  }, this.agents.get(agent.name) === agent && agent.streamId === myStreamId ? streamRouting.logical : undefined);
11226
12256
  this.emitTrace({
11227
12257
  type: 'inference:exhausted',
11228
12258
  agentName: agent.name,
11229
12259
  error: err.message,
11230
12260
  // Drives the poison-history breaker. `retryable` is kept for
11231
- // observability, but the breakers gate on `errorType` — only an
11232
- // `invalid_request` (a 400-class rejection of the history
11233
- // itself) means retrying the same context can never succeed.
12261
+ // observability, but the breaker gates on
12262
+ // shedMayHelp(errorType, httpStatus): only an `invalid_request`
12263
+ // at HTTP 400, 413 or 422 or with no status, or a
12264
+ // `context_length` at 413, suggests the provider refused the
12265
+ // request's content or size, which a shed of the newest
12266
+ // exchange may fix.
11234
12267
  ...this.classifyInferenceError(err),
11235
12268
  });
11236
12269
  if (this.agents.get(agent.name) === agent && agent.streamId === myStreamId) this.eventGate?.onInferenceEnded(agent.name);
@@ -11456,6 +12489,20 @@ export class AgentFramework {
11456
12489
  lifecyclePhase = 'failed'; agent.reset(); if (this.agents.get(agent.name) === agent && agent.streamId === myStreamId) this.eventGate?.onInferenceEnded(agent.name);
11457
12490
  return;
11458
12491
  }
12492
+ // The same rule for a turn that doesn't own provider admission as at
12493
+ // the other two failure sites: a conversation agent's stated wait is
12494
+ // recorded and its turn held; an ephemeral run whose wait it won't sleep
12495
+ // says why it ends.
12496
+ const statedWait = ownsProviderGate ? undefined : this.nonOwningProviderWait(agent, err, trigger);
12497
+ if (statedWait === 'held') {
12498
+ this.emitTrace({ type: 'inference:failed', agentName: agent.name, error: err.message, stack: err.stack });
12499
+ this.logInference({ timestamp: startTime, agentName: agent.name, requestId, success: false,
12500
+ error: `Provider wait: ${err.message}`,
12501
+ request: compiledRequest ?? { note: 'streaming request rate-limited' }, durationMs });
12502
+ this.abortAgentScript(agent.name, 'provider wait');
12503
+ lifecyclePhase = 'failed'; agent.reset(); if (this.agents.get(agent.name) === agent && agent.streamId === myStreamId) this.eventGate?.onInferenceEnded(agent.name);
12504
+ return;
12505
+ }
11459
12506
  this.emitTrace({
11460
12507
  type: 'inference:failed',
11461
12508
  agentName: agent.name,
@@ -11465,7 +12512,7 @@ export class AgentFramework {
11465
12512
  this.settleAgent(agent.name, {
11466
12513
  stopReason: 'exhausted',
11467
12514
  speech: '',
11468
- error: err.message,
12515
+ error: statedWait === 'fail' ? this.statedWaitNotSlept(err) : err.message,
11469
12516
  }, streamRouting.logical);
11470
12517
  this.emitTrace({
11471
12518
  type: 'inference:exhausted',
@@ -12897,6 +13944,9 @@ export class AgentFramework {
12897
13944
  private logInference(entry: InferenceLogEntry): void {
12898
13945
  // Store large request/response as blobs
12899
13946
  const entryToStore = { ...entry };
13947
+ // The failure text is stored inline in every record; the request that
13948
+ // produced it is already stored separately as a blob.
13949
+ if (typeof entryToStore.error === 'string') entryToStore.error = boundFailureText(entryToStore.error);
12900
13950
 
12901
13951
  // Blob threshold: 10KB - typical context-heavy requests exceed this
12902
13952
  const BLOB_THRESHOLD = 10000;
@@ -13481,6 +14531,10 @@ export class AgentFramework {
13481
14531
  (event.error as string) ?? 'unknown error',
13482
14532
  event.retryable as boolean | undefined,
13483
14533
  event.errorType as string | undefined,
14534
+ {
14535
+ httpStatus: event.httpStatus as number | undefined,
14536
+ providerErrorCode: event.providerErrorCode as string | undefined,
14537
+ },
13484
14538
  );
13485
14539
  } else if (event.type === 'inference:completed') {
13486
14540
  // A successful response — even mid-turn between tool calls — proves the
@@ -13499,8 +14553,26 @@ export class AgentFramework {
13499
14553
  ...event,
13500
14554
  timestamp: Date.now(),
13501
14555
  } as TraceEvent;
13502
-
13503
- for (const listener of this.traceListeners) {
14556
+ // Listeners (web UI, subagent observers, host loggers) receive bounded
14557
+ // failure text. An event's `error` and `stack` strings are whatever its
14558
+ // source's Error carried, which for a provider failure can be an echo of
14559
+ // the whole request.
14560
+ const bounded = traceEvent as unknown as Record<string, unknown>;
14561
+ if (typeof bounded.error === 'string') bounded.error = boundFailureText(bounded.error);
14562
+ if (typeof bounded.stack === 'string') bounded.stack = boundFailureText(bounded.stack);
14563
+
14564
+ // An event goes to the subscriptions present when its delivery starts,
14565
+ // each one only if it is still active when its turn comes, as the DOM's
14566
+ // EventTarget delivers: a subscription made during delivery waits for the
14567
+ // next event, and one removed during delivery is skipped if its turn
14568
+ // hasn't come. Each onTrace call is a subscription of its own, so a
14569
+ // function subscribed twice is called twice. Iterating the live array let
14570
+ // an unsubscribe during delivery make the listener after the one being
14571
+ // called miss the event, since splice shifts the rest under the iterator.
14572
+ for (const registration of [...this.traceListeners]) {
14573
+ if (!registration.active) continue;
14574
+ // Called as a plain function, as before, so the registration isn't its `this`.
14575
+ const { listener } = registration;
13504
14576
  try {
13505
14577
  listener(traceEvent);
13506
14578
  } catch (error) {
@@ -13523,15 +14595,18 @@ export class AgentFramework {
13523
14595
  * inference, so this never causes a retry/wake loop.
13524
14596
  * 3. escalation — after N consecutive failures the agent is hard-down;
13525
14597
  * log that loudly (a repeated identical failure is the textbook signal).
13526
- * 4. breaker — when the failures are an `invalid_request` (a 400-class
13527
- * rejection of the history itself, e.g. corrupted tool_use/tool_result
13528
- * pairing or an oversized attachment), retrying the same context can
13529
- * never succeed: every new push event wakes the agent onto the same
13530
- * poisoned history forever. At the hard-down threshold, automatically
13531
- * quarantine: shed the newest complete exchange (the same forced-rewind
13532
- * primitive `/unstick` uses — shedNewestTurn never orphans a
13533
- * tool_use/thinking block) and retry, bounded by the same rewind cap so
13534
- * the breaker can never eat the whole history.
14598
+ * 4. breaker — when a failure at or past the hard-down threshold suggests
14599
+ * the provider refused the request's content or size (shedMayHelp:
14600
+ * an `invalid_request` at HTTP 400, 413 or 422 or with no status, or a
14601
+ * `context_length` at 413, such as a provider's answer to a corrupted
14602
+ * tool_use/tool_result pairing or an oversized attachment), retrying
14603
+ * the same context is likely to fail the same way, and every new push
14604
+ * event would wake the agent onto that same history. At the hard-down
14605
+ * threshold, automatically quarantine: shed the newest complete
14606
+ * exchange (the same forced-rewind primitive `/unstick` uses —
14607
+ * shedNewestTurn never orphans a tool_use/thinking block) and retry,
14608
+ * bounded by the same rewind cap so the breaker can never eat the
14609
+ * whole history.
13535
14610
  */
13536
14611
  /**
13537
14612
  * Live health snapshot for /healthz and doctor tooling: gate state, queued
@@ -13556,11 +14631,14 @@ export class AgentFramework {
13556
14631
  primaryPending: gate?.primaryPending ?? false,
13557
14632
  auxiliaryInFlight: gate?.auxiliaryInFlight ?? 0,
13558
14633
  auxiliaryDeferred: gate?.deferredAuxiliary ?? 0,
13559
- cooldownUntil: cooldown?.until ?? null,
14634
+ // Null while none, and while a provider wait holds until released.
14635
+ cooldownUntil: cooldown !== undefined && Number.isFinite(cooldown.until) ? cooldown.until : null,
13560
14636
  cooldownReason: cooldown?.reason ?? null,
13561
14637
  hostHold: cooldown?.hostHoldError !== undefined,
13562
14638
  heldRequests: cooldown?.heldRequests.length ?? 0,
13563
14639
  lastRecovery: this.providerAccelerationLastRecovery?.get(name) ?? null,
14640
+ /** Provider waits (stated retry-after) binding this agent now, per model. */
14641
+ providerWaits: this.providerWaitSnapshot(name).map(({ agent: _agent, ...wait }) => wait),
13564
14642
  };
13565
14643
  })(),
13566
14644
  lastInference: this.lastInferenceAt.get(name) ?? null,
@@ -13576,10 +14654,14 @@ export class AgentFramework {
13576
14654
  * and kind-specific fields (additive only — doctor parses by regex). */
13577
14655
  private logFailure(record: Record<string, unknown>): void {
13578
14656
  try {
14657
+ // Every field is bounded: `reason` is conventionally free text, and any
14658
+ // caller's field may carry an error message, or an Error whose own
14659
+ // fields include the whole request.
14660
+ const bounded = boundFailureFields(record);
13579
14661
  mkdirSync('logs', { recursive: true });
13580
14662
  appendFileSync(
13581
14663
  'logs/failures.log',
13582
- JSON.stringify({ at: new Date().toISOString(), ...record }) + '\n',
14664
+ JSON.stringify({ at: new Date().toISOString(), ...bounded }) + '\n',
13583
14665
  );
13584
14666
  } catch { /* best-effort */ }
13585
14667
  }
@@ -13612,13 +14694,27 @@ export class AgentFramework {
13612
14694
  private opsAlert(
13613
14695
  kind: string,
13614
14696
  agentName: string,
13615
- message: string,
14697
+ rawMessage: string,
13616
14698
  opts?: { data?: Record<string, unknown>; skipLog?: boolean },
13617
14699
  ): void {
14700
+ // A JavaScript caller of notifyOps or notifyOpsAlert can pass a message
14701
+ // that isn't a string; it's described as a thrown value is, rather than
14702
+ // thrown on (a string is its own description).
14703
+ const wholeMessage = failureDescription(rawMessage);
14704
+ const message = boundFailureText(wholeMessage);
14705
+ // The caller's data goes to failures.log and to trace listeners, bounded
14706
+ // the same way for both. A JavaScript caller's null is no data, and data
14707
+ // that isn't a plain object (a string, an array, a Buffer) is kept as one
14708
+ // field, `data`, rather than spread into a field per character or element.
14709
+ const rawData: unknown = opts?.data;
14710
+ const plain = typeof rawData === 'object' && rawData !== null &&
14711
+ [Object.prototype, null].includes(Object.getPrototypeOf(rawData));
14712
+ const data = rawData == null ? undefined
14713
+ : boundFailureFields(plain ? rawData as Record<string, unknown> : { data: rawData });
13618
14714
  if (!opts?.skipLog) {
13619
- this.logFailure({ agent: agentName, kind, reason: message, ...opts?.data });
14715
+ this.logFailure({ agent: agentName, kind, reason: message, ...data });
13620
14716
  }
13621
- this.emitTrace({ type: 'ops:alert', kind, agentName, message, data: opts?.data });
14717
+ this.emitTrace({ type: 'ops:alert', kind, agentName, message, data });
13622
14718
 
13623
14719
  const hook = process.env.CONNECTOME_OPS_WEBHOOK;
13624
14720
  if (!hook) return;
@@ -13631,7 +14727,7 @@ export class AgentFramework {
13631
14727
  method: 'POST',
13632
14728
  headers: { 'content-type': 'application/json' },
13633
14729
  body: JSON.stringify({
13634
- content: `\u{1F6A8} **${agentName}** ${kind}: ${message.slice(0, 500)}`,
14730
+ content: `\u{1F6A8} **${agentName}** ${kind}: ${boundFailureText(wholeMessage, OPS_WEBHOOK_MESSAGE_MAX_CHARS)}`,
13635
14731
  }),
13636
14732
  }).catch((err) => console.error(
13637
14733
  `[ops-alert] webhook post failed (${key}):`,
@@ -13684,9 +14780,12 @@ export class AgentFramework {
13684
14780
 
13685
14781
  /**
13686
14782
  * Classify a terminal inference error for the `inference:exhausted` trace.
13687
- * Single-sourced for every emit site: the downstream breakers (poison-history
13688
- * quarantine, OverBudget drain kick) gate on `errorType`, so classification
13689
- * drift between sites would silently disable a safety net.
14783
+ * Single-sourced for every emit site that has an error to classify (a
14784
+ * stream the framework aborted has none): the downstream breakers gate on
14785
+ * what it returns (the poison-history quarantine on `errorType` and
14786
+ * `httpStatus`, through shedMayHelp; the OverBudget drain kick on
14787
+ * `errorType`), so classification drift between sites would silently
14788
+ * change when a safety net acts.
13690
14789
  *
13691
14790
  * context-manager's refusal errors are matched by a real `instanceof` since
13692
14791
  * CM exports them from its package root (context-manager#41 / #71; floor
@@ -13697,9 +14796,23 @@ export class AgentFramework {
13697
14796
  * Deliberately NOT a message match: the message wording belongs to CM and
13698
14797
  * can be reworded without warning.
13699
14798
  */
13700
- private classifyInferenceError(err: Error): { retryable?: boolean; errorType?: string } {
13701
- if (err instanceof MembraneError) {
13702
- return { retryable: err.retryable, errorType: err.type };
14799
+ private classifyInferenceError(err: Error): {
14800
+ retryable?: boolean; errorType?: string; httpStatus?: number; providerErrorCode?: string;
14801
+ } {
14802
+ // From either copy of membrane (isMembraneError). The fields are read as
14803
+ // data, and the type is bounded like describeFailure's, for every sink the
14804
+ // classification reaches (the trace, the marker, the logs).
14805
+ if (isMembraneError(err)) {
14806
+ const fields = err as Error & {
14807
+ retryable?: unknown; type?: unknown; httpStatus?: unknown; providerErrorCode?: unknown;
14808
+ };
14809
+ const providerErrorCode = boundProviderErrorCode(fields.providerErrorCode);
14810
+ return {
14811
+ ...(typeof fields.retryable === 'boolean' ? { retryable: fields.retryable } : {}),
14812
+ ...(typeof fields.type === 'string' ? { errorType: boundFailureText(fields.type, PROVIDER_ERROR_CODE_MAX_CHARS) } : {}),
14813
+ ...(typeof fields.httpStatus === 'number' ? { httpStatus: fields.httpStatus } : {}),
14814
+ ...(providerErrorCode !== undefined ? { providerErrorCode } : {}),
14815
+ };
13703
14816
  }
13704
14817
  if (err instanceof OverBudgetError || err.name === 'OverBudgetError') {
13705
14818
  return { errorType: 'over_budget' };
@@ -13727,25 +14840,70 @@ export class AgentFramework {
13727
14840
 
13728
14841
  private noteInferenceExhausted(
13729
14842
  agentName: string,
13730
- reason: string,
14843
+ rawReason: string,
13731
14844
  retryable?: boolean,
13732
- errorType?: string,
14845
+ rawErrorType?: string,
14846
+ detail?: { httpStatus?: number; providerErrorCode?: string },
13733
14847
  ): void {
14848
+ // The reason is the failure's own message, and a provider can echo the
14849
+ // whole rejected request in it: one household's [inference-failed]
14850
+ // marker carried ~1.7 MB of its own request back into its context. Every
14851
+ // sink below gets an excerpt cut from the whole reason at its own bound
14852
+ // (boundFailureText, so its marker states the whole length): the stderr
14853
+ // line, failures.log and the hard-down alert FAILURE_TEXT_MAX_CHARS, the
14854
+ // [inference-failed] marker FAILURE_MARKER_REASON_MAX_CHARS, the status's
14855
+ // lastError and the compile-refused alert FAILURE_SUMMARY_MAX_CHARS, and
14856
+ // the rewind marker REWIND_MARKER_REASON_MAX_CHARS. The classification
14857
+ // travels as structured fields beside the text.
14858
+ //
14859
+ // A reason that isn't a string (the stream paths always pass a message)
14860
+ // is described rather than allowed to throw before anything is recorded.
14861
+ // Its whole description is the whole reason, cut once by each sink.
14862
+ const wholeReason: string = typeof rawReason === 'string' ? rawReason : failureDescription(rawReason);
14863
+ // The type is bounded wherever it's written, as classifyInferenceError
14864
+ // bounds the types it reads, since a caller can pass one it didn't classify.
14865
+ const errorType = typeof rawErrorType === 'string'
14866
+ ? boundFailureText(rawErrorType, PROVIDER_ERROR_CODE_MAX_CHARS) : rawErrorType;
14867
+ const reason = boundFailureText(wholeReason);
14868
+ const markerReason = boundFailureText(wholeReason, FAILURE_MARKER_REASON_MAX_CHARS);
14869
+ const httpStatus = typeof detail?.httpStatus === 'number' ? detail.httpStatus : undefined;
14870
+ const providerErrorCode = boundProviderErrorCode(detail?.providerErrorCode);
14871
+ const classification: Record<string, unknown> = {
14872
+ ...(errorType !== undefined ? { errorType } : {}),
14873
+ ...(retryable !== undefined ? { retryable } : {}),
14874
+ ...(httpStatus !== undefined ? { httpStatus } : {}),
14875
+ ...(providerErrorCode !== undefined ? { providerErrorCode } : {}),
14876
+ };
14877
+ // Each excerpt says how long the whole was when it was cut.
14878
+ const cutAt = (excerpt: string): Record<string, unknown> =>
14879
+ excerpt === wholeReason ? {} : { reasonChars: wholeReason.length };
13734
14880
  const streak = (this.consecutiveInferenceFailures.get(agentName) ?? 0) + 1;
13735
14881
  this.consecutiveInferenceFailures.set(agentName, streak);
13736
- this.lastInferenceAt.set(agentName, { ...this.lastInferenceAt.get(agentName), failedAt: Date.now(), lastError: reason.slice(0, 300) });
14882
+ this.lastInferenceAt.set(agentName, { ...this.lastInferenceAt.get(agentName), failedAt: Date.now(), lastError: boundFailureText(wholeReason, FAILURE_SUMMARY_MAX_CHARS) });
13737
14883
 
13738
14884
  // (1) Durable stderr line — works in headless/daemon mode with no client.
13739
- console.error(`[inference-failed] agent=${agentName} consecutive=${streak}: ${reason}`);
14885
+ const labels = [
14886
+ errorType !== undefined ? `type=${errorType}` : undefined,
14887
+ httpStatus !== undefined ? `status=${httpStatus}` : undefined,
14888
+ providerErrorCode !== undefined ? `code=${providerErrorCode}` : undefined,
14889
+ ].filter((label): label is string => label !== undefined);
14890
+ console.error(
14891
+ `[inference-failed] agent=${agentName} consecutive=${streak}` +
14892
+ `${labels.length > 0 ? ` ${labels.join(' ')}` : ''}: ${reason}`,
14893
+ );
13740
14894
 
13741
14895
  // (1b) Machine-greppable durable record, independent of journald/unit log
13742
14896
  // redirects: logs/failures.log under the host's working directory. This is
13743
- // what connectome-doctor reads. Legacy fields kept; `kind` is additive.
13744
- this.logFailure({ agent: agentName, consecutive: streak, reason, kind: 'inference-exhausted' });
14897
+ // what connectome-doctor reads. Legacy fields kept; `kind` and the
14898
+ // classification fields are additive.
14899
+ this.logFailure({ agent: agentName, consecutive: streak, reason, kind: 'inference-exhausted', ...classification, ...cutAt(reason) });
13745
14900
 
13746
14901
  // (2) Agent-facing chronicle marker (no inference triggered → no loop).
13747
14902
  const agent = this.agents.get(agentName);
13748
14903
  if (agent && process.env.SUPPRESS_INFERENCE_FAILED_MARKER !== '1') {
14904
+ const cause = [errorType, httpStatus !== undefined ? `HTTP ${httpStatus}` : undefined]
14905
+ .filter((part): part is string => part !== undefined)
14906
+ .join(', ');
13749
14907
  try {
13750
14908
  agent.getContextManager().addMessage(
13751
14909
  'user',
@@ -13753,15 +14911,15 @@ export class AgentFramework {
13753
14911
  type: 'text',
13754
14912
  text:
13755
14913
  `[inference-failed] Your previous turn did not complete: the model ` +
13756
- `call failed and produced no response, so nothing was sent. Reason: ` +
13757
- `${reason}. If this recurs with the same cause, change approach ` +
14914
+ `call failed${cause ? ` (${cause})` : ''} and produced no response, so nothing was sent. Reason: ` +
14915
+ `${markerReason}. If this recurs with the same cause, change approach ` +
13758
14916
  `rather than retrying identically (e.g. drop an oversized attachment ` +
13759
14917
  `or an unsupported setting).`,
13760
14918
  }],
13761
- { system: true, kind: 'inference-failed', reason, consecutive: streak },
14919
+ { system: true, kind: 'inference-failed', reason: markerReason, consecutive: streak, ...classification, ...cutAt(markerReason) },
13762
14920
  );
13763
14921
  } catch (err) {
13764
- console.error(`[inference-failed] could not record chronicle marker for ${agentName}:`, err);
14922
+ console.error(`[inference-failed] could not record chronicle marker for ${agentName}: ${describeFailureWithFrames(err)}`);
13765
14923
  }
13766
14924
  }
13767
14925
 
@@ -13780,10 +14938,16 @@ export class AgentFramework {
13780
14938
  // (e.g. a reason string that crossed a serialization boundary). It matches
13781
14939
  // CM's current OverBudgetError wording and MAY rot if CM rewords it — the
13782
14940
  // errorType gate is the one that's load-bearing.
14941
+ //
14942
+ // The fallback applies only to an unclassified reason. A classified
14943
+ // failure (a provider's 400, say) already says what it is, and its
14944
+ // message is the provider's text, which can quote anything, including
14945
+ // these phrases from the agent's own context.
13783
14946
  const overBudget =
13784
14947
  errorType === 'over_budget' ||
13785
14948
  errorType === 'context_refusal' ||
13786
- /exceed hard budget|no summary covers/i.test(reason);
14949
+ // Decided on the whole reason: presentation bounds never change it.
14950
+ (errorType === undefined && /exceed hard budget|no summary covers/i.test(wholeReason));
13787
14951
  // Observability: a refused compile means the agent cannot think AT ALL
13788
14952
  // this turn — surface it on the ops-alert pipeline (ops:alert trace +
13789
14953
  // webhook → fleet-watch) immediately, not only at the hard-down streak.
@@ -13793,7 +14957,7 @@ export class AgentFramework {
13793
14957
  this.opsAlert(
13794
14958
  'context-refusal',
13795
14959
  agentName,
13796
- `compile refused (${errorType ?? 'unclassified'}): ${reason.slice(0, 300)} — ` +
14960
+ `compile refused (${errorType ?? 'unclassified'}): ${boundFailureText(wholeReason, FAILURE_SUMMARY_MAX_CHARS)} — ` +
13797
14961
  `drain breaker ${agent ? 'kicking' : 'unavailable (no agent handle)'}; ` +
13798
14962
  `if this persists, raise contextBudgetTokens or overBudgetGraceRatio.`,
13799
14963
  { skipLog: true },
@@ -13811,7 +14975,7 @@ export class AgentFramework {
13811
14975
  }
13812
14976
  console.error(`[inference-failed] drain kicked for ${agentName} (OverBudget breaker, ${ticks} ticks)`);
13813
14977
  } catch (err) {
13814
- console.error(`[inference-failed] drain kick failed for ${agentName} after ${ticks} ticks:`, err);
14978
+ console.error(`[inference-failed] drain kick failed for ${agentName} after ${ticks} ticks: ${describeFailureWithFrames(err)}`);
13815
14979
  } finally {
13816
14980
  this.overBudgetDrainInFlight.delete(agentName);
13817
14981
  }
@@ -13828,24 +14992,32 @@ export class AgentFramework {
13828
14992
  // Ops alert (webhook + ops:alert trace), throttled per (agent, kind) —
13829
14993
  // a hard-down agent re-posts every ~15 min, not on every failed retry.
13830
14994
  // failures.log already got the per-exhaustion record above.
14995
+ // The excerpt leaves room for the prefix, so the alert's own bound
14996
+ // never cuts it a second time.
14997
+ const hardDown = `${streak} consecutive inference failures. Last reason: `;
13831
14998
  this.opsAlert(
13832
14999
  'hard-down',
13833
15000
  agentName,
13834
- `${streak} consecutive inference failures. Last reason: ${reason}`,
15001
+ hardDown + boundFailureText(wholeReason, FAILURE_TEXT_MAX_CHARS - hardDown.length),
13835
15002
  { skipLog: true },
13836
15003
  );
13837
15004
 
13838
- // (4) Poison-history breaker: ONLY for `invalid_request` — a 400-class
13839
- // rejection of the history itself. Deliberately NOT keyed on
13840
- // `retryable === false`, which membrane also returns for auth (expired
13841
- // key), abort (deliberate cancel), context_length (compression's job,
13842
- // and shedding newest is the wrong direction), safety and unsupported —
13843
- // none of which mean the history is poisoned, and all of which would
13844
- // otherwise shed good exchanges and stamp a false "the API kept
13845
- // rejecting your history" marker. `retryable` is kept only for the trace.
15005
+ // (4) Poison-history breaker: ONLY when shedMayHelp(errorType,
15006
+ // httpStatus) holds, for an `invalid_request` at HTTP 400, 413 or 422 or
15007
+ // with no status, or a `context_length` at 413, which suggests the
15008
+ // provider refused the request's content or size. The current failure
15009
+ // decides, whatever failed before it in the streak. Deliberately NOT
15010
+ // keyed on `retryable === false`, which membrane also returns for auth
15011
+ // (expired key), abort (deliberate cancel), a context_length at 400 or
15012
+ // 422 (a token overflow, which the context budget exists to prevent;
15013
+ // shedding the newest exchange is the wrong direction for it), safety
15014
+ // and unsupported. For none of them is shedding the newest exchange
15015
+ // likely to help, and each would otherwise shed good exchanges and
15016
+ // stamp a rewind marker saying the API kept refusing them.
15017
+ // `retryable` is kept only for the trace.
13846
15018
  void retryable;
13847
- if (errorType === 'invalid_request' && agent) {
13848
- this.quarantinePoisonedHistory(agent, reason);
15019
+ if (agent && shedMayHelp(errorType, httpStatus)) {
15020
+ this.quarantinePoisonedHistory(agent, wholeReason);
13849
15021
  }
13850
15022
  }
13851
15023
  }
@@ -13856,14 +15028,14 @@ export class AgentFramework {
13856
15028
  * primitives as the refusal auto-rewind / `/unstick` (shedNewestTurn +
13857
15029
  * the single consolidated episode marker) — and queues a retry so the
13858
15030
  * rewound history is verified immediately instead of waiting for the next
13859
- * push event to wake the agent onto the same poisoned context.
15031
+ * push event to wake the agent onto the same context.
13860
15032
  *
13861
15033
  * Bounded: at most `refusalHandling.maxRewinds` (default 3, hard cap 10)
13862
15034
  * sheds per failure episode; the budget resets on any successful inference.
13863
15035
  * At the cap (or with nothing left to shed) it stops — the agent stays up
13864
15036
  * and hard-down logging continues, but no further history is consumed.
13865
15037
  */
13866
- private quarantinePoisonedHistory(agent: Agent, reason: string): void {
15038
+ private quarantinePoisonedHistory(agent: Agent, wholeReason: string): void {
13867
15039
  const cap = Math.max(1, Math.min(10, agent.refusalHandling?.maxRewinds ?? 3));
13868
15040
  const used = this.exhaustionRewinds.get(agent.name) ?? 0;
13869
15041
  if (used >= cap) {
@@ -13884,7 +15056,8 @@ export class AgentFramework {
13884
15056
  }
13885
15057
 
13886
15058
  this.exhaustionRewinds.set(agent.name, used + 1);
13887
- const count = this.updateRewindMarker(agent, truncateReason(reason), 'inference-failure');
15059
+ const count = this.updateRewindMarker(
15060
+ agent, boundFailureText(wholeReason, REWIND_MARKER_REASON_MAX_CHARS), 'inference-failure');
13888
15061
  console.error(
13889
15062
  `[inference-rewind] agent=${agent.name} auto-quarantined ${rec.descriptor} ` +
13890
15063
  `(${count} turn(s) shed this episode, cap ${cap}) after non-retryable ` +
@@ -14124,7 +15297,11 @@ export class AgentFramework {
14124
15297
  await this.refreshMcplTools();
14125
15298
  }
14126
15299
 
14127
- /** Reconcile the durable ledger with Chronicle, then deliver every server's work. */
15300
+ /**
15301
+ * Reconcile the durable ledger with Chronicle, then deliver every server's
15302
+ * work. A drain still running is waited for and then followed by a fresh
15303
+ * pass (drainForGeneration).
15304
+ */
14128
15305
  async syncDiscordAwarenessMarkers(_onlyServerId?: string): Promise<void> {
14129
15306
  if (!this.discordAwarenessOutbox) return;
14130
15307
  // A targeted retry may discover pending work for other Discord servers
@@ -14339,46 +15516,70 @@ export class AgentFramework {
14339
15516
  throw new DiscordAwarenessAccountingError('branch reconciliation', error);
14340
15517
  }
14341
15518
  const pending = this.readDiscordAwarenessPending();
14342
- if (pending.length === 0) {
15519
+ // A drain that is still running holds operations it read before this
15520
+ // reconciliation, and it sends them whatever the ledger now says. A branch
15521
+ // move can hide them from `pending`: restoring a rollback's source makes
15522
+ // its marks undesired, and an add whose reply has not arrived is not yet
15523
+ // present, so the ledger shows nothing to do while the drain goes on to
15524
+ // send its saved adds and then the removals they need. So a generation
15525
+ // accounts for every running drain as well as for every pending
15526
+ // operation, and waits for those drains instead of releasing the data
15527
+ // planes beside them (drainForGeneration).
15528
+ const serverIds = [...new Set([
15529
+ ...pending.map((operation) => operation.ref.serverId),
15530
+ ...this.discordAwarenessDrains.keys(),
15531
+ ])];
15532
+ if (serverIds.length === 0) {
14343
15533
  return {
14344
15534
  requiresBarrier: false,
14345
15535
  promise: Promise.resolve({ status: 'delivered', delivered: 0, failed: 0 }),
14346
15536
  };
14347
15537
  }
14348
- const serverIds = [...new Set(pending.map((operation) => operation.ref.serverId))];
14349
15538
  const promise = Promise.all(
14350
- serverIds.map((serverId) => this.drainDiscordAwarenessOutbox(serverId)),
14351
- ).then((outcomes): DiscordAwarenessDrainOutcome => {
14352
- if (outcomes.every((outcome) => outcome.status === 'unavailable')) {
14353
- return {
14354
- status: 'unavailable',
14355
- accounted: outcomes.reduce(
14356
- (count, outcome) => count + (outcome.status === 'unavailable' ? outcome.accounted : 0),
14357
- 0,
14358
- ),
14359
- };
14360
- }
14361
- return {
14362
- status: 'delivered',
14363
- delivered: outcomes.reduce(
14364
- (count, outcome) => count + (outcome.status === 'delivered' ? outcome.delivered : 0),
14365
- 0,
14366
- ),
14367
- failed: outcomes.reduce(
14368
- (count, outcome) => count + (outcome.status === 'delivered' ? outcome.failed : 0),
14369
- 0,
14370
- ),
14371
- };
14372
- });
15539
+ serverIds.map((serverId) => this.drainForGeneration(serverId)),
15540
+ ).then(combineDrainOutcomes);
14373
15541
  return { requiresBarrier: true, promise };
14374
15542
  }
14375
15543
 
15544
+ /**
15545
+ * One server's drain for a new generation: a gate completes only after
15546
+ * every operation pending when it was armed has been attempted. A drain
15547
+ * already running can't promise that. It never re-sends a key it has
15548
+ * attempted, so an operation re-pended while it runs (a rollback, its undo,
15549
+ * then a redo, all mid-drain) stays pending, and it may already have made
15550
+ * its last read of the ledger, missing a batch activated since. So a
15551
+ * generation that finds a drain running waits for it, then drains again
15552
+ * with a fresh pass that reads the ledger anew. Two generations waiting on
15553
+ * the same drain share the fresh pass that follows it.
15554
+ */
15555
+ private drainForGeneration(serverId: string): Promise<DiscordAwarenessDrainOutcome> {
15556
+ const running = this.discordAwarenessDrains.get(serverId);
15557
+ if (!running) return this.drainDiscordAwarenessOutbox(serverId);
15558
+ return running.then((first) => this.drainDiscordAwarenessOutbox(serverId)
15559
+ .then((fresh) => combineDrainOutcomes([first, fresh])));
15560
+ }
15561
+
15562
+ /**
15563
+ * Wait out the awareness gate for an inbound event: the generation current
15564
+ * now, and any generation installed while waiting, until none is current
15565
+ * or the current one is one already waited for. A handler that waited only
15566
+ * for the generation it first saw would go on behind a newer one armed
15567
+ * meanwhile. Rejects as the generation it waited for rejects.
15568
+ */
15569
+ private async awaitDiscordAwarenessBarriers(): Promise<void> {
15570
+ let waited: DiscordAwarenessBarrier | null = null;
15571
+ for (let barrier = this.discordAwarenessBarrier; barrier && barrier !== waited; barrier = this.discordAwarenessBarrier) {
15572
+ waited = barrier;
15573
+ await barrier.promise;
15574
+ }
15575
+ }
15576
+
14376
15577
  /**
14377
15578
  * Install one framework-global Discord-awareness generation before releasing
14378
- * any inference-bearing event. With pending work, every MCPL data plane is
14379
- * paused while all control planes remain live. With no pending work, ready()
14380
- * runs synchronously so request responders preserve their historical
14381
- * same-stack behavior.
15579
+ * any inference-bearing event. With pending work or a drain still running,
15580
+ * every MCPL data plane is paused while all control planes remain live.
15581
+ * With neither, ready() runs synchronously so request responders preserve
15582
+ * their historical same-stack behavior.
14382
15583
  */
14383
15584
  private installMcplDataPlaneGate(): DiscordAwarenessBarrier {
14384
15585
  for (const connection of this.mcplServerRegistry?.getAllServers() ?? []) {
@@ -14481,6 +15682,14 @@ export class AgentFramework {
14481
15682
  throw new Error('MCPL subsystem is not initialized');
14482
15683
  }
14483
15684
 
15685
+ // Every connect needs a readable awareness ledger: a runtime connect's
15686
+ // awareness barrier reconciles it, and a stdio child taking the
15687
+ // protective default derives that from it. Read it first, so an
15688
+ // unreadable ledger refuses the connect before anything is registered
15689
+ // under its id or spawned (a reconnecting server would otherwise be left
15690
+ // registered as a closed stub). With no outbox configured nothing is read.
15691
+ this.getPlacedReactionBaseline();
15692
+
14484
15693
  // Register prefix + config for tool dispatch routing (idempotent with
14485
15694
  // the pre-registration pass in initializeMcpl).
14486
15695
  const prefix = config.toolPrefix ?? `mcpl--${config.id}`;
@@ -14496,7 +15705,33 @@ export class AgentFramework {
14496
15705
  );
14497
15706
  }
14498
15707
 
14499
- const connection = await this.mcplServerRegistry.addServer(config, this.mcplHostCapabilities);
15708
+ // A stdio child's protective reaction default, DISCORD_SUPPRESSED_REACTIONS_BASELINE
15709
+ // (WebSocket servers own their environment outside this process). The
15710
+ // caller's config stays unchanged, so a runtime restart derives afresh.
15711
+ // The default is derived at each spawn: the env below is a getter, which
15712
+ // the transport reads at first connect, at every automatic reconnect and
15713
+ // at a stub's retry, so retained markers added since the last spawn are
15714
+ // covered.
15715
+ let connectionConfig = config;
15716
+ if (!isWebSocketTransport(config)) {
15717
+ const declared = this.declaredReactionBaseline(config);
15718
+ if (declared) {
15719
+ console.error(
15720
+ `[mcpl] ${config.id}: DISCORD_SUPPRESSED_REACTIONS_BASELINE ${declared.source === 'env' ? 'from its env' : 'inherited from the host'} ` +
15721
+ `replaces the framework's protective default`,
15722
+ );
15723
+ }
15724
+ connectionConfig = { ...config };
15725
+ Object.defineProperty(connectionConfig, 'env', {
15726
+ enumerable: true,
15727
+ configurable: true,
15728
+ get: () => ({
15729
+ ...config.env,
15730
+ DISCORD_SUPPRESSED_REACTIONS_BASELINE: declared?.value ?? this.getPlacedReactionBaseline().join(','),
15731
+ }),
15732
+ });
15733
+ }
15734
+ const connection = await this.mcplServerRegistry.addServer(connectionConfig, this.mcplHostCapabilities);
14500
15735
 
14501
15736
  // Wire listeners before either startup staging or the runtime global gate
14502
15737
  // releases control traffic needed for registration and marker service.
@@ -14862,6 +16097,92 @@ export class AgentFramework {
14862
16097
  // Runtime MCPL server lifecycle (agent-facing hot deploy/restart/unload)
14863
16098
  // ==========================================================================
14864
16099
 
16100
+ /**
16101
+ * Snapshot the protective Discord reaction default for this deployment, as
16102
+ * suppression tokens: the forms the Discord adapter's suppression matcher
16103
+ * reads, not marker payloads for an add, a remove or a replay. It covers
16104
+ * the reactions the framework places: refusal markers, the adapter's own
16105
+ * default awareness marker, the configured awareness marker, and retained
16106
+ * outbox markers (including offline recovery choices and marks that can
16107
+ * later be removed or replayed). A custom emoji appears as its bare ID,
16108
+ * which suppression matches even after a rename; stored markers and the
16109
+ * actual add/remove payloads stay unchanged. Explicit adapter or operator
16110
+ * suppression settings still take precedence. Throws on an unreadable
16111
+ * ledger rather than reporting a partial baseline.
16112
+ */
16113
+ getPlacedReactionBaseline(): readonly string[] {
16114
+ let retained: string[];
16115
+ try {
16116
+ retained = this.discordAwarenessOutbox?.batches().map((batch) => batch.emoji) ?? [];
16117
+ } catch (error) {
16118
+ throw new DiscordAwarenessAccountingError('reaction-baseline ledger read', error);
16119
+ }
16120
+ // Discord events carry the custom emoji's name and ID separately. Full
16121
+ // marker forms belong in add/remove calls, but suppression matches their
16122
+ // stable ID (including after a rename), rather than the composite token.
16123
+ const suppressionToken = (emoji: string): string => {
16124
+ const custom = /^(?:<a?:\w+:(\d{17,20})>|(?:a:)?\w+:(\d{17,20}))$/.exec(emoji.trim());
16125
+ return custom?.[1] ?? custom?.[2] ?? emoji;
16126
+ };
16127
+ return [...new Set([
16128
+ ...REFUSAL_REACTION_BASELINE,
16129
+ DEFAULT_DISCORD_AWARENESS_EMOJI,
16130
+ this.discordAwarenessEmoji,
16131
+ ...retained,
16132
+ ].map(suppressionToken))].filter((emoji) => emoji.length > 0);
16133
+ }
16134
+
16135
+ /**
16136
+ * The reaction baseline an operator chose for a stdio server, if any: a
16137
+ * value declared in its env, or one inherited from the host with
16138
+ * `inheritEnv`, the declared one first. A value with no tokens, as
16139
+ * discord-mcpl's parser reads them, counts as unset: the adapter splits on
16140
+ * commas and keeps a token only when its normalizer leaves something of it
16141
+ * (blank, `:`, `::` and a lone U+FE0F leave nothing), so such a value (an
16142
+ * empty systemd `Environment=`, say, or `${A},${B}` or `:${NAME}:` with the
16143
+ * variables empty) would otherwise turn the protective default off without
16144
+ * anyone choosing that.
16145
+ *
16146
+ * Windows names are matched in both sources: Workers expose a case-
16147
+ * sensitive `process.env` copy. Within one source, Node's spawn keeps the
16148
+ * first spelling in sorted order, so each source is resolved that way.
16149
+ *
16150
+ * A value is read as the child will see it. A config loaded from JSON can
16151
+ * carry a non-string there despite the type, and spawn writes every
16152
+ * defined value as its string form, so a number is a declared value, not a
16153
+ * failed connect. A null counts as unset, like a blank: written in JSON it
16154
+ * means no value, and read as spawn's "null" it would replace the
16155
+ * protective default with a list that suppresses nothing.
16156
+ */
16157
+ private declaredReactionBaseline(
16158
+ config: import('./mcpl/types.js').McplServerConfig,
16159
+ ): { value: string; source: 'env' | 'inherited' } | undefined {
16160
+ // A token as discord-mcpl's parser keeps one: `parseSuppressionEnvTokens`
16161
+ // trims each comma-separated token and keeps it when
16162
+ // `normalizeReactionEmoji` (discord-mcpl `src/filters.ts`) leaves
16163
+ // something, after it strips U+FE0F, trims, and strips one leading and one
16164
+ // trailing colon. The same steps in the same order, so the two can be
16165
+ // compared when either changes.
16166
+ const isToken = (token: string): boolean =>
16167
+ token.trim().replace(/\uFE0F/g, '').trim().replace(/^:|:$/g, '') !== '';
16168
+ const valueIn = (env: NodeJS.ProcessEnv | undefined): string | undefined => {
16169
+ let raw: unknown;
16170
+ if (process.platform !== 'win32') raw = env?.DISCORD_SUPPRESSED_REACTIONS_BASELINE;
16171
+ else {
16172
+ const key = Object.keys(env ?? {}).sort()
16173
+ .find((name) => name.toUpperCase() === 'DISCORD_SUPPRESSED_REACTIONS_BASELINE');
16174
+ raw = key === undefined ? undefined : env?.[key];
16175
+ }
16176
+ if (raw === undefined || raw === null) return undefined;
16177
+ const value = String(raw);
16178
+ return value.split(',').some(isToken) ? value : undefined;
16179
+ };
16180
+ const declared = valueIn(config.env);
16181
+ if (declared !== undefined) return { value: declared, source: 'env' };
16182
+ const inherited = config.inheritEnv ? valueIn(process.env) : undefined;
16183
+ return inherited !== undefined ? { value: inherited, source: 'inherited' } : undefined;
16184
+ }
16185
+
14865
16186
  /**
14866
16187
  * Connect a new MCPL server at runtime. Refreshes the tool list and
14867
16188
  * notifies the agent of newly available tools. Throws if the MCPL
@@ -14944,6 +16265,12 @@ export class AgentFramework {
14944
16265
  if (!config) {
14945
16266
  throw new Error(`MCPL server "${id}" is not configured`);
14946
16267
  }
16268
+ // Every connect reads the awareness ledger: its awareness barrier
16269
+ // reconciles it, and a stdio child taking the protective default derives
16270
+ // that from it. Read it before disconnecting, so an unreadable ledger
16271
+ // refuses the restart and leaves the running child up, whatever the
16272
+ // server's baseline or transport. (No outbox configured: nothing to read.)
16273
+ this.getPlacedReactionBaseline();
14947
16274
  await this.disconnectMcplServer(id);
14948
16275
  await this.connectMcplServer(config);
14949
16276
  }
@@ -15073,8 +16400,10 @@ export class AgentFramework {
15073
16400
  ) => {
15074
16401
  const answer = answerOnce(responder, 'push/event', connection.id);
15075
16402
  try {
15076
- const barrier = this.discordAwarenessBarrier;
15077
- if (barrier) await barrier.promise;
16403
+ // Only when a generation is current: with none, the event is handled in
16404
+ // the turn it arrives, as it always was (the push coalescer's ordering of
16405
+ // a push against its retraction depends on that).
16406
+ if (this.discordAwarenessBarrier) await this.awaitDiscordAwarenessBarriers();
15078
16407
  // Checked here, after the barrier and before anything records the
15079
16408
  // push as accepted: a stopped host refuses it.
15080
16409
  if (this.queue?.isClosed) {
@@ -15104,8 +16433,7 @@ export class AgentFramework {
15104
16433
  if (this.inferenceRouter && responder) {
15105
16434
  const answer = answerOnce(responder, 'inference/request', connection.id);
15106
16435
  try {
15107
- const barrier = this.discordAwarenessBarrier;
15108
- if (barrier) await barrier.promise;
16436
+ if (this.discordAwarenessBarrier) await this.awaitDiscordAwarenessBarriers();
15109
16437
  // A stopping host starts no inference for a server: refused, like a
15110
16438
  // push, before the router runs.
15111
16439
  if (this.queue?.isClosed) {
@@ -15220,8 +16548,7 @@ export class AgentFramework {
15220
16548
  ) => {
15221
16549
  const answer = answerOnce(responder, 'channels/incoming', connection.id);
15222
16550
  try {
15223
- const barrier = this.discordAwarenessBarrier;
15224
- if (barrier) await barrier.promise;
16551
+ if (this.discordAwarenessBarrier) await this.awaitDiscordAwarenessBarriers();
15225
16552
  if (this.queue?.isClosed) {
15226
16553
  this.refuseServerItem('channels/incoming', connection.id, answer, { kind: 'refused' });
15227
16554
  return;
@@ -15240,8 +16567,7 @@ export class AgentFramework {
15240
16567
  if (!responder) return;
15241
16568
  const answer = answerOnce(responder, 'host/command', connection.id);
15242
16569
  try {
15243
- const barrier = this.discordAwarenessBarrier;
15244
- if (barrier) await barrier.promise;
16570
+ if (this.discordAwarenessBarrier) await this.awaitDiscordAwarenessBarriers();
15245
16571
  // A stopping host runs no host command: refused like the requests
15246
16572
  // above, before its handler runs.
15247
16573
  if (this.queue?.isClosed) {
@@ -15288,7 +16614,7 @@ export class AgentFramework {
15288
16614
  // grant) before the data-plane barrier releases — otherwise the first
15289
16615
  // post-reconnect events land while the grant is still empty and are
15290
16616
  // rejected fail-closed instead of delivered.
15291
- void (async () => {
16617
+ const reregistered = (async () => {
15292
16618
  try {
15293
16619
  const config = this.mcplServerConfigs.get(connection.id);
15294
16620
  if (config) {
@@ -15303,7 +16629,11 @@ export class AgentFramework {
15303
16629
  this.releaseMcplDataPlaneGate(awarenessBarrier);
15304
16630
  this.handleToolsListChanged(connection.id);
15305
16631
  })();
15306
- void awarenessBarrier.promise.then(() => {
16632
+ // The gate completes, and the data planes go ready, only once both the
16633
+ // awareness drain and the re-registration have finished. A drain can
16634
+ // settle first (one already running, or little to send), and the grant
16635
+ // must exist before the first data-plane event flows.
16636
+ void Promise.all([awarenessBarrier.promise, reregistered]).then(() => {
15307
16637
  if (
15308
16638
  awarenessBarrier.requiresBarrier
15309
16639
  && !this.completeMcplDataPlaneGate(awarenessBarrier)
@@ -16423,6 +17753,7 @@ export class AgentFramework {
16423
17753
  | {
16424
17754
  kind: 'reference';
16425
17755
  refId: string;
17756
+ /** The type as the stub shows it (a label), lowercased: `image/…`. */
16426
17757
  mediaType: string;
16427
17758
  toolCallId: string;
16428
17759
  toolName: string;
@@ -16476,10 +17807,22 @@ export class AgentFramework {
16476
17807
  };
16477
17808
  }
16478
17809
  if (candidate.kind === 'reference') {
17810
+ // Point at fetch_reference only where it can deliver: not for a
17811
+ // reference this process holds no record of (fetch_reference
17812
+ // looks the id up in the same registry), nor, as the stub does,
17813
+ // for an unfetched one the fetcher would refuse as expired.
17814
+ const record = referenceRegistry.get(candidate.refId);
17815
+ const expiry = record && !record.fetchedPath ? referenceExpiry(record.testimony) : null;
16479
17816
  throw new Error(
16480
17817
  `save_recent_image: image at index ${position} is a reference (${candidate.refId}, ${candidate.mediaType}, ` +
16481
17818
  `from ${describeTool(candidate)}) — its bytes are not held here. Nothing written. ` +
16482
- 'Fetch it with fetch_reference and save the materialized file instead.',
17819
+ (!record
17820
+ ? 'This process holds no record of it (a restart or an eviction drops records, and a stub quoted ' +
17821
+ "from elsewhere never had one), so fetch_reference can't fetch it either. If its tool can produce " +
17822
+ 'it again, re-run that tool for a fresh reference.'
17823
+ : expiry
17824
+ ? `It can't be fetched either — ${EXPIRY_REASON[expiry]}.`
17825
+ : 'Fetch it with fetch_reference and save the materialized file instead.'),
16483
17826
  );
16484
17827
  }
16485
17828
  if (candidate.ref === null) {
@@ -16591,13 +17934,15 @@ export class AgentFramework {
16591
17934
  consider({ kind: 'tool', ref: ph.ref, toolCallId, toolName, mediaType: ph.mediaType, messagesBack });
16592
17935
  continue;
16593
17936
  }
16594
- // Reference stub: the registry's testimony (when this process
16595
- // still has the record) says whether it was an image; the
16596
- // sniffed stub text is the fallback. Non-image references
16597
- // are not slots.
16598
- const testimony = referenceRegistry.get(ph.refId)?.testimony.mimeType?.toLowerCase() ?? ph.mediaType;
16599
- if (!testimony || !testimony.startsWith('image/')) continue;
16600
- consider({ kind: 'reference', refId: ph.refId, mediaType: testimony, toolCallId, toolName, messagesBack });
17937
+ // Reference stub: the registry's claimed type (when this process
17938
+ // still has the record), else the first image/ token in the
17939
+ // stub's text, which can come from its name, says whether it was
17940
+ // an image. Either is read as the stub shows a type, sanitized
17941
+ // and bounded. Non-image references are not slots.
17942
+ const claimed = referenceRegistry.get(ph.refId)?.testimony.mimeType ?? ph.mediaType ?? undefined;
17943
+ const mediaType = referenceTypeLabel(claimed).toLowerCase();
17944
+ if (!mediaType.startsWith('image/')) continue;
17945
+ consider({ kind: 'reference', refId: ph.refId, mediaType, toolCallId, toolName, messagesBack });
16601
17946
  }
16602
17947
  };
16603
17948
 
@@ -16650,10 +17995,14 @@ export class AgentFramework {
16650
17995
  };
16651
17996
  if (block?.type === 'image') {
16652
17997
  const inline = block.source?.type === 'base64' && typeof block.source.data === 'string';
17998
+ // The sender's word, shown in the receipt and stored with the
17999
+ // file, so it's taken as a label; with nothing left of it, it
18000
+ // counts as absent.
18001
+ const claimed = block.source?.mediaType;
16653
18002
  consider({
16654
18003
  kind: 'attachment',
16655
18004
  data: inline ? block.source!.data! : null,
16656
- mediaType: block.source?.mediaType ?? 'image/png',
18005
+ mediaType: (typeof claimed === 'string' ? sanitizeLabel(claimed, REFERENCE_LIMITS.mimeType) : '') || 'image/png',
16657
18006
  messagesBack,
16658
18007
  });
16659
18008
  } else if (block?.type === 'tool_result' && typeof block.content === 'string') {