@animalabs/agent-framework 0.22.0 → 0.22.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (418) hide show
  1. package/README.md +12 -5
  2. package/dist/src/agent.d.ts +192 -19
  3. package/dist/src/agent.d.ts.map +1 -1
  4. package/dist/src/agent.js +407 -66
  5. package/dist/src/agent.js.map +1 -1
  6. package/dist/src/api/mcp-server-core.d.ts +41 -0
  7. package/dist/src/api/mcp-server-core.d.ts.map +1 -0
  8. package/dist/src/api/mcp-server-core.js +593 -0
  9. package/dist/src/api/mcp-server-core.js.map +1 -0
  10. package/dist/src/api/mcp-server.d.ts +2 -36
  11. package/dist/src/api/mcp-server.d.ts.map +1 -1
  12. package/dist/src/api/mcp-server.js +15 -595
  13. package/dist/src/api/mcp-server.js.map +1 -1
  14. package/dist/src/api/server.d.ts +9 -1
  15. package/dist/src/api/server.d.ts.map +1 -1
  16. package/dist/src/api/server.js +84 -23
  17. package/dist/src/api/server.js.map +1 -1
  18. package/dist/src/api/types.d.ts +9 -1
  19. package/dist/src/api/types.d.ts.map +1 -1
  20. package/dist/src/code-execution/py-runner.d.ts.map +1 -1
  21. package/dist/src/code-execution/py-runner.js.map +1 -1
  22. package/dist/src/framework.d.ts +682 -103
  23. package/dist/src/framework.d.ts.map +1 -1
  24. package/dist/src/framework.js +3202 -718
  25. package/dist/src/framework.js.map +1 -1
  26. package/dist/src/gate/event-gate.d.ts.map +1 -1
  27. package/dist/src/gate/gate-script.d.ts.map +1 -1
  28. package/dist/src/gate/gate-script.js.map +1 -1
  29. package/dist/src/index.d.ts +4 -4
  30. package/dist/src/index.d.ts.map +1 -1
  31. package/dist/src/index.js +2 -2
  32. package/dist/src/index.js.map +1 -1
  33. package/dist/src/mcpl/capability-grant.d.ts.map +1 -1
  34. package/dist/src/mcpl/channel-registry.d.ts +26 -7
  35. package/dist/src/mcpl/channel-registry.d.ts.map +1 -1
  36. package/dist/src/mcpl/channel-registry.js +102 -22
  37. package/dist/src/mcpl/channel-registry.js.map +1 -1
  38. package/dist/src/mcpl/checkpoint-manager.d.ts.map +1 -1
  39. package/dist/src/mcpl/checkpoint-manager.js.map +1 -1
  40. package/dist/src/mcpl/conversation-router.d.ts.map +1 -1
  41. package/dist/src/mcpl/errors.js.map +1 -1
  42. package/dist/src/mcpl/feature-set-manager.d.ts.map +1 -1
  43. package/dist/src/mcpl/feature-set-manager.js +4 -3
  44. package/dist/src/mcpl/feature-set-manager.js.map +1 -1
  45. package/dist/src/mcpl/hook-orchestrator.d.ts.map +1 -1
  46. package/dist/src/mcpl/index.d.ts +2 -0
  47. package/dist/src/mcpl/index.d.ts.map +1 -1
  48. package/dist/src/mcpl/index.js.map +1 -1
  49. package/dist/src/mcpl/inference-router.d.ts.map +1 -1
  50. package/dist/src/mcpl/log-value.d.ts +54 -0
  51. package/dist/src/mcpl/log-value.d.ts.map +1 -0
  52. package/dist/src/mcpl/log-value.js +109 -0
  53. package/dist/src/mcpl/log-value.js.map +1 -0
  54. package/dist/src/mcpl/prose-stream-router.d.ts +6 -6
  55. package/dist/src/mcpl/prose-stream-router.d.ts.map +1 -1
  56. package/dist/src/mcpl/prose-stream-router.js +4 -4
  57. package/dist/src/mcpl/prose-stream-router.js.map +1 -1
  58. package/dist/src/mcpl/push-coalescer.d.ts +38 -3
  59. package/dist/src/mcpl/push-coalescer.d.ts.map +1 -1
  60. package/dist/src/mcpl/push-coalescer.js +81 -16
  61. package/dist/src/mcpl/push-coalescer.js.map +1 -1
  62. package/dist/src/mcpl/push-handler.d.ts +26 -3
  63. package/dist/src/mcpl/push-handler.d.ts.map +1 -1
  64. package/dist/src/mcpl/push-handler.js +87 -23
  65. package/dist/src/mcpl/push-handler.js.map +1 -1
  66. package/dist/src/mcpl/reference-fetcher.d.ts +2 -1
  67. package/dist/src/mcpl/reference-fetcher.d.ts.map +1 -1
  68. package/dist/src/mcpl/reference-fetcher.js +20 -9
  69. package/dist/src/mcpl/reference-fetcher.js.map +1 -1
  70. package/dist/src/mcpl/references.d.ts +29 -6
  71. package/dist/src/mcpl/references.d.ts.map +1 -1
  72. package/dist/src/mcpl/references.js +57 -16
  73. package/dist/src/mcpl/references.js.map +1 -1
  74. package/dist/src/mcpl/refusal-lines.d.ts +37 -0
  75. package/dist/src/mcpl/refusal-lines.d.ts.map +1 -0
  76. package/dist/src/mcpl/refusal-lines.js +61 -0
  77. package/dist/src/mcpl/refusal-lines.js.map +1 -0
  78. package/dist/src/mcpl/scope-manager.d.ts.map +1 -1
  79. package/dist/src/mcpl/server-connection.d.ts +46 -3
  80. package/dist/src/mcpl/server-connection.d.ts.map +1 -1
  81. package/dist/src/mcpl/server-connection.js +98 -15
  82. package/dist/src/mcpl/server-connection.js.map +1 -1
  83. package/dist/src/mcpl/server-registry.d.ts.map +1 -1
  84. package/dist/src/mcpl/tool-classes.d.ts +1 -1
  85. package/dist/src/mcpl/tool-classes.d.ts.map +1 -1
  86. package/dist/src/mcpl/tool-classes.js +3 -2
  87. package/dist/src/mcpl/tool-classes.js.map +1 -1
  88. package/dist/src/mcpl/tool-lifecycle.d.ts.map +1 -1
  89. package/dist/src/mcpl/tool-lifecycle.js.map +1 -1
  90. package/dist/src/mcpl/transport.d.ts +23 -5
  91. package/dist/src/mcpl/transport.d.ts.map +1 -1
  92. package/dist/src/mcpl/transport.js +103 -10
  93. package/dist/src/mcpl/transport.js.map +1 -1
  94. package/dist/src/mcpl/types.d.ts +30 -26
  95. package/dist/src/mcpl/types.d.ts.map +1 -1
  96. package/dist/src/mcpl/types.js.map +1 -1
  97. package/dist/src/module-registry.d.ts +9 -3
  98. package/dist/src/module-registry.d.ts.map +1 -1
  99. package/dist/src/module-registry.js +3 -2
  100. package/dist/src/module-registry.js.map +1 -1
  101. package/dist/src/modules/api/index.d.ts.map +1 -1
  102. package/dist/src/modules/discord/discord-js-client.d.ts.map +1 -1
  103. package/dist/src/modules/discord/discord-js-client.js.map +1 -1
  104. package/dist/src/modules/discord/index.d.ts.map +1 -1
  105. package/dist/src/modules/discord/index.js +4 -2
  106. package/dist/src/modules/discord/index.js.map +1 -1
  107. package/dist/src/modules/health/index.d.ts.map +1 -1
  108. package/dist/src/modules/health/index.js.map +1 -1
  109. package/dist/src/modules/history/index.d.ts.map +1 -1
  110. package/dist/src/modules/history/index.js.map +1 -1
  111. package/dist/src/modules/history/semantic.d.ts.map +1 -1
  112. package/dist/src/modules/history/semantic.js.map +1 -1
  113. package/dist/src/modules/workspace/branch-intent.d.ts.map +1 -1
  114. package/dist/src/modules/workspace/branch-intent.js.map +1 -1
  115. package/dist/src/modules/workspace/disk-agreement.d.ts +22 -4
  116. package/dist/src/modules/workspace/disk-agreement.d.ts.map +1 -1
  117. package/dist/src/modules/workspace/disk-agreement.js +63 -25
  118. package/dist/src/modules/workspace/disk-agreement.js.map +1 -1
  119. package/dist/src/modules/workspace/effects.d.ts.map +1 -1
  120. package/dist/src/modules/workspace/index.d.ts +8 -4
  121. package/dist/src/modules/workspace/index.d.ts.map +1 -1
  122. package/dist/src/modules/workspace/index.js +85 -16
  123. package/dist/src/modules/workspace/index.js.map +1 -1
  124. package/dist/src/modules/workspace/observe.d.ts +56 -11
  125. package/dist/src/modules/workspace/observe.d.ts.map +1 -1
  126. package/dist/src/modules/workspace/observe.js +288 -43
  127. package/dist/src/modules/workspace/observe.js.map +1 -1
  128. package/dist/src/modules/workspace/reconcile.d.ts +31 -1
  129. package/dist/src/modules/workspace/reconcile.d.ts.map +1 -1
  130. package/dist/src/modules/workspace/reconcile.js +63 -12
  131. package/dist/src/modules/workspace/reconcile.js.map +1 -1
  132. package/dist/src/modules/workspace/types.d.ts +27 -2
  133. package/dist/src/modules/workspace/types.d.ts.map +1 -1
  134. package/dist/src/modules/workspace/types.js.map +1 -1
  135. package/dist/src/modules/workspace/watcher.d.ts.map +1 -1
  136. package/dist/src/modules/workspace/watcher.js.map +1 -1
  137. package/dist/src/operator-log.d.ts +41 -0
  138. package/dist/src/operator-log.d.ts.map +1 -1
  139. package/dist/src/operator-log.js.map +1 -1
  140. package/dist/src/provider-waits.d.ts +216 -0
  141. package/dist/src/provider-waits.d.ts.map +1 -0
  142. package/dist/src/provider-waits.js +482 -0
  143. package/dist/src/provider-waits.js.map +1 -0
  144. package/dist/src/queue.d.ts.map +1 -1
  145. package/dist/src/record-journal.d.ts.map +1 -1
  146. package/dist/src/record-journal.js.map +1 -1
  147. package/dist/src/recovery/discord-awareness-outbox.d.ts.map +1 -1
  148. package/dist/src/recovery/discord-awareness-outbox.js.map +1 -1
  149. package/dist/src/recovery/offline-branch.d.ts.map +1 -1
  150. package/dist/src/recovery/offline-branch.js +29 -24
  151. package/dist/src/recovery/offline-branch.js.map +1 -1
  152. package/dist/src/recovery/recover-cli.js.map +1 -1
  153. package/dist/src/refusal-reactions.d.ts +14 -12
  154. package/dist/src/refusal-reactions.d.ts.map +1 -1
  155. package/dist/src/refusal-reactions.js +14 -12
  156. package/dist/src/refusal-reactions.js.map +1 -1
  157. package/dist/src/runtime/liveness-watchdog.d.ts.map +1 -1
  158. package/dist/src/runtime/liveness-watchdog.js.map +1 -1
  159. package/dist/src/shutdown.d.ts.map +1 -1
  160. package/dist/src/shutdown.js.map +1 -1
  161. package/dist/src/stored-removal.d.ts +73 -0
  162. package/dist/src/stored-removal.d.ts.map +1 -0
  163. package/dist/src/stored-removal.js +100 -0
  164. package/dist/src/stored-removal.js.map +1 -0
  165. package/dist/src/thinking-token-stamp.js.map +1 -1
  166. package/dist/src/tool-image-ledger.d.ts.map +1 -1
  167. package/dist/src/tool-image-ledger.js.map +1 -1
  168. package/dist/src/tool-presentation.d.ts.map +1 -1
  169. package/dist/src/tool-presentation.js.map +1 -1
  170. package/dist/src/tool-result-guard.d.ts +30 -13
  171. package/dist/src/tool-result-guard.d.ts.map +1 -1
  172. package/dist/src/tool-result-guard.js +71 -25
  173. package/dist/src/tool-result-guard.js.map +1 -1
  174. package/dist/src/tune-out/coordinator.d.ts.map +1 -1
  175. package/dist/src/tune-out/coordinator.js.map +1 -1
  176. package/dist/src/tune-out/tools.d.ts +1 -1
  177. package/dist/src/tune-out/tools.d.ts.map +1 -1
  178. package/dist/src/types/agent.d.ts +42 -6
  179. package/dist/src/types/agent.d.ts.map +1 -1
  180. package/dist/src/types/events.d.ts +12 -0
  181. package/dist/src/types/events.d.ts.map +1 -1
  182. package/dist/src/types/framework.d.ts +10 -0
  183. package/dist/src/types/framework.d.ts.map +1 -1
  184. package/dist/src/types/module.d.ts +20 -8
  185. package/dist/src/types/module.d.ts.map +1 -1
  186. package/dist/src/types/trace.d.ts +75 -13
  187. package/dist/src/types/trace.d.ts.map +1 -1
  188. package/dist/src/usage/usage-tracker.d.ts.map +1 -1
  189. package/dist/test/agent-runtime-settings.test.js +1139 -3
  190. package/dist/test/agent-runtime-settings.test.js.map +1 -1
  191. package/dist/test/batched-wake-locus.test.js.map +1 -1
  192. package/dist/test/channel-label-history.test.js.map +1 -1
  193. package/dist/test/code-execution.test.js.map +1 -1
  194. package/dist/test/conversation-fork-injection-scoping.test.js.map +1 -1
  195. package/dist/test/conversation-router.test.js.map +1 -1
  196. package/dist/test/conversation-routing.test.js +240 -6
  197. package/dist/test/conversation-routing.test.js.map +1 -1
  198. package/dist/test/discord-awareness-outbox.test.js +107 -0
  199. package/dist/test/discord-awareness-outbox.test.js.map +1 -1
  200. package/dist/test/discord-typing-keepalive.test.d.ts +2 -0
  201. package/dist/test/discord-typing-keepalive.test.d.ts.map +1 -0
  202. package/dist/test/discord-typing-keepalive.test.js +77 -0
  203. package/dist/test/discord-typing-keepalive.test.js.map +1 -0
  204. package/dist/test/ephemeral-settle.test.js.map +1 -1
  205. package/dist/test/event-gate-passthrough.test.js.map +1 -1
  206. package/dist/test/event-gate-policy-mutation.test.js.map +1 -1
  207. package/dist/test/event-gate-rate-limit.test.js.map +1 -1
  208. package/dist/test/event-gate-script.test.js.map +1 -1
  209. package/dist/test/event-gate-selfwake.test.js.map +1 -1
  210. package/dist/test/event-gate-tags.test.js.map +1 -1
  211. package/dist/test/event-gate.test.js.map +1 -1
  212. package/dist/test/explicit-prose-routing.test.js +363 -44
  213. package/dist/test/explicit-prose-routing.test.js.map +1 -1
  214. package/dist/test/framework.test.js +160 -1
  215. package/dist/test/framework.test.js.map +1 -1
  216. package/dist/test/helpers/coalescing-fixture.d.ts +2 -0
  217. package/dist/test/helpers/coalescing-fixture.d.ts.map +1 -1
  218. package/dist/test/helpers/coalescing-fixture.js +11 -4
  219. package/dist/test/helpers/coalescing-fixture.js.map +1 -1
  220. package/dist/test/helpers/mock-membrane.d.ts.map +1 -1
  221. package/dist/test/helpers/mock-membrane.js.map +1 -1
  222. package/dist/test/hide-then-turn.test.d.ts +2 -0
  223. package/dist/test/hide-then-turn.test.d.ts.map +1 -0
  224. package/dist/test/hide-then-turn.test.js +101 -0
  225. package/dist/test/hide-then-turn.test.js.map +1 -0
  226. package/dist/test/history-module-overview.test.js.map +1 -1
  227. package/dist/test/history-module-semantic.test.js.map +1 -1
  228. package/dist/test/history-module.test.js.map +1 -1
  229. package/dist/test/history-semantic-review-findings.test.js.map +1 -1
  230. package/dist/test/host-quiesce.test.js +125 -0
  231. package/dist/test/host-quiesce.test.js.map +1 -1
  232. package/dist/test/inference-exhaustion-rewind.test.js +220 -14
  233. package/dist/test/inference-exhaustion-rewind.test.js.map +1 -1
  234. package/dist/test/inference-failure-bounds.test.d.ts +2 -0
  235. package/dist/test/inference-failure-bounds.test.d.ts.map +1 -0
  236. package/dist/test/inference-failure-bounds.test.js +1202 -0
  237. package/dist/test/inference-failure-bounds.test.js.map +1 -0
  238. package/dist/test/kv-unified-open-flight-retry.test.js.map +1 -1
  239. package/dist/test/late-events-after-stop.test.d.ts +2 -0
  240. package/dist/test/late-events-after-stop.test.d.ts.map +1 -0
  241. package/dist/test/late-events-after-stop.test.js +122 -0
  242. package/dist/test/late-events-after-stop.test.js.map +1 -0
  243. package/dist/test/liveness-watchdog.test.js.map +1 -1
  244. package/dist/test/maintenance-timer.test.js +1564 -2
  245. package/dist/test/maintenance-timer.test.js.map +1 -1
  246. package/dist/test/mcp-server-bin.test.d.ts +2 -0
  247. package/dist/test/mcp-server-bin.test.d.ts.map +1 -0
  248. package/dist/test/mcp-server-bin.test.js +142 -0
  249. package/dist/test/mcp-server-bin.test.js.map +1 -0
  250. package/dist/test/mcpl-awareness-barrier.test.js +141 -0
  251. package/dist/test/mcpl-awareness-barrier.test.js.map +1 -1
  252. package/dist/test/mcpl-child-env.test.js +1 -1
  253. package/dist/test/mcpl-child-env.test.js.map +1 -1
  254. package/dist/test/mcpl-coalesced-content-parity.test.d.ts +2 -0
  255. package/dist/test/mcpl-coalesced-content-parity.test.d.ts.map +1 -0
  256. package/dist/test/mcpl-coalesced-content-parity.test.js +532 -0
  257. package/dist/test/mcpl-coalesced-content-parity.test.js.map +1 -0
  258. package/dist/test/mcpl-coalescing.test.js +78 -16
  259. package/dist/test/mcpl-coalescing.test.js.map +1 -1
  260. package/dist/test/mcpl-config-diagnostics.test.js +6 -3
  261. package/dist/test/mcpl-config-diagnostics.test.js.map +1 -1
  262. package/dist/test/mcpl-gate-roundtrip.test.js.map +1 -1
  263. package/dist/test/mcpl-manifest-changed.test.js +2 -1
  264. package/dist/test/mcpl-manifest-changed.test.js.map +1 -1
  265. package/dist/test/mcpl-reconnect-events.test.js.map +1 -1
  266. package/dist/test/mcpl-reconnect-reregistration.test.js +2 -0
  267. package/dist/test/mcpl-reconnect-reregistration.test.js.map +1 -1
  268. package/dist/test/mcpl-reference-fetcher.test.js +232 -4
  269. package/dist/test/mcpl-reference-fetcher.test.js.map +1 -1
  270. package/dist/test/mcpl-reference-fields.test.js +69 -2
  271. package/dist/test/mcpl-reference-fields.test.js.map +1 -1
  272. package/dist/test/mcpl-rejection-log-lines.test.d.ts +2 -0
  273. package/dist/test/mcpl-rejection-log-lines.test.d.ts.map +1 -0
  274. package/dist/test/mcpl-rejection-log-lines.test.js +349 -0
  275. package/dist/test/mcpl-rejection-log-lines.test.js.map +1 -0
  276. package/dist/test/mcpl-stderr-tail.test.d.ts +2 -0
  277. package/dist/test/mcpl-stderr-tail.test.d.ts.map +1 -0
  278. package/dist/test/mcpl-stderr-tail.test.js +210 -0
  279. package/dist/test/mcpl-stderr-tail.test.js.map +1 -0
  280. package/dist/test/mcpl-tool-lifecycle-e2e.test.js.map +1 -1
  281. package/dist/test/mcpl-tool-lifecycle-emitter.test.js.map +1 -1
  282. package/dist/test/mcpl-tool-lifecycle-script.test.js.map +1 -1
  283. package/dist/test/mcpl-tool-lifecycle.test.js.map +1 -1
  284. package/dist/test/mcpl-transport-close.test.d.ts +2 -0
  285. package/dist/test/mcpl-transport-close.test.d.ts.map +1 -0
  286. package/dist/test/mcpl-transport-close.test.js +373 -0
  287. package/dist/test/mcpl-transport-close.test.js.map +1 -0
  288. package/dist/test/mcpl-transport-spawn.test.d.ts +2 -0
  289. package/dist/test/mcpl-transport-spawn.test.d.ts.map +1 -0
  290. package/dist/test/mcpl-transport-spawn.test.js +58 -0
  291. package/dist/test/mcpl-transport-spawn.test.js.map +1 -0
  292. package/dist/test/mcpl-ws-transport.test.js +104 -0
  293. package/dist/test/mcpl-ws-transport.test.js.map +1 -1
  294. package/dist/test/offline-recovery-branch.test.js +344 -0
  295. package/dist/test/offline-recovery-branch.test.js.map +1 -1
  296. package/dist/test/operator-surgery.test.js +197 -0
  297. package/dist/test/operator-surgery.test.js.map +1 -1
  298. package/dist/test/ops-alert.test.js +2 -1
  299. package/dist/test/ops-alert.test.js.map +1 -1
  300. package/dist/test/overbudget-drain-breaker.test.js +1 -0
  301. package/dist/test/overbudget-drain-breaker.test.js.map +1 -1
  302. package/dist/test/pack-installs-alone.test.d.ts +2 -0
  303. package/dist/test/pack-installs-alone.test.d.ts.map +1 -0
  304. package/dist/test/pack-installs-alone.test.js +284 -0
  305. package/dist/test/pack-installs-alone.test.js.map +1 -0
  306. package/dist/test/physical-window-restart.test.d.ts +6 -4
  307. package/dist/test/physical-window-restart.test.d.ts.map +1 -1
  308. package/dist/test/physical-window-restart.test.js +275 -10
  309. package/dist/test/physical-window-restart.test.js.map +1 -1
  310. package/dist/test/placed-reaction-baseline.test.d.ts +2 -0
  311. package/dist/test/placed-reaction-baseline.test.d.ts.map +1 -0
  312. package/dist/test/placed-reaction-baseline.test.js +312 -0
  313. package/dist/test/placed-reaction-baseline.test.js.map +1 -0
  314. package/dist/test/present-while-acting.test.js +949 -6
  315. package/dist/test/present-while-acting.test.js.map +1 -1
  316. package/dist/test/provider-acceleration-cooldown.test.js +5 -1
  317. package/dist/test/provider-acceleration-cooldown.test.js.map +1 -1
  318. package/dist/test/provider-waits.test.d.ts +2 -0
  319. package/dist/test/provider-waits.test.d.ts.map +1 -0
  320. package/dist/test/provider-waits.test.js +2259 -0
  321. package/dist/test/provider-waits.test.js.map +1 -0
  322. package/dist/test/puppet-tool-call.test.js +2 -0
  323. package/dist/test/puppet-tool-call.test.js.map +1 -1
  324. package/dist/test/reaction-baseline-platform.test.d.ts +2 -0
  325. package/dist/test/reaction-baseline-platform.test.d.ts.map +1 -0
  326. package/dist/test/reaction-baseline-platform.test.js +124 -0
  327. package/dist/test/reaction-baseline-platform.test.js.map +1 -0
  328. package/dist/test/record-journal.test.js.map +1 -1
  329. package/dist/test/refusal-reaction-baseline.test.js +10 -3
  330. package/dist/test/refusal-reaction-baseline.test.js.map +1 -1
  331. package/dist/test/refusal-rewind.test.js +9 -0
  332. package/dist/test/refusal-rewind.test.js.map +1 -1
  333. package/dist/test/refusal-single-call.test.js.map +1 -1
  334. package/dist/test/release-bump.test.d.ts +2 -0
  335. package/dist/test/release-bump.test.d.ts.map +1 -0
  336. package/dist/test/release-bump.test.js +248 -0
  337. package/dist/test/release-bump.test.js.map +1 -0
  338. package/dist/test/release-changelog.test.js.map +1 -1
  339. package/dist/test/release-holds.test.d.ts +2 -0
  340. package/dist/test/release-holds.test.d.ts.map +1 -0
  341. package/dist/test/release-holds.test.js +456 -0
  342. package/dist/test/release-holds.test.js.map +1 -0
  343. package/dist/test/same-round-think-policy.test.js.map +1 -1
  344. package/dist/test/save-recent-image.test.js +138 -2
  345. package/dist/test/save-recent-image.test.js.map +1 -1
  346. package/dist/test/shutdown-errors.test.js.map +1 -1
  347. package/dist/test/silent-heartbeat-rows.test.js.map +1 -1
  348. package/dist/test/stop-with-host-operations.test.d.ts +20 -0
  349. package/dist/test/stop-with-host-operations.test.d.ts.map +1 -0
  350. package/dist/test/stop-with-host-operations.test.js +1174 -0
  351. package/dist/test/stop-with-host-operations.test.js.map +1 -0
  352. package/dist/test/stop-with-work-in-flight.test.js +162 -50
  353. package/dist/test/stop-with-work-in-flight.test.js.map +1 -1
  354. package/dist/test/surgery-marks-receipt.test.d.ts +2 -0
  355. package/dist/test/surgery-marks-receipt.test.d.ts.map +1 -0
  356. package/dist/test/surgery-marks-receipt.test.js +540 -0
  357. package/dist/test/surgery-marks-receipt.test.js.map +1 -0
  358. package/dist/test/tool-batch-hook.test.js.map +1 -1
  359. package/dist/test/tool-result-guard.test.js +352 -16
  360. package/dist/test/tool-result-guard.test.js.map +1 -1
  361. package/dist/test/tool-result-spill.test.js.map +1 -1
  362. package/dist/test/tool-wrapper-prose-guard-integration.test.js.map +1 -1
  363. package/dist/test/trunk-channel-routing.test.js +330 -7
  364. package/dist/test/trunk-channel-routing.test.js.map +1 -1
  365. package/dist/test/tune-out-e2e.test.js.map +1 -1
  366. package/dist/test/unconvertible-tool-result.test.js.map +1 -1
  367. package/dist/test/unsent-compiles.test.d.ts +2 -0
  368. package/dist/test/unsent-compiles.test.d.ts.map +1 -0
  369. package/dist/test/unsent-compiles.test.js +139 -0
  370. package/dist/test/unsent-compiles.test.js.map +1 -0
  371. package/dist/test/workspace-branch-continuation.test.js.map +1 -1
  372. package/dist/test/workspace-materialize-freshness.test.js.map +1 -1
  373. package/dist/test/workspace-read-text.test.js.map +1 -1
  374. package/dist/test/workspace-three-way.test.js +1022 -7
  375. package/dist/test/workspace-three-way.test.js.map +1 -1
  376. package/dist/test/workspace-watcher.test.js.map +1 -1
  377. package/dist/test/xml-round-persist.test.js.map +1 -1
  378. package/dist/tsconfig.tsbuildinfo +1 -1
  379. package/package.json +8 -6
  380. package/src/agent.ts +415 -67
  381. package/src/api/mcp-server-core.ts +681 -0
  382. package/src/api/mcp-server.ts +16 -685
  383. package/src/api/server.ts +87 -26
  384. package/src/api/types.ts +9 -1
  385. package/src/framework.ts +3389 -782
  386. package/src/index.ts +9 -4
  387. package/src/mcpl/channel-registry.ts +126 -31
  388. package/src/mcpl/feature-set-manager.ts +4 -3
  389. package/src/mcpl/index.ts +2 -0
  390. package/src/mcpl/log-value.ts +116 -0
  391. package/src/mcpl/prose-stream-router.ts +6 -6
  392. package/src/mcpl/push-coalescer.ts +76 -12
  393. package/src/mcpl/push-handler.ts +95 -23
  394. package/src/mcpl/reference-fetcher.ts +18 -8
  395. package/src/mcpl/references.ts +57 -13
  396. package/src/mcpl/refusal-lines.ts +59 -0
  397. package/src/mcpl/server-connection.ts +125 -15
  398. package/src/mcpl/tool-classes.ts +3 -2
  399. package/src/mcpl/transport.ts +105 -14
  400. package/src/mcpl/types.ts +5 -1
  401. package/src/module-registry.ts +17 -9
  402. package/src/modules/discord/index.ts +4 -2
  403. package/src/modules/workspace/disk-agreement.ts +74 -18
  404. package/src/modules/workspace/index.ts +85 -15
  405. package/src/modules/workspace/observe.ts +295 -43
  406. package/src/modules/workspace/reconcile.ts +90 -12
  407. package/src/modules/workspace/types.ts +26 -1
  408. package/src/operator-log.ts +30 -0
  409. package/src/provider-waits.ts +523 -0
  410. package/src/recovery/offline-branch.ts +32 -28
  411. package/src/refusal-reactions.ts +14 -12
  412. package/src/stored-removal.ts +126 -0
  413. package/src/tool-result-guard.ts +72 -24
  414. package/src/types/agent.ts +42 -6
  415. package/src/types/events.ts +12 -0
  416. package/src/types/framework.ts +10 -0
  417. package/src/types/module.ts +20 -8
  418. package/src/types/trace.ts +76 -14
@@ -0,0 +1,2259 @@
1
+ /**
2
+ * Provider waits: a provider's stated retry-after is a lower bound on the next
3
+ * call to that model, for that agent, across restarts and branch switches.
4
+ *
5
+ * Unit cases drive ProviderWaits on a real Chronicle store; framework cases
6
+ * drive a real AgentFramework whose membrane answers with classified errors
7
+ * carrying retryAfterMs.
8
+ */
9
+ import { describe, it, test } from 'node:test';
10
+ import assert from 'node:assert/strict';
11
+ import { mkdtempSync, rmSync } from 'node:fs';
12
+ import { tmpdir } from 'node:os';
13
+ import { join } from 'node:path';
14
+ import { JsStore } from '@animalabs/chronicle';
15
+ import { MembraneError } from '@animalabs/membrane';
16
+ import { AgentFramework, ApiServer, ProviderWaitNotSleptError } from '../src/index.js';
17
+ import { ProviderWaits, waitDeadline, PROVIDER_WAIT_RECORD_TYPE } from '../src/provider-waits.js';
18
+ import { MockMembrane, MockYieldingStream, createMockResponse } from './helpers/mock-membrane.js';
19
+ function withStoreDir(fn) {
20
+ const dir = mkdtempSync(join(tmpdir(), 'provider-waits-'));
21
+ return Promise.resolve(fn(join(dir, 'store'))).finally(() => rmSync(dir, { recursive: true, force: true }));
22
+ }
23
+ /** A clock tests can move. */
24
+ function clock(start = 1_700_000_000_000) {
25
+ let now = start;
26
+ return { now: () => now, advance: (ms) => { now += ms; } };
27
+ }
28
+ const quiet = { log: () => { } };
29
+ describe('waitDeadline', () => {
30
+ it('is the absolute instant of a usable wait', () => {
31
+ assert.equal(waitDeadline(1_500, 1_000), 2_500);
32
+ assert.equal(waitDeadline(0, 1_000), 1_000);
33
+ assert.equal(waitDeadline(0.4, 1_000), 1_001, 'rounded up, never down');
34
+ });
35
+ it('is null (held until released) for a wait it cannot represent', () => {
36
+ for (const unusable of [Number.NaN, Number.POSITIVE_INFINITY, 1e20, '60', undefined]) {
37
+ assert.equal(waitDeadline(unusable, 1_000), null, String(unusable));
38
+ }
39
+ assert.equal(waitDeadline(8.64e15 - 1_000, 1_000), 8.64e15, 'the last representable instant');
40
+ assert.equal(waitDeadline(8.64e15, 1_000), null, 'one past it');
41
+ });
42
+ it('is the moment it was stated for a negative wait: already over, never held (PR #251 review)', () => {
43
+ // -Infinity too: as negative as any, so over, never held.
44
+ for (const negative of [-1, -1_000, -0.4, -0, -1e20, -Number.MAX_VALUE, Number.NEGATIVE_INFINITY]) {
45
+ assert.equal(waitDeadline(negative, 1_000), 1_000, String(negative));
46
+ }
47
+ });
48
+ });
49
+ describe('ProviderWaits', () => {
50
+ it('binds (agent, model) until the wait passes', () => withStoreDir((path) => {
51
+ const store = JsStore.openOrCreate({ path });
52
+ try {
53
+ const time = clock();
54
+ const waits = new ProviderWaits(store, { now: time.now, ...quiet });
55
+ waits.set('ada', 'zz-model', 60_000, 'zz 429');
56
+ assert.equal(waits.active('ada', 'zz-model')?.until, time.now() + 60_000);
57
+ assert.equal(waits.active('ada', 'zz-other'), undefined, 'another model is not bound');
58
+ assert.equal(waits.active('bo', 'zz-model'), undefined, 'another agent is not bound');
59
+ time.advance(59_999);
60
+ assert.ok(waits.active('ada', 'zz-model'));
61
+ time.advance(1);
62
+ assert.equal(waits.active('ada', 'zz-model'), undefined);
63
+ }
64
+ finally {
65
+ store.close();
66
+ }
67
+ }));
68
+ it('never lets a later, shorter wait shorten an outstanding one; indefinite stays indefinite', () => withStoreDir((path) => {
69
+ const store = JsStore.openOrCreate({ path });
70
+ try {
71
+ const time = clock();
72
+ const waits = new ProviderWaits(store, { now: time.now, ...quiet });
73
+ const long = waits.set('ada', 'zz-model', 600_000, 'long');
74
+ assert.equal(long.changed, true);
75
+ const short = waits.set('ada', 'zz-model', 10_000, 'short');
76
+ assert.equal(short.changed, false);
77
+ assert.equal(short.wait.until, time.now() + 600_000);
78
+ assert.equal(short.wait.reason, 'long');
79
+ const longer = waits.set('ada', 'zz-model', 900_000, 'longer');
80
+ assert.equal(longer.changed, true);
81
+ assert.equal(waits.active('ada', 'zz-model')?.until, time.now() + 900_000);
82
+ waits.set('ada', 'zz-model', Number.NaN, 'unusable');
83
+ assert.equal(waits.active('ada', 'zz-model')?.until, null);
84
+ waits.set('ada', 'zz-model', 5_000, 'finite after indefinite');
85
+ assert.equal(waits.active('ada', 'zz-model')?.until, null);
86
+ time.advance(10 * 365 * 24 * 3_600_000);
87
+ assert.equal(waits.active('ada', 'zz-model')?.until, null, 'an indefinite wait does not pass');
88
+ }
89
+ finally {
90
+ store.close();
91
+ }
92
+ }));
93
+ it('survives a reopen, a checkpoint and a branch switch, with the same lower bound', () => withStoreDir((path) => {
94
+ const time = clock();
95
+ let store = JsStore.openOrCreate({ path });
96
+ try {
97
+ const waits = new ProviderWaits(store, { now: time.now, ...quiet });
98
+ waits.set('ada', 'zz-model', 600_000, 'long');
99
+ waits.set('ada', 'zz-forever', Number.POSITIVE_INFINITY, 'unusable');
100
+ // Enough entries to force a checkpoint, then shorter hints after it.
101
+ for (let i = 0; i < 70; i++)
102
+ waits.set(`agent-${i}`, 'zz-model', 1_000 + i, `hint ${i}`);
103
+ waits.set('ada', 'zz-model', 10_000, 'short, after the checkpoint');
104
+ waits.set('ada', 'zz-forever', 10_000, 'finite, after the checkpoint');
105
+ assert.ok(store.getRecordIdsByType(`${PROVIDER_WAIT_RECORD_TYPE}/checkpoint`).length >= 1, 'a checkpoint was written');
106
+ // A rollback-shaped branch switch: back before every wait was recorded.
107
+ const main = store.currentBranch().name;
108
+ store.createBranchAt('rollback', main, 0);
109
+ store.switchBranch('rollback');
110
+ store.close();
111
+ store = JsStore.openOrCreate({ path });
112
+ assert.equal(store.currentBranch().name, 'rollback');
113
+ const reopened = new ProviderWaits(store, { now: time.now, ...quiet });
114
+ assert.equal(reopened.active('ada', 'zz-model')?.until, time.now() + 600_000, 'the longer wait still binds');
115
+ assert.equal(reopened.active('ada', 'zz-model')?.reason, 'long');
116
+ assert.equal(reopened.active('ada', 'zz-forever')?.until, null, 'the indefinite wait still binds');
117
+ reopened.set('ada', 'zz-model', 20_000, 'shorter again');
118
+ assert.equal(reopened.active('ada', 'zz-model')?.until, time.now() + 600_000);
119
+ }
120
+ finally {
121
+ if (!store.isClosed())
122
+ store.close();
123
+ }
124
+ }));
125
+ it('a recorded release outlives a reopen; releasing one model leaves the others', () => withStoreDir((path) => {
126
+ const time = clock();
127
+ let store = JsStore.openOrCreate({ path });
128
+ try {
129
+ const waits = new ProviderWaits(store, { now: time.now, ...quiet });
130
+ waits.set('ada', 'zz-a', 600_000, 'a');
131
+ waits.set('ada', 'zz-b', 600_000, 'b');
132
+ waits.set('bo', 'zz-a', 600_000, 'bo');
133
+ assert.deepEqual(waits.release('ada', 'zz-a', 'operator').map((r) => r.wait.model), ['zz-a']);
134
+ assert.equal(waits.active('ada', 'zz-a'), undefined);
135
+ assert.ok(waits.active('ada', 'zz-b'));
136
+ store.close();
137
+ store = JsStore.openOrCreate({ path });
138
+ const reopened = new ProviderWaits(store, { now: time.now, ...quiet });
139
+ assert.equal(reopened.active('ada', 'zz-a'), undefined, 'released stays released');
140
+ assert.ok(reopened.active('ada', 'zz-b'));
141
+ assert.deepEqual(reopened.release('ada', undefined, 'operator').map((r) => r.wait.model), ['zz-b']);
142
+ assert.ok(reopened.active('bo', 'zz-a'), "another agent's wait is untouched");
143
+ assert.deepEqual(reopened.release('ada', undefined, 'operator'), [], 'nothing left to release');
144
+ }
145
+ finally {
146
+ if (!store.isClosed())
147
+ store.close();
148
+ }
149
+ }));
150
+ // A negative stated wait (a stream error frame's `retry_after_ms: -1000`, or
151
+ // an HTTP date already past) means "retry now". It used to fall into the
152
+ // indefinite case and hold the model until an operator released it, across
153
+ // reopens, and one arriving during a finite wait made that wait indefinite
154
+ // (PR #251 review, 2026-10-08).
155
+ it('a negative stated wait binds nothing, before or after a reopen, and leaves an outstanding wait as it was', () => withStoreDir((path) => {
156
+ const time = clock();
157
+ let store = JsStore.openOrCreate({ path });
158
+ try {
159
+ const waits = new ProviderWaits(store, { now: time.now, ...quiet });
160
+ for (const stated of [-1, -1_000, 0]) {
161
+ const set = waits.set('ada', 'zz-model', stated, `retry now (${stated})`);
162
+ assert.equal(set.changed, false, `${stated} records nothing`);
163
+ assert.equal(waits.active('ada', 'zz-model'), undefined, `${stated} binds nothing`);
164
+ }
165
+ assert.equal(store.getRecordIdsByType(PROVIDER_WAIT_RECORD_TYPE).length, 0, 'no record for a wait already over');
166
+ waits.set('ada', 'zz-held', 600_000, 'long');
167
+ const after = waits.set('ada', 'zz-held', -1, 'retry now');
168
+ assert.equal(after.changed, false);
169
+ assert.equal(waits.active('ada', 'zz-held')?.until, time.now() + 600_000, 'the outstanding wait stands, neither shortened nor made indefinite');
170
+ store.close();
171
+ store = JsStore.openOrCreate({ path });
172
+ const reopened = new ProviderWaits(store, { now: time.now, ...quiet });
173
+ assert.equal(reopened.active('ada', 'zz-model'), undefined, 'nothing binds after a reopen, at the same instant');
174
+ assert.deepEqual(reopened.list('ada').map((wait) => [wait.model, wait.until]), [['zz-held', time.now() + 600_000]]);
175
+ }
176
+ finally {
177
+ if (!store.isClosed())
178
+ store.close();
179
+ }
180
+ }));
181
+ });
182
+ /** The store's journal surface, with failures a test can arm. */
183
+ function flaky(store) {
184
+ const fail = { appendJson: 0, appendAfter: 0, checkpoint: 0, sync: 0, read: false };
185
+ const surface = {
186
+ getRecordIdsByType: (type) => { if (fail.read)
187
+ throw new Error('zz unreadable record'); return store.getRecordIdsByType(type); },
188
+ getRecord: (id) => store.getRecord(id),
189
+ appendJson: (type, data) => {
190
+ if (fail.appendJson > 0) {
191
+ fail.appendJson--;
192
+ throw new Error('zz append refused');
193
+ }
194
+ // Written, then thrown: as when the store's periodic sync fails after its log has the record.
195
+ if (fail.appendAfter > 0) {
196
+ fail.appendAfter--;
197
+ store.appendJson(type, data);
198
+ throw new Error('zz append threw after writing');
199
+ }
200
+ if (fail.checkpoint > 0 && type !== PROVIDER_WAIT_RECORD_TYPE) {
201
+ fail.checkpoint--;
202
+ throw new Error('zz checkpoint refused');
203
+ }
204
+ return store.appendJson(type, data);
205
+ },
206
+ sync: () => { if (fail.sync > 0) {
207
+ fail.sync--;
208
+ throw new Error('zz disk full');
209
+ } store.sync(); },
210
+ };
211
+ return { surface, fail };
212
+ }
213
+ describe('ProviderWaits: durable and pending acts (room-225 #46276)', () => {
214
+ it('a release made after an ambiguous write stays released: through reconciliation, a checkpoint and a reopen', () => withStoreDir((path) => {
215
+ const time = clock();
216
+ let store = JsStore.openOrCreate({ path });
217
+ try {
218
+ const { surface, fail } = flaky(store);
219
+ const waits = new ProviderWaits(surface, { now: time.now, ...quiet });
220
+ fail.sync = 1; // the set reaches the store, its barrier fails: ambiguous
221
+ waits.set('ada', 'zz-model', 600_000, 'zz 429');
222
+ assert.ok(waits.active('ada', 'zz-model'), 'binds here while its record is unknown');
223
+ const receipt = waits.release('ada', 'zz-model', 'operator');
224
+ assert.deepEqual(receipt.map((r) => [r.wait.model, r.release]), [['zz-model', 'recorded']]);
225
+ assert.equal(waits.active('ada', 'zz-model'), undefined, 'reconciling the set does not undo the release');
226
+ for (let i = 0; i < 70; i++)
227
+ waits.set(`zz-agent-${i}`, 'zz-model', 600_000, 'unrelated'); // past a checkpoint
228
+ assert.equal(waits.active('ada', 'zz-model'), undefined);
229
+ store.close();
230
+ store = JsStore.openOrCreate({ path });
231
+ const reopened = new ProviderWaits(store, { now: time.now, ...quiet });
232
+ assert.equal(reopened.active('ada', 'zz-model'), undefined, 'released stays released after the checkpoint and a reopen');
233
+ assert.ok(reopened.active('zz-agent-69', 'zz-model'));
234
+ }
235
+ finally {
236
+ if (!store.isClosed())
237
+ store.close();
238
+ }
239
+ }));
240
+ it('a binding wait not known to be recorded is recorded at the next act, even one that leaves the binding unchanged', () => withStoreDir((path) => {
241
+ const time = clock();
242
+ let store = JsStore.openOrCreate({ path });
243
+ try {
244
+ const { surface, fail } = flaky(store);
245
+ const waits = new ProviderWaits(surface, { now: time.now, ...quiet });
246
+ const recorded = () => store.getRecordIdsByType(PROVIDER_WAIT_RECORD_TYPE).length;
247
+ fail.sync = 1; // the set reaches the store, its barrier fails: not known to be recorded
248
+ assert.equal(waits.set('ada', 'zz-model', Number.POSITIVE_INFINITY, 'zz indefinite').wait.until, null);
249
+ assert.equal(recorded(), 1, 'its record reached the store, unconfirmed');
250
+ const shorter = waits.set('ada', 'zz-model', 1_000, 'zz shorter');
251
+ assert.equal(shorter.changed, false);
252
+ assert.equal(shorter.wait.until, null, 'the indefinite wait still binds');
253
+ assert.equal(recorded(), 2, 'the later act offered the pending set again, and it was recorded');
254
+ store.close();
255
+ store = JsStore.openOrCreate({ path });
256
+ assert.equal(new ProviderWaits(store, { now: time.now, ...quiet }).active('ada', 'zz-model')?.until, null, 'and survives a reopen');
257
+ }
258
+ finally {
259
+ if (!store.isClosed())
260
+ store.close();
261
+ }
262
+ }));
263
+ it('a release replayed while its record is pending lists nothing, and offers the record again at once (PR #251 review)', () => withStoreDir((path) => {
264
+ const time = clock();
265
+ let store = JsStore.openOrCreate({ path });
266
+ try {
267
+ const { surface, fail } = flaky(store);
268
+ const waits = new ProviderWaits(surface, { now: time.now, ...quiet });
269
+ const recorded = () => store.getRecordIdsByType(PROVIDER_WAIT_RECORD_TYPE).length;
270
+ waits.set('ada', 'zz-model', 600_000, 'zz 429');
271
+ fail.sync = 1; // the release reaches the store, its barrier fails: not known to be recorded
272
+ assert.deepEqual(waits.release('ada', 'zz-model', 'operator').map((r) => [r.wait.model, r.release]), [['zz-model', 'pending']]);
273
+ const before = recorded();
274
+ assert.deepEqual(waits.release('ada', 'zz-model', 'operator'), [], 'nothing binds to lift');
275
+ assert.equal(recorded(), before + 1, 'the replay offered the pending release again at once');
276
+ store.close();
277
+ store = JsStore.openOrCreate({ path });
278
+ assert.equal(new ProviderWaits(store, { now: time.now, ...quiet }).active('ada', 'zz-model'), undefined, 'the replay recorded the release, so a restart keeps it');
279
+ }
280
+ finally {
281
+ if (!store.isClosed())
282
+ store.close();
283
+ }
284
+ }));
285
+ it('absent any later act, a wait not known to be recorded is offered again from admission checks', () => withStoreDir((path) => {
286
+ const time = clock();
287
+ const store = JsStore.openOrCreate({ path });
288
+ try {
289
+ const { surface, fail } = flaky(store);
290
+ const waits = new ProviderWaits(surface, { now: time.now, ...quiet });
291
+ const recorded = () => store.getRecordIdsByType(PROVIDER_WAIT_RECORD_TYPE).length;
292
+ fail.sync = 1; // the set reaches the store, its barrier fails: not known to be recorded
293
+ waits.set('ada', 'zz-model', 3_600_000, 'zz 429');
294
+ const before = recorded();
295
+ waits.active('ada', 'zz-model');
296
+ assert.equal(recorded(), before, 'not retried at once');
297
+ time.advance(30_000);
298
+ waits.active('ada', 'zz-model');
299
+ assert.equal(recorded(), before + 1, 'retried after the interval');
300
+ }
301
+ finally {
302
+ store.close();
303
+ }
304
+ }));
305
+ it('after an append that throws inside the store, waits and releases hold in this process only, said once, and nothing more is written', () => withStoreDir((path) => {
306
+ const time = clock();
307
+ let store = JsStore.openOrCreate({ path });
308
+ try {
309
+ const lines = [];
310
+ const { surface, fail } = flaky(store);
311
+ const waits = new ProviderWaits(surface, { now: time.now, log: (line) => lines.push(line) });
312
+ const recorded = () => store.getRecordIdsByType(PROVIDER_WAIT_RECORD_TYPE).length;
313
+ waits.set('ada', 'zz-recorded', 600_000, 'zz recorded first');
314
+ fail.appendJson = 1; // the store's append throws: the journal writes nothing until the store is reopened
315
+ waits.set('ada', 'zz-model', 600_000, 'zz 429');
316
+ assert.ok(waits.active('ada', 'zz-model'), 'the wait binds in this process');
317
+ waits.set('bob', 'zz-model', 600_000, 'zz later');
318
+ assert.deepEqual(waits.release('ada', 'zz-recorded', 'operator').map((r) => [r.wait.model, r.release]), [['zz-recorded', 'pending']]);
319
+ time.advance(30_000);
320
+ assert.ok(waits.active('bob', 'zz-model'), 'a later wait binds in this process too');
321
+ assert.equal(waits.active('ada', 'zz-recorded'), undefined, 'and the release lifts its wait here');
322
+ assert.equal(recorded(), 1, 'nothing is written after the failed append, not even from the 30 s re-offer');
323
+ assert.equal(lines.filter((line) => line.includes('until the store is reopened')).length, 1, 'said once');
324
+ assert.equal(lines.filter((line) => line.includes('could not record')).length, 0, 'not once per act');
325
+ store.close();
326
+ store = JsStore.openOrCreate({ path });
327
+ const reopened = new ProviderWaits(store, { now: time.now, ...quiet });
328
+ assert.equal(reopened.active('ada', 'zz-model'), undefined, 'a reopened store has none of the acts that held in process only');
329
+ assert.equal(reopened.active('bob', 'zz-model'), undefined);
330
+ assert.ok(reopened.active('ada', 'zz-recorded'), 'and a release not recorded no longer lifts its wait');
331
+ }
332
+ finally {
333
+ if (!store.isClosed())
334
+ store.close();
335
+ }
336
+ }));
337
+ it('after an append that throws once its record reached the store, the next start has that act, and only it', () => withStoreDir((path) => {
338
+ const time = clock();
339
+ let store = JsStore.openOrCreate({ path });
340
+ try {
341
+ const lines = [];
342
+ const { surface, fail } = flaky(store);
343
+ const waits = new ProviderWaits(surface, { now: time.now, log: (line) => lines.push(line) });
344
+ fail.appendAfter = 1;
345
+ waits.set('ada', 'zz-landed', 600_000, 'zz written, then the append threw');
346
+ waits.set('ada', 'zz-later', 600_000, 'zz after the throw');
347
+ assert.ok(waits.active('ada', 'zz-landed'));
348
+ assert.ok(waits.active('ada', 'zz-later'), 'both bind in this process');
349
+ assert.equal(lines.filter((line) => line.includes("may still be in the store's log")).length, 1, 'said once');
350
+ store.close();
351
+ store = JsStore.openOrCreate({ path });
352
+ const reopened = new ProviderWaits(store, { now: time.now, ...quiet });
353
+ assert.ok(reopened.active('ada', 'zz-landed'), 'the act whose append threw after writing: the next start has it');
354
+ assert.equal(reopened.active('ada', 'zz-later'), undefined, 'a later act held in this process only');
355
+ }
356
+ finally {
357
+ if (!store.isClosed())
358
+ store.close();
359
+ }
360
+ }));
361
+ it('after a checkpoint that throws inside the store, nothing more is written, and that is said once', () => withStoreDir((path) => {
362
+ const time = clock();
363
+ let store = JsStore.openOrCreate({ path });
364
+ try {
365
+ const lines = [];
366
+ const { surface, fail } = flaky(store);
367
+ const waits = new ProviderWaits(surface, { now: time.now, log: (line) => lines.push(line) });
368
+ const recorded = () => store.getRecordIdsByType(PROVIDER_WAIT_RECORD_TYPE).length;
369
+ fail.checkpoint = 1; // the 64th entry's flush checkpoints, and the store's append of it throws
370
+ for (let i = 0; i < 64; i++)
371
+ waits.set(`zz-agent-${i}`, 'zz-model', 600_000, 'zz 429');
372
+ assert.deepEqual(waits.release('zz-nobody', 'zz-model', 'operator'), [], 'an act with nothing to record tries no checkpoint again');
373
+ waits.set('ada', 'zz-model', 600_000, 'zz after');
374
+ time.advance(30_000);
375
+ assert.ok(waits.active('ada', 'zz-model'), 'a later wait binds in this process');
376
+ assert.equal(recorded(), 64, 'the entries before the checkpoint are recorded, and nothing after it');
377
+ assert.equal(lines.filter((line) => line.includes('until the store is reopened')).length, 1, 'said once');
378
+ assert.equal(lines.filter((line) => line.includes('could not')).length, 0, 'not as a failed checkpoint or once per act');
379
+ store.close();
380
+ store = JsStore.openOrCreate({ path });
381
+ const reopened = new ProviderWaits(store, { now: time.now, ...quiet });
382
+ assert.ok(reopened.active('zz-agent-63', 'zz-model'), 'a reopened store has the entries');
383
+ assert.equal(reopened.active('ada', 'zz-model'), undefined, 'and not the wait that held in process only');
384
+ }
385
+ finally {
386
+ if (!store.isClosed())
387
+ store.close();
388
+ }
389
+ }));
390
+ });
391
+ describe('ProviderWaits: unreadable history fails closed (room-225 #46189, #46449)', () => {
392
+ it('holds every (agent, model) with a reason, lifts exactly what a release names, and applies that release in order once readable', () => withStoreDir((path) => {
393
+ const time = clock();
394
+ let store = JsStore.openOrCreate({ path });
395
+ try {
396
+ new ProviderWaits(store, { now: time.now, ...quiet }).set('ada', 'zz-a', Number.POSITIVE_INFINITY, 'zz recorded indefinite');
397
+ store.close();
398
+ store = JsStore.openOrCreate({ path });
399
+ const { surface, fail } = flaky(store);
400
+ fail.read = true;
401
+ const lines = [];
402
+ const waits = new ProviderWaits(surface, { now: time.now, log: (line) => lines.push(line) });
403
+ assert.equal(lines.length, 1);
404
+ assert.match(lines[0], /could not be read \(zz unreadable record\): no provider call is admitted/);
405
+ for (const [agent, model] of [['ada', 'zz-a'], ['ada', 'zz-b'], ['bo', 'zz-a']]) {
406
+ const held = waits.active(agent, model);
407
+ assert.equal(held?.until, null, `${agent}/${model} held: absence is unknown, not "no waits"`);
408
+ assert.match(held.reason, /recorded provider waits could not be read \(zz unreadable record\)/);
409
+ }
410
+ assert.deepEqual(waits.list('ada').map((w) => [w.model, w.until]), [['*', null]], 'listed for inspection');
411
+ const narrow = waits.release('ada', 'zz-b', 'operator');
412
+ assert.deepEqual(narrow.map((r) => [r.wait.model, r.release]), [['zz-b', 'in-process override']]);
413
+ assert.equal(waits.active('ada', 'zz-b'), undefined, 'the named model is lifted');
414
+ assert.equal(waits.active('ada', 'zz-a')?.until, null, 'another model of the agent stays held');
415
+ assert.equal(waits.active('bo', 'zz-b')?.until, null, 'another agent stays held');
416
+ // Readable again in this process: the recorded wait returns, and the
417
+ // release already made applies after it, in order.
418
+ fail.read = false;
419
+ time.advance(30_000);
420
+ assert.equal(waits.active('ada', 'zz-a')?.reason, 'zz recorded indefinite', 'the recorded wait, not the stand-in');
421
+ assert.equal(waits.active('ada', 'zz-b'), undefined, 'the release made meanwhile still holds');
422
+ assert.equal(waits.active('bo', 'zz-a'), undefined, 'no stand-in once readable');
423
+ store.close();
424
+ store = JsStore.openOrCreate({ path });
425
+ const again = flaky(store);
426
+ again.fail.read = true;
427
+ const restarted = new ProviderWaits(again.surface, { now: time.now, ...quiet });
428
+ assert.equal(restarted.active('ada', 'zz-b')?.until, null, 'a restart that still cannot read holds again');
429
+ const all = restarted.release('ada', undefined, 'operator');
430
+ assert.deepEqual(all.map((r) => [r.wait.model, r.release]), [['*', 'in-process override']]);
431
+ assert.equal(restarted.active('ada', 'zz-a'), undefined, 'omitting the model lifts every model of the agent');
432
+ assert.equal(restarted.active('bo', 'zz-a')?.until, null);
433
+ }
434
+ finally {
435
+ if (!store.isClosed())
436
+ store.close();
437
+ }
438
+ }));
439
+ it("readable again, the log counts what binds after this process's unrecorded release (PR #251 review)", () => withStoreDir((path) => {
440
+ const time = clock();
441
+ let store = JsStore.openOrCreate({ path });
442
+ try {
443
+ new ProviderWaits(store, { now: time.now, ...quiet }).set('ada', 'zz-a', 600_000, 'zz recorded');
444
+ store.close();
445
+ store = JsStore.openOrCreate({ path });
446
+ const { surface, fail } = flaky(store);
447
+ fail.read = true;
448
+ const lines = [];
449
+ const waits = new ProviderWaits(surface, { now: time.now, log: (line) => lines.push(line) });
450
+ fail.appendJson = 1; // the release's record is refused: it stays this process's act
451
+ assert.deepEqual(waits.release('ada', undefined, 'operator').map((r) => r.release), ['in-process override']);
452
+ fail.read = false;
453
+ time.advance(30_000);
454
+ assert.equal(waits.active('ada', 'zz-a'), undefined, 'the release made meanwhile lifts the recorded wait');
455
+ assert.ok(lines.some((line) => /readable again; 0 bind$/.test(line)), lines.join('\n'));
456
+ }
457
+ finally {
458
+ if (!store.isClosed())
459
+ store.close();
460
+ }
461
+ }));
462
+ it('reports each episode of unreadable records once, and a failing report leaves the hold standing (PR #251 review)', () => withStoreDir((path) => {
463
+ const time = clock();
464
+ const store = JsStore.openOrCreate({ path });
465
+ try {
466
+ const { surface, fail } = flaky(store);
467
+ fail.read = true;
468
+ const reported = [];
469
+ const onUnreadable = (error) => { reported.push(error); };
470
+ const waits = new ProviderWaits(surface, { now: time.now, ...quiet, onUnreadable });
471
+ assert.deepEqual(reported, ['zz unreadable record'], 'reported as the hold begins');
472
+ time.advance(30_000);
473
+ assert.equal(waits.active('ada', 'zz-a')?.until, null);
474
+ assert.equal(reported.length, 1, 'a re-read that fails again is the same episode');
475
+ fail.read = false;
476
+ time.advance(30_000);
477
+ assert.equal(waits.active('ada', 'zz-a'), undefined, 'readable again');
478
+ fail.read = true;
479
+ new ProviderWaits(surface, { now: time.now, ...quiet, onUnreadable });
480
+ assert.equal(reported.length, 2, 'a restart that still cannot read is a new episode');
481
+ const unreported = new ProviderWaits(surface, { now: time.now, ...quiet, onUnreadable: () => { throw new Error('zz alert sink down'); } });
482
+ assert.equal(unreported.active('ada', 'zz-a')?.until, null, 'a report that throws changes nothing');
483
+ }
484
+ finally {
485
+ store.close();
486
+ }
487
+ }));
488
+ });
489
+ describe('ProviderWaits: a record that parses but is not a provider wait fails closed (PR #251 review)', () => {
490
+ const CHECKPOINT = `${PROVIDER_WAIT_RECORD_TYPE}/checkpoint`;
491
+ const wait = { agent: 'ada', model: 'zz-a', until: 1_700_000_600_000, reason: 'zz', setAt: 1_700_000_000_000 };
492
+ // Each writes one malformed record after a well-formed recorded wait; a
493
+ // checkpoint's `through` covers that wait, so reading it as empty drops it.
494
+ const malformed = [
495
+ ['a checkpoint whose snapshot is {}', (store, through) => { store.appendJson(CHECKPOINT, { through, snapshot: {} }); }],
496
+ ['a checkpoint whose snapshot is null', (store, through) => { store.appendJson(CHECKPOINT, { through, snapshot: null }); }],
497
+ ['a checkpoint wait without its deadline', (store, through) => {
498
+ const { until: _until, ...partial } = wait;
499
+ store.appendJson(CHECKPOINT, { through, snapshot: { waits: [partial] } });
500
+ }],
501
+ ['a checkpoint holding one (agent, model) twice, the shorter last', (store, through) => {
502
+ store.appendJson(CHECKPOINT, { through, snapshot: { waits: [wait, { ...wait, until: wait.setAt + 1 }] } });
503
+ }],
504
+ ['an entry of a kind this reader does not know', (store) => {
505
+ store.appendJson(PROVIDER_WAIT_RECORD_TYPE, { kind: 'paused', agent: 'ada', model: 'zz-a', at: 1 });
506
+ }],
507
+ ['a set whose deadline is not an instant', (store) => {
508
+ store.appendJson(PROVIDER_WAIT_RECORD_TYPE, { kind: 'set', agent: 'ada', model: 'zz-a', until: 'later', reason: 'zz', at: 1 });
509
+ }],
510
+ ['a set whose deadline is past any Date', (store) => {
511
+ store.appendJson(PROVIDER_WAIT_RECORD_TYPE, { kind: 'set', agent: 'ada', model: 'zz-a', until: 1e300, reason: 'zz', at: 1 });
512
+ }],
513
+ ['a release that names no agent', (store) => {
514
+ store.appendJson(PROVIDER_WAIT_RECORD_TYPE, { kind: 'released', model: null, by: 'zz', at: 1 });
515
+ }],
516
+ ];
517
+ for (const [shape, write] of malformed) {
518
+ it(`${shape}: every (agent, model) is held, not read as no waits`, () => withStoreDir((path) => {
519
+ const time = clock();
520
+ let store = JsStore.openOrCreate({ path });
521
+ try {
522
+ new ProviderWaits(store, { now: time.now, ...quiet }).set('ada', 'zz-a', 600_000, 'zz recorded');
523
+ const [through] = store.getRecordIdsByType(PROVIDER_WAIT_RECORD_TYPE);
524
+ write(store, through);
525
+ store.close();
526
+ store = JsStore.openOrCreate({ path });
527
+ const waits = new ProviderWaits(store, { now: time.now, ...quiet });
528
+ for (const [agent, model] of [['ada', 'zz-a'], ['bo', 'zz-b']]) {
529
+ const held = waits.active(agent, model);
530
+ assert.equal(held?.until, null, `${agent}/${model} held`);
531
+ assert.match(held.reason, /could not be read \(.*malformed\)/);
532
+ }
533
+ }
534
+ finally {
535
+ if (!store.isClosed())
536
+ store.close();
537
+ }
538
+ }));
539
+ }
540
+ it("a release's `by` is never read back, so an odd requester does not make the history unreadable", () => withStoreDir((path) => {
541
+ const time = clock();
542
+ let store = JsStore.openOrCreate({ path });
543
+ try {
544
+ const waits = new ProviderWaits(store, { now: time.now, ...quiet });
545
+ waits.set('ada', 'zz-a', 600_000, 'zz recorded');
546
+ waits.set('ada', 'zz-b', 600_000, 'zz recorded');
547
+ waits.release('ada', 'zz-a', 42);
548
+ store.close();
549
+ store = JsStore.openOrCreate({ path });
550
+ const reopened = new ProviderWaits(store, { now: time.now, ...quiet });
551
+ assert.equal(reopened.active('ada', 'zz-a'), undefined, 'released, and the history reads');
552
+ assert.equal(reopened.active('ada', 'zz-b')?.reason, 'zz recorded');
553
+ }
554
+ finally {
555
+ if (!store.isClosed())
556
+ store.close();
557
+ }
558
+ }));
559
+ });
560
+ describe("ProviderWaits: '*' is every model, everywhere (room-225 #47372)", () => {
561
+ it("releasing the listed '*' is the same release as omitting the model: in process, in the record, and after recovery", () => withStoreDir((path) => {
562
+ const time = clock();
563
+ let store = JsStore.openOrCreate({ path });
564
+ try {
565
+ new ProviderWaits(store, { now: time.now, ...quiet }).set('r', 'zz-a', 600_000, 'zz recorded');
566
+ store.close();
567
+ store = JsStore.openOrCreate({ path });
568
+ const { surface, fail } = flaky(store);
569
+ fail.read = true;
570
+ const waits = new ProviderWaits(surface, { now: time.now, ...quiet });
571
+ const listed = waits.list('r');
572
+ assert.deepEqual(listed.map((w) => w.model), ['*']);
573
+ const receipt = waits.release('r', listed[0].model, 'operator');
574
+ assert.deepEqual(receipt.map((r) => [r.wait.model, r.release]), [['*', 'in-process override']]);
575
+ assert.equal(waits.active('r', 'zz-a'), undefined, 'lifted for every model in process');
576
+ fail.read = false;
577
+ time.advance(30_000);
578
+ assert.equal(waits.active('r', 'zz-a'), undefined, 'and the recorded release covers every model once readable');
579
+ store.close();
580
+ store = JsStore.openOrCreate({ path });
581
+ assert.equal(new ProviderWaits(store, { now: time.now, ...quiet }).active('r', 'zz-a'), undefined, 'and after a reopen');
582
+ }
583
+ finally {
584
+ if (!store.isClosed())
585
+ store.close();
586
+ }
587
+ }));
588
+ it('a specific-model release stays specific while the records are unreadable', () => withStoreDir((path) => {
589
+ const time = clock();
590
+ const store = JsStore.openOrCreate({ path });
591
+ try {
592
+ const { surface, fail } = flaky(store);
593
+ fail.read = true;
594
+ const waits = new ProviderWaits(surface, { now: time.now, ...quiet });
595
+ waits.release('r', 'zz-a', 'operator');
596
+ assert.equal(waits.active('r', 'zz-a'), undefined);
597
+ assert.equal(waits.active('r', 'zz-b')?.until, null, 'another model stays held');
598
+ assert.deepEqual(waits.list('r').map((w) => w.model), ['*'], 'the every-model hold is still listed');
599
+ }
600
+ finally {
601
+ store.close();
602
+ }
603
+ }));
604
+ });
605
+ // ---------------------------------------------------------------------------
606
+ // The framework: a real store, context manager and admission.
607
+ // ---------------------------------------------------------------------------
608
+ class InputModule {
609
+ name = 'input';
610
+ async start(_ctx) { }
611
+ async stop() { }
612
+ getTools() { return []; }
613
+ async handleToolCall(_call) { return { success: false, isError: true, error: 'none' }; }
614
+ async onProcess(event, _state) {
615
+ if (event.type !== 'external-message')
616
+ return {};
617
+ return { addMessages: [{ participant: 'User', content: [{ type: 'text', text: String(event.content) }] }], requestInference: true };
618
+ }
619
+ }
620
+ class ErrorStream {
621
+ error;
622
+ isWaitingForTools = false;
623
+ pendingToolCallIds = [];
624
+ toolDepth = 0;
625
+ isCancelled = false;
626
+ constructor(error) {
627
+ this.error = error;
628
+ }
629
+ provideToolResults() { throw new Error('not waiting'); }
630
+ cancel() { this.isCancelled = true; }
631
+ async *[Symbol.asyncIterator]() {
632
+ yield { type: 'error', error: this.error };
633
+ }
634
+ }
635
+ const rateLimited = (retryAfterMs, request) => new MembraneError({
636
+ type: 'rate_limit', retryable: true, httpStatus: 429, retryAfterMs,
637
+ message: 'zz rate limit reached for requests', rawError: { status: 429 }, rawRequest: request,
638
+ });
639
+ /** Primary turns fail with a stated wait `failures` times, then succeed. */
640
+ class WaitingMembrane extends MockMembrane {
641
+ failures;
642
+ retryAfterMs;
643
+ primary = 0;
644
+ auxiliary = [];
645
+ auxiliaryFailure;
646
+ constructor(failures, retryAfterMs) {
647
+ super();
648
+ this.failures = failures;
649
+ this.retryAfterMs = retryAfterMs;
650
+ }
651
+ streamYielding(request) {
652
+ this.calls.push(request);
653
+ this.primary++;
654
+ if (this.primary <= this.failures)
655
+ return new ErrorStream(rateLimited(this.retryAfterMs, request));
656
+ return new MockYieldingStream([createMockResponse([{ type: 'text', text: 'zz-recovered' }])]);
657
+ }
658
+ async complete(request) {
659
+ this.auxiliary.push(request);
660
+ if (this.auxiliaryFailure)
661
+ throw this.auxiliaryFailure;
662
+ return createMockResponse([{ type: 'text', text: 'zz-summary' }]);
663
+ }
664
+ }
665
+ async function framework(path, membrane) {
666
+ const fw = await AgentFramework.create({
667
+ storePath: path, membrane: membrane.asMembrane(),
668
+ agents: [{ name: 'resident', model: 'zz-model', systemPrompt: 'system' }],
669
+ modules: [new InputModule()], syncIntervalMs: 0, maintenanceIntervalMs: 0,
670
+ });
671
+ return { fw, internal: fw };
672
+ }
673
+ const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
674
+ const auxRequest = (model) => ({
675
+ messages: [{ participant: 'User', content: [{ type: 'text', text: 'zz-compress' }] }],
676
+ config: { model, maxTokens: 16 },
677
+ });
678
+ function silence() {
679
+ const lines = [];
680
+ const orig = console.error;
681
+ console.error = (...a) => { lines.push(a.map(String).join(' ')); };
682
+ return { lines, restore: () => { console.error = orig; } };
683
+ }
684
+ test('a primary 429 with retry-after holds the turn, without a failure marker, and resumes once after it', async () => {
685
+ await withStoreDir(async (path) => {
686
+ const log = silence();
687
+ const membrane = new WaitingMembrane(1, 400);
688
+ const { fw, internal } = await framework(path, membrane);
689
+ try {
690
+ fw.pushEvent({ type: 'external-message', source: 'test', content: 'zz-first', metadata: {} });
691
+ await fw.runUntilIdle();
692
+ assert.equal(membrane.primary, 1, 'no retry inside the wait');
693
+ assert.equal(internal.consecutiveInferenceFailures.get('resident') ?? 0, 0, 'capacity is not a failure streak');
694
+ const health = fw.healthSnapshot();
695
+ assert.equal(health.agents[0].providerAdmission.providerWaits[0].model, 'zz-model');
696
+ fw.pushEvent({ type: 'external-message', source: 'test', content: 'zz-arrived while waiting', metadata: {} });
697
+ await fw.runUntilIdle();
698
+ assert.equal(membrane.primary, 1, 'an arrival during the wait is held, not inferred');
699
+ await waitFor(fw, () => membrane.primary >= 2, 'the fresh compile after the wait');
700
+ assert.equal(membrane.primary, 2, 'one fresh compile after the wait');
701
+ const text = membrane.calls[1].messages.flatMap((m) => m.content).map((b) => b.text ?? '').join('\n');
702
+ assert.match(text, /zz-first/);
703
+ assert.match(text, /zz-arrived while waiting/);
704
+ const markers = fw.getAgent('resident').getContextManager().queryMessages({}).messages
705
+ .filter((m) => m.metadata?.kind === 'inference-failed');
706
+ assert.equal(markers.length, 0);
707
+ }
708
+ finally {
709
+ await fw.stop();
710
+ log.restore();
711
+ }
712
+ });
713
+ });
714
+ test('a stated wait past the framework cap is honoured in full, until an operator releases it', async () => {
715
+ await withStoreDir(async (path) => {
716
+ const log = silence();
717
+ const twentyMinutes = 20 * 60_000;
718
+ const membrane = new WaitingMembrane(1, twentyMinutes);
719
+ const { fw, internal } = await framework(path, membrane);
720
+ try {
721
+ const before = Date.now();
722
+ fw.pushEvent({ type: 'external-message', source: 'test', content: 'zz-first', metadata: {} });
723
+ await fw.runUntilIdle();
724
+ const cooldown = internal.providerAccelerationCooldowns.get('resident');
725
+ assert.ok(cooldown.until >= before + twentyMinutes, 'not shortened to the 10-minute cap');
726
+ assert.equal(cooldown.waitModel, 'zz-model');
727
+ const released = await internal.handleHostCommand('zz-surface', { command: 'release-provider-wait', agentName: 'resident', model: 'zz-model', requesterName: 'zz-operator' });
728
+ assert.equal(released.ok, true);
729
+ assert.deepEqual(released.released.map((w) => w.model), ['zz-model']);
730
+ await fw.runUntilIdle();
731
+ assert.equal(membrane.primary, 2, 'the held turn runs once released');
732
+ assert.ok(log.lines.some((l) => l.includes('released by zz-operator')));
733
+ }
734
+ finally {
735
+ await fw.stop();
736
+ log.restore();
737
+ }
738
+ });
739
+ });
740
+ test("a non-retryable failure worded as organization acceleration keeps its terminal disposition", async () => {
741
+ await withStoreDir(async (path) => {
742
+ const log = silence();
743
+ const membrane = new WaitingMembrane(1, 400);
744
+ const refuse = (request) => new MembraneError({
745
+ type: 'rate_limit', retryable: false, httpStatus: 429, retryAfterMs: 400,
746
+ // Worded as framework.ts's isOrganizationAccelerationRateLimit reads one.
747
+ message: 'zz your organization acceleration limit was reached', rawError: { status: 429 }, rawRequest: request,
748
+ });
749
+ const stream = membrane.streamYielding.bind(membrane);
750
+ membrane.streamYielding = (request) => {
751
+ if (membrane.primary === 0) {
752
+ membrane.calls.push(request);
753
+ membrane.primary++;
754
+ return new ErrorStream(refuse(request));
755
+ }
756
+ return stream(request);
757
+ };
758
+ const { fw, internal } = await framework(path, membrane);
759
+ try {
760
+ fw.pushEvent({ type: 'external-message', source: 'test', content: 'zz-first', metadata: {} });
761
+ await fw.runUntilIdle();
762
+ const markers = fw.getAgent('resident').getContextManager().queryMessages({}).messages
763
+ .filter((m) => m.metadata?.kind === 'inference-failed');
764
+ assert.equal(markers.length, 1, 'terminal: its marker is written');
765
+ assert.equal(internal.providerAccelerationCooldowns.get('resident')?.heldRequests.length ?? 0, 0, 'not retained for retry');
766
+ }
767
+ finally {
768
+ await fw.stop();
769
+ log.restore();
770
+ }
771
+ });
772
+ });
773
+ test('a turn arriving at a recorded wait is held without counting a failure', async () => {
774
+ await withStoreDir(async (path) => {
775
+ const log = silence();
776
+ const membrane = new WaitingMembrane(1, 60_000);
777
+ let opened = await framework(path, membrane);
778
+ const failures = (internal) => internal.providerAccelerationCooldowns.get('resident')?.failures;
779
+ try {
780
+ opened.fw.pushEvent({ type: 'external-message', source: 'test', content: 'zz-first', metadata: {} });
781
+ await opened.fw.runUntilIdle();
782
+ assert.equal(failures(opened.internal), 1, 'the failed call counts once');
783
+ }
784
+ finally {
785
+ await opened.fw.stop();
786
+ }
787
+ // After a restart, the recorded wait holds the next turn at admission:
788
+ // no call failed in this process.
789
+ opened = await framework(path, membrane);
790
+ try {
791
+ opened.fw.pushEvent({ type: 'external-message', source: 'test', content: 'zz-after restart', metadata: {} });
792
+ await opened.fw.runUntilIdle();
793
+ assert.equal(membrane.primary, 1, 'held, not called');
794
+ assert.equal(failures(opened.internal), 0, 'an arrival at the hold is not a failure');
795
+ }
796
+ finally {
797
+ await opened.fw.stop();
798
+ log.restore();
799
+ }
800
+ });
801
+ });
802
+ test('auxiliary admission: the same model is refused without a call; another model proceeds', async () => {
803
+ await withStoreDir(async (path) => {
804
+ const log = silence();
805
+ const membrane = new WaitingMembrane(1, 60_000);
806
+ const { fw, internal } = await framework(path, membrane);
807
+ try {
808
+ fw.pushEvent({ type: 'external-message', source: 'test', content: 'zz-first', metadata: {} });
809
+ await fw.runUntilIdle();
810
+ const aux = internal.auxiliaryMembraneFor('resident');
811
+ const refused = await aux.complete(auxRequest('zz-model')).then(() => undefined, (e) => e);
812
+ assert.ok(refused instanceof MembraneError);
813
+ assert.equal(refused.type, 'rate_limit');
814
+ assert.ok(refused.retryAfterMs !== undefined && refused.retryAfterMs > 50_000 && refused.retryAfterMs <= 60_000);
815
+ assert.match(refused.message, /no call was made/);
816
+ assert.equal(refused.providerAdmission, 'deferred', 'marked as a deferral, so a pacing caller does not count it as a failed call');
817
+ assert.equal(membrane.auxiliary.length, 0, 'the provider was not called');
818
+ await aux.complete(auxRequest('zz-compression-model'));
819
+ assert.equal(membrane.auxiliary.length, 1, 'a model the provider did not limit keeps working');
820
+ }
821
+ finally {
822
+ await fw.stop();
823
+ log.restore();
824
+ }
825
+ });
826
+ });
827
+ test("an auxiliary call's stated wait binds that model only", async () => {
828
+ await withStoreDir(async (path) => {
829
+ const log = silence();
830
+ const membrane = new WaitingMembrane(0, undefined);
831
+ membrane.auxiliaryFailure = rateLimited(60_000);
832
+ const { fw, internal } = await framework(path, membrane);
833
+ try {
834
+ const aux = internal.auxiliaryMembraneFor('resident');
835
+ await assert.rejects(aux.complete(auxRequest('zz-compression-model')), /zz rate limit/);
836
+ membrane.auxiliaryFailure = undefined;
837
+ await assert.rejects(aux.complete(auxRequest('zz-compression-model')), /Provider wait/);
838
+ assert.equal(membrane.auxiliary.length, 1, 'the second call never reached the provider');
839
+ fw.pushEvent({ type: 'external-message', source: 'test', content: 'zz-turn', metadata: {} });
840
+ await fw.runUntilIdle();
841
+ assert.equal(membrane.primary, 1, 'the primary model is not bound by the compression model wait');
842
+ }
843
+ finally {
844
+ await fw.stop();
845
+ log.restore();
846
+ }
847
+ });
848
+ });
849
+ test('a restart honours the wait, even after a switch to a branch from before it', async () => {
850
+ await withStoreDir(async (path) => {
851
+ const log = silence();
852
+ const membrane = new WaitingMembrane(1, 60_000);
853
+ let opened = await framework(path, membrane);
854
+ try {
855
+ opened.fw.pushEvent({ type: 'external-message', source: 'test', content: 'zz-first', metadata: {} });
856
+ await opened.fw.runUntilIdle();
857
+ assert.equal(membrane.primary, 1);
858
+ }
859
+ finally {
860
+ await opened.fw.stop();
861
+ }
862
+ // Roll the store back to before the wait existed, then restart on it.
863
+ const store = JsStore.openOrCreate({ path });
864
+ const main = store.currentBranch().name;
865
+ store.createBranchAt('zz-rollback', main, 0);
866
+ store.switchBranch('zz-rollback');
867
+ store.close();
868
+ opened = await framework(path, membrane);
869
+ try {
870
+ opened.fw.pushEvent({ type: 'external-message', source: 'test', content: 'zz-after restart', metadata: {} });
871
+ await opened.fw.runUntilIdle();
872
+ assert.equal(membrane.primary, 1, 'no call before the provider said');
873
+ const waits = opened.fw.providerWaitSnapshot('resident');
874
+ assert.equal(waits.length, 1);
875
+ assert.equal(waits[0].model, 'zz-model');
876
+ opened.fw.releaseProviderWait('resident');
877
+ await opened.fw.runUntilIdle();
878
+ assert.equal(membrane.primary, 2, 'released by the operator, the held turn runs');
879
+ }
880
+ finally {
881
+ await opened.fw.stop();
882
+ log.restore();
883
+ }
884
+ });
885
+ });
886
+ for (const [delay, label] of [[3_000_000_000, 'a 34-day delay'], [Number.POSITIVE_INFINITY, 'an infinite delay']]) {
887
+ test(`a retry policy delay past one timer is waited in full, and stop() ends the wait: ${label}`, async () => {
888
+ await withStoreDir(async (path) => {
889
+ const log = silence();
890
+ // A non-membrane error keeps provider admission out of it: only the policy decides.
891
+ class ThrowingMembrane extends WaitingMembrane {
892
+ streamYielding(request) {
893
+ this.calls.push(request);
894
+ this.primary++;
895
+ return new ErrorStream(new Error('zz transient'));
896
+ }
897
+ }
898
+ const membrane = new ThrowingMembrane(0, undefined);
899
+ const fw = await AgentFramework.create({
900
+ storePath: path, membrane: membrane.asMembrane(),
901
+ agents: [{ name: 'resident', model: 'zz-model', systemPrompt: 'system' }],
902
+ modules: [new InputModule()], syncIntervalMs: 0, maintenanceIntervalMs: 0,
903
+ errorPolicy: {
904
+ maxRetries: 2,
905
+ onInferenceError: (_e, _a, attempt) => (attempt < 2 ? { retry: true, delayMs: delay } : { retry: false }),
906
+ },
907
+ });
908
+ let stopped = false;
909
+ try {
910
+ fw.pushEvent({ type: 'external-message', source: 'test', content: 'zz-first', metadata: {} });
911
+ void fw.runUntilIdle();
912
+ // The event the assertion rests on: the turn has failed once and is
913
+ // asleep in its retry wait.
914
+ const asleep = fw;
915
+ for (let i = 0; i < 500 && asleep.retryWaitWakers.size === 0; i++)
916
+ await sleep(10);
917
+ assert.equal(asleep.retryWaitWakers.size, 1, 'asleep in the retry wait');
918
+ assert.equal(membrane.primary, 1, `${label} does not fire almost at once`);
919
+ const stopping = fw.stop();
920
+ await Promise.race([stopping, sleep(5_000).then(() => { throw new Error('stop() did not end the wait'); })]);
921
+ stopped = true;
922
+ await sleep(200);
923
+ assert.equal(membrane.primary, 1, 'stop() cancels the retry: no call after it (room-225 #46745)');
924
+ }
925
+ finally {
926
+ if (!stopped)
927
+ await fw.stop();
928
+ log.restore();
929
+ }
930
+ });
931
+ });
932
+ }
933
+ /** The longest delay one setTimeout holds (2^31 - 1 ms, about 24.8 days). */
934
+ const ONE_TIMER_MS = 2_147_483_647;
935
+ test('a retry wait past one timer waits in steps of at most one timer, and only the last step ends it (PR #251 review)', async (t) => {
936
+ await withStoreDir(async (path) => {
937
+ const log = silence();
938
+ const { fw } = await framework(path, new WaitingMembrane(0, undefined));
939
+ const stepping = fw;
940
+ t.mock.timers.enable({ apis: ['setTimeout'] });
941
+ const mocked = globalThis.setTimeout;
942
+ const delays = [];
943
+ globalThis.setTimeout = ((fn, ms, ...rest) => {
944
+ delays.push(ms ?? 0);
945
+ return mocked(fn, ms, ...rest);
946
+ });
947
+ try {
948
+ let done = false;
949
+ const waiting = stepping.waitForRetry(ONE_TIMER_MS + 1_000).then(() => { done = true; });
950
+ t.mock.timers.tick(ONE_TIMER_MS);
951
+ for (let i = 0; i < 20; i++)
952
+ await Promise.resolve();
953
+ assert.equal(done, false, "still waiting after one timer's worth");
954
+ t.mock.timers.tick(1_000);
955
+ await waiting;
956
+ assert.ok(delays.includes(ONE_TIMER_MS), `a whole timer first (${delays.join(', ')})`);
957
+ assert.ok(delays.every((ms) => ms <= ONE_TIMER_MS), 'no step longer than one timer holds');
958
+ }
959
+ finally {
960
+ globalThis.setTimeout = mocked;
961
+ t.mock.timers.reset();
962
+ await fw.stop();
963
+ log.restore();
964
+ }
965
+ });
966
+ });
967
+ test('a hold on a wait longer than one timer is armed for at most one timer, and re-arms rather than releasing while the wait binds (PR #251 review)', async () => {
968
+ await withStoreDir(async (path) => {
969
+ const log = silence();
970
+ const membrane = new WaitingMembrane(1, 30 * 86_400_000); // 30 days
971
+ const { fw, internal } = await framework(path, membrane);
972
+ const real = globalThis.setTimeout;
973
+ try {
974
+ fw.pushEvent({ type: 'external-message', source: 'test', content: 'zz-first', metadata: {} });
975
+ await fw.runUntilIdle();
976
+ const hold = internal.providerAccelerationCooldowns.get('resident');
977
+ assert.equal(hold?.waitModel, 'zz-model');
978
+ const until = hold.until;
979
+ // Its timer fires one timer's worth in, about 24.8 days: the wait still
980
+ // binds, so the hold arms again rather than releasing the turn.
981
+ const delays = [];
982
+ globalThis.setTimeout = ((fn, ms, ...rest) => {
983
+ delays.push(ms ?? 0);
984
+ return real(fn, ms, ...rest);
985
+ });
986
+ internal.releaseProviderAccelerationCooldown('resident');
987
+ globalThis.setTimeout = real;
988
+ assert.equal(internal.providerAccelerationCooldowns.get('resident')?.until, until, 'still held, to the same deadline');
989
+ assert.deepEqual(delays, [ONE_TIMER_MS], 'armed again for one timer');
990
+ await fw.runUntilIdle();
991
+ assert.equal(membrane.primary, 1, 'and no call is made');
992
+ }
993
+ finally {
994
+ globalThis.setTimeout = real;
995
+ await fw.stop();
996
+ log.restore();
997
+ }
998
+ });
999
+ });
1000
+ test("an ephemeral run's cap includes its edge: a stated wait of exactly 600 s is left to the policy, and one past it ends the run (PR #251 review)", async () => {
1001
+ await withStoreDir(async (path) => {
1002
+ const log = silence();
1003
+ const { fw, internal } = await framework(path, new WaitingMembrane(0, undefined));
1004
+ const runs = internal;
1005
+ const name = 'zz-ephemeral-edge';
1006
+ try {
1007
+ const created = await fw.createEphemeralAgent({ name, model: 'zz-model', systemPrompt: 'Do the task.' });
1008
+ runs.ephemeralRuns.set(name, {});
1009
+ assert.equal(runs.nonOwningProviderWait(created.agent, rateLimited(600_000)), undefined, 'exactly the cap: slept by the error policy');
1010
+ assert.equal(runs.nonOwningProviderWait(created.agent, rateLimited(600_001)), 'fail', 'past the cap: the run ends now');
1011
+ }
1012
+ finally {
1013
+ runs.ephemeralRuns.delete(name);
1014
+ await fw.stop();
1015
+ log.restore();
1016
+ }
1017
+ });
1018
+ });
1019
+ test('stop() cancels a retry wait on the path that does not own provider admission (a conversation agent)', async () => {
1020
+ await withStoreDir(async (path) => {
1021
+ const log = silence();
1022
+ class ThrowingMembrane extends WaitingMembrane {
1023
+ streamYielding(request) {
1024
+ this.calls.push(request);
1025
+ this.primary++;
1026
+ return new ErrorStream(new Error('zz transient'));
1027
+ }
1028
+ }
1029
+ const membrane = new ThrowingMembrane(0, undefined);
1030
+ const fw = await AgentFramework.create({
1031
+ storePath: path, membrane: membrane.asMembrane(),
1032
+ agents: [{ name: 'resident', model: 'zz-model', systemPrompt: 'system' }],
1033
+ modules: [new InputModule()], syncIntervalMs: 0, maintenanceIntervalMs: 0,
1034
+ errorPolicy: {
1035
+ maxRetries: 2,
1036
+ onInferenceError: (_e, _a, attempt) => (attempt < 2 ? { retry: true, delayMs: 3_000_000_000 } : { retry: false }),
1037
+ },
1038
+ });
1039
+ const internal = fw;
1040
+ let stopped = false;
1041
+ try {
1042
+ internal.conversationAgentHomes.set('resident', 'world:zz');
1043
+ fw.getAgent('resident').getContextManager().addMessage('User', [{ type: 'text', text: 'zz-direct' }]);
1044
+ const run = internal.startAgentStream(fw.getAgent('resident'), { agentName: 'resident', reason: 'conversation', source: 'test', timestamp: Date.now() });
1045
+ for (let i = 0; i < 300 && internal.retryWaitWakers.size === 0; i++)
1046
+ await sleep(10);
1047
+ assert.equal(membrane.primary, 1);
1048
+ assert.ok(internal.retryWaitWakers.size > 0, 'parked in the policy delay');
1049
+ await fw.stop();
1050
+ stopped = true;
1051
+ await run;
1052
+ await sleep(200);
1053
+ assert.equal(membrane.primary, 1, 'no call after stop()');
1054
+ }
1055
+ finally {
1056
+ if (!stopped)
1057
+ await fw.stop();
1058
+ log.restore();
1059
+ }
1060
+ });
1061
+ });
1062
+ // ---------------------------------------------------------------------------
1063
+ // Corrective round (room-225 #46189, #46418, #46371).
1064
+ // ---------------------------------------------------------------------------
1065
+ test('a non-retryable failure with a stated wait keeps its terminal disposition; the wait binds the calls after it', async () => {
1066
+ await withStoreDir(async (path) => {
1067
+ const log = silence();
1068
+ const membrane = new WaitingMembrane(1, 400);
1069
+ // The first primary call is refused outright (not retryable), with a hint.
1070
+ const refuse = (request) => new MembraneError({
1071
+ type: 'invalid_request', retryable: false, httpStatus: 400, retryAfterMs: 400,
1072
+ message: 'zz request refused', rawError: { status: 400 }, rawRequest: request,
1073
+ });
1074
+ const stream = membrane.streamYielding.bind(membrane);
1075
+ membrane.streamYielding = (request) => {
1076
+ if (membrane.primary === 0) {
1077
+ membrane.calls.push(request);
1078
+ membrane.primary++;
1079
+ return new ErrorStream(refuse(request));
1080
+ }
1081
+ return stream(request);
1082
+ };
1083
+ const { fw, internal } = await framework(path, membrane);
1084
+ try {
1085
+ fw.pushEvent({ type: 'external-message', source: 'test', content: 'zz-first', metadata: {} });
1086
+ await fw.runUntilIdle();
1087
+ assert.equal(membrane.primary, 1);
1088
+ const markers = () => fw.getAgent('resident').getContextManager().queryMessages({}).messages
1089
+ .filter((m) => m.metadata?.kind === 'inference-failed');
1090
+ assert.equal(markers().length, 1, 'the failure is terminal: its marker is written');
1091
+ assert.equal(internal.providerAccelerationCooldowns.get('resident')?.heldRequests.length ?? 0, 0, 'the failed request is not retained for retry');
1092
+ fw.pushEvent({ type: 'external-message', source: 'test', content: 'zz-during the wait', metadata: {} });
1093
+ await fw.runUntilIdle();
1094
+ assert.equal(membrane.primary, 1, 'a later wake is held by the stated wait');
1095
+ await waitFor(fw, () => membrane.primary >= 2, 'the held wake after the wait');
1096
+ assert.equal(membrane.primary, 2, 'and runs once after it');
1097
+ }
1098
+ finally {
1099
+ await fw.stop();
1100
+ log.restore();
1101
+ }
1102
+ });
1103
+ });
1104
+ test('a primary wake parked behind an auxiliary call is held by the wait that call\'s failure records', async () => {
1105
+ await withStoreDir(async (path) => {
1106
+ const log = silence();
1107
+ let entered;
1108
+ let fail;
1109
+ const inFlight = new Promise((resolve) => { entered = resolve; });
1110
+ const held = new Promise((_resolve, reject) => { fail = reject; });
1111
+ const membrane = new WaitingMembrane(0, undefined);
1112
+ membrane.complete = async (request) => { membrane.auxiliary.push(request); entered(); return held; };
1113
+ const { fw, internal } = await framework(path, membrane);
1114
+ try {
1115
+ const aux = internal.auxiliaryMembraneFor('resident').complete(auxRequest('zz-model')).catch((error) => error);
1116
+ await inFlight;
1117
+ fw.pushEvent({ type: 'external-message', source: 'test', content: 'zz-parked', metadata: {} });
1118
+ const run = fw.runUntilIdle();
1119
+ const gates = fw.providerGates;
1120
+ for (let i = 0; i < 300 && !((gates.get('resident')?.primaryDepth ?? 0) > 0); i++)
1121
+ await sleep(10);
1122
+ assert.ok((gates.get('resident')?.primaryDepth ?? 0) > 0, 'the primary wake parked behind the auxiliary call');
1123
+ fail(rateLimited(400));
1124
+ await aux;
1125
+ // The run started above drives the parked wake to the boundary.
1126
+ for (let i = 0; i < 500 && !log.lines.some((line) => /binds at admission/.test(line)); i++)
1127
+ await sleep(10);
1128
+ assert.ok(log.lines.some((line) => /binds at admission/.test(line)), 'the parked wake reached the boundary');
1129
+ assert.equal(membrane.primary, 0, 'not started into the wait its parked-behind call recorded');
1130
+ // The hold's timer releases it after the wait, and queues the wake again.
1131
+ for (let i = 0; i < 500 && internal.providerAccelerationCooldowns.has('resident'); i++)
1132
+ await sleep(10);
1133
+ assert.equal(internal.providerAccelerationCooldowns.has('resident'), false, 'the hold ended with the wait');
1134
+ await run;
1135
+ await fw.runUntilIdle();
1136
+ assert.equal(membrane.primary, 1, 'the held wake runs once after the wait');
1137
+ }
1138
+ finally {
1139
+ await fw.stop();
1140
+ log.restore();
1141
+ }
1142
+ });
1143
+ });
1144
+ test("an operator release reaches the context strategy's compression lane, naming the model", async () => {
1145
+ await withStoreDir(async (path) => {
1146
+ const log = silence();
1147
+ const membrane = new WaitingMembrane(1, 60_000);
1148
+ const { fw, internal } = await framework(path, membrane);
1149
+ try {
1150
+ fw.pushEvent({ type: 'external-message', source: 'test', content: 'zz-first', metadata: {} });
1151
+ await fw.runUntilIdle();
1152
+ const strategy = fw.getAgent('resident').getContextManager().getStrategy();
1153
+ const asked = [];
1154
+ strategy.releaseCompressionPause = (model) => { asked.push(model); return true; };
1155
+ await internal.handleHostCommand('zz-surface', { command: 'release-provider-wait', agentName: 'resident', model: 'zz-model', requesterName: 'zz-operator' });
1156
+ assert.deepEqual(asked, ['zz-model']);
1157
+ await internal.handleHostCommand('zz-surface', { command: 'release-provider-wait', agentName: 'resident', requesterName: 'zz-operator' });
1158
+ assert.deepEqual(asked, ['zz-model', undefined], 'an explicit release reaches the lane even with nothing left to release here');
1159
+ membrane.primary = 0; // fail the next primary again: a new wait
1160
+ fw.pushEvent({ type: 'external-message', source: 'test', content: 'zz-second', metadata: {} });
1161
+ await fw.runUntilIdle();
1162
+ await internal.handleHostCommand('zz-surface', { command: 'release-provider-wait', agentName: 'resident', model: '*', requesterName: 'zz-operator' });
1163
+ assert.deepEqual(asked, ['zz-model', undefined, undefined], "'*' reaches the lane as every model");
1164
+ assert.ok(log.lines.some((line) => line.includes("compression lane's provider wait released")));
1165
+ }
1166
+ finally {
1167
+ await fw.stop();
1168
+ log.restore();
1169
+ }
1170
+ });
1171
+ });
1172
+ // ---------------------------------------------------------------------------
1173
+ // Review round on PR #251: held-turn recovery, acceleration release, release
1174
+ // model scope.
1175
+ // ---------------------------------------------------------------------------
1176
+ /** Swap in waits whose reads fail until `fail.read` is cleared, re-read every 50 ms. */
1177
+ function unreadableWaits(internal) {
1178
+ const { surface, fail } = flaky(internal.store);
1179
+ fail.read = true;
1180
+ internal.providerWaits = new ProviderWaits(surface, { retryReadMs: 50, ...quiet });
1181
+ return fail;
1182
+ }
1183
+ test('a hold for unreadable records ends by itself once they read again: its own re-check finds them readable', async () => {
1184
+ await withStoreDir(async (path) => {
1185
+ const log = silence();
1186
+ const membrane = new WaitingMembrane(0, undefined);
1187
+ const { fw, internal } = await framework(path, membrane);
1188
+ try {
1189
+ const fail = unreadableWaits(internal);
1190
+ fw.pushEvent({ type: 'external-message', source: 'test', content: 'zz-first', metadata: {} });
1191
+ await fw.runUntilIdle();
1192
+ assert.equal(membrane.primary, 0, 'held: the absence of recorded waits is unknown');
1193
+ assert.equal(internal.providerAccelerationCooldowns.get('resident')?.until, Number.POSITIVE_INFINITY);
1194
+ await sleep(200);
1195
+ await fw.runUntilIdle();
1196
+ assert.equal(membrane.primary, 0, 'still held while they cannot be read');
1197
+ // No health call and no operator release: only the hold's own timer.
1198
+ fail.read = false;
1199
+ await sleep(200);
1200
+ await fw.runUntilIdle();
1201
+ assert.equal(membrane.primary, 1, 'the held turn runs once the records read again');
1202
+ assert.equal(internal.providerAccelerationCooldowns.size, 0);
1203
+ }
1204
+ finally {
1205
+ await fw.stop();
1206
+ log.restore();
1207
+ }
1208
+ });
1209
+ });
1210
+ test('an auxiliary call naming no model is held by the unreadable-records hold, and only by it', async () => {
1211
+ await withStoreDir(async (path) => {
1212
+ const log = silence();
1213
+ const membrane = new WaitingMembrane(1, 60_000);
1214
+ const { fw, internal } = await framework(path, membrane);
1215
+ try {
1216
+ fw.pushEvent({ type: 'external-message', source: 'test', content: 'zz-first', metadata: {} });
1217
+ await fw.runUntilIdle(); // a wait is recorded for the primary model
1218
+ const aux = internal.auxiliaryMembraneFor('resident');
1219
+ const unnamed = { messages: auxRequest('x').messages, config: { maxTokens: 16 } };
1220
+ await aux.complete(unnamed);
1221
+ assert.equal(membrane.auxiliary.length, 1, "a recorded model's wait can't be matched to a call naming no model");
1222
+ unreadableWaits(internal);
1223
+ const refused = await aux.complete(unnamed).then(() => undefined, (e) => e);
1224
+ assert.ok(refused instanceof MembraneError, 'the hold that stands for unreadable records binds it');
1225
+ assert.match(refused.message, /\* is held for resident/);
1226
+ assert.equal(membrane.auxiliary.length, 1, 'the provider was not called');
1227
+ }
1228
+ finally {
1229
+ await fw.stop();
1230
+ log.restore();
1231
+ }
1232
+ });
1233
+ });
1234
+ test("once the records read again, a hold takes the recorded wait's deadline: it neither stays indefinite nor ends early", async () => {
1235
+ await withStoreDir(async (path) => {
1236
+ const log = silence();
1237
+ const membrane = new WaitingMembrane(0, undefined);
1238
+ const { fw, internal } = await framework(path, membrane);
1239
+ try {
1240
+ const recorded = internal.providerWaits.set('resident', 'zz-model', 900, 'zz recorded before the reads failed').wait;
1241
+ const fail = unreadableWaits(internal);
1242
+ fw.pushEvent({ type: 'external-message', source: 'test', content: 'zz-first', metadata: {} });
1243
+ await fw.runUntilIdle();
1244
+ assert.equal(internal.providerAccelerationCooldowns.get('resident')?.until, Number.POSITIVE_INFINITY);
1245
+ await sleep(100);
1246
+ fail.read = false;
1247
+ await sleep(200);
1248
+ await fw.runUntilIdle();
1249
+ assert.equal(internal.providerAccelerationCooldowns.get('resident')?.until, recorded.until, 'the recorded deadline, not the stand-in');
1250
+ assert.equal(membrane.primary, 0, 'and not before it');
1251
+ await waitFor(fw, () => membrane.primary >= 1, 'the held turn after that wait');
1252
+ assert.equal(membrane.primary, 1, 'the held turn runs once that wait passes');
1253
+ }
1254
+ finally {
1255
+ await fw.stop();
1256
+ log.restore();
1257
+ }
1258
+ });
1259
+ });
1260
+ const accelerationLimited = (retryAfterMs, request) => new MembraneError({
1261
+ type: 'rate_limit', retryable: true, httpStatus: 429, retryAfterMs,
1262
+ message: "This request would exceed your organization's maximum usage increase rate for input tokens per minute",
1263
+ rawError: { status: 429 }, rawRequest: request,
1264
+ });
1265
+ test("releasing the stated wait of an organization-acceleration hold leaves AF's own pacing; another model's release leaves the hold", async () => {
1266
+ await withStoreDir(async (path) => {
1267
+ const log = silence();
1268
+ const twentyMinutes = 20 * 60_000;
1269
+ const membrane = new WaitingMembrane(0, undefined);
1270
+ const stream = membrane.streamYielding.bind(membrane);
1271
+ membrane.streamYielding = (request) => {
1272
+ if (membrane.primary === 0) {
1273
+ membrane.calls.push(request);
1274
+ membrane.primary++;
1275
+ return new ErrorStream(accelerationLimited(twentyMinutes, request));
1276
+ }
1277
+ return stream(request);
1278
+ };
1279
+ const { fw, internal } = await framework(path, membrane);
1280
+ internal.providerAccelerationDefaultCooldownMs = 1_000;
1281
+ internal.providerAccelerationJitterMs = 0;
1282
+ try {
1283
+ const before = Date.now();
1284
+ fw.pushEvent({ type: 'external-message', source: 'test', content: 'zz-first', metadata: {} });
1285
+ await fw.runUntilIdle();
1286
+ const armedBy = Date.now();
1287
+ assert.equal(membrane.primary, 1);
1288
+ const hold = internal.providerAccelerationCooldowns.get('resident');
1289
+ assert.equal(hold.waitModel, undefined, 'an acceleration hold parks the whole agent');
1290
+ const statedUntil = hold.until;
1291
+ assert.ok(statedUntil >= before + twentyMinutes, 'its length is the stated wait');
1292
+ const other = await internal.handleHostCommand('zz-surface', { command: 'release-provider-wait', agentName: 'resident', model: 'zz-other', requesterName: 'zz-operator' });
1293
+ assert.equal(other.ok, true);
1294
+ assert.equal(internal.providerAccelerationCooldowns.get('resident')?.until, statedUntil, "another model's release leaves the hold");
1295
+ const released = await internal.handleHostCommand('zz-surface', { command: 'release-provider-wait', agentName: 'resident', model: 'zz-model', requesterName: 'zz-operator' });
1296
+ assert.deepEqual(released.released.map((w) => w.model), ['zz-model']);
1297
+ const paced = internal.providerAccelerationCooldowns.get('resident');
1298
+ assert.ok(paced, "AF's own pacing for the failure still holds");
1299
+ assert.ok(paced.until >= before + 1_000 && paced.until <= armedBy + 1_000, `held to AF's own pacing (${paced.until - before} ms after the failure)`);
1300
+ await fw.runUntilIdle();
1301
+ assert.equal(membrane.primary, 1, 'not before that pacing passes');
1302
+ await waitFor(fw, () => membrane.primary >= 2, 'the held turn after that pacing');
1303
+ assert.equal(membrane.primary, 2, 'the held turn runs once it has');
1304
+ }
1305
+ finally {
1306
+ await fw.stop();
1307
+ log.restore();
1308
+ }
1309
+ });
1310
+ });
1311
+ test("releasing a provider wait leaves a host hold the host's", async () => {
1312
+ await withStoreDir(async (path) => {
1313
+ const log = silence();
1314
+ const membrane = new WaitingMembrane(1, 20 * 60_000);
1315
+ const fw = await AgentFramework.create({
1316
+ storePath: path, membrane: membrane.asMembrane(),
1317
+ agents: [{ name: 'resident', model: 'zz-model', systemPrompt: 'system' }],
1318
+ modules: [new InputModule()], syncIntervalMs: 0, maintenanceIntervalMs: 0,
1319
+ providerHold: () => ({ holdMs: 60_000, reason: 'zz weekly quota spent' }),
1320
+ });
1321
+ const internal = fw;
1322
+ try {
1323
+ fw.pushEvent({ type: 'external-message', source: 'test', content: 'zz-first', metadata: {} });
1324
+ await fw.runUntilIdle();
1325
+ const hold = internal.providerAccelerationCooldowns.get('resident');
1326
+ assert.ok(hold.hostHoldError, 'a host hold');
1327
+ const until = hold.until;
1328
+ assert.deepEqual(fw.providerWaitSnapshot('resident').map((w) => w.model), ['zz-model'], 'the stated wait is recorded beside it');
1329
+ const released = await internal.handleHostCommand('zz-surface', { command: 'release-provider-wait', agentName: 'resident', model: 'zz-model', requesterName: 'zz-operator' });
1330
+ assert.deepEqual(released.released.map((w) => w.model), ['zz-model']);
1331
+ assert.deepEqual(fw.providerWaitSnapshot('resident'), [], 'the stated wait is released');
1332
+ assert.equal(internal.providerAccelerationCooldowns.get('resident')?.until, until, 'the host hold stands, unchanged');
1333
+ await fw.runUntilIdle();
1334
+ assert.equal(membrane.primary, 1, 'and still holds the turn');
1335
+ }
1336
+ finally {
1337
+ await fw.stop();
1338
+ log.restore();
1339
+ }
1340
+ });
1341
+ });
1342
+ test('a malformed release model is refused at both operator surfaces, never widened to every model', async () => {
1343
+ await withStoreDir(async (path) => {
1344
+ const log = silence();
1345
+ const membrane = new WaitingMembrane(1, 60_000);
1346
+ const { fw, internal } = await framework(path, membrane);
1347
+ try {
1348
+ fw.pushEvent({ type: 'external-message', source: 'test', content: 'zz-first', metadata: {} });
1349
+ await fw.runUntilIdle();
1350
+ assert.deepEqual(fw.providerWaitSnapshot('resident').map((w) => w.model), ['zz-model']);
1351
+ const api = new ApiServer(fw);
1352
+ for (const model of ['', null, 42, ['zz-model'], {}, 'zz-model\n[provider-wait] agent=resident model=zz-forged released by nobody']) {
1353
+ const label = JSON.stringify(model);
1354
+ const refused = await internal.handleHostCommand('zz-surface', { command: 'release-provider-wait', agentName: 'resident', model, requesterName: 'zz-operator' });
1355
+ assert.equal(refused.ok, false, `host command, model ${label}`);
1356
+ assert.match(String(refused.error), /model must be a model name/);
1357
+ await assert.rejects(api.executeCommand(undefined, 'host.releaseProviderWait', { agentName: 'resident', model }), /model must be a model name/, `WS verb, model ${label}`);
1358
+ }
1359
+ assert.deepEqual(fw.providerWaitSnapshot('resident').map((w) => w.model), ['zz-model'], 'nothing was released');
1360
+ const every = await api.executeCommand(undefined, 'host.releaseProviderWait', { agentName: 'resident', model: '*' });
1361
+ assert.deepEqual(every.released.map((w) => w.model), ['zz-model'], "'*' still releases every model");
1362
+ }
1363
+ finally {
1364
+ await fw.stop();
1365
+ log.restore();
1366
+ }
1367
+ });
1368
+ });
1369
+ test('a release names one of the agents, the API verb takes the admin token, and a requester is marked and cleaned (PR #251 review)', async () => {
1370
+ await withStoreDir(async (path) => {
1371
+ const log = silence();
1372
+ const membrane = new WaitingMembrane(1, 60_000);
1373
+ const { fw, internal } = await framework(path, membrane);
1374
+ try {
1375
+ fw.pushEvent({ type: 'external-message', source: 'test', content: 'zz-first', metadata: {} });
1376
+ await fw.runUntilIdle();
1377
+ const api = new ApiServer(fw, { adminToken: 'zz-token' });
1378
+ const records = () => internal.store.getRecordIdsByType(PROVIDER_WAIT_RECORD_TYPE).length;
1379
+ const before = records();
1380
+ await assert.rejects(api.executeCommand(undefined, 'host.releaseProviderWait', { agentName: 'resident' }), /adminToken/, 'no token, no release');
1381
+ // '*' and 'framework' are how the hold for unreadable records is listed and alerted.
1382
+ for (const agentName of ['zz-typo', '*', 'framework']) {
1383
+ await assert.rejects(api.executeCommand(undefined, 'host.releaseProviderWait', { agentName, adminToken: 'zz-token' }), /Unknown agent/, `WS verb, ${agentName}`);
1384
+ assert.throws(() => fw.releaseProviderWait(agentName), /Unknown agent/, `framework, ${agentName}`);
1385
+ const refused = await internal.handleHostCommand('zz-surface', { command: 'release-provider-wait', agentName, requesterName: 'zz-operator' });
1386
+ assert.equal(refused.ok, false, `host command, ${agentName}`);
1387
+ }
1388
+ const unnamed = await internal.handleHostCommand('zz-surface', { command: 'release-provider-wait', requesterName: 'zz-operator' });
1389
+ assert.equal(unnamed.ok, false, 'a host command that names no agent releases nothing, not the first agent');
1390
+ assert.match(String(unnamed.error), /agentName must name the agent/);
1391
+ assert.equal(records(), before, 'nothing was recorded');
1392
+ assert.deepEqual(fw.providerWaitSnapshot('resident').map((w) => w.model), ['zz-model'], 'nothing was released');
1393
+ await assert.rejects(api.executeCommand(undefined, 'host.providerWaits', { agentName: 42 }), /agentName must be an agent name/, 'a malformed filter is refused, never widened to every agent');
1394
+ const listed = await api.executeCommand(undefined, 'host.providerWaits', {});
1395
+ assert.deepEqual(listed.waits.map((w) => w.agent), ['resident'], 'a read, like host.status: no token needed');
1396
+ log.lines.length = 0;
1397
+ const forged = 'zz-ops\n[provider-wait] agent=resident model=zz-forged released by nobody';
1398
+ const released = await api.executeCommand(undefined, 'host.releaseProviderWait', { agentName: 'resident', adminToken: 'zz-token', requester: forged });
1399
+ assert.deepEqual(released.released.map((w) => w.model), ['zz-model']);
1400
+ assert.ok(log.lines.every((line) => !/[\r\n]/.test(line)), 'no line carries a line break');
1401
+ const said = log.lines.filter((line) => line.includes('released by'));
1402
+ assert.equal(said.length, 1, said.join('\n'));
1403
+ assert.match(said[0], /released by api:zz-ops \[provider-wait\] agent=resident model=zz-forged released by nobody \(recorded\)$/, 'the requester is marked as the API\'s and kept on one line');
1404
+ }
1405
+ finally {
1406
+ await fw.stop();
1407
+ log.restore();
1408
+ }
1409
+ });
1410
+ });
1411
+ test('a release whose context strategy throws still releases the hold it lifted (PR #251 review)', async () => {
1412
+ await withStoreDir(async (path) => {
1413
+ const log = silence();
1414
+ const membrane = new WaitingMembrane(1, 60_000);
1415
+ const { fw, internal } = await framework(path, membrane);
1416
+ const cm = fw.getAgent('resident').getContextManager();
1417
+ const getStrategy = cm.getStrategy;
1418
+ try {
1419
+ fw.pushEvent({ type: 'external-message', source: 'test', content: 'zz-first', metadata: {} });
1420
+ await fw.runUntilIdle();
1421
+ assert.equal(internal.providerAccelerationCooldowns.get('resident')?.waitModel, 'zz-model');
1422
+ cm.getStrategy = () => { throw new Error('zz strategy unavailable'); };
1423
+ const released = fw.releaseProviderWait('resident', 'zz-model', 'zz-operator');
1424
+ cm.getStrategy = getStrategy;
1425
+ assert.deepEqual(released.map((w) => w.model), ['zz-model']);
1426
+ assert.equal(internal.providerAccelerationCooldowns.has('resident'), false, 'the hold is released');
1427
+ await waitFor(fw, () => membrane.primary >= 2, 'the held turn after the release');
1428
+ }
1429
+ finally {
1430
+ cm.getStrategy = getStrategy;
1431
+ await fw.stop();
1432
+ log.restore();
1433
+ }
1434
+ });
1435
+ });
1436
+ test('an explicit release reaches the compression lane with exactly its scope, even when no wait binds here', async () => {
1437
+ await withStoreDir(async (path) => {
1438
+ const log = silence();
1439
+ const membrane = new WaitingMembrane(0, undefined);
1440
+ const { fw, internal } = await framework(path, membrane);
1441
+ try {
1442
+ // The lane holds a pause the framework no longer has: no wait binds here
1443
+ // (as after the records read again, or a release in another process).
1444
+ const paused = new Set(['zz-model', 'zz-compression-model']);
1445
+ const asked = [];
1446
+ const strategy = fw.getAgent('resident').getContextManager().getStrategy();
1447
+ strategy.releaseCompressionPause = (model) => {
1448
+ asked.push(model);
1449
+ if (model !== undefined)
1450
+ return paused.delete(model);
1451
+ const had = paused.size > 0;
1452
+ paused.clear();
1453
+ return had;
1454
+ };
1455
+ const release = (model) => internal.handleHostCommand('zz-surface', {
1456
+ command: 'release-provider-wait', agentName: 'resident', requesterName: 'zz-operator', ...(model === undefined ? {} : { model }),
1457
+ });
1458
+ assert.deepEqual(fw.providerWaitSnapshot('resident'), [], 'no wait binds here');
1459
+ const other = await release('zz-other');
1460
+ assert.deepEqual(other.released, [], 'the receipt lists only waits released here');
1461
+ assert.deepEqual(asked, ['zz-other'], 'the lane is asked with exactly the named model');
1462
+ assert.deepEqual([...paused].sort(), ['zz-compression-model', 'zz-model'], "other models' pauses stand");
1463
+ const named = await release('zz-model');
1464
+ assert.deepEqual(named.released, []);
1465
+ assert.deepEqual(asked, ['zz-other', 'zz-model']);
1466
+ assert.deepEqual([...paused], ['zz-compression-model'], 'the named pause is released, and only it');
1467
+ assert.ok(log.lines.some((line) => line.includes("model=zz-model compression lane's provider wait released")));
1468
+ const refused = await release('');
1469
+ assert.equal(refused.ok, false);
1470
+ assert.deepEqual(asked, ['zz-other', 'zz-model'], 'a refused release never reaches the lane');
1471
+ await release();
1472
+ assert.deepEqual(asked, ['zz-other', 'zz-model', undefined], 'an omitted model reaches it as every model');
1473
+ assert.equal(paused.size, 0);
1474
+ }
1475
+ finally {
1476
+ await fw.stop();
1477
+ log.restore();
1478
+ }
1479
+ });
1480
+ });
1481
+ // ---------------------------------------------------------------------------
1482
+ // PR #251 review (2026-10-08): a negative stated wait means "retry now", and
1483
+ // a hold a person has to judge is raised as an ops alert.
1484
+ // ---------------------------------------------------------------------------
1485
+ test('a primary 429 stating a negative retry-after is retried, not held until released', async () => {
1486
+ await withStoreDir(async (path) => {
1487
+ const log = silence();
1488
+ const membrane = new WaitingMembrane(1, -1_000);
1489
+ const { fw, internal } = await framework(path, membrane);
1490
+ try {
1491
+ fw.pushEvent({ type: 'external-message', source: 'test', content: 'zz-first', metadata: {} });
1492
+ await fw.runUntilIdle();
1493
+ assert.equal(internal.providerAccelerationCooldowns.has('resident'), false, 'no hold');
1494
+ assert.deepEqual(fw.providerWaitSnapshot('resident'), [], 'no wait binds');
1495
+ assert.equal(membrane.primary, 2, 'the error policy retried, and the retry recovered');
1496
+ assert.ok(!log.lines.some((line) => /\[provider-wait\].* holds /.test(line)), 'and nothing is logged as held');
1497
+ }
1498
+ finally {
1499
+ await fw.stop();
1500
+ log.restore();
1501
+ }
1502
+ });
1503
+ });
1504
+ test("an organization-acceleration 429 stating a negative retry-after is paced as a stated 0, not by AF's default", async () => {
1505
+ await withStoreDir(async (path) => {
1506
+ const log = silence();
1507
+ const membrane = new WaitingMembrane(0, undefined);
1508
+ const stream = membrane.streamYielding.bind(membrane);
1509
+ membrane.streamYielding = (request) => {
1510
+ if (membrane.primary === 0) {
1511
+ membrane.calls.push(request);
1512
+ membrane.primary++;
1513
+ return new ErrorStream(accelerationLimited(-1_000, request));
1514
+ }
1515
+ return stream(request);
1516
+ };
1517
+ const { fw, internal } = await framework(path, membrane);
1518
+ internal.providerAccelerationDefaultCooldownMs = 60_000;
1519
+ internal.providerAccelerationJitterMs = 0;
1520
+ try {
1521
+ const before = Date.now();
1522
+ fw.pushEvent({ type: 'external-message', source: 'test', content: 'zz-first', metadata: {} });
1523
+ await fw.runUntilIdle();
1524
+ const armedBy = Date.now();
1525
+ const hold = internal.providerAccelerationCooldowns.get('resident');
1526
+ assert.ok(hold.until >= before + 1_000 && hold.until <= armedBy + 1_000, `held ${hold.until - before} ms: the 1 s floor, not the 60 s default`);
1527
+ assert.deepEqual(fw.providerWaitSnapshot('resident'), [], 'no wait binds');
1528
+ await waitFor(fw, () => membrane.primary >= 2, 'the held turn after the hold');
1529
+ assert.equal(membrane.primary, 2, 'the held turn runs once the hold passes');
1530
+ }
1531
+ finally {
1532
+ await fw.stop();
1533
+ log.restore();
1534
+ }
1535
+ });
1536
+ });
1537
+ test('a stated wait held until released, or for more than an hour, raises an ops alert when it is recorded', async () => {
1538
+ await withStoreDir(async (path) => {
1539
+ const log = silence();
1540
+ const membrane = new WaitingMembrane(0, undefined);
1541
+ const { fw, internal } = await framework(path, membrane);
1542
+ const alerts = [];
1543
+ fw.onTrace((event) => { if (event.type === 'ops:alert')
1544
+ alerts.push(event); });
1545
+ const logged = [];
1546
+ fw.logFailure = (record) => { logged.push(record); };
1547
+ const aux = internal.auxiliaryMembraneFor('resident');
1548
+ const stated = async (model, retryAfterMs) => {
1549
+ membrane.auxiliaryFailure = rateLimited(retryAfterMs);
1550
+ return aux.complete(auxRequest(model)).then(() => undefined, (error) => error);
1551
+ };
1552
+ try {
1553
+ await stated('zz-minute', 60_000);
1554
+ await stated('zz-hour', 3_600_000);
1555
+ await stated('zz-retry-now', -1_000);
1556
+ assert.equal(alerts.length, 0, 'a wait that passes within the hour is pacing: no alert');
1557
+ const before = Date.now();
1558
+ await stated('zz-day', 24 * 3_600_000);
1559
+ await stated('zz-beyond-a-date', 1e20);
1560
+ await stated('zz-nan', Number.NaN);
1561
+ assert.deepEqual(alerts.map((alert) => [alert.kind, alert.agentName, alert.data?.model]), [
1562
+ ['provider-wait', 'resident', 'zz-day'],
1563
+ ['provider-wait', 'resident', 'zz-beyond-a-date'],
1564
+ ['provider-wait', 'resident', 'zz-nan'],
1565
+ ]);
1566
+ assert.ok(Date.parse(String(alerts[0].data?.until)) >= before + 24 * 3_600_000, 'the day-long wait names its end');
1567
+ assert.equal(alerts[1].data?.until, null, 'held until released');
1568
+ assert.equal(alerts[2].data?.until, null, 'held until released');
1569
+ assert.match(alerts[0].message, /release-provider-wait/);
1570
+ assert.deepEqual(logged.map((record) => [record.kind, record.model]), [
1571
+ ['provider-wait', 'zz-day'], ['provider-wait', 'zz-beyond-a-date'], ['provider-wait', 'zz-nan'],
1572
+ ], 'each is also a failures.log record');
1573
+ const refused = await stated('zz-day', 24 * 3_600_000);
1574
+ assert.equal(refused.providerAdmission, 'deferred', 'a held model is not called again');
1575
+ assert.equal(alerts.length, 3, 'so its alert is raised once');
1576
+ }
1577
+ finally {
1578
+ await fw.stop();
1579
+ log.restore();
1580
+ }
1581
+ });
1582
+ });
1583
+ test('unreadable recorded waits raise an ops alert for the whole framework as the hold begins', async () => {
1584
+ await withStoreDir(async (path) => {
1585
+ const log = silence();
1586
+ // A record from a reader this one does not know (a newer framework, say):
1587
+ // re-reading never fixes it, so only a person can end the hold.
1588
+ const store = JsStore.openOrCreate({ path });
1589
+ store.appendJson(PROVIDER_WAIT_RECORD_TYPE, { kind: 'paused', agent: 'resident', model: 'zz-model', at: 1 });
1590
+ store.close();
1591
+ // Construction reads the records before any trace listener can attach:
1592
+ // failures.log (and the webhook) carry it then.
1593
+ const proto = AgentFramework.prototype;
1594
+ const logFailure = proto.logFailure;
1595
+ const logged = [];
1596
+ proto.logFailure = (record) => { logged.push(record); };
1597
+ let opened;
1598
+ try {
1599
+ opened = await framework(path, new WaitingMembrane(0, undefined));
1600
+ const alerts = logged.filter((record) => record.kind === 'provider-wait');
1601
+ assert.equal(alerts.length, 1);
1602
+ assert.equal(alerts[0].agent, 'framework');
1603
+ assert.equal(alerts[0].model, '*');
1604
+ assert.equal(alerts[0].until, null);
1605
+ assert.match(String(alerts[0].reason), /could not be read \(provider-wait record .* is malformed\): no provider call is admitted for any agent/);
1606
+ assert.deepEqual(opened.fw.providerWaitSnapshot('resident').map((wait) => [wait.model, wait.until]), [['*', null]], 'and the hold stands');
1607
+ }
1608
+ finally {
1609
+ proto.logFailure = logFailure;
1610
+ await opened?.fw.stop();
1611
+ log.restore();
1612
+ }
1613
+ });
1614
+ });
1615
+ // ---------------------------------------------------------------------------
1616
+ // The paths that don't own provider admission (membrane #56 review, M3): a
1617
+ // conversation agent's turn and an ephemeral run never sleep a stated wait
1618
+ // for hours, silently. Each is reached at both retry sites: driveStream's
1619
+ // error case (a stream's error event) and beginAgentTurn's catch (a failure
1620
+ // before the stream exists, here thrown from streamYielding).
1621
+ // ---------------------------------------------------------------------------
1622
+ /** A stream whose iteration throws, instead of yielding an error event:
1623
+ * driveStream's catch, the third failure site. */
1624
+ class ThrowingStream extends ErrorStream {
1625
+ thrown;
1626
+ constructor(thrown) {
1627
+ super(thrown);
1628
+ this.thrown = thrown;
1629
+ }
1630
+ async *[Symbol.asyncIterator]() {
1631
+ throw this.thrown;
1632
+ }
1633
+ }
1634
+ /** Fails its first `failing` calls with `fail()` at `site`, then succeeds. */
1635
+ class FailingAt extends WaitingMembrane {
1636
+ failing;
1637
+ site;
1638
+ fail;
1639
+ constructor(failing, site, fail) {
1640
+ super(0, undefined);
1641
+ this.failing = failing;
1642
+ this.site = site;
1643
+ this.fail = fail;
1644
+ }
1645
+ streamYielding(request) {
1646
+ this.calls.push(request);
1647
+ this.primary++;
1648
+ if (this.primary <= this.failing) {
1649
+ if (this.site === 'setup')
1650
+ throw this.fail();
1651
+ if (this.site === 'throw')
1652
+ return new ThrowingStream(this.fail());
1653
+ return new ErrorStream(this.fail());
1654
+ }
1655
+ return new MockYieldingStream([createMockResponse([{ type: 'text', text: 'zz-recovered' }])]);
1656
+ }
1657
+ }
1658
+ /** The framework's one in-flight runUntilIdle, shared: two at once would
1659
+ * run its event loop concurrently. */
1660
+ const driving = new WeakMap();
1661
+ function drive(fw) {
1662
+ let run = driving.get(fw);
1663
+ if (!run) {
1664
+ run = fw.runUntilIdle().catch(() => { }).finally(() => { driving.delete(fw); });
1665
+ driving.set(fw, run);
1666
+ }
1667
+ return run;
1668
+ }
1669
+ /** Poll for `done`, driving the framework, with time only as an outer bound:
1670
+ * a turn asleep in a retry wait keeps the drive busy, and that must fail the
1671
+ * test, not hang it. */
1672
+ async function waitFor(fw, done, what, boundMs = 5_000) {
1673
+ const deadline = Date.now() + boundMs;
1674
+ while (!done()) {
1675
+ const left = deadline - Date.now();
1676
+ if (left <= 0)
1677
+ throw new Error(`zz-timeout: ${what} did not happen within ${boundMs} ms`);
1678
+ // Not awaited: the poll's own sleep lets timers run (a drive with nothing
1679
+ // to do settles in microtasks, which would starve them).
1680
+ void drive(fw);
1681
+ await sleep(Math.min(20, left));
1682
+ }
1683
+ }
1684
+ async function conversationTurn(fw, internal, text) {
1685
+ internal.conversationAgentHomes.set('resident', 'world:zz');
1686
+ fw.getAgent('resident').getContextManager().addMessage('User', [{ type: 'text', text }]);
1687
+ await internal.startAgentStream(fw.getAgent('resident'), { agentName: 'resident', reason: 'conversation', source: 'test', timestamp: Date.now() });
1688
+ }
1689
+ for (const site of ['stream', 'setup', 'throw']) {
1690
+ test(`a conversation agent treats a stated wait as a resident does (${site}): recorded, held without a retry in the turn, resumed once after it`, async () => {
1691
+ await withStoreDir(async (path) => {
1692
+ const log = silence();
1693
+ const membrane = new FailingAt(1, site, () => rateLimited(400));
1694
+ const { fw, internal } = await framework(path, membrane);
1695
+ const nonOwning = internal;
1696
+ const exhausted = [];
1697
+ fw.onTrace((event) => { if (event.type === 'inference:exhausted')
1698
+ exhausted.push(event); });
1699
+ try {
1700
+ await conversationTurn(fw, nonOwning, 'zz-first');
1701
+ await waitFor(fw, () => internal.providerAccelerationCooldowns.has('resident'), 'the hold');
1702
+ assert.equal(internal.providerAccelerationCooldowns.get('resident')?.waitModel, 'zz-model');
1703
+ assert.equal(membrane.primary, 1, 'no retry slept inside the turn');
1704
+ assert.deepEqual(fw.providerWaitSnapshot('resident').map((wait) => wait.model), ['zz-model'], 'the wait is recorded');
1705
+ await drive(fw);
1706
+ assert.deepEqual(exhausted, [], 'a held turn is not settled as exhausted');
1707
+ await waitFor(fw, () => membrane.primary >= 2, 'the fresh compile after the wait');
1708
+ assert.equal(membrane.primary, 2, 'exactly one');
1709
+ assert.match(membrane.calls[1].messages.flatMap((m) => m.content).map((b) => b.text ?? '').join('\n'), /zz-first/);
1710
+ }
1711
+ finally {
1712
+ await fw.stop();
1713
+ log.restore();
1714
+ }
1715
+ });
1716
+ });
1717
+ }
1718
+ test("a conversation agent's stated wait past an hour raises one ops alert and holds until an operator releases it", async () => {
1719
+ await withStoreDir(async (path) => {
1720
+ const log = silence();
1721
+ const membrane = new FailingAt(1, 'stream', () => rateLimited(2 * 3_600_000));
1722
+ const { fw, internal } = await framework(path, membrane);
1723
+ const alerts = [];
1724
+ fw.onTrace((event) => { if (event.type === 'ops:alert')
1725
+ alerts.push(event); });
1726
+ try {
1727
+ await conversationTurn(fw, internal, 'zz-first');
1728
+ await waitFor(fw, () => internal.providerAccelerationCooldowns.has('resident'), 'the hold');
1729
+ assert.equal(membrane.primary, 1, 'two hours are not slept in the turn');
1730
+ assert.deepEqual(alerts.map((alert) => [alert.kind, alert.agentName]), [['provider-wait', 'resident']]);
1731
+ assert.match(alerts[0].message, /release-provider-wait/);
1732
+ const released = await internal.handleHostCommand('zz-surface', { command: 'release-provider-wait', agentName: 'resident', model: 'zz-model', requesterName: 'zz-operator' });
1733
+ assert.equal(released.ok, true);
1734
+ await waitFor(fw, () => membrane.primary >= 2, 'the held turn after the release');
1735
+ assert.equal(membrane.primary, 2);
1736
+ }
1737
+ finally {
1738
+ await fw.stop();
1739
+ log.restore();
1740
+ }
1741
+ });
1742
+ });
1743
+ test("a conversation agent's non-retryable failure that states a wait records it and isn't held; the wait holds the wake after it", async () => {
1744
+ await withStoreDir(async (path) => {
1745
+ const log = silence();
1746
+ const refused = () => new MembraneError({
1747
+ type: 'rate_limit', retryable: false, httpStatus: 429, retryAfterMs: 400,
1748
+ message: 'zz refused, with a hint', rawError: { status: 429 },
1749
+ });
1750
+ const membrane = new FailingAt(1, 'stream', refused);
1751
+ const { fw, internal } = await framework(path, membrane);
1752
+ const nonOwning = internal;
1753
+ try {
1754
+ await conversationTurn(fw, nonOwning, 'zz-first');
1755
+ await waitFor(fw, () => fw.providerWaitSnapshot('resident').length > 0, 'the recorded wait');
1756
+ await drive(fw);
1757
+ assert.equal(internal.providerAccelerationCooldowns.has('resident'), false, 'a terminal failure is not held');
1758
+ assert.equal(membrane.primary, 1, 'nor retried');
1759
+ nonOwning.pendingRequests.push({ agentName: 'resident', reason: 'conversation', source: 'test', timestamp: Date.now() });
1760
+ await drive(fw);
1761
+ assert.equal(internal.providerAccelerationCooldowns.get('resident')?.waitModel, 'zz-model', 'the next wake is held by the recorded wait');
1762
+ assert.equal(membrane.primary, 1);
1763
+ await waitFor(fw, () => membrane.primary >= 2, 'the held wake after the wait');
1764
+ }
1765
+ finally {
1766
+ await fw.stop();
1767
+ log.restore();
1768
+ }
1769
+ });
1770
+ });
1771
+ test('a conversation agent closed during a stated-wait hold takes the hold with it: nothing is requeued for it past the deadline', async () => {
1772
+ await withStoreDir(async (path) => {
1773
+ const log = silence();
1774
+ const membrane = new FailingAt(1, 'stream', () => rateLimited(400));
1775
+ const { fw, internal } = await framework(path, membrane);
1776
+ const nonOwning = internal;
1777
+ try {
1778
+ await conversationTurn(fw, nonOwning, 'zz-first');
1779
+ await waitFor(fw, () => internal.providerAccelerationCooldowns.has('resident'), 'the hold');
1780
+ const until = internal.providerAccelerationCooldowns.get('resident').until;
1781
+ nonOwning.disposeConversationAgent('resident');
1782
+ assert.equal(internal.providerAccelerationCooldowns.has('resident'), false, 'the hold goes with the agent');
1783
+ // Past the moment its timer would have released it, plus a few turns.
1784
+ while (Date.now() <= until + 50)
1785
+ await sleep(25);
1786
+ await sleep(50);
1787
+ assert.deepEqual(nonOwning.pendingRequests.filter((r) => r.agentName === 'resident'), [], 'nothing requeued for an agent that is gone');
1788
+ assert.equal(membrane.primary, 1);
1789
+ }
1790
+ finally {
1791
+ await fw.stop();
1792
+ log.restore();
1793
+ }
1794
+ });
1795
+ });
1796
+ test("a conversation agent's retry is held at the last boundary by a wait recorded while it slept", async () => {
1797
+ await withStoreDir(async (path) => {
1798
+ const log = silence();
1799
+ // A failure with no stated wait: the error policy retries after 1 s.
1800
+ const flaky = () => new MembraneError({
1801
+ type: 'server', retryable: true, httpStatus: 500,
1802
+ message: 'zz overloaded, no hint', rawError: { status: 500 },
1803
+ });
1804
+ const membrane = new FailingAt(1, 'stream', flaky);
1805
+ const { fw, internal } = await framework(path, membrane);
1806
+ const nonOwning = internal;
1807
+ try {
1808
+ void conversationTurn(fw, nonOwning, 'zz-first').catch(() => { });
1809
+ await waitFor(fw, () => nonOwning.retryWaitWakers.size > 0, 'the turn asleep before its retry');
1810
+ // While the turn sleeps before its retry, a wait is recorded for its
1811
+ // model, as its own compression call's stated wait would be.
1812
+ internal.providerWaits.set('resident', 'zz-model', 2_500, 'zz recorded while the turn slept');
1813
+ await waitFor(fw, () => membrane.primary >= 2 || internal.providerAccelerationCooldowns.has('resident'), 'the retry, sent or held');
1814
+ assert.equal(membrane.primary, 1, 'the retry is not sent inside the wait');
1815
+ assert.equal(internal.providerAccelerationCooldowns.get('resident')?.waitModel, 'zz-model', 'the retry is held by the wait');
1816
+ await waitFor(fw, () => membrane.primary >= 2, 'the held turn after the wait');
1817
+ assert.equal(membrane.primary, 2, 'exactly one call after the wait');
1818
+ }
1819
+ finally {
1820
+ await fw.stop();
1821
+ log.restore();
1822
+ }
1823
+ });
1824
+ });
1825
+ for (const who of ['resident', 'conversation agent']) {
1826
+ test(`a ${who}'s retry held at the last boundary leaves no inference open at the event gate`, async () => {
1827
+ await withStoreDir(async (path) => {
1828
+ const log = silence();
1829
+ // A setup failure with no stated wait: the error policy retries after 1 s.
1830
+ const flaky = () => new MembraneError({
1831
+ type: 'server', retryable: true, httpStatus: 500,
1832
+ message: 'zz overloaded, no hint', rawError: { status: 500 },
1833
+ });
1834
+ const membrane = new FailingAt(1, 'setup', flaky);
1835
+ const fw = await AgentFramework.create({
1836
+ storePath: path, membrane: membrane.asMembrane(),
1837
+ agents: [{ name: 'resident', model: 'zz-model', systemPrompt: 'system' }],
1838
+ modules: [new InputModule()], syncIntervalMs: 0, maintenanceIntervalMs: 0,
1839
+ gate: { config: { policies: [], default: 'skip' } },
1840
+ });
1841
+ const internal = fw;
1842
+ try {
1843
+ if (who === 'resident') {
1844
+ // A scheduler wake (the gate's 'skip' default would drop an event).
1845
+ fw.getAgent('resident').getContextManager().addMessage('User', [{ type: 'text', text: 'zz-first' }]);
1846
+ internal.pendingRequests.push({ agentName: 'resident', reason: 'test', source: 'test', timestamp: Date.now() });
1847
+ }
1848
+ else
1849
+ void conversationTurn(fw, internal, 'zz-first').catch(() => { });
1850
+ await waitFor(fw, () => internal.retryWaitWakers.size > 0, 'the turn asleep before its retry');
1851
+ internal.providerWaits.set('resident', 'zz-model', 3_000, 'zz recorded while the turn slept');
1852
+ await waitFor(fw, () => membrane.primary >= 2 || internal.providerAccelerationCooldowns.has('resident'), 'the retry, sent or held');
1853
+ assert.equal(membrane.primary, 1, 'the retry is held, not sent');
1854
+ // While a turn is open at the gate, every gate event buffers: a held
1855
+ // turn that left one open would stall every agent's wakes for the wait.
1856
+ assert.deepEqual(internal.eventGate.inferenceDiagnostics().inferring, [], 'the held turn leaves no inference open');
1857
+ }
1858
+ finally {
1859
+ await fw.stop();
1860
+ log.restore();
1861
+ }
1862
+ });
1863
+ });
1864
+ }
1865
+ /** A direct message on an MCPL channel: with conversation routing, it spawns
1866
+ * a conversation agent (a fork of the template) for its channel. */
1867
+ function directMessage(channelId, text) {
1868
+ return {
1869
+ type: 'mcpl:channel-incoming', serverId: 'zz-server', channelId,
1870
+ messageId: `zz-${Math.random().toString(36).slice(2)}`,
1871
+ author: { id: 'U1', name: 'zz-alice' },
1872
+ content: [{ type: 'text', text }],
1873
+ timestamp: new Date().toISOString(),
1874
+ metadata: { mentioned: false, channel_type: 'im' },
1875
+ triggerInference: true,
1876
+ };
1877
+ }
1878
+ test("a conversation agent's compression is refused by its own recorded wait, and records the wait its own call states, without a provider gate", async () => {
1879
+ await withStoreDir(async (path) => {
1880
+ const log = silence();
1881
+ // Conversation routing is deprecated (#235): creating it warns once.
1882
+ const warn = console.warn;
1883
+ console.warn = () => { };
1884
+ const membrane = new WaitingMembrane(0, undefined);
1885
+ const fw = await AgentFramework.create({
1886
+ storePath: path, membrane: membrane.asMembrane(),
1887
+ agents: [{ name: 'resident', model: 'zz-model', systemPrompt: 'system' }],
1888
+ modules: [new InputModule()], syncIntervalMs: 0, maintenanceIntervalMs: 0,
1889
+ conversations: { templateAgent: 'resident' },
1890
+ }).finally(() => { console.warn = warn; });
1891
+ const internal = fw;
1892
+ try {
1893
+ fw.pushEvent(directMessage('zz:D1', 'zz-hello'));
1894
+ await fw.runUntilIdle();
1895
+ const fork = 'conversation-zz-D1-g1';
1896
+ const agent = fw.getAgent(fork);
1897
+ assert.ok(agent, 'the direct message spawned a conversation agent');
1898
+ const compression = agent.getContextManager().membrane;
1899
+ internal.providerWaits.set(fork, 'zz-model', 60_000, 'zz held for the fork');
1900
+ const sent = membrane.auxiliary.length;
1901
+ await assert.rejects(compression.complete(auxRequest('zz-model')), (error) => error.providerAdmission === 'deferred');
1902
+ assert.equal(membrane.auxiliary.length, sent, 'no call is made to the held model');
1903
+ await compression.complete(auxRequest('zz-other-model'));
1904
+ assert.equal(membrane.auxiliary.length, sent + 1, 'another model proceeds');
1905
+ membrane.auxiliaryFailure = rateLimited(60_000);
1906
+ await assert.rejects(compression.complete(auxRequest('zz-compression-model')), /zz rate limit reached/);
1907
+ assert.ok(fw.providerWaitSnapshot(fork).some((wait) => wait.model === 'zz-compression-model'), "the compression call's stated wait is recorded for the fork");
1908
+ assert.equal(internal.providerGates.has(fork), false, 'the fork takes no provider gate');
1909
+ }
1910
+ finally {
1911
+ await fw.stop();
1912
+ log.restore();
1913
+ }
1914
+ });
1915
+ });
1916
+ async function ephemeralRun(fw, name) {
1917
+ const created = await fw.createEphemeralAgent({ name, model: 'zz-model', systemPrompt: 'Do the task.' });
1918
+ created.contextManager.addMessage('User', [{ type: 'text', text: 'zz-task' }]);
1919
+ const run = fw.runEphemeralToCompletion(created.agent, created.contextManager);
1920
+ const outcome = run.then((result) => ({ result }), (error) => ({ error: error }));
1921
+ // Not awaited: a turn asleep in a retry wait would hold it until the wait ends.
1922
+ void drive(fw);
1923
+ // A run still asleep after this long is the defect itself: fail, don't hang.
1924
+ return Promise.race([outcome, sleep(5_000).then(() => ({ error: new Error('zz-unsettled: the run did not end within 5 s') }))]);
1925
+ }
1926
+ for (const site of ['stream', 'setup', 'throw']) {
1927
+ test(`an ephemeral run whose stated wait is past the framework cap ends at once (${site}), saying why, and is never retried`, async () => {
1928
+ await withStoreDir(async (path) => {
1929
+ const log = silence();
1930
+ const membrane = new FailingAt(1, site, () => rateLimited(20 * 60_000));
1931
+ const { fw } = await framework(path, membrane);
1932
+ try {
1933
+ const outcome = await ephemeralRun(fw, `zz-ephemeral-long-${site}`);
1934
+ assert.ok('error' in outcome, 'the run ends with an error');
1935
+ assert.match(outcome.error.message, /zz rate limit reached/);
1936
+ assert.match(outcome.error.message, /the provider asked to wait 1200 s; an ephemeral run doesn't sleep a stated wait longer than 600 s, so it ends now/);
1937
+ // Marked, so a host needn't read the words.
1938
+ assert.ok(outcome.error instanceof ProviderWaitNotSleptError, 'the rejection is a ProviderWaitNotSleptError');
1939
+ assert.equal(outcome.error.name, 'ProviderWaitNotSleptError');
1940
+ assert.equal(outcome.error.retryAfterMs, 20 * 60_000, 'it carries the stated wait');
1941
+ await sleep(100);
1942
+ assert.equal(membrane.primary, 1, 'no retry before the provider\'s time');
1943
+ assert.deepEqual(fw.providerWaitSnapshot(`zz-ephemeral-long-${site}`), [], 'nothing recorded for a single-use name');
1944
+ }
1945
+ finally {
1946
+ await fw.stop();
1947
+ log.restore();
1948
+ }
1949
+ });
1950
+ });
1951
+ }
1952
+ // Each ending's words as published: a host may match them (connectome-host
1953
+ // 0.10.2 does, one pattern per ending), so they don't change.
1954
+ for (const [stated, says] of [
1955
+ [Number.NaN, /the provider's stated wait is not a number, and an ephemeral run sleeps only a stated wait of 600 s or less, so it ends now\)$/],
1956
+ [Number.POSITIVE_INFINITY, /the provider asked to wait longer than can be represented; an ephemeral run doesn't sleep a stated wait longer than 600 s, so it ends now\)$/],
1957
+ ]) {
1958
+ test(`an ephemeral run whose stated wait can't be represented (${stated}) ends at once too`, async () => {
1959
+ await withStoreDir(async (path) => {
1960
+ const log = silence();
1961
+ const membrane = new FailingAt(1, 'stream', () => rateLimited(stated));
1962
+ const { fw } = await framework(path, membrane);
1963
+ try {
1964
+ const outcome = await ephemeralRun(fw, `zz-ephemeral-${stated}`);
1965
+ assert.ok('error' in outcome, 'the run ends with an error');
1966
+ assert.match(outcome.error.message, says);
1967
+ assert.ok(outcome.error instanceof ProviderWaitNotSleptError);
1968
+ assert.equal(outcome.error.name, 'ProviderWaitNotSleptError');
1969
+ assert.ok(Object.is(outcome.error.retryAfterMs, stated), `it carries ${stated}, not ${outcome.error.retryAfterMs}`);
1970
+ assert.equal(membrane.primary, 1);
1971
+ }
1972
+ finally {
1973
+ await fw.stop();
1974
+ log.restore();
1975
+ }
1976
+ });
1977
+ });
1978
+ }
1979
+ for (const [label, failure] of [
1980
+ ['refused', () => new MembraneError({
1981
+ type: 'invalid_request', retryable: false, httpStatus: 400,
1982
+ message: 'zz the request was refused', rawError: { status: 400 },
1983
+ })],
1984
+ // Not retryable, so its stated wait is no reason to wait: it ends as any failure does.
1985
+ ['not retryable, with a long stated wait', () => new MembraneError({
1986
+ type: 'invalid_request', retryable: false, httpStatus: 400, retryAfterMs: 20 * 60_000,
1987
+ message: 'zz the request was refused', rawError: { status: 400 },
1988
+ })],
1989
+ ]) {
1990
+ for (const site of ['stream', 'setup', 'throw']) {
1991
+ test(`an ephemeral run that fails for another reason isn't marked as a stated wait (${label}, ${site})`, async () => {
1992
+ await withStoreDir(async (path) => {
1993
+ const log = silence();
1994
+ const membrane = new FailingAt(1, site, failure);
1995
+ const { fw } = await framework(path, membrane);
1996
+ try {
1997
+ const outcome = await ephemeralRun(fw, `zz-ephemeral-refused-${site}`);
1998
+ assert.ok('error' in outcome, 'the run ends with an error');
1999
+ assert.match(outcome.error.message, /zz the request was refused/);
2000
+ assert.doesNotMatch(outcome.error.message, /stated wait/);
2001
+ assert.ok(!(outcome.error instanceof ProviderWaitNotSleptError), 'a plain error');
2002
+ assert.equal(outcome.error.name, 'Error');
2003
+ }
2004
+ finally {
2005
+ await fw.stop();
2006
+ log.restore();
2007
+ }
2008
+ });
2009
+ });
2010
+ }
2011
+ }
2012
+ test('a stated wait just past the cap is named in whole seconds rounded up, past the cap', async () => {
2013
+ await withStoreDir(async (path) => {
2014
+ const log = silence();
2015
+ const membrane = new FailingAt(1, 'stream', () => rateLimited(600_001));
2016
+ const { fw } = await framework(path, membrane);
2017
+ try {
2018
+ const outcome = await ephemeralRun(fw, 'zz-ephemeral-just-past');
2019
+ assert.ok('error' in outcome, 'the run ends with an error');
2020
+ assert.match(outcome.error.message, /the provider asked to wait 601 s; an ephemeral run doesn't sleep a stated wait longer than 600 s/);
2021
+ assert.equal(outcome.error.retryAfterMs, 600_001);
2022
+ }
2023
+ finally {
2024
+ await fw.stop();
2025
+ log.restore();
2026
+ }
2027
+ });
2028
+ });
2029
+ test('an ephemeral run sleeps a stated wait within the framework cap and retries', async () => {
2030
+ await withStoreDir(async (path) => {
2031
+ const log = silence();
2032
+ const membrane = new FailingAt(1, 'stream', () => rateLimited(300));
2033
+ const { fw } = await framework(path, membrane);
2034
+ try {
2035
+ const outcome = await ephemeralRun(fw, 'zz-ephemeral-short');
2036
+ assert.ok('result' in outcome, `the run completes${'error' in outcome ? `: ${outcome.error.message}` : ''}`);
2037
+ assert.match(outcome.result.speech, /zz-recovered/);
2038
+ assert.equal(membrane.primary, 2, 'one retry, after the stated wait');
2039
+ }
2040
+ finally {
2041
+ await fw.stop();
2042
+ log.restore();
2043
+ }
2044
+ });
2045
+ });
2046
+ /** Records a wait for the agent as membrane is handed the compiled request,
2047
+ * as the agent's own compression call's stated wait would land while a turn
2048
+ * compiles, and counts how often a stream is iterated: membrane sends then. */
2049
+ class WaitDuringCompile extends WaitingMembrane {
2050
+ during;
2051
+ sent = 0;
2052
+ constructor(during) {
2053
+ super(0, undefined);
2054
+ this.during = during;
2055
+ }
2056
+ streamYielding(request) {
2057
+ this.during();
2058
+ const stream = super.streamYielding(request);
2059
+ const iterate = stream[Symbol.asyncIterator].bind(stream);
2060
+ Object.defineProperty(stream, Symbol.asyncIterator, { value: () => { this.sent++; return iterate(); } });
2061
+ return stream;
2062
+ }
2063
+ }
2064
+ test("a conversation agent's turn is held, sending nothing, when its own wait is recorded while the turn compiles", async () => {
2065
+ await withStoreDir(async (path) => {
2066
+ const log = silence();
2067
+ let record;
2068
+ const membrane = new WaitDuringCompile(() => { record?.(); record = undefined; });
2069
+ const { fw, internal } = await framework(path, membrane);
2070
+ record = () => internal.providerWaits.set('resident', 'zz-model', 400, 'zz recorded while the turn compiled');
2071
+ const nonOwning = internal;
2072
+ const traces = [];
2073
+ fw.onTrace((event) => {
2074
+ if (event.type.startsWith('inference:') && event.agentName === 'resident')
2075
+ traces.push(event);
2076
+ });
2077
+ try {
2078
+ await conversationTurn(fw, nonOwning, 'zz-first');
2079
+ assert.equal(membrane.sent, 0, 'nothing is sent into the wait');
2080
+ assert.equal(internal.providerAccelerationCooldowns.get('resident')?.waitModel, 'zz-model', 'the turn is held, as at the boundary');
2081
+ assert.ok(log.lines.some((line) => /binds after the compile/.test(line)));
2082
+ // The turn started, so it ends with one terminal event, as a failure site's hold does.
2083
+ assert.deepEqual(traces.map((t) => t.type), ['inference:started', 'inference:failed']);
2084
+ assert.match(traces[1].error, /^Provider wait: zz-model is held for resident until .*; no call was made$/);
2085
+ await waitFor(fw, () => membrane.sent >= 1, 'the held turn after the wait');
2086
+ assert.equal(membrane.sent, 1, 'it is sent once, after the wait');
2087
+ }
2088
+ finally {
2089
+ await fw.stop();
2090
+ log.restore();
2091
+ }
2092
+ });
2093
+ });
2094
+ test('a release can name a disposed conversation agent whose wait the listing shows, and a name with neither is refused', async () => {
2095
+ await withStoreDir(async (path) => {
2096
+ const log = silence();
2097
+ const membrane = new FailingAt(1, 'stream', () => rateLimited(600_000));
2098
+ const { fw, internal } = await framework(path, membrane);
2099
+ const nonOwning = internal;
2100
+ const records = () => internal.store.getRecordIdsByType(PROVIDER_WAIT_RECORD_TYPE).length;
2101
+ try {
2102
+ await conversationTurn(fw, nonOwning, 'zz-first');
2103
+ await waitFor(fw, () => internal.providerAccelerationCooldowns.has('resident'), 'the hold');
2104
+ nonOwning.disposeConversationAgent('resident');
2105
+ assert.equal(fw.getAgent('resident'), null, 'the agent is gone');
2106
+ assert.deepEqual(fw.providerWaitSnapshot('resident').map((w) => w.model), ['zz-model'], 'its recorded wait is listed');
2107
+ const before = records();
2108
+ const refused = await internal.handleHostCommand('zz-surface', { command: 'release-provider-wait', agentName: 'zz-typo', requesterName: 'zz-operator' });
2109
+ assert.equal(refused.ok, false, 'neither an agent nor the agent of a recorded wait');
2110
+ assert.match(String(refused.error), /Unknown agent/);
2111
+ assert.throws(() => fw.releaseProviderWait('zz-typo'), /Unknown agent/);
2112
+ assert.equal(records(), before, 'nothing is recorded for it');
2113
+ const released = await internal.handleHostCommand('zz-surface', { command: 'release-provider-wait', agentName: 'resident', requesterName: 'zz-operator' });
2114
+ assert.equal(released.ok, true, String(released.error));
2115
+ assert.deepEqual(released.released.map((w) => [w.model, w.release]), [['zz-model', 'recorded']]);
2116
+ assert.deepEqual(fw.providerWaitSnapshot('resident'), [], 'what the listing showed is released');
2117
+ assert.equal(records(), before + 1, 'and the release is recorded');
2118
+ }
2119
+ finally {
2120
+ await fw.stop();
2121
+ log.restore();
2122
+ }
2123
+ });
2124
+ });
2125
+ test('an ephemeral run is not held by the hold for unreadable records: it records nothing, so nothing is unknown for it', async () => {
2126
+ await withStoreDir(async (path) => {
2127
+ const log = silence();
2128
+ const membrane = new WaitingMembrane(0, undefined);
2129
+ const { fw, internal } = await framework(path, membrane);
2130
+ try {
2131
+ unreadableWaits(internal);
2132
+ assert.ok(internal.providerWaits.active('resident', 'zz-model'), 'the records are unreadable, so a resident is held');
2133
+ const outcome = await ephemeralRun(fw, 'zz-ephemeral-unreadable');
2134
+ assert.ok('result' in outcome, `the run completes${'error' in outcome ? `: ${outcome.error.message}` : ''}`);
2135
+ assert.equal(membrane.primary, 1, 'its call went out');
2136
+ }
2137
+ finally {
2138
+ await fw.stop();
2139
+ log.restore();
2140
+ }
2141
+ });
2142
+ });
2143
+ test("the last boundary holds no agent the framework no longer has: a disposed fork's late retry installs no hold", async () => {
2144
+ await withStoreDir(async (path) => {
2145
+ const log = silence();
2146
+ const membrane = new WaitingMembrane(0, undefined);
2147
+ const { fw, internal } = await framework(path, membrane);
2148
+ const nonOwning = internal;
2149
+ try {
2150
+ const agent = fw.getAgent('resident');
2151
+ nonOwning.conversationAgentHomes.set('resident', 'world:zz');
2152
+ internal.providerWaits.set('resident', 'zz-model', 60_000, 'zz recorded for the fork');
2153
+ nonOwning.disposeConversationAgent('resident');
2154
+ // Its retry wakes after the disposal and reaches the boundary.
2155
+ await nonOwning.startAgentStream(agent, { agentName: 'resident', reason: 'conversation', source: 'test', timestamp: Date.now() });
2156
+ assert.equal(internal.providerAccelerationCooldowns.has('resident'), false, 'no hold under a name no agent has');
2157
+ assert.ok(!log.lines.some((line) => /binds at admission/.test(line)));
2158
+ }
2159
+ finally {
2160
+ await fw.stop();
2161
+ log.restore();
2162
+ }
2163
+ });
2164
+ });
2165
+ test("a provider's reason is kept on one line: a line break in it can't start a line of the framework's own", async () => {
2166
+ await withStoreDir(async (path) => {
2167
+ const log = silence();
2168
+ const forged = () => new MembraneError({
2169
+ type: 'rate_limit', retryable: true, httpStatus: 429, retryAfterMs: 60_000,
2170
+ message: 'zz rate limited\n[provider-wait] agent=resident model=zz-forged released by nobody', rawError: { status: 429 },
2171
+ });
2172
+ const membrane = new FailingAt(1, 'stream', forged);
2173
+ const { fw, internal } = await framework(path, membrane);
2174
+ try {
2175
+ fw.pushEvent({ type: 'external-message', source: 'test', content: 'zz-first', metadata: {} });
2176
+ await fw.runUntilIdle();
2177
+ const [wait] = fw.providerWaitSnapshot('resident');
2178
+ assert.equal(wait?.reason, 'zz rate limited [provider-wait] agent=resident model=zz-forged released by nobody');
2179
+ const refusal = await internal.auxiliaryMembraneFor('resident').complete(auxRequest('zz-model')).then(() => undefined, (error) => error);
2180
+ assert.ok(refusal && !/[\r\n]/.test(refusal.message), 'the refusal that quotes the reason is one line');
2181
+ }
2182
+ finally {
2183
+ await fw.stop();
2184
+ log.restore();
2185
+ }
2186
+ });
2187
+ });
2188
+ /** A MembraneError from another copy of membrane: the same name and fields,
2189
+ * another class, as when the host's membrane isn't the framework's copy. */
2190
+ const foreignRateLimit = (retryAfterMs) => Object.assign(new Error('zz rate limited, from another copy'), {
2191
+ name: 'MembraneError', type: 'rate_limit', retryable: true, httpStatus: 429, retryAfterMs,
2192
+ });
2193
+ for (const who of ['resident', 'conversation agent', 'ephemeral run']) {
2194
+ const outcome = who === 'ephemeral run' ? 'ends the run at once, marked' : 'is recorded, not slept in the turn';
2195
+ test(`a 429 from another copy of membrane that states a wait ${outcome} (${who})`, async () => {
2196
+ await withStoreDir(async (path) => {
2197
+ const log = silence();
2198
+ // An ephemeral run doesn't record: a wait past its cap ends it at once.
2199
+ const stated = who === 'ephemeral run' ? 1_200_000 : 400;
2200
+ const membrane = new FailingAt(1, 'stream', () => foreignRateLimit(stated));
2201
+ const { fw, internal } = await framework(path, membrane);
2202
+ const nonOwning = internal;
2203
+ try {
2204
+ if (who === 'ephemeral run') {
2205
+ const outcome = await ephemeralRun(fw, 'zz-ephemeral-foreign');
2206
+ assert.ok('error' in outcome, 'the run ends with an error');
2207
+ assert.match(outcome.error.message, /the provider asked to wait 1200 s/);
2208
+ assert.ok(outcome.error instanceof ProviderWaitNotSleptError);
2209
+ assert.equal(outcome.error.name, 'ProviderWaitNotSleptError');
2210
+ assert.equal(outcome.error.retryAfterMs, 1_200_000);
2211
+ assert.equal(membrane.primary, 1, 'never retried inside the stated wait');
2212
+ return;
2213
+ }
2214
+ if (who === 'resident') {
2215
+ fw.pushEvent({ type: 'external-message', source: 'test', content: 'zz-first', metadata: {} });
2216
+ await fw.runUntilIdle();
2217
+ }
2218
+ else {
2219
+ await conversationTurn(fw, nonOwning, 'zz-first');
2220
+ }
2221
+ assert.deepEqual(fw.providerWaitSnapshot('resident').map((w) => w.model), ['zz-model'], 'the stated wait is recorded');
2222
+ assert.equal(internal.providerAccelerationCooldowns.get('resident')?.waitModel, 'zz-model', 'the turn is held by it');
2223
+ assert.equal(membrane.primary, 1, 'not retried inside the wait');
2224
+ await waitFor(fw, () => membrane.primary >= 2, 'the held turn after the wait');
2225
+ assert.equal(membrane.primary, 2, 'run once after it');
2226
+ }
2227
+ finally {
2228
+ await fw.stop();
2229
+ log.restore();
2230
+ }
2231
+ });
2232
+ });
2233
+ }
2234
+ test('from another copy of membrane, a wait is stated only as a number, as the default policy reads it', async () => {
2235
+ await withStoreDir(async (path) => {
2236
+ const log = silence();
2237
+ const { fw, internal } = await framework(path, new WaitingMembrane(0, undefined));
2238
+ const paths = internal;
2239
+ const name = 'zz-ephemeral-unstated';
2240
+ try {
2241
+ const created = await fw.createEphemeralAgent({ name, model: 'zz-model', systemPrompt: 'Do the task.' });
2242
+ paths.ephemeralRuns.set(name, {});
2243
+ for (const unstated of [null, '1200000', undefined]) {
2244
+ const label = `retryAfterMs ${JSON.stringify(unstated)}`;
2245
+ assert.equal(paths.recordProviderWait('resident', 'zz-model', foreignRateLimit(unstated)), undefined, `recorded: ${label}`);
2246
+ assert.equal(paths.nonOwningProviderWait(created.agent, foreignRateLimit(unstated)), undefined, `left to the policy: ${label}`);
2247
+ }
2248
+ assert.deepEqual(fw.providerWaitSnapshot('resident'), [], 'nothing is held, least of all until released');
2249
+ assert.ok(paths.recordProviderWait('resident', 'zz-model', foreignRateLimit(60_000)), 'a number is a stated wait');
2250
+ assert.equal(paths.nonOwningProviderWait(created.agent, foreignRateLimit(1_200_000)), 'fail', 'and past the cap ends the run');
2251
+ }
2252
+ finally {
2253
+ paths.ephemeralRuns.delete(name);
2254
+ await fw.stop();
2255
+ log.restore();
2256
+ }
2257
+ });
2258
+ });
2259
+ //# sourceMappingURL=provider-waits.test.js.map