@caupulican/pi-adaptative 0.86.3 → 0.86.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (226) hide show
  1. package/CHANGELOG.md +32 -0
  2. package/dist/core/agent-paths.d.ts +13 -0
  3. package/dist/core/agent-paths.d.ts.map +1 -1
  4. package/dist/core/agent-paths.js +23 -0
  5. package/dist/core/agent-paths.js.map +1 -1
  6. package/dist/core/autonomy/bounded-completion.d.ts +7 -0
  7. package/dist/core/autonomy/bounded-completion.d.ts.map +1 -1
  8. package/dist/core/autonomy/bounded-completion.js +12 -4
  9. package/dist/core/autonomy/bounded-completion.js.map +1 -1
  10. package/dist/core/autonomy/lane-tool-surface.d.ts +2 -0
  11. package/dist/core/autonomy/lane-tool-surface.d.ts.map +1 -1
  12. package/dist/core/autonomy/lane-tool-surface.js +6 -3
  13. package/dist/core/autonomy/lane-tool-surface.js.map +1 -1
  14. package/dist/core/autonomy/subagent-prompt.js +1 -1
  15. package/dist/core/autonomy/subagent-prompt.js.map +1 -1
  16. package/dist/core/background-lane-controller.d.ts +25 -2
  17. package/dist/core/background-lane-controller.d.ts.map +1 -1
  18. package/dist/core/background-lane-controller.js +67 -0
  19. package/dist/core/background-lane-controller.js.map +1 -1
  20. package/dist/core/delegation/replay-safe-mailbox-bounds.d.ts +4 -0
  21. package/dist/core/delegation/replay-safe-mailbox-bounds.d.ts.map +1 -0
  22. package/dist/core/delegation/replay-safe-mailbox-bounds.js +20 -0
  23. package/dist/core/delegation/replay-safe-mailbox-bounds.js.map +1 -0
  24. package/dist/core/delegation/sanitized-context-fork.d.ts +25 -0
  25. package/dist/core/delegation/sanitized-context-fork.d.ts.map +1 -0
  26. package/dist/core/delegation/sanitized-context-fork.js +299 -0
  27. package/dist/core/delegation/sanitized-context-fork.js.map +1 -0
  28. package/dist/core/delegation/session-root-mailbox.d.ts +75 -0
  29. package/dist/core/delegation/session-root-mailbox.d.ts.map +1 -0
  30. package/dist/core/delegation/session-root-mailbox.js +627 -0
  31. package/dist/core/delegation/session-root-mailbox.js.map +1 -0
  32. package/dist/core/delegation/worker-agent-control-coordinator.d.ts +102 -9
  33. package/dist/core/delegation/worker-agent-control-coordinator.d.ts.map +1 -1
  34. package/dist/core/delegation/worker-agent-control-coordinator.js +1307 -125
  35. package/dist/core/delegation/worker-agent-control-coordinator.js.map +1 -1
  36. package/dist/core/delegation/worker-agent-control.d.ts +183 -8
  37. package/dist/core/delegation/worker-agent-control.d.ts.map +1 -1
  38. package/dist/core/delegation/worker-agent-control.js +778 -48
  39. package/dist/core/delegation/worker-agent-control.js.map +1 -1
  40. package/dist/core/delegation/worker-attempt-executor.d.ts +9 -0
  41. package/dist/core/delegation/worker-attempt-executor.d.ts.map +1 -1
  42. package/dist/core/delegation/worker-attempt-executor.js +479 -214
  43. package/dist/core/delegation/worker-attempt-executor.js.map +1 -1
  44. package/dist/core/delegation/worker-authority-resolver.d.ts +9 -0
  45. package/dist/core/delegation/worker-authority-resolver.d.ts.map +1 -1
  46. package/dist/core/delegation/worker-authority-resolver.js +21 -2
  47. package/dist/core/delegation/worker-authority-resolver.js.map +1 -1
  48. package/dist/core/delegation/worker-context-fork-store.d.ts +71 -0
  49. package/dist/core/delegation/worker-context-fork-store.d.ts.map +1 -0
  50. package/dist/core/delegation/worker-context-fork-store.js +437 -0
  51. package/dist/core/delegation/worker-context-fork-store.js.map +1 -0
  52. package/dist/core/delegation/worker-context-inheritance-policy.d.ts +18 -0
  53. package/dist/core/delegation/worker-context-inheritance-policy.d.ts.map +1 -0
  54. package/dist/core/delegation/worker-context-inheritance-policy.js +28 -0
  55. package/dist/core/delegation/worker-context-inheritance-policy.js.map +1 -0
  56. package/dist/core/delegation/worker-conversation-revision.d.ts +31 -0
  57. package/dist/core/delegation/worker-conversation-revision.d.ts.map +1 -0
  58. package/dist/core/delegation/worker-conversation-revision.js +203 -0
  59. package/dist/core/delegation/worker-conversation-revision.js.map +1 -0
  60. package/dist/core/delegation/worker-conversation-store.d.ts +130 -18
  61. package/dist/core/delegation/worker-conversation-store.d.ts.map +1 -1
  62. package/dist/core/delegation/worker-conversation-store.js +1263 -79
  63. package/dist/core/delegation/worker-conversation-store.js.map +1 -1
  64. package/dist/core/delegation/worker-delegation-controller.d.ts +29 -2
  65. package/dist/core/delegation/worker-delegation-controller.d.ts.map +1 -1
  66. package/dist/core/delegation/worker-delegation-controller.js +625 -111
  67. package/dist/core/delegation/worker-delegation-controller.js.map +1 -1
  68. package/dist/core/delegation/worker-delegation-request.d.ts +2 -0
  69. package/dist/core/delegation/worker-delegation-request.d.ts.map +1 -1
  70. package/dist/core/delegation/worker-delegation-request.js.map +1 -1
  71. package/dist/core/delegation/worker-dispatch-scheduler.d.ts +20 -1
  72. package/dist/core/delegation/worker-dispatch-scheduler.d.ts.map +1 -1
  73. package/dist/core/delegation/worker-dispatch-scheduler.js +210 -37
  74. package/dist/core/delegation/worker-dispatch-scheduler.js.map +1 -1
  75. package/dist/core/delegation/worker-fleet-limits.d.ts +38 -0
  76. package/dist/core/delegation/worker-fleet-limits.d.ts.map +1 -0
  77. package/dist/core/delegation/worker-fleet-limits.js +113 -0
  78. package/dist/core/delegation/worker-fleet-limits.js.map +1 -0
  79. package/dist/core/delegation/worker-lane-projection.d.ts +1 -0
  80. package/dist/core/delegation/worker-lane-projection.d.ts.map +1 -1
  81. package/dist/core/delegation/worker-lane-projection.js +6 -2
  82. package/dist/core/delegation/worker-lane-projection.js.map +1 -1
  83. package/dist/core/delegation/worker-lifecycle.d.ts +20 -3
  84. package/dist/core/delegation/worker-lifecycle.d.ts.map +1 -1
  85. package/dist/core/delegation/worker-lifecycle.js +42 -18
  86. package/dist/core/delegation/worker-lifecycle.js.map +1 -1
  87. package/dist/core/delegation/worker-notification-coordinator.d.ts +4 -0
  88. package/dist/core/delegation/worker-notification-coordinator.d.ts.map +1 -1
  89. package/dist/core/delegation/worker-notification-coordinator.js +45 -4
  90. package/dist/core/delegation/worker-notification-coordinator.js.map +1 -1
  91. package/dist/core/delegation/worker-provider-turn-protocol.d.ts +50 -0
  92. package/dist/core/delegation/worker-provider-turn-protocol.d.ts.map +1 -0
  93. package/dist/core/delegation/worker-provider-turn-protocol.js +185 -0
  94. package/dist/core/delegation/worker-provider-turn-protocol.js.map +1 -0
  95. package/dist/core/delegation/worker-recovery-coordinator.d.ts +45 -4
  96. package/dist/core/delegation/worker-recovery-coordinator.d.ts.map +1 -1
  97. package/dist/core/delegation/worker-recovery-coordinator.js +223 -17
  98. package/dist/core/delegation/worker-recovery-coordinator.js.map +1 -1
  99. package/dist/core/delegation/worker-retry-policy.d.ts +36 -0
  100. package/dist/core/delegation/worker-retry-policy.d.ts.map +1 -0
  101. package/dist/core/delegation/worker-retry-policy.js +41 -0
  102. package/dist/core/delegation/worker-retry-policy.js.map +1 -0
  103. package/dist/core/delegation/worker-runner.d.ts +3 -1
  104. package/dist/core/delegation/worker-runner.d.ts.map +1 -1
  105. package/dist/core/delegation/worker-runner.js +4 -2
  106. package/dist/core/delegation/worker-runner.js.map +1 -1
  107. package/dist/core/delegation/worker-task-view.d.ts +43 -0
  108. package/dist/core/delegation/worker-task-view.d.ts.map +1 -0
  109. package/dist/core/delegation/worker-task-view.js +194 -0
  110. package/dist/core/delegation/worker-task-view.js.map +1 -0
  111. package/dist/core/delegation/worker-terminal-handoff-coordinator.d.ts +55 -0
  112. package/dist/core/delegation/worker-terminal-handoff-coordinator.d.ts.map +1 -0
  113. package/dist/core/delegation/worker-terminal-handoff-coordinator.js +179 -0
  114. package/dist/core/delegation/worker-terminal-handoff-coordinator.js.map +1 -0
  115. package/dist/core/delegation/worker-tree-budget-coordinator.d.ts +2 -2
  116. package/dist/core/delegation/worker-tree-budget-coordinator.d.ts.map +1 -1
  117. package/dist/core/delegation/worker-tree-budget-coordinator.js +12 -46
  118. package/dist/core/delegation/worker-tree-budget-coordinator.js.map +1 -1
  119. package/dist/core/keybindings.d.ts +10 -0
  120. package/dist/core/keybindings.d.ts.map +1 -1
  121. package/dist/core/keybindings.js +2 -0
  122. package/dist/core/keybindings.js.map +1 -1
  123. package/dist/core/orchestration/attempt-ordering.d.ts +9 -0
  124. package/dist/core/orchestration/attempt-ordering.d.ts.map +1 -0
  125. package/dist/core/orchestration/attempt-ordering.js +29 -0
  126. package/dist/core/orchestration/attempt-ordering.js.map +1 -0
  127. package/dist/core/orchestration/capability-gateway.d.ts +19 -0
  128. package/dist/core/orchestration/capability-gateway.d.ts.map +1 -1
  129. package/dist/core/orchestration/capability-gateway.js +19 -2
  130. package/dist/core/orchestration/capability-gateway.js.map +1 -1
  131. package/dist/core/orchestration/contracts.d.ts +43 -3
  132. package/dist/core/orchestration/contracts.d.ts.map +1 -1
  133. package/dist/core/orchestration/contracts.js +24 -0
  134. package/dist/core/orchestration/contracts.js.map +1 -1
  135. package/dist/core/orchestration/delegation-ledger.d.ts +8 -3
  136. package/dist/core/orchestration/delegation-ledger.d.ts.map +1 -1
  137. package/dist/core/orchestration/delegation-ledger.js +148 -55
  138. package/dist/core/orchestration/delegation-ledger.js.map +1 -1
  139. package/dist/core/orchestration/event-store.d.ts +6 -3
  140. package/dist/core/orchestration/event-store.d.ts.map +1 -1
  141. package/dist/core/orchestration/event-store.js +228 -30
  142. package/dist/core/orchestration/event-store.js.map +1 -1
  143. package/dist/core/orchestration/task-runtime-codecs.d.ts +31 -0
  144. package/dist/core/orchestration/task-runtime-codecs.d.ts.map +1 -0
  145. package/dist/core/orchestration/task-runtime-codecs.js +1471 -0
  146. package/dist/core/orchestration/task-runtime-codecs.js.map +1 -0
  147. package/dist/core/orchestration/task-runtime-projection.d.ts +30 -0
  148. package/dist/core/orchestration/task-runtime-projection.d.ts.map +1 -0
  149. package/dist/core/orchestration/task-runtime-projection.js +340 -0
  150. package/dist/core/orchestration/task-runtime-projection.js.map +1 -0
  151. package/dist/core/orchestration/task-runtime-reducer.d.ts +38 -0
  152. package/dist/core/orchestration/task-runtime-reducer.d.ts.map +1 -0
  153. package/dist/core/orchestration/task-runtime-reducer.js +1270 -0
  154. package/dist/core/orchestration/task-runtime-reducer.js.map +1 -0
  155. package/dist/core/orchestration/task-runtime-state.d.ts +142 -0
  156. package/dist/core/orchestration/task-runtime-state.d.ts.map +1 -0
  157. package/dist/core/orchestration/task-runtime-state.js +14 -0
  158. package/dist/core/orchestration/task-runtime-state.js.map +1 -0
  159. package/dist/core/orchestration/task-runtime.d.ts +44 -98
  160. package/dist/core/orchestration/task-runtime.d.ts.map +1 -1
  161. package/dist/core/orchestration/task-runtime.js +357 -1113
  162. package/dist/core/orchestration/task-runtime.js.map +1 -1
  163. package/dist/core/orchestration/worker-context-fork-reference.d.ts +20 -0
  164. package/dist/core/orchestration/worker-context-fork-reference.d.ts.map +1 -0
  165. package/dist/core/orchestration/worker-context-fork-reference.js +53 -0
  166. package/dist/core/orchestration/worker-context-fork-reference.js.map +1 -0
  167. package/dist/core/research/research-runner.d.ts +2 -0
  168. package/dist/core/research/research-runner.d.ts.map +1 -1
  169. package/dist/core/research/research-runner.js +1 -0
  170. package/dist/core/research/research-runner.js.map +1 -1
  171. package/dist/core/runtime-builder.d.ts.map +1 -1
  172. package/dist/core/runtime-builder.js +8 -0
  173. package/dist/core/runtime-builder.js.map +1 -1
  174. package/dist/core/secrets/credential-project.d.ts.map +1 -1
  175. package/dist/core/secrets/credential-project.js +24 -2
  176. package/dist/core/secrets/credential-project.js.map +1 -1
  177. package/dist/core/security/secret-text.d.ts +3 -0
  178. package/dist/core/security/secret-text.d.ts.map +1 -1
  179. package/dist/core/security/secret-text.js +15 -0
  180. package/dist/core/security/secret-text.js.map +1 -1
  181. package/dist/core/settings-manager.d.ts +3 -0
  182. package/dist/core/settings-manager.d.ts.map +1 -1
  183. package/dist/core/settings-manager.js +160 -8
  184. package/dist/core/settings-manager.js.map +1 -1
  185. package/dist/core/tools/delegate.d.ts +31 -4
  186. package/dist/core/tools/delegate.d.ts.map +1 -1
  187. package/dist/core/tools/delegate.js +749 -103
  188. package/dist/core/tools/delegate.js.map +1 -1
  189. package/dist/main.d.ts.map +1 -1
  190. package/dist/main.js +1 -0
  191. package/dist/main.js.map +1 -1
  192. package/dist/modes/interactive/components/activity-lane.d.ts +5 -0
  193. package/dist/modes/interactive/components/activity-lane.d.ts.map +1 -1
  194. package/dist/modes/interactive/components/activity-lane.js +133 -49
  195. package/dist/modes/interactive/components/activity-lane.js.map +1 -1
  196. package/dist/modes/interactive/components/agents-overlay.d.ts +40 -0
  197. package/dist/modes/interactive/components/agents-overlay.d.ts.map +1 -0
  198. package/dist/modes/interactive/components/agents-overlay.js +153 -0
  199. package/dist/modes/interactive/components/agents-overlay.js.map +1 -0
  200. package/dist/modes/interactive/components/settings-selector.d.ts.map +1 -1
  201. package/dist/modes/interactive/components/settings-selector.js +18 -13
  202. package/dist/modes/interactive/components/settings-selector.js.map +1 -1
  203. package/dist/modes/interactive/interactive-event-controller.d.ts +1 -1
  204. package/dist/modes/interactive/interactive-event-controller.d.ts.map +1 -1
  205. package/dist/modes/interactive/interactive-event-controller.js +6 -3
  206. package/dist/modes/interactive/interactive-event-controller.js.map +1 -1
  207. package/dist/modes/interactive/interactive-mode.d.ts +5 -0
  208. package/dist/modes/interactive/interactive-mode.d.ts.map +1 -1
  209. package/dist/modes/interactive/interactive-mode.js +51 -0
  210. package/dist/modes/interactive/interactive-mode.js.map +1 -1
  211. package/dist/modes/interactive/key-handlers.d.ts +1 -0
  212. package/dist/modes/interactive/key-handlers.d.ts.map +1 -1
  213. package/dist/modes/interactive/key-handlers.js +1 -0
  214. package/dist/modes/interactive/key-handlers.js.map +1 -1
  215. package/docs/harness-architecture.md +61 -17
  216. package/docs/settings.md +17 -9
  217. package/docs/worker-profiles.md +6 -2
  218. package/examples/extensions/custom-provider-anthropic/package-lock.json +2 -2
  219. package/examples/extensions/custom-provider-anthropic/package.json +1 -1
  220. package/examples/extensions/custom-provider-gitlab-duo/package.json +1 -1
  221. package/examples/extensions/sandbox/package-lock.json +2 -2
  222. package/examples/extensions/sandbox/package.json +1 -1
  223. package/examples/extensions/with-deps/package-lock.json +2 -2
  224. package/examples/extensions/with-deps/package.json +1 -1
  225. package/npm-shrinkwrap.json +12 -12
  226. package/package.json +4 -4
@@ -0,0 +1 @@
1
+ {"version":3,"file":"worker-provider-turn-protocol.js","sourceRoot":"","sources":["../../../src/core/delegation/worker-provider-turn-protocol.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,iCAAiC,CAAC;AAI7E,MAAM,OAAO,6BAA8B,SAAQ,KAAK;IACvD,YAAY,OAAe;QAC1B,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,+BAA+B,CAAC;IAC7C,CAAC;CACD;AAED,SAAS,UAAU,CAAC,KAAY;IAC/B,OAAO;QACN,WAAW,EAAE,KAAK,CAAC,KAAK;QACxB,YAAY,EAAE,KAAK,CAAC,MAAM;QAC1B,eAAe,EAAE,KAAK,CAAC,SAAS;QAChC,gBAAgB,EAAE,KAAK,CAAC,UAAU;QAClC,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,KAAK;KACzB,CAAC;AACH,CAAC;AAED,SAAS,kBAAkB,CAAC,QAAe,EAAE,SAAgB;IAC5D,OAAO;QACN,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC;QAC1D,YAAY,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC;QAC7D,eAAe,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,SAAS,GAAG,SAAS,CAAC,SAAS,CAAC;QACtE,gBAAgB,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,UAAU,GAAG,SAAS,CAAC,UAAU,CAAC;QACzE,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,WAAW,GAAG,SAAS,CAAC,WAAW,CAAC;QACtE,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC;KAChE,CAAC;AACH,CAAC;AAED,SAAS,QAAQ,CAAC,KAAkC;IACnD,OAAO,CACN,KAAK,CAAC,WAAW,KAAK,CAAC;QACvB,KAAK,CAAC,YAAY,KAAK,CAAC;QACxB,KAAK,CAAC,eAAe,KAAK,CAAC;QAC3B,KAAK,CAAC,gBAAgB,KAAK,CAAC;QAC5B,KAAK,CAAC,WAAW,KAAK,CAAC;QACvB,KAAK,CAAC,OAAO,KAAK,CAAC,CACnB,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,OAAO,0BAA0B;IAetC,YAAY,OAKX;QAfgB,2BAAsB,GAAG,gBAAgB,EAAE,CAAC;QAErD,iBAAY,GAAG,KAAK,CAAC;QACrB,4BAAuB,GAAG,KAAK,CAAC;QAChC,aAAQ,GAAG,KAAK,CAAC;QACjB,eAAU,GAAG,CAAC,CAAC;QACf,WAAM,GAAG,KAAK,CAAC;QACf,yBAAoB,GAAG,CAAC,CAAC;QACzB,uBAAkB,GAAG,CAAC,CAAC;QAQ9B,IAAI,CAAC,kBAAkB,GAAG,OAAO,CAAC,kBAAkB,CAAC;QACrD,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;QACvC,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;QAC7B,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;IACpC,CAAC;IAED,KAAK,CAAC,gBAAgB;QACrB,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,QAAQ,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC;YACvE,MAAM,KAAK,GAAG,IAAI,6BAA6B,CAC9C,gFAAgF,CAChF,CAAC;YACF,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;YACrB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YACtB,IAAI,CAAC,sBAAsB,EAAE,CAAC;YAC9B,MAAM,KAAK,CAAC;QACb,CAAC;QACD,6FAA6F;QAC7F,iDAAiD;QACjD,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAC9B,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,MAAM,UAAU,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC;QACrC,IAAI,QAA+C,CAAC;QACpD,IAAI,CAAC;YACJ,QAAQ,GAAG,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC3C,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC;YAC7B,IAAI,IAAI,CAAC,MAAM,IAAI,UAAU,KAAK,IAAI,CAAC,UAAU,EAAE,CAAC;gBACnD,MAAM,IAAI,6BAA6B,CACtC,iFAAiF,CACjF,CAAC;YACH,CAAC;YACD,IAAI,CAAC,IAAI,GAAG,QAAQ,CAAC;YACrB,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;YAC1B,IAAI,CAAC,uBAAuB,GAAG,KAAK,CAAC;YACrC,QAAQ,GAAG,SAAS,CAAC;YACrB,IAAI,CAAC,oBAAoB,IAAI,CAAC,CAAC;YAC/B,OAAO,EAAE,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;QAC3C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YACtB,QAAQ,EAAE,OAAO,EAAE,CAAC;YACpB,IAAI,CAAC,sBAAsB,EAAE,CAAC;YAC9B,MAAM,KAAK,CAAC;QACb,CAAC;gBAAS,CAAC;YACV,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QACvB,CAAC;IACF,CAAC;IAED,qBAAqB,CAAC,KAAY;QACjC,IAAI,CAAC,4BAA4B,EAAE,CAAC;QACpC,IAAI,IAAI,CAAC,uBAAuB,EAAE,CAAC;YAClC,IAAI,CAAC,IAAI,CACR,IAAI,6BAA6B,CAChC,8FAA8F,CAC9F,CACD,CAAC;QACH,CAAC;QACD,IAAI,CAAC;YACJ,IAAI,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC;YACtC,QAAQ,CAAC,IAAI,CAAC,sBAAsB,EAAE,KAAK,CAAC,CAAC;YAC7C,IAAI,CAAC,uBAAuB,GAAG,IAAI,CAAC;QACrC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YACtB,MAAM,KAAK,CAAC;QACb,CAAC;IACF,CAAC;IAED,iGAAiG;IACjG,8BAA8B;QAC7B,IAAI,CAAC,yBAAyB,EAAE,CAAC;QACjC,IAAI,CAAC,sBAAsB,EAAE,CAAC;IAC/B,CAAC;IAED,sGAAsG;IACtG,+BAA+B;QAC9B,IAAI,CAAC,yBAAyB,EAAE,CAAC;IAClC,CAAC;IAED,sCAAsC;QACrC,IACC,CAAC,IAAI,CAAC,QAAQ;YACd,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,YAAY,CAAC;YACjC,IAAI,CAAC,kBAAkB,KAAK,IAAI,CAAC,oBAAoB,EACpD,CAAC;YACF,OAAO;QACR,CAAC;QACD,IAAI,CAAC,IAAI,CACR,IAAI,6BAA6B,CAChC,kGAAkG,CAClG,CACD,CAAC;IACH,CAAC;IAED,6BAA6B,CAAC,IAAI,GAAgC,YAAY;QAC7E,IAAI,IAAI,CAAC,oBAAoB,GAAG,CAAC;YAAE,OAAO;QAC1C,IAAI,CAAC,IAAI,CACR,IAAI,6BAA6B,CAChC,IAAI,KAAK,YAAY;YACpB,CAAC,CAAC,sFAAsF;YACxF,CAAC,CAAC,sFAAsF,CACzF,CACD,CAAC;IACH,CAAC;IAED,sBAAsB;QACrB,OAAO,IAAI,CAAC,oBAAoB,GAAG,CAAC,CAAC;IACtC,CAAC;IAED,kCAAkC;QACjC,OAAO,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC;IACtD,CAAC;IAED,qGAAqG;IACrG,iCAAiC,CAAC,QAAe;QAChD,MAAM,KAAK,GAAG,kBAAkB,CAAC,QAAQ,EAAE,IAAI,CAAC,sBAAsB,CAAC,CAAC;QACxE,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,IAAI,IAAI,CAAC,WAAW,IAAI,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YACzC,IAAI,CAAC;gBACJ,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;gBACxB,QAAQ,GAAG,IAAI,CAAC;YACjB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAChB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;gBACtB,MAAM,KAAK,CAAC;YACb,CAAC;QACF,CAAC;QACD,OAAO,QAAQ,CAAC;IACjB,CAAC;IAED,KAAK;QACJ,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YAClB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;YACnB,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;QACtB,CAAC;QACD,IAAI,CAAC,sBAAsB,EAAE,CAAC;IAC/B,CAAC;IAEO,yBAAyB;QAChC,IAAI,CAAC,4BAA4B,EAAE,CAAC;QACpC,IAAI,CAAC,IAAI,CAAC,uBAAuB,EAAE,CAAC;YACnC,IAAI,CAAC,IAAI,CACR,IAAI,6BAA6B,CAChC,8FAA8F,CAC9F,CACD,CAAC;QACH,CAAC;QACD,IAAI,CAAC,kBAAkB,IAAI,CAAC,CAAC;QAC7B,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;IAC1B,CAAC;IAEO,4BAA4B;QACnC,IAAI,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,YAAY;YAAE,OAAO;QAC5C,IAAI,CAAC,IAAI,CACR,IAAI,6BAA6B,CAChC,0FAA0F,CAC1F,CACD,CAAC;IACH,CAAC;IAEO,sBAAsB;QAC7B,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;QACvB,IAAI,CAAC,IAAI,GAAG,SAAS,CAAC;QACtB,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC1B,IAAI,CAAC,uBAAuB,GAAG,KAAK,CAAC;QACrC,IAAI,EAAE,OAAO,EAAE,CAAC;IACjB,CAAC;IAEO,IAAI,CAAC,KAAoC;QAChD,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QACtB,MAAM,KAAK,CAAC;IACb,CAAC;CACD","sourcesContent":["import { addUsage, createEmptyUsage } from \"@caupulican/pi-agent-core/usage\";\nimport type { Usage } from \"@caupulican/pi-ai\";\nimport type { GatewayUsageDelta, ProviderBudgetReservation } from \"../orchestration/capability-gateway.ts\";\n\nexport class WorkerCompletionProtocolError extends Error {\n\tconstructor(message: string) {\n\t\tsuper(message);\n\t\tthis.name = \"WorkerCompletionProtocolError\";\n\t}\n}\n\nfunction usageDelta(usage: Usage): Required<GatewayUsageDelta> {\n\treturn {\n\t\tinputTokens: usage.input,\n\t\toutputTokens: usage.output,\n\t\tcacheReadTokens: usage.cacheRead,\n\t\tcacheWriteTokens: usage.cacheWrite,\n\t\ttotalTokens: usage.totalTokens,\n\t\tcostUsd: usage.cost.total,\n\t};\n}\n\nfunction positiveUsageDelta(reported: Usage, accounted: Usage): Required<GatewayUsageDelta> {\n\treturn {\n\t\tinputTokens: Math.max(0, reported.input - accounted.input),\n\t\toutputTokens: Math.max(0, reported.output - accounted.output),\n\t\tcacheReadTokens: Math.max(0, reported.cacheRead - accounted.cacheRead),\n\t\tcacheWriteTokens: Math.max(0, reported.cacheWrite - accounted.cacheWrite),\n\t\ttotalTokens: Math.max(0, reported.totalTokens - accounted.totalTokens),\n\t\tcostUsd: Math.max(0, reported.cost.total - accounted.cost.total),\n\t};\n}\n\nfunction hasUsage(delta: Required<GatewayUsageDelta>): boolean {\n\treturn (\n\t\tdelta.inputTokens !== 0 ||\n\t\tdelta.outputTokens !== 0 ||\n\t\tdelta.cacheReadTokens !== 0 ||\n\t\tdelta.cacheWriteTokens !== 0 ||\n\t\tdelta.totalTokens !== 0 ||\n\t\tdelta.costUsd !== 0\n\t);\n}\n\n/**\n * Owns one provider request's reservation epochs and usage evidence. The caller remains responsible\n * for persisting the assistant and checkpoint between accounting and consuming an epoch.\n */\nexport class WorkerProviderTurnProtocol {\n\tprivate readonly acquireReservation: () => Promise<ProviderBudgetReservation>;\n\tprivate readonly recordUsage: ((delta: GatewayUsageDelta) => void) | undefined;\n\tprivate readonly signal: AbortSignal;\n\tprivate readonly onFailure: (error: unknown) => void;\n\tprivate readonly callbackAccountedUsage = createEmptyUsage();\n\tprivate held: ProviderBudgetReservation | undefined;\n\tprivate heldConsumed = false;\n\tprivate assistantUsageAccounted = false;\n\tprivate inFlight = false;\n\tprivate generation = 0;\n\tprivate closed = false;\n\tprivate successfulPreflights = 0;\n\tprivate consumedPreflights = 0;\n\n\tconstructor(options: {\n\t\tacquireReservation(): Promise<ProviderBudgetReservation>;\n\t\trecordUsage?(delta: GatewayUsageDelta): void;\n\t\tsignal: AbortSignal;\n\t\tonFailure(error: unknown): void;\n\t}) {\n\t\tthis.acquireReservation = options.acquireReservation;\n\t\tthis.recordUsage = options.recordUsage;\n\t\tthis.signal = options.signal;\n\t\tthis.onFailure = options.onFailure;\n\t}\n\n\tasync requestPreflight(): Promise<{ maxTokens: number }> {\n\t\tif (this.closed || this.inFlight || (this.held && !this.heldConsumed)) {\n\t\t\tconst error = new WorkerCompletionProtocolError(\n\t\t\t\t\"Worker completion attempted an overlapping or out-of-order provider preflight.\",\n\t\t\t);\n\t\t\tthis.generation += 1;\n\t\t\tthis.onFailure(error);\n\t\t\tthis.releaseHeldReservation();\n\t\t\tthrow error;\n\t\t}\n\t\t// A terminal assistant retains capacity until either its result is reconciled or the adapter\n\t\t// proves that another provider turn is starting.\n\t\tthis.releaseHeldReservation();\n\t\tthis.inFlight = true;\n\t\tconst generation = ++this.generation;\n\t\tlet acquired: ProviderBudgetReservation | undefined;\n\t\ttry {\n\t\t\tacquired = await this.acquireReservation();\n\t\t\tthis.signal.throwIfAborted();\n\t\t\tif (this.closed || generation !== this.generation) {\n\t\t\t\tthrow new WorkerCompletionProtocolError(\n\t\t\t\t\t\"Worker completion provider preflight resolved after its ownership fence closed.\",\n\t\t\t\t);\n\t\t\t}\n\t\t\tthis.held = acquired;\n\t\t\tthis.heldConsumed = false;\n\t\t\tthis.assistantUsageAccounted = false;\n\t\t\tacquired = undefined;\n\t\t\tthis.successfulPreflights += 1;\n\t\t\treturn { maxTokens: this.held.maxTokens };\n\t\t} catch (error) {\n\t\t\tthis.onFailure(error);\n\t\t\tacquired?.release();\n\t\t\tthis.releaseHeldReservation();\n\t\t\tthrow error;\n\t\t} finally {\n\t\t\tthis.inFlight = false;\n\t\t}\n\t}\n\n\taccountAssistantUsage(usage: Usage): void {\n\t\tthis.assertOutstandingReservation();\n\t\tif (this.assistantUsageAccounted) {\n\t\t\tthis.fail(\n\t\t\t\tnew WorkerCompletionProtocolError(\n\t\t\t\t\t\"Worker completion attempted to account more than one assistant under one provider preflight.\",\n\t\t\t\t),\n\t\t\t);\n\t\t}\n\t\ttry {\n\t\t\tthis.recordUsage?.(usageDelta(usage));\n\t\t\taddUsage(this.callbackAccountedUsage, usage);\n\t\t\tthis.assistantUsageAccounted = true;\n\t\t} catch (error) {\n\t\t\tthis.onFailure(error);\n\t\t\tthrow error;\n\t\t}\n\t}\n\n\t/** Account usage before releasing a tool-request turn for nested or subsequent provider work. */\n\tconsumeToolAssistantAndRelease(): void {\n\t\tthis.consumeAccountedAssistant();\n\t\tthis.releaseHeldReservation();\n\t}\n\n\t/** Account usage while retaining a terminal turn's remaining capacity through result verification. */\n\tconsumeTerminalAssistantAndHold(): void {\n\t\tthis.consumeAccountedAssistant();\n\t}\n\n\tassertEverySuccessfulPreflightConsumed(): void {\n\t\tif (\n\t\t\t!this.inFlight &&\n\t\t\t(!this.held || this.heldConsumed) &&\n\t\t\tthis.consumedPreflights === this.successfulPreflights\n\t\t) {\n\t\t\treturn;\n\t\t}\n\t\tthis.fail(\n\t\t\tnew WorkerCompletionProtocolError(\n\t\t\t\t\"Worker completion returned with a provider preflight authority epoch that no assistant consumed.\",\n\t\t\t),\n\t\t);\n\t}\n\n\tassertProviderOutputPreflight(kind: \"completion\" | \"compaction\" = \"completion\"): void {\n\t\tif (this.successfulPreflights > 0) return;\n\t\tthis.fail(\n\t\t\tnew WorkerCompletionProtocolError(\n\t\t\t\tkind === \"compaction\"\n\t\t\t\t\t? \"Worker compaction returned provider output without a successful authority preflight.\"\n\t\t\t\t\t: \"Worker completion returned provider output without a successful authority preflight.\",\n\t\t\t),\n\t\t);\n\t}\n\n\thasSuccessfulPreflight(): boolean {\n\t\treturn this.successfulPreflights > 0;\n\t}\n\n\thasOutstandingAssistantReservation(): boolean {\n\t\treturn this.held !== undefined && !this.heldConsumed;\n\t}\n\n\t/** Conservatively charge only provider-result usage not already evidenced by assistant callbacks. */\n\taccountUnverifiedResultUsageDelta(reported: Usage): boolean {\n\t\tconst delta = positiveUsageDelta(reported, this.callbackAccountedUsage);\n\t\tlet recorded = false;\n\t\tif (this.recordUsage && hasUsage(delta)) {\n\t\t\ttry {\n\t\t\t\tthis.recordUsage(delta);\n\t\t\t\trecorded = true;\n\t\t\t} catch (error) {\n\t\t\t\tthis.onFailure(error);\n\t\t\t\tthrow error;\n\t\t\t}\n\t\t}\n\t\treturn recorded;\n\t}\n\n\tclose(): void {\n\t\tif (!this.closed) {\n\t\t\tthis.closed = true;\n\t\t\tthis.generation += 1;\n\t\t}\n\t\tthis.releaseHeldReservation();\n\t}\n\n\tprivate consumeAccountedAssistant(): void {\n\t\tthis.assertOutstandingReservation();\n\t\tif (!this.assistantUsageAccounted) {\n\t\t\tthis.fail(\n\t\t\t\tnew WorkerCompletionProtocolError(\n\t\t\t\t\t\"Worker completion attempted to consume an assistant before its provider usage was accounted.\",\n\t\t\t\t),\n\t\t\t);\n\t\t}\n\t\tthis.consumedPreflights += 1;\n\t\tthis.heldConsumed = true;\n\t}\n\n\tprivate assertOutstandingReservation(): void {\n\t\tif (this.held && !this.heldConsumed) return;\n\t\tthis.fail(\n\t\t\tnew WorkerCompletionProtocolError(\n\t\t\t\t\"Worker completion attempted to persist an assistant without a held provider reservation.\",\n\t\t\t),\n\t\t);\n\t}\n\n\tprivate releaseHeldReservation(): void {\n\t\tconst held = this.held;\n\t\tthis.held = undefined;\n\t\tthis.heldConsumed = false;\n\t\tthis.assistantUsageAccounted = false;\n\t\theld?.release();\n\t}\n\n\tprivate fail(error: WorkerCompletionProtocolError): never {\n\t\tthis.onFailure(error);\n\t\tthrow error;\n\t}\n}\n"]}
@@ -13,14 +13,29 @@ export type WorkerRecoveryDispatchResult = {
13
13
  started: false;
14
14
  skipReason: string;
15
15
  };
16
+ export type WorkerRetryScheduleResult = {
17
+ scheduled: true;
18
+ record: LaneRecord;
19
+ retriesUsed: number;
20
+ notBefore: string;
21
+ } | {
22
+ scheduled: false;
23
+ reason: string;
24
+ };
16
25
  export interface WorkerRecoveryCoordinatorOptions {
17
26
  lifecycle: WorkerLifecycle;
18
- scheduler: Pick<WorkerDispatchScheduler, "enqueue">;
27
+ scheduler: Pick<WorkerDispatchScheduler, "enqueue"> & Partial<Pick<WorkerDispatchScheduler, "onQueueCapacityAvailable">>;
19
28
  recoverWriteReservations(): void;
20
29
  publishTerminalRecord(record: LaneRecord): void;
30
+ publishTerminalRecords?(records: readonly LaneRecord[]): void;
21
31
  dispatchVerification(recovery: Extract<PendingVerificationRecovery, {
22
32
  action: "dispatch";
23
33
  }>): WorkerRecoveryDispatchResult;
34
+ recoverTaskBearingMailboxTurns(): void;
35
+ recoverSessionRootReplies(): void;
36
+ /** Wake the scheduler and UI after a retained retry deadline becomes eligible. */
37
+ retryReady?(): void;
38
+ now?(): number;
24
39
  warn(message: string): void;
25
40
  }
26
41
  /**
@@ -33,21 +48,47 @@ export declare class WorkerRecoveryCoordinator {
33
48
  private queueRecovered;
34
49
  private recoveringDurableState;
35
50
  private readonly verificationRecoveryFailures;
51
+ private readonly retryTimers;
52
+ private readonly unsubscribeQueueCapacity;
36
53
  constructor(options: WorkerRecoveryCoordinatorOptions);
37
- /** Idempotently rebuild in-process queue state and replay only durable terminal handoffs. */
54
+ /** Idempotently rebuild in-process queues, terminal outboxes, and task-bearing mailboxes. */
38
55
  recover(): void;
56
+ /** Evaluate, persist, and arm one retry through the same durable path used by restart recovery. */
57
+ scheduleAttemptRetry(args: {
58
+ laneId: string;
59
+ agentId: string;
60
+ ownerId: string;
61
+ request: WorkerDelegationRequest;
62
+ outcome: {
63
+ laneStatus: string;
64
+ reasonCode: string;
65
+ reasonDetail?: string;
66
+ };
67
+ provider: string;
68
+ maxAttempts?: number;
69
+ }): WorkerRetryScheduleResult;
70
+ /** Cancel only process-local alarms; the retained deadline remains authoritative on disk. */
71
+ dispose(): void;
72
+ clearScheduledRetry(laneId: string): void;
73
+ /** Keep an externally requested wake queued while the durable retry deadline is still retained. */
74
+ deferRetryIfNeeded(record: LaneRecord, request: WorkerDelegationRequest): boolean;
75
+ private enqueueOrDeferRetry;
76
+ private onRetryTimer;
77
+ private tryEnqueueDueRetry;
78
+ private flushDueRetries;
79
+ private now;
39
80
  /** Rebuild task metadata from its durable dispatch; no current profile/model selection is consulted. */
40
81
  recoveredRequest(attempt: AttemptRuntimeState, verificationOfTaskId?: string): WorkerDelegationRequest;
41
82
  /** Append deterministic unknown outcomes only for tool calls not durably followed by a result. */
42
83
  repairInterruptedToolResults(conversation: WorkerConversation): void;
43
84
  /** Return an already-persisted final assistant response so recovery cannot duplicate provider work. */
44
- recoveredTerminalCompletion(conversation: WorkerConversation): {
85
+ recoveredTerminalCompletion(conversation: WorkerConversation, attemptId: string): {
45
86
  text: string;
46
87
  usage: Usage;
47
88
  stopReason: string;
48
89
  } | undefined;
49
90
  /** Legacy raw transcripts predate durable usage checkpoints; reconstruct their bounded cumulative baseline. */
50
- initialUsage(conversation: WorkerConversation, checkpointUsage: AttemptUsageSnapshot | undefined): AttemptUsageSnapshot;
91
+ initialUsage(conversation: WorkerConversation, checkpointUsage: AttemptUsageSnapshot | undefined, attemptId: string): AttemptUsageSnapshot;
51
92
  /** Fail closed when the current owner profile no longer permits a persisted grant. */
52
93
  durableGrantIsStillPermitted(grant: ExecutionGrant, plan: WorkerExecutionPlan, selectedResources: readonly ResourcePointer[]): boolean;
53
94
  }
@@ -1 +1 @@
1
- {"version":3,"file":"worker-recovery-coordinator.d.ts","sourceRoot":"","sources":["../../../src/core/delegation/worker-recovery-coordinator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAW,KAAK,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAC;AAG9D,OAAO,KAAK,EAAE,oBAAoB,EAAE,cAAc,EAAE,eAAe,EAAc,MAAM,+BAA+B,CAAC;AACvH,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,kCAAkC,CAAC;AAC5E,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,gCAAgC,CAAC;AACzE,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,gCAAgC,CAAC;AAC9E,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,gCAAgC,CAAC;AAC9E,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,8BAA8B,CAAC;AACxE,OAAO,KAAK,EAAE,2BAA2B,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAE1F,MAAM,MAAM,4BAA4B,GAAG;IAAE,OAAO,EAAE,IAAI,CAAA;CAAE,GAAG;IAAE,OAAO,EAAE,KAAK,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,CAAC;AAEtG,MAAM,WAAW,gCAAgC;IAChD,SAAS,EAAE,eAAe,CAAC;IAC3B,SAAS,EAAE,IAAI,CAAC,uBAAuB,EAAE,SAAS,CAAC,CAAC;IACpD,wBAAwB,IAAI,IAAI,CAAC;IACjC,qBAAqB,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI,CAAC;IAChD,oBAAoB,CACnB,QAAQ,EAAE,OAAO,CAAC,2BAA2B,EAAE;QAAE,MAAM,EAAE,UAAU,CAAA;KAAE,CAAC,GACpE,4BAA4B,CAAC;IAChC,IAAI,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5B;AAED;;;;GAIG;AACH,qBAAa,yBAAyB;IACrC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmC;IAC3D,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,sBAAsB,CAAS;IACvC,OAAO,CAAC,QAAQ,CAAC,4BAA4B,CAA6B;IAE1E,YAAY,OAAO,EAAE,gCAAgC,EAEpD;IAED,6FAA6F;IAC7F,OAAO,IAAI,IAAI,CAqEd;IAED,wGAAwG;IACxG,gBAAgB,CAAC,OAAO,EAAE,mBAAmB,EAAE,oBAAoB,CAAC,EAAE,MAAM,GAAG,uBAAuB,CAkBrG;IAED,kGAAkG;IAClG,4BAA4B,CAAC,YAAY,EAAE,kBAAkB,GAAG,IAAI,CA0BnE;IAED,uGAAuG;IACvG,2BAA2B,CAC1B,YAAY,EAAE,kBAAkB,GAC9B;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,KAAK,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,GAAG,SAAS,CAahE;IAED,+GAA+G;IAC/G,YAAY,CACX,YAAY,EAAE,kBAAkB,EAChC,eAAe,EAAE,oBAAoB,GAAG,SAAS,GAC/C,oBAAoB,CAGtB;IAED,sFAAsF;IACtF,4BAA4B,CAC3B,KAAK,EAAE,cAAc,EACrB,IAAI,EAAE,mBAAmB,EACzB,iBAAiB,EAAE,SAAS,eAAe,EAAE,GAC3C,OAAO,CAqCT;CACD"}
1
+ {"version":3,"file":"worker-recovery-coordinator.d.ts","sourceRoot":"","sources":["../../../src/core/delegation/worker-recovery-coordinator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAW,KAAK,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAC;AAG9D,OAAO,KAAK,EAAE,oBAAoB,EAAE,cAAc,EAAE,eAAe,EAAc,MAAM,+BAA+B,CAAC;AACvH,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,kCAAkC,CAAC;AAC5E,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,gCAAgC,CAAC;AACzE,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,gCAAgC,CAAC;AAC9E,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,gCAAgC,CAAC;AAC9E,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,8BAA8B,CAAC;AACxE,OAAO,KAAK,EAAE,2BAA2B,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAG1F,MAAM,MAAM,4BAA4B,GAAG;IAAE,OAAO,EAAE,IAAI,CAAA;CAAE,GAAG;IAAE,OAAO,EAAE,KAAK,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,CAAC;AAEtG,MAAM,MAAM,yBAAyB,GAClC;IAAE,SAAS,EAAE,IAAI,CAAC;IAAC,MAAM,EAAE,UAAU,CAAC;IAAC,WAAW,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GAC/E;IAAE,SAAS,EAAE,KAAK,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAUxC,MAAM,WAAW,gCAAgC;IAChD,SAAS,EAAE,eAAe,CAAC;IAC3B,SAAS,EAAE,IAAI,CAAC,uBAAuB,EAAE,SAAS,CAAC,GAClD,OAAO,CAAC,IAAI,CAAC,uBAAuB,EAAE,0BAA0B,CAAC,CAAC,CAAC;IACpE,wBAAwB,IAAI,IAAI,CAAC;IACjC,qBAAqB,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI,CAAC;IAChD,sBAAsB,CAAC,CAAC,OAAO,EAAE,SAAS,UAAU,EAAE,GAAG,IAAI,CAAC;IAC9D,oBAAoB,CACnB,QAAQ,EAAE,OAAO,CAAC,2BAA2B,EAAE;QAAE,MAAM,EAAE,UAAU,CAAA;KAAE,CAAC,GACpE,4BAA4B,CAAC;IAChC,8BAA8B,IAAI,IAAI,CAAC;IACvC,yBAAyB,IAAI,IAAI,CAAC;IAClC,kFAAkF;IAClF,UAAU,CAAC,IAAI,IAAI,CAAC;IACpB,GAAG,CAAC,IAAI,MAAM,CAAC;IACf,IAAI,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5B;AAcD;;;;GAIG;AACH,qBAAa,yBAAyB;IACrC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmC;IAC3D,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,sBAAsB,CAAS;IACvC,OAAO,CAAC,QAAQ,CAAC,4BAA4B,CAA6B;IAC1E,OAAO,CAAC,QAAQ,CAAC,WAAW,CAA2C;IACvE,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAA2B;IAEpE,YAAY,OAAO,EAAE,gCAAgC,EAQpD;IAED,6FAA6F;IAC7F,OAAO,IAAI,IAAI,CAyGd;IAED,mGAAmG;IACnG,oBAAoB,CAAC,IAAI,EAAE;QAC1B,MAAM,EAAE,MAAM,CAAC;QACf,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,EAAE,uBAAuB,CAAC;QACjC,OAAO,EAAE;YAAE,UAAU,EAAE,MAAM,CAAC;YAAC,UAAU,EAAE,MAAM,CAAC;YAAC,YAAY,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;QAC3E,QAAQ,EAAE,MAAM,CAAC;QACjB,WAAW,CAAC,EAAE,MAAM,CAAC;KACrB,GAAG,yBAAyB,CAoC5B;IAED,6FAA6F;IAC7F,OAAO,IAAI,IAAI,CAMd;IAED,mBAAmB,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAMxC;IAED,mGAAmG;IACnG,kBAAkB,CAAC,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,uBAAuB,GAAG,OAAO,CAYhF;IAED,OAAO,CAAC,mBAAmB;IAqC3B,OAAO,CAAC,YAAY;IAOpB,OAAO,CAAC,kBAAkB;IAqC1B,OAAO,CAAC,eAAe;IAOvB,OAAO,CAAC,GAAG;IAIX,wGAAwG;IACxG,gBAAgB,CAAC,OAAO,EAAE,mBAAmB,EAAE,oBAAoB,CAAC,EAAE,MAAM,GAAG,uBAAuB,CAmBrG;IAED,kGAAkG;IAClG,4BAA4B,CAAC,YAAY,EAAE,kBAAkB,GAAG,IAAI,CA0BnE;IAED,uGAAuG;IACvG,2BAA2B,CAC1B,YAAY,EAAE,kBAAkB,EAChC,SAAS,EAAE,MAAM,GACf;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,KAAK,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,GAAG,SAAS,CAahE;IAED,+GAA+G;IAC/G,YAAY,CACX,YAAY,EAAE,kBAAkB,EAChC,eAAe,EAAE,oBAAoB,GAAG,SAAS,EACjD,SAAS,EAAE,MAAM,GACf,oBAAoB,CAGtB;IAED,sFAAsF;IACtF,4BAA4B,CAC3B,KAAK,EAAE,cAAc,EACrB,IAAI,EAAE,mBAAmB,EACzB,iBAAiB,EAAE,SAAS,eAAe,EAAE,GAC3C,OAAO,CAqCT;CACD"}
@@ -1,5 +1,12 @@
1
1
  import { isPathWithinScope } from "../autonomy/path-scope.js";
2
2
  import { reconcileAttemptUsage } from "../orchestration/attempt-usage.js";
3
+ import { evaluateWorkerRetry } from "./worker-retry-policy.js";
4
+ function restartResumableSuspension(attempt, newlySuspendedAttemptIds) {
5
+ return (newlySuspendedAttemptIds.has(attempt.attemptId) ||
6
+ attempt.reasonCode === "agent_process_recovered_after_owner_exit" ||
7
+ attempt.reasonCode === "agent_process_interrupted" ||
8
+ attempt.reasonCode?.startsWith("retry_scheduled:") === true);
9
+ }
3
10
  /**
4
11
  * Owns restart-only worker state repair. It has no provider or tool execution authority: it
5
12
  * reconstructs bounded durable requests, requeues them through the owner scheduler, and supplies
@@ -10,17 +17,24 @@ export class WorkerRecoveryCoordinator {
10
17
  this.queueRecovered = false;
11
18
  this.recoveringDurableState = false;
12
19
  this.verificationRecoveryFailures = new Map();
20
+ this.retryTimers = new Map();
13
21
  this.options = options;
22
+ this.unsubscribeQueueCapacity = options.scheduler.onQueueCapacityAvailable?.(() => {
23
+ this.flushDueRetries();
24
+ // Mandatory verifier dispatches remain derivable from durable subject state even after the
25
+ // ordinary queue was fully recovered. Every released slot must replay that retained demand.
26
+ this.recover();
27
+ });
14
28
  }
15
- /** Idempotently rebuild in-process queue state and replay only durable terminal handoffs. */
29
+ /** Idempotently rebuild in-process queues, terminal outboxes, and task-bearing mailboxes. */
16
30
  recover() {
17
31
  if (this.recoveringDurableState)
18
32
  return;
19
33
  this.recoveringDurableState = true;
20
- const { lifecycle, scheduler } = this.options;
34
+ const { lifecycle } = this.options;
21
35
  try {
22
36
  if (!this.queueRecovered) {
23
- const suspendedAttemptIds = new Set(lifecycle.suspendBoundInProcessAttemptsForRestart());
37
+ const newlySuspendedAttemptIds = new Set(lifecycle.suspendBoundInProcessAttemptsForRestart());
24
38
  try {
25
39
  this.options.recoverWriteReservations();
26
40
  }
@@ -29,10 +43,13 @@ export class WorkerRecoveryCoordinator {
29
43
  }
30
44
  const queued = lifecycle.recoverQueued();
31
45
  const snapshot = lifecycle.getTaskRuntimeSnapshot();
32
- for (const attemptId of suspendedAttemptIds) {
33
- const attempt = snapshot.attempts[attemptId];
34
- if (!attempt || attempt.dispatch.executionKind === "managed-process")
46
+ for (const attempt of Object.values(snapshot.attempts)) {
47
+ if (attempt.status !== "suspended" ||
48
+ !attempt.agentId ||
49
+ attempt.dispatch.executionKind === "managed-process" ||
50
+ !restartResumableSuspension(attempt, newlySuspendedAttemptIds)) {
35
51
  continue;
52
+ }
36
53
  const record = lifecycle.getRecord(attempt.taskId);
37
54
  const task = snapshot.tasks[attempt.taskId]?.task;
38
55
  if (!record || !task)
@@ -43,10 +60,23 @@ export class WorkerRecoveryCoordinator {
43
60
  ...(task.verificationOfTaskId ? { verificationOfTaskId: task.verificationOfTaskId } : {}),
44
61
  });
45
62
  }
63
+ let recoveredAll = true;
46
64
  for (const { attempt, record, verificationOfTaskId } of queued) {
47
- scheduler.enqueue(record, this.recoveredRequest(attempt, verificationOfTaskId), true, verificationOfTaskId !== undefined);
65
+ let retained;
66
+ try {
67
+ retained = this.enqueueOrDeferRetry(record, attempt, this.recoveredRequest(attempt, verificationOfTaskId), verificationOfTaskId !== undefined);
68
+ }
69
+ catch (error) {
70
+ recoveredAll = false;
71
+ this.options.warn(`Failed to recover queued worker ${record.laneId}; retaining durable dispatch: ${error instanceof Error ? error.message : String(error)}`);
72
+ continue;
73
+ }
74
+ if (retained)
75
+ continue;
76
+ recoveredAll = false;
77
+ break;
48
78
  }
49
- this.queueRecovered = true;
79
+ this.queueRecovered = recoveredAll;
50
80
  }
51
81
  for (const recovery of lifecycle.getPendingVerificationRecoveries()) {
52
82
  if (recovery.action === "reconcile") {
@@ -59,7 +89,19 @@ export class WorkerRecoveryCoordinator {
59
89
  }
60
90
  continue;
61
91
  }
62
- const started = this.options.dispatchVerification(recovery);
92
+ let started;
93
+ try {
94
+ started = this.options.dispatchVerification(recovery);
95
+ }
96
+ catch (error) {
97
+ const reason = error instanceof Error ? error.message : String(error);
98
+ const failureKey = `dispatch_threw:${reason}`;
99
+ if (this.verificationRecoveryFailures.get(recovery.subjectTaskId) !== failureKey) {
100
+ this.verificationRecoveryFailures.set(recovery.subjectTaskId, failureKey);
101
+ this.options.warn(`Recovered verification for ${recovery.subjectTaskId} threw during dispatch: ${reason}`);
102
+ }
103
+ continue;
104
+ }
63
105
  if (started.started) {
64
106
  this.verificationRecoveryFailures.delete(recovery.subjectTaskId);
65
107
  continue;
@@ -69,22 +111,186 @@ export class WorkerRecoveryCoordinator {
69
111
  this.verificationRecoveryFailures.set(recovery.subjectTaskId, started.skipReason);
70
112
  this.options.warn(`Recovered verification for ${recovery.subjectTaskId} did not start: ${started.skipReason}`);
71
113
  }
72
- for (const notification of lifecycle.getPendingTerminalNotifications()) {
73
- this.options.publishTerminalRecord(notification.record);
114
+ const terminalRecords = lifecycle.getPendingTerminalNotifications().map((notification) => notification.record);
115
+ if (terminalRecords.length > 0) {
116
+ if (this.options.publishTerminalRecords)
117
+ this.options.publishTerminalRecords(terminalRecords);
118
+ else
119
+ for (const record of terminalRecords)
120
+ this.options.publishTerminalRecord(record);
74
121
  }
122
+ this.options.recoverSessionRootReplies();
123
+ this.options.recoverTaskBearingMailboxTurns();
75
124
  }
76
125
  finally {
77
126
  this.recoveringDurableState = false;
78
127
  }
79
128
  }
129
+ /** Evaluate, persist, and arm one retry through the same durable path used by restart recovery. */
130
+ scheduleAttemptRetry(args) {
131
+ const attempt = this.options.lifecycle.getActiveAttempt(args.laneId);
132
+ const record = this.options.lifecycle.getRecord(args.laneId);
133
+ if (!attempt || !record)
134
+ return { scheduled: false, reason: "attempt_missing" };
135
+ const retriesUsed = attempt.retry?.retriesUsed ?? 0;
136
+ const decision = evaluateWorkerRetry({
137
+ laneStatus: args.outcome.laneStatus,
138
+ reasonCode: args.outcome.reasonCode,
139
+ ...(args.outcome.reasonDetail ? { reasonDetail: args.outcome.reasonDetail } : {}),
140
+ provider: args.provider,
141
+ retriesUsed,
142
+ ...(args.maxAttempts !== undefined ? { maxAttempts: args.maxAttempts } : {}),
143
+ });
144
+ if (!decision.retry)
145
+ return { scheduled: false, reason: decision.reason };
146
+ const nextRetriesUsed = retriesUsed + 1;
147
+ const notBefore = new Date(this.now() + decision.delayMs).toISOString();
148
+ let suspended;
149
+ try {
150
+ suspended = this.options.lifecycle.scheduleAgentRetry({
151
+ laneId: args.laneId,
152
+ agentId: args.agentId,
153
+ ownerId: args.ownerId,
154
+ reasonCode: `retry_scheduled:${decision.reason}`,
155
+ retry: { retriesUsed: nextRetriesUsed, notBefore },
156
+ });
157
+ }
158
+ catch (error) {
159
+ this.options.warn(`Worker ${args.laneId} retry suspension failed; terminalizing instead: ${error instanceof Error ? error.message : String(error)}`);
160
+ return { scheduled: false, reason: "retry_suspension_failed" };
161
+ }
162
+ this.options.warn(`Worker ${args.laneId} failed (${decision.reason}); retrying from the persisted transcript in ${Math.ceil(decision.delayMs / 1000)}s (attempt ${nextRetriesUsed + 1}).`);
163
+ this.enqueueOrDeferRetry(record, suspended, args.request, args.request.verificationOfTaskId !== undefined);
164
+ return { scheduled: true, record, retriesUsed: nextRetriesUsed, notBefore };
165
+ }
166
+ /** Cancel only process-local alarms; the retained deadline remains authoritative on disk. */
167
+ dispose() {
168
+ this.unsubscribeQueueCapacity?.();
169
+ for (const entry of this.retryTimers.values()) {
170
+ if (entry.timer)
171
+ clearTimeout(entry.timer);
172
+ }
173
+ this.retryTimers.clear();
174
+ }
175
+ clearScheduledRetry(laneId) {
176
+ for (const [attemptId, entry] of this.retryTimers) {
177
+ if (entry.laneId !== laneId)
178
+ continue;
179
+ if (entry.timer)
180
+ clearTimeout(entry.timer);
181
+ this.retryTimers.delete(attemptId);
182
+ }
183
+ }
184
+ /** Keep an externally requested wake queued while the durable retry deadline is still retained. */
185
+ deferRetryIfNeeded(record, request) {
186
+ const attempt = this.options.lifecycle.getActiveAttempt(record.laneId);
187
+ if (!attempt ||
188
+ attempt.status !== "suspended" ||
189
+ !attempt.retry ||
190
+ Date.parse(attempt.retry.notBefore) <= this.now()) {
191
+ return false;
192
+ }
193
+ this.enqueueOrDeferRetry(record, attempt, request, request.verificationOfTaskId !== undefined);
194
+ return true;
195
+ }
196
+ enqueueOrDeferRetry(record, attempt, request, priority) {
197
+ const retry = attempt.retry;
198
+ if (!retry) {
199
+ try {
200
+ this.options.scheduler.enqueue(record, request, true, priority);
201
+ return true;
202
+ }
203
+ catch (error) {
204
+ if (error instanceof Error && error.message === "worker_dispatch_queue_full")
205
+ return false;
206
+ throw error;
207
+ }
208
+ }
209
+ const existing = this.retryTimers.get(attempt.attemptId);
210
+ if (existing?.notBefore === retry.notBefore)
211
+ return true;
212
+ if (existing?.timer)
213
+ clearTimeout(existing.timer);
214
+ const entry = {
215
+ laneId: record.laneId,
216
+ notBefore: retry.notBefore,
217
+ request,
218
+ priority,
219
+ };
220
+ this.retryTimers.set(attempt.attemptId, entry);
221
+ const remainingMs = Date.parse(retry.notBefore) - this.now();
222
+ if (remainingMs <= 0) {
223
+ this.tryEnqueueDueRetry(attempt.attemptId, entry);
224
+ return true;
225
+ }
226
+ const timer = setTimeout(() => this.onRetryTimer(attempt.attemptId), Math.min(remainingMs, 2_147_000_000));
227
+ if (typeof timer === "object" && "unref" in timer)
228
+ timer.unref();
229
+ entry.timer = timer;
230
+ return true;
231
+ }
232
+ onRetryTimer(attemptId) {
233
+ const entry = this.retryTimers.get(attemptId);
234
+ if (!entry)
235
+ return;
236
+ delete entry.timer;
237
+ this.tryEnqueueDueRetry(attemptId, entry);
238
+ }
239
+ tryEnqueueDueRetry(attemptId, entry) {
240
+ const attempt = this.options.lifecycle.getActiveAttempt(entry.laneId);
241
+ if (!attempt ||
242
+ attempt.attemptId !== attemptId ||
243
+ attempt.status !== "suspended" ||
244
+ attempt.retry?.notBefore !== entry.notBefore) {
245
+ this.retryTimers.delete(attemptId);
246
+ return "stale";
247
+ }
248
+ const record = this.options.lifecycle.getRecord(entry.laneId);
249
+ if (!record) {
250
+ this.retryTimers.delete(attemptId);
251
+ return "stale";
252
+ }
253
+ const remainingMs = Date.parse(entry.notBefore) - this.now();
254
+ if (remainingMs > 0) {
255
+ const timer = setTimeout(() => this.onRetryTimer(attemptId), Math.min(remainingMs, 2_147_000_000));
256
+ if (typeof timer === "object" && "unref" in timer)
257
+ timer.unref();
258
+ entry.timer = timer;
259
+ return "stale";
260
+ }
261
+ try {
262
+ this.options.scheduler.enqueue(record, entry.request, true, entry.priority);
263
+ }
264
+ catch (error) {
265
+ if (error instanceof Error && error.message === "worker_dispatch_queue_full")
266
+ return "full";
267
+ this.options.warn(`Worker ${entry.laneId} retry enqueue failed; retaining the durable retry: ${error instanceof Error ? error.message : String(error)}`);
268
+ return "full";
269
+ }
270
+ this.retryTimers.delete(attemptId);
271
+ this.options.retryReady?.();
272
+ return "enqueued";
273
+ }
274
+ flushDueRetries() {
275
+ for (const [attemptId, entry] of this.retryTimers) {
276
+ if (entry.timer || Date.parse(entry.notBefore) > this.now())
277
+ continue;
278
+ if (this.tryEnqueueDueRetry(attemptId, entry) === "full")
279
+ break;
280
+ }
281
+ }
282
+ now() {
283
+ return this.options.now?.() ?? Date.now();
284
+ }
80
285
  /** Rebuild task metadata from its durable dispatch; no current profile/model selection is consulted. */
81
286
  recoveredRequest(attempt, verificationOfTaskId) {
82
287
  const task = this.options.lifecycle.getTask(attempt.taskId)?.task;
288
+ const durableVerificationOfTaskId = verificationOfTaskId ?? task?.verificationOfTaskId;
83
289
  return {
84
290
  instructions: attempt.dispatch.instructions,
85
291
  profileId: attempt.dispatch.profileId,
86
292
  ...(attempt.dispatch.parentAgentId ? { parentAgentId: attempt.dispatch.parentAgentId } : {}),
87
- ...(verificationOfTaskId ? { verificationOfTaskId } : {}),
293
+ ...(durableVerificationOfTaskId ? { verificationOfTaskId: durableVerificationOfTaskId } : {}),
88
294
  ...(task
89
295
  ? {
90
296
  taskContext: {
@@ -128,8 +334,8 @@ export class WorkerRecoveryCoordinator {
128
334
  }
129
335
  }
130
336
  /** Return an already-persisted final assistant response so recovery cannot duplicate provider work. */
131
- recoveredTerminalCompletion(conversation) {
132
- const last = conversation.getProviderMessages().at(-1);
337
+ recoveredTerminalCompletion(conversation, attemptId) {
338
+ const last = conversation.getLastAttemptMessage(attemptId);
133
339
  if (!last || last.role !== "assistant" || last.stopReason === "error" || last.stopReason === "aborted") {
134
340
  return undefined;
135
341
  }
@@ -139,11 +345,11 @@ export class WorkerRecoveryCoordinator {
139
345
  .filter((content) => content.type === "text")
140
346
  .map((content) => content.text)
141
347
  .join("");
142
- return text ? { text, usage: last.usage, stopReason: last.stopReason } : undefined;
348
+ return { text, usage: last.usage, stopReason: last.stopReason };
143
349
  }
144
350
  /** Legacy raw transcripts predate durable usage checkpoints; reconstruct their bounded cumulative baseline. */
145
- initialUsage(conversation, checkpointUsage) {
146
- const transcriptUsage = conversation.getRawTranscriptUsage();
351
+ initialUsage(conversation, checkpointUsage, attemptId) {
352
+ const transcriptUsage = conversation.getRawTranscriptUsage(attemptId);
147
353
  return checkpointUsage ? reconcileAttemptUsage(checkpointUsage, transcriptUsage) : transcriptUsage;
148
354
  }
149
355
  /** Fail closed when the current owner profile no longer permits a persisted grant. */
@@ -1 +1 @@
1
- {"version":3,"file":"worker-recovery-coordinator.js","sourceRoot":"","sources":["../../../src/core/delegation/worker-recovery-coordinator.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AAsB1E;;;;GAIG;AACH,MAAM,OAAO,yBAAyB;IAMrC,YAAY,OAAyC;QAJ7C,mBAAc,GAAG,KAAK,CAAC;QACvB,2BAAsB,GAAG,KAAK,CAAC;QACtB,iCAA4B,GAAG,IAAI,GAAG,EAAkB,CAAC;QAGzE,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IACxB,CAAC;IAED,6FAA6F;IAC7F,OAAO;QACN,IAAI,IAAI,CAAC,sBAAsB;YAAE,OAAO;QACxC,IAAI,CAAC,sBAAsB,GAAG,IAAI,CAAC;QACnC,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC;QAC9C,IAAI,CAAC;YACJ,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;gBAC1B,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,uCAAuC,EAAE,CAAC,CAAC;gBACzF,IAAI,CAAC;oBACJ,IAAI,CAAC,OAAO,CAAC,wBAAwB,EAAE,CAAC;gBACzC,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBAChB,IAAI,CAAC,OAAO,CAAC,IAAI,CAChB,oDAAoD,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAC5G,CAAC;gBACH,CAAC;gBACD,MAAM,MAAM,GAAG,SAAS,CAAC,aAAa,EAAE,CAAC;gBACzC,MAAM,QAAQ,GAAG,SAAS,CAAC,sBAAsB,EAAE,CAAC;gBACpD,KAAK,MAAM,SAAS,IAAI,mBAAmB,EAAE,CAAC;oBAC7C,MAAM,OAAO,GAAG,QAAQ,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;oBAC7C,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,QAAQ,CAAC,aAAa,KAAK,iBAAiB;wBAAE,SAAS;oBAC/E,MAAM,MAAM,GAAG,SAAS,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;oBACnD,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC;oBAClD,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI;wBAAE,SAAS;oBAC/B,MAAM,CAAC,IAAI,CAAC;wBACX,MAAM;wBACN,OAAO;wBACP,GAAG,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC,EAAE,oBAAoB,EAAE,IAAI,CAAC,oBAAoB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;qBACzF,CAAC,CAAC;gBACJ,CAAC;gBACD,KAAK,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,oBAAoB,EAAE,IAAI,MAAM,EAAE,CAAC;oBAChE,SAAS,CAAC,OAAO,CAChB,MAAM,EACN,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,oBAAoB,CAAC,EACpD,IAAI,EACJ,oBAAoB,KAAK,SAAS,CAClC,CAAC;gBACH,CAAC;gBACD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;YAC5B,CAAC;YAED,KAAK,MAAM,QAAQ,IAAI,SAAS,CAAC,gCAAgC,EAAE,EAAE,CAAC;gBACrE,IAAI,QAAQ,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;oBACrC,IAAI,CAAC;wBACJ,IAAI,CAAC,OAAO,CAAC,qBAAqB,CAAC,SAAS,CAAC,qBAAqB,CAAC,QAAQ,CAAC,CAAC,CAAC;wBAC9E,IAAI,CAAC,4BAA4B,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;oBAClE,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBAChB,IAAI,CAAC,OAAO,CAAC,IAAI,CAChB,kDAAkD,QAAQ,CAAC,aAAa,KAAK,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CACrI,CAAC;oBACH,CAAC;oBACD,SAAS;gBACV,CAAC;gBACD,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,oBAAoB,CAAC,QAAQ,CAAC,CAAC;gBAC5D,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;oBACrB,IAAI,CAAC,4BAA4B,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;oBACjE,SAAS;gBACV,CAAC;gBACD,IAAI,IAAI,CAAC,4BAA4B,CAAC,GAAG,CAAC,QAAQ,CAAC,aAAa,CAAC,KAAK,OAAO,CAAC,UAAU;oBAAE,SAAS;gBACnG,IAAI,CAAC,4BAA4B,CAAC,GAAG,CAAC,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;gBAClF,IAAI,CAAC,OAAO,CAAC,IAAI,CAChB,8BAA8B,QAAQ,CAAC,aAAa,mBAAmB,OAAO,CAAC,UAAU,EAAE,CAC3F,CAAC;YACH,CAAC;YAED,KAAK,MAAM,YAAY,IAAI,SAAS,CAAC,+BAA+B,EAAE,EAAE,CAAC;gBACxE,IAAI,CAAC,OAAO,CAAC,qBAAqB,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;YACzD,CAAC;QACF,CAAC;gBAAS,CAAC;YACV,IAAI,CAAC,sBAAsB,GAAG,KAAK,CAAC;QACrC,CAAC;IACF,CAAC;IAED,wGAAwG;IACxG,gBAAgB,CAAC,OAA4B,EAAE,oBAA6B;QAC3E,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC;QAClE,OAAO;YACN,YAAY,EAAE,OAAO,CAAC,QAAQ,CAAC,YAAY;YAC3C,SAAS,EAAE,OAAO,CAAC,QAAQ,CAAC,SAAS;YACrC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,OAAO,CAAC,QAAQ,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC5F,GAAG,CAAC,oBAAoB,CAAC,CAAC,CAAC,EAAE,oBAAoB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACzD,GAAG,CAAC,IAAI;gBACP,CAAC,CAAC;oBACA,WAAW,EAAE;wBACZ,cAAc,EAAE,OAAO,CAAC,QAAQ,CAAC,cAAc,IAAI,EAAE;wBACrD,gBAAgB,EAAE,IAAI,CAAC,SAAS;wBAChC,sBAAsB,EAAE,IAAI,CAAC,sBAAsB;wBACnD,kBAAkB,EAAE,OAAO,CAAC,QAAQ,CAAC,kBAAkB;qBACvD;iBACD;gBACF,CAAC,CAAC,EAAE,CAAC;SACN,CAAC;IACH,CAAC;IAED,kGAAkG;IAClG,4BAA4B,CAAC,YAAgC;QAC5D,MAAM,QAAQ,GAAG,YAAY,CAAC,mBAAmB,EAAE,CAAC;QACpD,MAAM,oBAAoB,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/C,IAAI,KAAK,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;QAChC,OAAO,KAAK,IAAI,CAAC,IAAI,QAAQ,CAAC,KAAK,CAAC,EAAE,IAAI,KAAK,YAAY,EAAE,CAAC;YAC7D,oBAAoB,CAAC,GAAG,CAAE,QAAQ,CAAC,KAAK,CAA8C,CAAC,UAAU,CAAC,CAAC;YACnG,KAAK,IAAI,CAAC,CAAC;QACZ,CAAC;QACD,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;QAClC,IAAI,CAAC,SAAS,IAAI,SAAS,CAAC,IAAI,KAAK,WAAW;YAAE,OAAO;QACzD,KAAK,MAAM,OAAO,IAAI,SAAS,CAAC,OAAO,EAAE,CAAC;YACzC,IAAI,OAAO,CAAC,IAAI,KAAK,UAAU,IAAI,oBAAoB,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;gBAAE,SAAS;YAClF,YAAY,CAAC,aAAa,CAAC;gBAC1B,IAAI,EAAE,YAAY;gBAClB,UAAU,EAAE,OAAO,CAAC,EAAE;gBACtB,QAAQ,EAAE,OAAO,CAAC,IAAI;gBACtB,OAAO,EAAE;oBACR;wBACC,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,uMAAuM;qBAC7M;iBACD;gBACD,OAAO,EAAE,IAAI;gBACb,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;aACrB,CAAC,CAAC;QACJ,CAAC;IACF,CAAC;IAED,uGAAuG;IACvG,2BAA2B,CAC1B,YAAgC;QAEhC,MAAM,IAAI,GAAG,YAAY,CAAC,mBAAmB,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACvD,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,KAAK,WAAW,IAAI,IAAI,CAAC,UAAU,KAAK,OAAO,IAAI,IAAI,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;YACxG,OAAO,SAAS,CAAC;QAClB,CAAC;QACD,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,KAAK,UAAU,CAAC;YAAE,OAAO,SAAS,CAAC;QAClF,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO;aACvB,MAAM,CACN,CAAC,OAAO,EAAuE,EAAE,CAAC,OAAO,CAAC,IAAI,KAAK,MAAM,CACzG;aACA,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;aAC9B,IAAI,CAAC,EAAE,CAAC,CAAC;QACX,OAAO,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IACpF,CAAC;IAED,+GAA+G;IAC/G,YAAY,CACX,YAAgC,EAChC,eAAiD;QAEjD,MAAM,eAAe,GAAG,YAAY,CAAC,qBAAqB,EAAE,CAAC;QAC7D,OAAO,eAAe,CAAC,CAAC,CAAC,qBAAqB,CAAC,eAAe,EAAE,eAAe,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC;IACpG,CAAC;IAED,sFAAsF;IACtF,4BAA4B,CAC3B,KAAqB,EACrB,IAAyB,EACzB,iBAA6C;QAE7C,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;QACtF,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;QAC/D,MAAM,qBAAqB,GAAG,CAAC,OAA0B,EAAE,SAA4B,EAAE,EAAE,CAC1F,OAAO,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,iBAAiB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC;QAC9F,MAAM,yBAAyB,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,EAAE,CACnE,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,iBAAiB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CACvE,CAAC;QACF,MAAM,sBAAsB,GAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAA6B,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;YACpG,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACnC,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACtC,OAAO,OAAO,KAAK,SAAS,IAAI,CAAC,SAAS,KAAK,SAAS,IAAI,SAAS,IAAI,OAAO,CAAC,CAAC;QACnF,CAAC,CAAC,CAAC;QACH,MAAM,wBAAwB,GAC7B,KAAK,CAAC,SAAS,CAAC,MAAM,KAAK,iBAAiB,CAAC,MAAM;YACnD,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE;gBACxC,MAAM,QAAQ,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;gBAC1C,OAAO,CACN,QAAQ,KAAK,SAAS;oBACtB,OAAO,CAAC,EAAE,KAAK,QAAQ,CAAC,EAAE;oBAC1B,OAAO,CAAC,IAAI,KAAK,QAAQ,CAAC,IAAI;oBAC9B,OAAO,CAAC,GAAG,KAAK,QAAQ,CAAC,GAAG;oBAC5B,OAAO,CAAC,QAAQ,KAAK,QAAQ,CAAC,QAAQ;oBACtC,OAAO,OAAO,CAAC,MAAM,KAAK,QAAQ;oBAClC,CAAC,QAAQ,CAAC,MAAM,KAAK,SAAS,IAAI,QAAQ,CAAC,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC;oBACrE,OAAO,CAAC,QAAQ,EAAE,IAAI,KAAK,QAAQ,CAAC,QAAQ,EAAE,IAAI,CAClD,CAAC;YACH,CAAC,CAAC,CAAC;QACJ,OAAO,CACN,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAClE,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,mBAAmB,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YAC7E,wBAAwB;YACxB,qBAAqB,CAAC,KAAK,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC;YACtD,qBAAqB,CAAC,KAAK,CAAC,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC;YACxD,yBAAyB;YACzB,sBAAsB,CACtB,CAAC;IACH,CAAC;CACD","sourcesContent":["import type { Message, Usage } from \"@caupulican/pi-ai\";\nimport type { LaneRecord } from \"../autonomy/lane-tracker.ts\";\nimport { isPathWithinScope } from \"../autonomy/path-scope.ts\";\nimport { reconcileAttemptUsage } from \"../orchestration/attempt-usage.ts\";\nimport type { AttemptUsageSnapshot, ExecutionGrant, ResourcePointer, RiskBudget } from \"../orchestration/contracts.ts\";\nimport type { AttemptRuntimeState } from \"../orchestration/task-runtime.ts\";\nimport type { WorkerConversation } from \"./worker-conversation-store.ts\";\nimport type { WorkerDelegationRequest } from \"./worker-delegation-request.ts\";\nimport type { WorkerDispatchScheduler } from \"./worker-dispatch-scheduler.ts\";\nimport type { WorkerExecutionPlan } from \"./worker-execution-policy.ts\";\nimport type { PendingVerificationRecovery, WorkerLifecycle } from \"./worker-lifecycle.ts\";\n\nexport type WorkerRecoveryDispatchResult = { started: true } | { started: false; skipReason: string };\n\nexport interface WorkerRecoveryCoordinatorOptions {\n\tlifecycle: WorkerLifecycle;\n\tscheduler: Pick<WorkerDispatchScheduler, \"enqueue\">;\n\trecoverWriteReservations(): void;\n\tpublishTerminalRecord(record: LaneRecord): void;\n\tdispatchVerification(\n\t\trecovery: Extract<PendingVerificationRecovery, { action: \"dispatch\" }>,\n\t): WorkerRecoveryDispatchResult;\n\twarn(message: string): void;\n}\n\n/**\n * Owns restart-only worker state repair. It has no provider or tool execution authority: it\n * reconstructs bounded durable requests, requeues them through the owner scheduler, and supplies\n * the execution layer with narrow transcript/usage/grant recovery decisions.\n */\nexport class WorkerRecoveryCoordinator {\n\tprivate readonly options: WorkerRecoveryCoordinatorOptions;\n\tprivate queueRecovered = false;\n\tprivate recoveringDurableState = false;\n\tprivate readonly verificationRecoveryFailures = new Map<string, string>();\n\n\tconstructor(options: WorkerRecoveryCoordinatorOptions) {\n\t\tthis.options = options;\n\t}\n\n\t/** Idempotently rebuild in-process queue state and replay only durable terminal handoffs. */\n\trecover(): void {\n\t\tif (this.recoveringDurableState) return;\n\t\tthis.recoveringDurableState = true;\n\t\tconst { lifecycle, scheduler } = this.options;\n\t\ttry {\n\t\t\tif (!this.queueRecovered) {\n\t\t\t\tconst suspendedAttemptIds = new Set(lifecycle.suspendBoundInProcessAttemptsForRestart());\n\t\t\t\ttry {\n\t\t\t\t\tthis.options.recoverWriteReservations();\n\t\t\t\t} catch (error) {\n\t\t\t\t\tthis.options.warn(\n\t\t\t\t\t\t`Worker write reservation recovery failed closed: ${error instanceof Error ? error.message : String(error)}`,\n\t\t\t\t\t);\n\t\t\t\t}\n\t\t\t\tconst queued = lifecycle.recoverQueued();\n\t\t\t\tconst snapshot = lifecycle.getTaskRuntimeSnapshot();\n\t\t\t\tfor (const attemptId of suspendedAttemptIds) {\n\t\t\t\t\tconst attempt = snapshot.attempts[attemptId];\n\t\t\t\t\tif (!attempt || attempt.dispatch.executionKind === \"managed-process\") continue;\n\t\t\t\t\tconst record = lifecycle.getRecord(attempt.taskId);\n\t\t\t\t\tconst task = snapshot.tasks[attempt.taskId]?.task;\n\t\t\t\t\tif (!record || !task) continue;\n\t\t\t\t\tqueued.push({\n\t\t\t\t\t\trecord,\n\t\t\t\t\t\tattempt,\n\t\t\t\t\t\t...(task.verificationOfTaskId ? { verificationOfTaskId: task.verificationOfTaskId } : {}),\n\t\t\t\t\t});\n\t\t\t\t}\n\t\t\t\tfor (const { attempt, record, verificationOfTaskId } of queued) {\n\t\t\t\t\tscheduler.enqueue(\n\t\t\t\t\t\trecord,\n\t\t\t\t\t\tthis.recoveredRequest(attempt, verificationOfTaskId),\n\t\t\t\t\t\ttrue,\n\t\t\t\t\t\tverificationOfTaskId !== undefined,\n\t\t\t\t\t);\n\t\t\t\t}\n\t\t\t\tthis.queueRecovered = true;\n\t\t\t}\n\n\t\t\tfor (const recovery of lifecycle.getPendingVerificationRecoveries()) {\n\t\t\t\tif (recovery.action === \"reconcile\") {\n\t\t\t\t\ttry {\n\t\t\t\t\t\tthis.options.publishTerminalRecord(lifecycle.reconcileVerification(recovery));\n\t\t\t\t\t\tthis.verificationRecoveryFailures.delete(recovery.subjectTaskId);\n\t\t\t\t\t} catch (error) {\n\t\t\t\t\t\tthis.options.warn(\n\t\t\t\t\t\t\t`Failed to reconcile recovered verification for ${recovery.subjectTaskId}: ${error instanceof Error ? error.message : String(error)}`,\n\t\t\t\t\t\t);\n\t\t\t\t\t}\n\t\t\t\t\tcontinue;\n\t\t\t\t}\n\t\t\t\tconst started = this.options.dispatchVerification(recovery);\n\t\t\t\tif (started.started) {\n\t\t\t\t\tthis.verificationRecoveryFailures.delete(recovery.subjectTaskId);\n\t\t\t\t\tcontinue;\n\t\t\t\t}\n\t\t\t\tif (this.verificationRecoveryFailures.get(recovery.subjectTaskId) === started.skipReason) continue;\n\t\t\t\tthis.verificationRecoveryFailures.set(recovery.subjectTaskId, started.skipReason);\n\t\t\t\tthis.options.warn(\n\t\t\t\t\t`Recovered verification for ${recovery.subjectTaskId} did not start: ${started.skipReason}`,\n\t\t\t\t);\n\t\t\t}\n\n\t\t\tfor (const notification of lifecycle.getPendingTerminalNotifications()) {\n\t\t\t\tthis.options.publishTerminalRecord(notification.record);\n\t\t\t}\n\t\t} finally {\n\t\t\tthis.recoveringDurableState = false;\n\t\t}\n\t}\n\n\t/** Rebuild task metadata from its durable dispatch; no current profile/model selection is consulted. */\n\trecoveredRequest(attempt: AttemptRuntimeState, verificationOfTaskId?: string): WorkerDelegationRequest {\n\t\tconst task = this.options.lifecycle.getTask(attempt.taskId)?.task;\n\t\treturn {\n\t\t\tinstructions: attempt.dispatch.instructions,\n\t\t\tprofileId: attempt.dispatch.profileId,\n\t\t\t...(attempt.dispatch.parentAgentId ? { parentAgentId: attempt.dispatch.parentAgentId } : {}),\n\t\t\t...(verificationOfTaskId ? { verificationOfTaskId } : {}),\n\t\t\t...(task\n\t\t\t\t? {\n\t\t\t\t\t\ttaskContext: {\n\t\t\t\t\t\t\trequirementIds: attempt.dispatch.requirementIds ?? [],\n\t\t\t\t\t\t\tdependsOnTaskIds: task.dependsOn,\n\t\t\t\t\t\t\tacceptanceCriterionIds: task.acceptanceCriterionIds,\n\t\t\t\t\t\t\tresourcePointerIds: attempt.dispatch.resourcePointerIds,\n\t\t\t\t\t\t},\n\t\t\t\t\t}\n\t\t\t\t: {}),\n\t\t};\n\t}\n\n\t/** Append deterministic unknown outcomes only for tool calls not durably followed by a result. */\n\trepairInterruptedToolResults(conversation: WorkerConversation): void {\n\t\tconst messages = conversation.getProviderMessages();\n\t\tconst completedToolCallIds = new Set<string>();\n\t\tlet index = messages.length - 1;\n\t\twhile (index >= 0 && messages[index]?.role === \"toolResult\") {\n\t\t\tcompletedToolCallIds.add((messages[index] as Extract<Message, { role: \"toolResult\" }>).toolCallId);\n\t\t\tindex -= 1;\n\t\t}\n\t\tconst assistant = messages[index];\n\t\tif (!assistant || assistant.role !== \"assistant\") return;\n\t\tfor (const content of assistant.content) {\n\t\t\tif (content.type !== \"toolCall\" || completedToolCallIds.has(content.id)) continue;\n\t\t\tconversation.appendMessage({\n\t\t\t\trole: \"toolResult\",\n\t\t\t\ttoolCallId: content.id,\n\t\t\t\ttoolName: content.name,\n\t\t\t\tcontent: [\n\t\t\t\t\t{\n\t\t\t\t\t\ttype: \"text\",\n\t\t\t\t\t\ttext: \"Execution outcome is unknown because this worker was interrupted before a durable tool result was recorded. Inspect the workspace and prior evidence before deciding whether to retry this operation.\",\n\t\t\t\t\t},\n\t\t\t\t],\n\t\t\t\tisError: true,\n\t\t\t\ttimestamp: Date.now(),\n\t\t\t});\n\t\t}\n\t}\n\n\t/** Return an already-persisted final assistant response so recovery cannot duplicate provider work. */\n\trecoveredTerminalCompletion(\n\t\tconversation: WorkerConversation,\n\t): { text: string; usage: Usage; stopReason: string } | undefined {\n\t\tconst last = conversation.getProviderMessages().at(-1);\n\t\tif (!last || last.role !== \"assistant\" || last.stopReason === \"error\" || last.stopReason === \"aborted\") {\n\t\t\treturn undefined;\n\t\t}\n\t\tif (last.content.some((content) => content.type === \"toolCall\")) return undefined;\n\t\tconst text = last.content\n\t\t\t.filter(\n\t\t\t\t(content): content is Extract<(typeof last.content)[number], { type: \"text\" }> => content.type === \"text\",\n\t\t\t)\n\t\t\t.map((content) => content.text)\n\t\t\t.join(\"\");\n\t\treturn text ? { text, usage: last.usage, stopReason: last.stopReason } : undefined;\n\t}\n\n\t/** Legacy raw transcripts predate durable usage checkpoints; reconstruct their bounded cumulative baseline. */\n\tinitialUsage(\n\t\tconversation: WorkerConversation,\n\t\tcheckpointUsage: AttemptUsageSnapshot | undefined,\n\t): AttemptUsageSnapshot {\n\t\tconst transcriptUsage = conversation.getRawTranscriptUsage();\n\t\treturn checkpointUsage ? reconcileAttemptUsage(checkpointUsage, transcriptUsage) : transcriptUsage;\n\t}\n\n\t/** Fail closed when the current owner profile no longer permits a persisted grant. */\n\tdurableGrantIsStillPermitted(\n\t\tgrant: ExecutionGrant,\n\t\tplan: WorkerExecutionPlan,\n\t\tselectedResources: readonly ResourcePointer[],\n\t): boolean {\n\t\tconst allowedTools = new Set(plan.toolManifests.map((manifest) => manifest.toolName));\n\t\tconst allowedCapabilities = new Set(plan.requiredCapabilities);\n\t\tconst scopesRemainPermitted = (granted: readonly string[], permitted: readonly string[]) =>\n\t\t\tgranted.every((scope) => permitted.some((candidate) => isPathWithinScope(scope, candidate)));\n\t\tconst liveDenialsRemainEnforced = plan.deniedPaths.every((denied) =>\n\t\t\tgrant.deniedPaths.some((granted) => isPathWithinScope(denied, granted)),\n\t\t);\n\t\tconst budgetRemainsPermitted = (Object.keys(plan.budget) as Array<keyof RiskBudget>).every((field) => {\n\t\t\tconst current = plan.budget[field];\n\t\t\tconst persisted = grant.budget[field];\n\t\t\treturn current === undefined || (persisted !== undefined && persisted <= current);\n\t\t});\n\t\tconst resourcesRemainPermitted =\n\t\t\tgrant.resources.length === selectedResources.length &&\n\t\t\tgrant.resources.every((granted, index) => {\n\t\t\t\tconst selected = selectedResources[index];\n\t\t\t\treturn (\n\t\t\t\t\tselected !== undefined &&\n\t\t\t\t\tgranted.id === selected.id &&\n\t\t\t\t\tgranted.kind === selected.kind &&\n\t\t\t\t\tgranted.uri === selected.uri &&\n\t\t\t\t\tgranted.readOnly === selected.readOnly &&\n\t\t\t\t\ttypeof granted.digest === \"string\" &&\n\t\t\t\t\t(selected.digest === undefined || selected.digest === granted.digest) &&\n\t\t\t\t\tgranted.metadata?.name === selected.metadata?.name\n\t\t\t\t);\n\t\t\t});\n\t\treturn (\n\t\t\tgrant.allowedTools.every((toolName) => allowedTools.has(toolName)) &&\n\t\t\tgrant.capabilities.every((capability) => allowedCapabilities.has(capability)) &&\n\t\t\tresourcesRemainPermitted &&\n\t\t\tscopesRemainPermitted(grant.readPaths, plan.readPaths) &&\n\t\t\tscopesRemainPermitted(grant.writePaths, plan.writePaths) &&\n\t\t\tliveDenialsRemainEnforced &&\n\t\t\tbudgetRemainsPermitted\n\t\t);\n\t}\n}\n"]}
1
+ {"version":3,"file":"worker-recovery-coordinator.js","sourceRoot":"","sources":["../../../src/core/delegation/worker-recovery-coordinator.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AAQ1E,OAAO,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAkC/D,SAAS,0BAA0B,CAClC,OAA4B,EAC5B,wBAA6C;IAE7C,OAAO,CACN,wBAAwB,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC;QAC/C,OAAO,CAAC,UAAU,KAAK,0CAA0C;QACjE,OAAO,CAAC,UAAU,KAAK,2BAA2B;QAClD,OAAO,CAAC,UAAU,EAAE,UAAU,CAAC,kBAAkB,CAAC,KAAK,IAAI,CAC3D,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,OAAO,yBAAyB;IAQrC,YAAY,OAAyC;QAN7C,mBAAc,GAAG,KAAK,CAAC;QACvB,2BAAsB,GAAG,KAAK,CAAC;QACtB,iCAA4B,GAAG,IAAI,GAAG,EAAkB,CAAC;QACzD,gBAAW,GAAG,IAAI,GAAG,EAAgC,CAAC;QAItE,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,wBAAwB,GAAG,OAAO,CAAC,SAAS,CAAC,wBAAwB,EAAE,CAAC,GAAG,EAAE;YACjF,IAAI,CAAC,eAAe,EAAE,CAAC;YACvB,2FAA2F;YAC3F,4FAA4F;YAC5F,IAAI,CAAC,OAAO,EAAE,CAAC;QAChB,CAAC,CAAC,CAAC;IACJ,CAAC;IAED,6FAA6F;IAC7F,OAAO;QACN,IAAI,IAAI,CAAC,sBAAsB;YAAE,OAAO;QACxC,IAAI,CAAC,sBAAsB,GAAG,IAAI,CAAC;QACnC,MAAM,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC;QACnC,IAAI,CAAC;YACJ,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;gBAC1B,MAAM,wBAAwB,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,uCAAuC,EAAE,CAAC,CAAC;gBAC9F,IAAI,CAAC;oBACJ,IAAI,CAAC,OAAO,CAAC,wBAAwB,EAAE,CAAC;gBACzC,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBAChB,IAAI,CAAC,OAAO,CAAC,IAAI,CAChB,oDAAoD,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAC5G,CAAC;gBACH,CAAC;gBACD,MAAM,MAAM,GAAG,SAAS,CAAC,aAAa,EAAE,CAAC;gBACzC,MAAM,QAAQ,GAAG,SAAS,CAAC,sBAAsB,EAAE,CAAC;gBACpD,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;oBACxD,IACC,OAAO,CAAC,MAAM,KAAK,WAAW;wBAC9B,CAAC,OAAO,CAAC,OAAO;wBAChB,OAAO,CAAC,QAAQ,CAAC,aAAa,KAAK,iBAAiB;wBACpD,CAAC,0BAA0B,CAAC,OAAO,EAAE,wBAAwB,CAAC,EAC7D,CAAC;wBACF,SAAS;oBACV,CAAC;oBACD,MAAM,MAAM,GAAG,SAAS,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;oBACnD,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC;oBAClD,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI;wBAAE,SAAS;oBAC/B,MAAM,CAAC,IAAI,CAAC;wBACX,MAAM;wBACN,OAAO;wBACP,GAAG,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC,EAAE,oBAAoB,EAAE,IAAI,CAAC,oBAAoB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;qBACzF,CAAC,CAAC;gBACJ,CAAC;gBACD,IAAI,YAAY,GAAG,IAAI,CAAC;gBACxB,KAAK,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,oBAAoB,EAAE,IAAI,MAAM,EAAE,CAAC;oBAChE,IAAI,QAAiB,CAAC;oBACtB,IAAI,CAAC;wBACJ,QAAQ,GAAG,IAAI,CAAC,mBAAmB,CAClC,MAAM,EACN,OAAO,EACP,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,oBAAoB,CAAC,EACpD,oBAAoB,KAAK,SAAS,CAClC,CAAC;oBACH,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBAChB,YAAY,GAAG,KAAK,CAAC;wBACrB,IAAI,CAAC,OAAO,CAAC,IAAI,CAChB,mCAAmC,MAAM,CAAC,MAAM,iCAAiC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CACzI,CAAC;wBACF,SAAS;oBACV,CAAC;oBACD,IAAI,QAAQ;wBAAE,SAAS;oBACvB,YAAY,GAAG,KAAK,CAAC;oBACrB,MAAM;gBACP,CAAC;gBACD,IAAI,CAAC,cAAc,GAAG,YAAY,CAAC;YACpC,CAAC;YAED,KAAK,MAAM,QAAQ,IAAI,SAAS,CAAC,gCAAgC,EAAE,EAAE,CAAC;gBACrE,IAAI,QAAQ,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;oBACrC,IAAI,CAAC;wBACJ,IAAI,CAAC,OAAO,CAAC,qBAAqB,CAAC,SAAS,CAAC,qBAAqB,CAAC,QAAQ,CAAC,CAAC,CAAC;wBAC9E,IAAI,CAAC,4BAA4B,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;oBAClE,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBAChB,IAAI,CAAC,OAAO,CAAC,IAAI,CAChB,kDAAkD,QAAQ,CAAC,aAAa,KAAK,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CACrI,CAAC;oBACH,CAAC;oBACD,SAAS;gBACV,CAAC;gBACD,IAAI,OAAqC,CAAC;gBAC1C,IAAI,CAAC;oBACJ,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,oBAAoB,CAAC,QAAQ,CAAC,CAAC;gBACvD,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBAChB,MAAM,MAAM,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;oBACtE,MAAM,UAAU,GAAG,kBAAkB,MAAM,EAAE,CAAC;oBAC9C,IAAI,IAAI,CAAC,4BAA4B,CAAC,GAAG,CAAC,QAAQ,CAAC,aAAa,CAAC,KAAK,UAAU,EAAE,CAAC;wBAClF,IAAI,CAAC,4BAA4B,CAAC,GAAG,CAAC,QAAQ,CAAC,aAAa,EAAE,UAAU,CAAC,CAAC;wBAC1E,IAAI,CAAC,OAAO,CAAC,IAAI,CAChB,8BAA8B,QAAQ,CAAC,aAAa,2BAA2B,MAAM,EAAE,CACvF,CAAC;oBACH,CAAC;oBACD,SAAS;gBACV,CAAC;gBACD,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;oBACrB,IAAI,CAAC,4BAA4B,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;oBACjE,SAAS;gBACV,CAAC;gBACD,IAAI,IAAI,CAAC,4BAA4B,CAAC,GAAG,CAAC,QAAQ,CAAC,aAAa,CAAC,KAAK,OAAO,CAAC,UAAU;oBAAE,SAAS;gBACnG,IAAI,CAAC,4BAA4B,CAAC,GAAG,CAAC,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;gBAClF,IAAI,CAAC,OAAO,CAAC,IAAI,CAChB,8BAA8B,QAAQ,CAAC,aAAa,mBAAmB,OAAO,CAAC,UAAU,EAAE,CAC3F,CAAC;YACH,CAAC;YAED,MAAM,eAAe,GAAG,SAAS,CAAC,+BAA+B,EAAE,CAAC,GAAG,CAAC,CAAC,YAAY,EAAE,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;YAC/G,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAChC,IAAI,IAAI,CAAC,OAAO,CAAC,sBAAsB;oBAAE,IAAI,CAAC,OAAO,CAAC,sBAAsB,CAAC,eAAe,CAAC,CAAC;;oBACzF,KAAK,MAAM,MAAM,IAAI,eAAe;wBAAE,IAAI,CAAC,OAAO,CAAC,qBAAqB,CAAC,MAAM,CAAC,CAAC;YACvF,CAAC;YACD,IAAI,CAAC,OAAO,CAAC,yBAAyB,EAAE,CAAC;YACzC,IAAI,CAAC,OAAO,CAAC,8BAA8B,EAAE,CAAC;QAC/C,CAAC;gBAAS,CAAC;YACV,IAAI,CAAC,sBAAsB,GAAG,KAAK,CAAC;QACrC,CAAC;IACF,CAAC;IAED,mGAAmG;IACnG,oBAAoB,CAAC,IAQpB;QACA,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACrE,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC7D,IAAI,CAAC,OAAO,IAAI,CAAC,MAAM;YAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,iBAAiB,EAAE,CAAC;QAChF,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,EAAE,WAAW,IAAI,CAAC,CAAC;QACpD,MAAM,QAAQ,GAAG,mBAAmB,CAAC;YACpC,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,UAAU;YACnC,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,UAAU;YACnC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACjF,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,WAAW;YACX,GAAG,CAAC,IAAI,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC5E,CAAC,CAAC;QACH,IAAI,CAAC,QAAQ,CAAC,KAAK;YAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC;QAC1E,MAAM,eAAe,GAAG,WAAW,GAAG,CAAC,CAAC;QACxC,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC;QACxE,IAAI,SAA8B,CAAC;QACnC,IAAI,CAAC;YACJ,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,kBAAkB,CAAC;gBACrD,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,UAAU,EAAE,mBAAmB,QAAQ,CAAC,MAAM,EAAE;gBAChD,KAAK,EAAE,EAAE,WAAW,EAAE,eAAe,EAAE,SAAS,EAAE;aAClD,CAAC,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,IAAI,CAAC,OAAO,CAAC,IAAI,CAChB,UAAU,IAAI,CAAC,MAAM,oDAAoD,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CACjI,CAAC;YACF,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,yBAAyB,EAAE,CAAC;QAChE,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,IAAI,CAChB,UAAU,IAAI,CAAC,MAAM,YAAY,QAAQ,CAAC,MAAM,gDAAgD,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,GAAG,IAAI,CAAC,cAAc,eAAe,GAAG,CAAC,IAAI,CACvK,CAAC;QACF,IAAI,CAAC,mBAAmB,CAAC,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,oBAAoB,KAAK,SAAS,CAAC,CAAC;QAC3G,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,eAAe,EAAE,SAAS,EAAE,CAAC;IAC7E,CAAC;IAED,6FAA6F;IAC7F,OAAO;QACN,IAAI,CAAC,wBAAwB,EAAE,EAAE,CAAC;QAClC,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,EAAE,CAAC;YAC/C,IAAI,KAAK,CAAC,KAAK;gBAAE,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC5C,CAAC;QACD,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;IAC1B,CAAC;IAED,mBAAmB,CAAC,MAAc;QACjC,KAAK,MAAM,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACnD,IAAI,KAAK,CAAC,MAAM,KAAK,MAAM;gBAAE,SAAS;YACtC,IAAI,KAAK,CAAC,KAAK;gBAAE,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC3C,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QACpC,CAAC;IACF,CAAC;IAED,mGAAmG;IACnG,kBAAkB,CAAC,MAAkB,EAAE,OAAgC;QACtE,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,gBAAgB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACvE,IACC,CAAC,OAAO;YACR,OAAO,CAAC,MAAM,KAAK,WAAW;YAC9B,CAAC,OAAO,CAAC,KAAK;YACd,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE,EAChD,CAAC;YACF,OAAO,KAAK,CAAC;QACd,CAAC;QACD,IAAI,CAAC,mBAAmB,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,oBAAoB,KAAK,SAAS,CAAC,CAAC;QAC/F,OAAO,IAAI,CAAC;IACb,CAAC;IAEO,mBAAmB,CAC1B,MAAkB,EAClB,OAA4B,EAC5B,OAAgC,EAChC,QAAiB;QAEjB,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;QAC5B,IAAI,CAAC,KAAK,EAAE,CAAC;YACZ,IAAI,CAAC;gBACJ,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC;gBAChE,OAAO,IAAI,CAAC;YACb,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAChB,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,OAAO,KAAK,4BAA4B;oBAAE,OAAO,KAAK,CAAC;gBAC3F,MAAM,KAAK,CAAC;YACb,CAAC;QACF,CAAC;QACD,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACzD,IAAI,QAAQ,EAAE,SAAS,KAAK,KAAK,CAAC,SAAS;YAAE,OAAO,IAAI,CAAC;QACzD,IAAI,QAAQ,EAAE,KAAK;YAAE,YAAY,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QAClD,MAAM,KAAK,GAAyB;YACnC,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,OAAO;YACP,QAAQ;SACR,CAAC;QACF,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;QAC/C,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7D,IAAI,WAAW,IAAI,CAAC,EAAE,CAAC;YACtB,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;YAClD,OAAO,IAAI,CAAC;QACb,CAAC;QACD,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,aAAa,CAAC,CAAC,CAAC;QAC3G,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,OAAO,IAAI,KAAK;YAAE,KAAK,CAAC,KAAK,EAAE,CAAC;QACjE,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC;QACpB,OAAO,IAAI,CAAC;IACb,CAAC;IAEO,YAAY,CAAC,SAAiB;QACrC,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAC9C,IAAI,CAAC,KAAK;YAAE,OAAO;QACnB,OAAO,KAAK,CAAC,KAAK,CAAC;QACnB,IAAI,CAAC,kBAAkB,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IAC3C,CAAC;IAEO,kBAAkB,CAAC,SAAiB,EAAE,KAA2B;QACxE,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,gBAAgB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACtE,IACC,CAAC,OAAO;YACR,OAAO,CAAC,SAAS,KAAK,SAAS;YAC/B,OAAO,CAAC,MAAM,KAAK,WAAW;YAC9B,OAAO,CAAC,KAAK,EAAE,SAAS,KAAK,KAAK,CAAC,SAAS,EAC3C,CAAC;YACF,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YACnC,OAAO,OAAO,CAAC;QAChB,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC9D,IAAI,CAAC,MAAM,EAAE,CAAC;YACb,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YACnC,OAAO,OAAO,CAAC;QAChB,CAAC;QACD,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7D,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;YACrB,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,aAAa,CAAC,CAAC,CAAC;YACnG,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,OAAO,IAAI,KAAK;gBAAE,KAAK,CAAC,KAAK,EAAE,CAAC;YACjE,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC;YACpB,OAAO,OAAO,CAAC;QAChB,CAAC;QACD,IAAI,CAAC;YACJ,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,EAAE,KAAK,CAAC,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC7E,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,OAAO,KAAK,4BAA4B;gBAAE,OAAO,MAAM,CAAC;YAC5F,IAAI,CAAC,OAAO,CAAC,IAAI,CAChB,UAAU,KAAK,CAAC,MAAM,uDAAuD,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CACrI,CAAC;YACF,OAAO,MAAM,CAAC;QACf,CAAC;QACD,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QACnC,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;QAC5B,OAAO,UAAU,CAAC;IACnB,CAAC;IAEO,eAAe;QACtB,KAAK,MAAM,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACnD,IAAI,KAAK,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE;gBAAE,SAAS;YACtE,IAAI,IAAI,CAAC,kBAAkB,CAAC,SAAS,EAAE,KAAK,CAAC,KAAK,MAAM;gBAAE,MAAM;QACjE,CAAC;IACF,CAAC;IAEO,GAAG;QACV,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;IAC3C,CAAC;IAED,wGAAwG;IACxG,gBAAgB,CAAC,OAA4B,EAAE,oBAA6B;QAC3E,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC;QAClE,MAAM,2BAA2B,GAAG,oBAAoB,IAAI,IAAI,EAAE,oBAAoB,CAAC;QACvF,OAAO;YACN,YAAY,EAAE,OAAO,CAAC,QAAQ,CAAC,YAAY;YAC3C,SAAS,EAAE,OAAO,CAAC,QAAQ,CAAC,SAAS;YACrC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,OAAO,CAAC,QAAQ,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC5F,GAAG,CAAC,2BAA2B,CAAC,CAAC,CAAC,EAAE,oBAAoB,EAAE,2BAA2B,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC7F,GAAG,CAAC,IAAI;gBACP,CAAC,CAAC;oBACA,WAAW,EAAE;wBACZ,cAAc,EAAE,OAAO,CAAC,QAAQ,CAAC,cAAc,IAAI,EAAE;wBACrD,gBAAgB,EAAE,IAAI,CAAC,SAAS;wBAChC,sBAAsB,EAAE,IAAI,CAAC,sBAAsB;wBACnD,kBAAkB,EAAE,OAAO,CAAC,QAAQ,CAAC,kBAAkB;qBACvD;iBACD;gBACF,CAAC,CAAC,EAAE,CAAC;SACN,CAAC;IACH,CAAC;IAED,kGAAkG;IAClG,4BAA4B,CAAC,YAAgC;QAC5D,MAAM,QAAQ,GAAG,YAAY,CAAC,mBAAmB,EAAE,CAAC;QACpD,MAAM,oBAAoB,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/C,IAAI,KAAK,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;QAChC,OAAO,KAAK,IAAI,CAAC,IAAI,QAAQ,CAAC,KAAK,CAAC,EAAE,IAAI,KAAK,YAAY,EAAE,CAAC;YAC7D,oBAAoB,CAAC,GAAG,CAAE,QAAQ,CAAC,KAAK,CAA8C,CAAC,UAAU,CAAC,CAAC;YACnG,KAAK,IAAI,CAAC,CAAC;QACZ,CAAC;QACD,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;QAClC,IAAI,CAAC,SAAS,IAAI,SAAS,CAAC,IAAI,KAAK,WAAW;YAAE,OAAO;QACzD,KAAK,MAAM,OAAO,IAAI,SAAS,CAAC,OAAO,EAAE,CAAC;YACzC,IAAI,OAAO,CAAC,IAAI,KAAK,UAAU,IAAI,oBAAoB,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;gBAAE,SAAS;YAClF,YAAY,CAAC,aAAa,CAAC;gBAC1B,IAAI,EAAE,YAAY;gBAClB,UAAU,EAAE,OAAO,CAAC,EAAE;gBACtB,QAAQ,EAAE,OAAO,CAAC,IAAI;gBACtB,OAAO,EAAE;oBACR;wBACC,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,uMAAuM;qBAC7M;iBACD;gBACD,OAAO,EAAE,IAAI;gBACb,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;aACrB,CAAC,CAAC;QACJ,CAAC;IACF,CAAC;IAED,uGAAuG;IACvG,2BAA2B,CAC1B,YAAgC,EAChC,SAAiB;QAEjB,MAAM,IAAI,GAAG,YAAY,CAAC,qBAAqB,CAAC,SAAS,CAAC,CAAC;QAC3D,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,KAAK,WAAW,IAAI,IAAI,CAAC,UAAU,KAAK,OAAO,IAAI,IAAI,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;YACxG,OAAO,SAAS,CAAC;QAClB,CAAC;QACD,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,KAAK,UAAU,CAAC;YAAE,OAAO,SAAS,CAAC;QAClF,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO;aACvB,MAAM,CACN,CAAC,OAAO,EAAuE,EAAE,CAAC,OAAO,CAAC,IAAI,KAAK,MAAM,CACzG;aACA,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;aAC9B,IAAI,CAAC,EAAE,CAAC,CAAC;QACX,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC;IACjE,CAAC;IAED,+GAA+G;IAC/G,YAAY,CACX,YAAgC,EAChC,eAAiD,EACjD,SAAiB;QAEjB,MAAM,eAAe,GAAG,YAAY,CAAC,qBAAqB,CAAC,SAAS,CAAC,CAAC;QACtE,OAAO,eAAe,CAAC,CAAC,CAAC,qBAAqB,CAAC,eAAe,EAAE,eAAe,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC;IACpG,CAAC;IAED,sFAAsF;IACtF,4BAA4B,CAC3B,KAAqB,EACrB,IAAyB,EACzB,iBAA6C;QAE7C,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;QACtF,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;QAC/D,MAAM,qBAAqB,GAAG,CAAC,OAA0B,EAAE,SAA4B,EAAE,EAAE,CAC1F,OAAO,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,iBAAiB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC;QAC9F,MAAM,yBAAyB,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,EAAE,CACnE,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,iBAAiB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CACvE,CAAC;QACF,MAAM,sBAAsB,GAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAA6B,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;YACpG,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACnC,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACtC,OAAO,OAAO,KAAK,SAAS,IAAI,CAAC,SAAS,KAAK,SAAS,IAAI,SAAS,IAAI,OAAO,CAAC,CAAC;QACnF,CAAC,CAAC,CAAC;QACH,MAAM,wBAAwB,GAC7B,KAAK,CAAC,SAAS,CAAC,MAAM,KAAK,iBAAiB,CAAC,MAAM;YACnD,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE;gBACxC,MAAM,QAAQ,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;gBAC1C,OAAO,CACN,QAAQ,KAAK,SAAS;oBACtB,OAAO,CAAC,EAAE,KAAK,QAAQ,CAAC,EAAE;oBAC1B,OAAO,CAAC,IAAI,KAAK,QAAQ,CAAC,IAAI;oBAC9B,OAAO,CAAC,GAAG,KAAK,QAAQ,CAAC,GAAG;oBAC5B,OAAO,CAAC,QAAQ,KAAK,QAAQ,CAAC,QAAQ;oBACtC,OAAO,OAAO,CAAC,MAAM,KAAK,QAAQ;oBAClC,CAAC,QAAQ,CAAC,MAAM,KAAK,SAAS,IAAI,QAAQ,CAAC,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC;oBACrE,OAAO,CAAC,QAAQ,EAAE,IAAI,KAAK,QAAQ,CAAC,QAAQ,EAAE,IAAI,CAClD,CAAC;YACH,CAAC,CAAC,CAAC;QACJ,OAAO,CACN,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAClE,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,mBAAmB,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YAC7E,wBAAwB;YACxB,qBAAqB,CAAC,KAAK,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC;YACtD,qBAAqB,CAAC,KAAK,CAAC,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC;YACxD,yBAAyB;YACzB,sBAAsB,CACtB,CAAC;IACH,CAAC;CACD","sourcesContent":["import type { Message, Usage } from \"@caupulican/pi-ai\";\nimport type { LaneRecord } from \"../autonomy/lane-tracker.ts\";\nimport { isPathWithinScope } from \"../autonomy/path-scope.ts\";\nimport { reconcileAttemptUsage } from \"../orchestration/attempt-usage.ts\";\nimport type { AttemptUsageSnapshot, ExecutionGrant, ResourcePointer, RiskBudget } from \"../orchestration/contracts.ts\";\nimport type { AttemptRuntimeState } from \"../orchestration/task-runtime.ts\";\nimport type { WorkerConversation } from \"./worker-conversation-store.ts\";\nimport type { WorkerDelegationRequest } from \"./worker-delegation-request.ts\";\nimport type { WorkerDispatchScheduler } from \"./worker-dispatch-scheduler.ts\";\nimport type { WorkerExecutionPlan } from \"./worker-execution-policy.ts\";\nimport type { PendingVerificationRecovery, WorkerLifecycle } from \"./worker-lifecycle.ts\";\nimport { evaluateWorkerRetry } from \"./worker-retry-policy.ts\";\n\nexport type WorkerRecoveryDispatchResult = { started: true } | { started: false; skipReason: string };\n\nexport type WorkerRetryScheduleResult =\n\t| { scheduled: true; record: LaneRecord; retriesUsed: number; notBefore: string }\n\t| { scheduled: false; reason: string };\n\ninterface PendingRetryDispatch {\n\tlaneId: string;\n\tnotBefore: string;\n\trequest: WorkerDelegationRequest;\n\tpriority: boolean;\n\ttimer?: ReturnType<typeof setTimeout>;\n}\n\nexport interface WorkerRecoveryCoordinatorOptions {\n\tlifecycle: WorkerLifecycle;\n\tscheduler: Pick<WorkerDispatchScheduler, \"enqueue\"> &\n\t\tPartial<Pick<WorkerDispatchScheduler, \"onQueueCapacityAvailable\">>;\n\trecoverWriteReservations(): void;\n\tpublishTerminalRecord(record: LaneRecord): void;\n\tpublishTerminalRecords?(records: readonly LaneRecord[]): void;\n\tdispatchVerification(\n\t\trecovery: Extract<PendingVerificationRecovery, { action: \"dispatch\" }>,\n\t): WorkerRecoveryDispatchResult;\n\trecoverTaskBearingMailboxTurns(): void;\n\trecoverSessionRootReplies(): void;\n\t/** Wake the scheduler and UI after a retained retry deadline becomes eligible. */\n\tretryReady?(): void;\n\tnow?(): number;\n\twarn(message: string): void;\n}\n\nfunction restartResumableSuspension(\n\tattempt: AttemptRuntimeState,\n\tnewlySuspendedAttemptIds: ReadonlySet<string>,\n): boolean {\n\treturn (\n\t\tnewlySuspendedAttemptIds.has(attempt.attemptId) ||\n\t\tattempt.reasonCode === \"agent_process_recovered_after_owner_exit\" ||\n\t\tattempt.reasonCode === \"agent_process_interrupted\" ||\n\t\tattempt.reasonCode?.startsWith(\"retry_scheduled:\") === true\n\t);\n}\n\n/**\n * Owns restart-only worker state repair. It has no provider or tool execution authority: it\n * reconstructs bounded durable requests, requeues them through the owner scheduler, and supplies\n * the execution layer with narrow transcript/usage/grant recovery decisions.\n */\nexport class WorkerRecoveryCoordinator {\n\tprivate readonly options: WorkerRecoveryCoordinatorOptions;\n\tprivate queueRecovered = false;\n\tprivate recoveringDurableState = false;\n\tprivate readonly verificationRecoveryFailures = new Map<string, string>();\n\tprivate readonly retryTimers = new Map<string, PendingRetryDispatch>();\n\tprivate readonly unsubscribeQueueCapacity: (() => void) | undefined;\n\n\tconstructor(options: WorkerRecoveryCoordinatorOptions) {\n\t\tthis.options = options;\n\t\tthis.unsubscribeQueueCapacity = options.scheduler.onQueueCapacityAvailable?.(() => {\n\t\t\tthis.flushDueRetries();\n\t\t\t// Mandatory verifier dispatches remain derivable from durable subject state even after the\n\t\t\t// ordinary queue was fully recovered. Every released slot must replay that retained demand.\n\t\t\tthis.recover();\n\t\t});\n\t}\n\n\t/** Idempotently rebuild in-process queues, terminal outboxes, and task-bearing mailboxes. */\n\trecover(): void {\n\t\tif (this.recoveringDurableState) return;\n\t\tthis.recoveringDurableState = true;\n\t\tconst { lifecycle } = this.options;\n\t\ttry {\n\t\t\tif (!this.queueRecovered) {\n\t\t\t\tconst newlySuspendedAttemptIds = new Set(lifecycle.suspendBoundInProcessAttemptsForRestart());\n\t\t\t\ttry {\n\t\t\t\t\tthis.options.recoverWriteReservations();\n\t\t\t\t} catch (error) {\n\t\t\t\t\tthis.options.warn(\n\t\t\t\t\t\t`Worker write reservation recovery failed closed: ${error instanceof Error ? error.message : String(error)}`,\n\t\t\t\t\t);\n\t\t\t\t}\n\t\t\t\tconst queued = lifecycle.recoverQueued();\n\t\t\t\tconst snapshot = lifecycle.getTaskRuntimeSnapshot();\n\t\t\t\tfor (const attempt of Object.values(snapshot.attempts)) {\n\t\t\t\t\tif (\n\t\t\t\t\t\tattempt.status !== \"suspended\" ||\n\t\t\t\t\t\t!attempt.agentId ||\n\t\t\t\t\t\tattempt.dispatch.executionKind === \"managed-process\" ||\n\t\t\t\t\t\t!restartResumableSuspension(attempt, newlySuspendedAttemptIds)\n\t\t\t\t\t) {\n\t\t\t\t\t\tcontinue;\n\t\t\t\t\t}\n\t\t\t\t\tconst record = lifecycle.getRecord(attempt.taskId);\n\t\t\t\t\tconst task = snapshot.tasks[attempt.taskId]?.task;\n\t\t\t\t\tif (!record || !task) continue;\n\t\t\t\t\tqueued.push({\n\t\t\t\t\t\trecord,\n\t\t\t\t\t\tattempt,\n\t\t\t\t\t\t...(task.verificationOfTaskId ? { verificationOfTaskId: task.verificationOfTaskId } : {}),\n\t\t\t\t\t});\n\t\t\t\t}\n\t\t\t\tlet recoveredAll = true;\n\t\t\t\tfor (const { attempt, record, verificationOfTaskId } of queued) {\n\t\t\t\t\tlet retained: boolean;\n\t\t\t\t\ttry {\n\t\t\t\t\t\tretained = this.enqueueOrDeferRetry(\n\t\t\t\t\t\t\trecord,\n\t\t\t\t\t\t\tattempt,\n\t\t\t\t\t\t\tthis.recoveredRequest(attempt, verificationOfTaskId),\n\t\t\t\t\t\t\tverificationOfTaskId !== undefined,\n\t\t\t\t\t\t);\n\t\t\t\t\t} catch (error) {\n\t\t\t\t\t\trecoveredAll = false;\n\t\t\t\t\t\tthis.options.warn(\n\t\t\t\t\t\t\t`Failed to recover queued worker ${record.laneId}; retaining durable dispatch: ${error instanceof Error ? error.message : String(error)}`,\n\t\t\t\t\t\t);\n\t\t\t\t\t\tcontinue;\n\t\t\t\t\t}\n\t\t\t\t\tif (retained) continue;\n\t\t\t\t\trecoveredAll = false;\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t\tthis.queueRecovered = recoveredAll;\n\t\t\t}\n\n\t\t\tfor (const recovery of lifecycle.getPendingVerificationRecoveries()) {\n\t\t\t\tif (recovery.action === \"reconcile\") {\n\t\t\t\t\ttry {\n\t\t\t\t\t\tthis.options.publishTerminalRecord(lifecycle.reconcileVerification(recovery));\n\t\t\t\t\t\tthis.verificationRecoveryFailures.delete(recovery.subjectTaskId);\n\t\t\t\t\t} catch (error) {\n\t\t\t\t\t\tthis.options.warn(\n\t\t\t\t\t\t\t`Failed to reconcile recovered verification for ${recovery.subjectTaskId}: ${error instanceof Error ? error.message : String(error)}`,\n\t\t\t\t\t\t);\n\t\t\t\t\t}\n\t\t\t\t\tcontinue;\n\t\t\t\t}\n\t\t\t\tlet started: WorkerRecoveryDispatchResult;\n\t\t\t\ttry {\n\t\t\t\t\tstarted = this.options.dispatchVerification(recovery);\n\t\t\t\t} catch (error) {\n\t\t\t\t\tconst reason = error instanceof Error ? error.message : String(error);\n\t\t\t\t\tconst failureKey = `dispatch_threw:${reason}`;\n\t\t\t\t\tif (this.verificationRecoveryFailures.get(recovery.subjectTaskId) !== failureKey) {\n\t\t\t\t\t\tthis.verificationRecoveryFailures.set(recovery.subjectTaskId, failureKey);\n\t\t\t\t\t\tthis.options.warn(\n\t\t\t\t\t\t\t`Recovered verification for ${recovery.subjectTaskId} threw during dispatch: ${reason}`,\n\t\t\t\t\t\t);\n\t\t\t\t\t}\n\t\t\t\t\tcontinue;\n\t\t\t\t}\n\t\t\t\tif (started.started) {\n\t\t\t\t\tthis.verificationRecoveryFailures.delete(recovery.subjectTaskId);\n\t\t\t\t\tcontinue;\n\t\t\t\t}\n\t\t\t\tif (this.verificationRecoveryFailures.get(recovery.subjectTaskId) === started.skipReason) continue;\n\t\t\t\tthis.verificationRecoveryFailures.set(recovery.subjectTaskId, started.skipReason);\n\t\t\t\tthis.options.warn(\n\t\t\t\t\t`Recovered verification for ${recovery.subjectTaskId} did not start: ${started.skipReason}`,\n\t\t\t\t);\n\t\t\t}\n\n\t\t\tconst terminalRecords = lifecycle.getPendingTerminalNotifications().map((notification) => notification.record);\n\t\t\tif (terminalRecords.length > 0) {\n\t\t\t\tif (this.options.publishTerminalRecords) this.options.publishTerminalRecords(terminalRecords);\n\t\t\t\telse for (const record of terminalRecords) this.options.publishTerminalRecord(record);\n\t\t\t}\n\t\t\tthis.options.recoverSessionRootReplies();\n\t\t\tthis.options.recoverTaskBearingMailboxTurns();\n\t\t} finally {\n\t\t\tthis.recoveringDurableState = false;\n\t\t}\n\t}\n\n\t/** Evaluate, persist, and arm one retry through the same durable path used by restart recovery. */\n\tscheduleAttemptRetry(args: {\n\t\tlaneId: string;\n\t\tagentId: string;\n\t\townerId: string;\n\t\trequest: WorkerDelegationRequest;\n\t\toutcome: { laneStatus: string; reasonCode: string; reasonDetail?: string };\n\t\tprovider: string;\n\t\tmaxAttempts?: number;\n\t}): WorkerRetryScheduleResult {\n\t\tconst attempt = this.options.lifecycle.getActiveAttempt(args.laneId);\n\t\tconst record = this.options.lifecycle.getRecord(args.laneId);\n\t\tif (!attempt || !record) return { scheduled: false, reason: \"attempt_missing\" };\n\t\tconst retriesUsed = attempt.retry?.retriesUsed ?? 0;\n\t\tconst decision = evaluateWorkerRetry({\n\t\t\tlaneStatus: args.outcome.laneStatus,\n\t\t\treasonCode: args.outcome.reasonCode,\n\t\t\t...(args.outcome.reasonDetail ? { reasonDetail: args.outcome.reasonDetail } : {}),\n\t\t\tprovider: args.provider,\n\t\t\tretriesUsed,\n\t\t\t...(args.maxAttempts !== undefined ? { maxAttempts: args.maxAttempts } : {}),\n\t\t});\n\t\tif (!decision.retry) return { scheduled: false, reason: decision.reason };\n\t\tconst nextRetriesUsed = retriesUsed + 1;\n\t\tconst notBefore = new Date(this.now() + decision.delayMs).toISOString();\n\t\tlet suspended: AttemptRuntimeState;\n\t\ttry {\n\t\t\tsuspended = this.options.lifecycle.scheduleAgentRetry({\n\t\t\t\tlaneId: args.laneId,\n\t\t\t\tagentId: args.agentId,\n\t\t\t\townerId: args.ownerId,\n\t\t\t\treasonCode: `retry_scheduled:${decision.reason}`,\n\t\t\t\tretry: { retriesUsed: nextRetriesUsed, notBefore },\n\t\t\t});\n\t\t} catch (error) {\n\t\t\tthis.options.warn(\n\t\t\t\t`Worker ${args.laneId} retry suspension failed; terminalizing instead: ${error instanceof Error ? error.message : String(error)}`,\n\t\t\t);\n\t\t\treturn { scheduled: false, reason: \"retry_suspension_failed\" };\n\t\t}\n\t\tthis.options.warn(\n\t\t\t`Worker ${args.laneId} failed (${decision.reason}); retrying from the persisted transcript in ${Math.ceil(decision.delayMs / 1000)}s (attempt ${nextRetriesUsed + 1}).`,\n\t\t);\n\t\tthis.enqueueOrDeferRetry(record, suspended, args.request, args.request.verificationOfTaskId !== undefined);\n\t\treturn { scheduled: true, record, retriesUsed: nextRetriesUsed, notBefore };\n\t}\n\n\t/** Cancel only process-local alarms; the retained deadline remains authoritative on disk. */\n\tdispose(): void {\n\t\tthis.unsubscribeQueueCapacity?.();\n\t\tfor (const entry of this.retryTimers.values()) {\n\t\t\tif (entry.timer) clearTimeout(entry.timer);\n\t\t}\n\t\tthis.retryTimers.clear();\n\t}\n\n\tclearScheduledRetry(laneId: string): void {\n\t\tfor (const [attemptId, entry] of this.retryTimers) {\n\t\t\tif (entry.laneId !== laneId) continue;\n\t\t\tif (entry.timer) clearTimeout(entry.timer);\n\t\t\tthis.retryTimers.delete(attemptId);\n\t\t}\n\t}\n\n\t/** Keep an externally requested wake queued while the durable retry deadline is still retained. */\n\tdeferRetryIfNeeded(record: LaneRecord, request: WorkerDelegationRequest): boolean {\n\t\tconst attempt = this.options.lifecycle.getActiveAttempt(record.laneId);\n\t\tif (\n\t\t\t!attempt ||\n\t\t\tattempt.status !== \"suspended\" ||\n\t\t\t!attempt.retry ||\n\t\t\tDate.parse(attempt.retry.notBefore) <= this.now()\n\t\t) {\n\t\t\treturn false;\n\t\t}\n\t\tthis.enqueueOrDeferRetry(record, attempt, request, request.verificationOfTaskId !== undefined);\n\t\treturn true;\n\t}\n\n\tprivate enqueueOrDeferRetry(\n\t\trecord: LaneRecord,\n\t\tattempt: AttemptRuntimeState,\n\t\trequest: WorkerDelegationRequest,\n\t\tpriority: boolean,\n\t): boolean {\n\t\tconst retry = attempt.retry;\n\t\tif (!retry) {\n\t\t\ttry {\n\t\t\t\tthis.options.scheduler.enqueue(record, request, true, priority);\n\t\t\t\treturn true;\n\t\t\t} catch (error) {\n\t\t\t\tif (error instanceof Error && error.message === \"worker_dispatch_queue_full\") return false;\n\t\t\t\tthrow error;\n\t\t\t}\n\t\t}\n\t\tconst existing = this.retryTimers.get(attempt.attemptId);\n\t\tif (existing?.notBefore === retry.notBefore) return true;\n\t\tif (existing?.timer) clearTimeout(existing.timer);\n\t\tconst entry: PendingRetryDispatch = {\n\t\t\tlaneId: record.laneId,\n\t\t\tnotBefore: retry.notBefore,\n\t\t\trequest,\n\t\t\tpriority,\n\t\t};\n\t\tthis.retryTimers.set(attempt.attemptId, entry);\n\t\tconst remainingMs = Date.parse(retry.notBefore) - this.now();\n\t\tif (remainingMs <= 0) {\n\t\t\tthis.tryEnqueueDueRetry(attempt.attemptId, entry);\n\t\t\treturn true;\n\t\t}\n\t\tconst timer = setTimeout(() => this.onRetryTimer(attempt.attemptId), Math.min(remainingMs, 2_147_000_000));\n\t\tif (typeof timer === \"object\" && \"unref\" in timer) timer.unref();\n\t\tentry.timer = timer;\n\t\treturn true;\n\t}\n\n\tprivate onRetryTimer(attemptId: string): void {\n\t\tconst entry = this.retryTimers.get(attemptId);\n\t\tif (!entry) return;\n\t\tdelete entry.timer;\n\t\tthis.tryEnqueueDueRetry(attemptId, entry);\n\t}\n\n\tprivate tryEnqueueDueRetry(attemptId: string, entry: PendingRetryDispatch): \"enqueued\" | \"full\" | \"stale\" {\n\t\tconst attempt = this.options.lifecycle.getActiveAttempt(entry.laneId);\n\t\tif (\n\t\t\t!attempt ||\n\t\t\tattempt.attemptId !== attemptId ||\n\t\t\tattempt.status !== \"suspended\" ||\n\t\t\tattempt.retry?.notBefore !== entry.notBefore\n\t\t) {\n\t\t\tthis.retryTimers.delete(attemptId);\n\t\t\treturn \"stale\";\n\t\t}\n\t\tconst record = this.options.lifecycle.getRecord(entry.laneId);\n\t\tif (!record) {\n\t\t\tthis.retryTimers.delete(attemptId);\n\t\t\treturn \"stale\";\n\t\t}\n\t\tconst remainingMs = Date.parse(entry.notBefore) - this.now();\n\t\tif (remainingMs > 0) {\n\t\t\tconst timer = setTimeout(() => this.onRetryTimer(attemptId), Math.min(remainingMs, 2_147_000_000));\n\t\t\tif (typeof timer === \"object\" && \"unref\" in timer) timer.unref();\n\t\t\tentry.timer = timer;\n\t\t\treturn \"stale\";\n\t\t}\n\t\ttry {\n\t\t\tthis.options.scheduler.enqueue(record, entry.request, true, entry.priority);\n\t\t} catch (error) {\n\t\t\tif (error instanceof Error && error.message === \"worker_dispatch_queue_full\") return \"full\";\n\t\t\tthis.options.warn(\n\t\t\t\t`Worker ${entry.laneId} retry enqueue failed; retaining the durable retry: ${error instanceof Error ? error.message : String(error)}`,\n\t\t\t);\n\t\t\treturn \"full\";\n\t\t}\n\t\tthis.retryTimers.delete(attemptId);\n\t\tthis.options.retryReady?.();\n\t\treturn \"enqueued\";\n\t}\n\n\tprivate flushDueRetries(): void {\n\t\tfor (const [attemptId, entry] of this.retryTimers) {\n\t\t\tif (entry.timer || Date.parse(entry.notBefore) > this.now()) continue;\n\t\t\tif (this.tryEnqueueDueRetry(attemptId, entry) === \"full\") break;\n\t\t}\n\t}\n\n\tprivate now(): number {\n\t\treturn this.options.now?.() ?? Date.now();\n\t}\n\n\t/** Rebuild task metadata from its durable dispatch; no current profile/model selection is consulted. */\n\trecoveredRequest(attempt: AttemptRuntimeState, verificationOfTaskId?: string): WorkerDelegationRequest {\n\t\tconst task = this.options.lifecycle.getTask(attempt.taskId)?.task;\n\t\tconst durableVerificationOfTaskId = verificationOfTaskId ?? task?.verificationOfTaskId;\n\t\treturn {\n\t\t\tinstructions: attempt.dispatch.instructions,\n\t\t\tprofileId: attempt.dispatch.profileId,\n\t\t\t...(attempt.dispatch.parentAgentId ? { parentAgentId: attempt.dispatch.parentAgentId } : {}),\n\t\t\t...(durableVerificationOfTaskId ? { verificationOfTaskId: durableVerificationOfTaskId } : {}),\n\t\t\t...(task\n\t\t\t\t? {\n\t\t\t\t\t\ttaskContext: {\n\t\t\t\t\t\t\trequirementIds: attempt.dispatch.requirementIds ?? [],\n\t\t\t\t\t\t\tdependsOnTaskIds: task.dependsOn,\n\t\t\t\t\t\t\tacceptanceCriterionIds: task.acceptanceCriterionIds,\n\t\t\t\t\t\t\tresourcePointerIds: attempt.dispatch.resourcePointerIds,\n\t\t\t\t\t\t},\n\t\t\t\t\t}\n\t\t\t\t: {}),\n\t\t};\n\t}\n\n\t/** Append deterministic unknown outcomes only for tool calls not durably followed by a result. */\n\trepairInterruptedToolResults(conversation: WorkerConversation): void {\n\t\tconst messages = conversation.getProviderMessages();\n\t\tconst completedToolCallIds = new Set<string>();\n\t\tlet index = messages.length - 1;\n\t\twhile (index >= 0 && messages[index]?.role === \"toolResult\") {\n\t\t\tcompletedToolCallIds.add((messages[index] as Extract<Message, { role: \"toolResult\" }>).toolCallId);\n\t\t\tindex -= 1;\n\t\t}\n\t\tconst assistant = messages[index];\n\t\tif (!assistant || assistant.role !== \"assistant\") return;\n\t\tfor (const content of assistant.content) {\n\t\t\tif (content.type !== \"toolCall\" || completedToolCallIds.has(content.id)) continue;\n\t\t\tconversation.appendMessage({\n\t\t\t\trole: \"toolResult\",\n\t\t\t\ttoolCallId: content.id,\n\t\t\t\ttoolName: content.name,\n\t\t\t\tcontent: [\n\t\t\t\t\t{\n\t\t\t\t\t\ttype: \"text\",\n\t\t\t\t\t\ttext: \"Execution outcome is unknown because this worker was interrupted before a durable tool result was recorded. Inspect the workspace and prior evidence before deciding whether to retry this operation.\",\n\t\t\t\t\t},\n\t\t\t\t],\n\t\t\t\tisError: true,\n\t\t\t\ttimestamp: Date.now(),\n\t\t\t});\n\t\t}\n\t}\n\n\t/** Return an already-persisted final assistant response so recovery cannot duplicate provider work. */\n\trecoveredTerminalCompletion(\n\t\tconversation: WorkerConversation,\n\t\tattemptId: string,\n\t): { text: string; usage: Usage; stopReason: string } | undefined {\n\t\tconst last = conversation.getLastAttemptMessage(attemptId);\n\t\tif (!last || last.role !== \"assistant\" || last.stopReason === \"error\" || last.stopReason === \"aborted\") {\n\t\t\treturn undefined;\n\t\t}\n\t\tif (last.content.some((content) => content.type === \"toolCall\")) return undefined;\n\t\tconst text = last.content\n\t\t\t.filter(\n\t\t\t\t(content): content is Extract<(typeof last.content)[number], { type: \"text\" }> => content.type === \"text\",\n\t\t\t)\n\t\t\t.map((content) => content.text)\n\t\t\t.join(\"\");\n\t\treturn { text, usage: last.usage, stopReason: last.stopReason };\n\t}\n\n\t/** Legacy raw transcripts predate durable usage checkpoints; reconstruct their bounded cumulative baseline. */\n\tinitialUsage(\n\t\tconversation: WorkerConversation,\n\t\tcheckpointUsage: AttemptUsageSnapshot | undefined,\n\t\tattemptId: string,\n\t): AttemptUsageSnapshot {\n\t\tconst transcriptUsage = conversation.getRawTranscriptUsage(attemptId);\n\t\treturn checkpointUsage ? reconcileAttemptUsage(checkpointUsage, transcriptUsage) : transcriptUsage;\n\t}\n\n\t/** Fail closed when the current owner profile no longer permits a persisted grant. */\n\tdurableGrantIsStillPermitted(\n\t\tgrant: ExecutionGrant,\n\t\tplan: WorkerExecutionPlan,\n\t\tselectedResources: readonly ResourcePointer[],\n\t): boolean {\n\t\tconst allowedTools = new Set(plan.toolManifests.map((manifest) => manifest.toolName));\n\t\tconst allowedCapabilities = new Set(plan.requiredCapabilities);\n\t\tconst scopesRemainPermitted = (granted: readonly string[], permitted: readonly string[]) =>\n\t\t\tgranted.every((scope) => permitted.some((candidate) => isPathWithinScope(scope, candidate)));\n\t\tconst liveDenialsRemainEnforced = plan.deniedPaths.every((denied) =>\n\t\t\tgrant.deniedPaths.some((granted) => isPathWithinScope(denied, granted)),\n\t\t);\n\t\tconst budgetRemainsPermitted = (Object.keys(plan.budget) as Array<keyof RiskBudget>).every((field) => {\n\t\t\tconst current = plan.budget[field];\n\t\t\tconst persisted = grant.budget[field];\n\t\t\treturn current === undefined || (persisted !== undefined && persisted <= current);\n\t\t});\n\t\tconst resourcesRemainPermitted =\n\t\t\tgrant.resources.length === selectedResources.length &&\n\t\t\tgrant.resources.every((granted, index) => {\n\t\t\t\tconst selected = selectedResources[index];\n\t\t\t\treturn (\n\t\t\t\t\tselected !== undefined &&\n\t\t\t\t\tgranted.id === selected.id &&\n\t\t\t\t\tgranted.kind === selected.kind &&\n\t\t\t\t\tgranted.uri === selected.uri &&\n\t\t\t\t\tgranted.readOnly === selected.readOnly &&\n\t\t\t\t\ttypeof granted.digest === \"string\" &&\n\t\t\t\t\t(selected.digest === undefined || selected.digest === granted.digest) &&\n\t\t\t\t\tgranted.metadata?.name === selected.metadata?.name\n\t\t\t\t);\n\t\t\t});\n\t\treturn (\n\t\t\tgrant.allowedTools.every((toolName) => allowedTools.has(toolName)) &&\n\t\t\tgrant.capabilities.every((capability) => allowedCapabilities.has(capability)) &&\n\t\t\tresourcesRemainPermitted &&\n\t\t\tscopesRemainPermitted(grant.readPaths, plan.readPaths) &&\n\t\t\tscopesRemainPermitted(grant.writePaths, plan.writePaths) &&\n\t\t\tliveDenialsRemainEnforced &&\n\t\t\tbudgetRemainsPermitted\n\t\t);\n\t}\n}\n"]}
@@ -0,0 +1,36 @@
1
+ import { type RetryPolicy } from "@caupulican/pi-agent-core/reliability";
2
+ /**
3
+ * Restart-durable, host-owned attempt ladder policy for delegated workers.
4
+ *
5
+ * The budget model has always carried `maxAttempts`, but nothing drove a second attempt: a
6
+ * failed worker terminalized immediately and "retry" meant the orchestrator model burning a
7
+ * turn to re-delegate blind, losing the worker's transcript. The ladder makes retry a host
8
+ * mechanism instead: a retryable bounded failure suspends the attempt (durable, fenced) and
9
+ * re-enqueues it through the owner scheduler after backoff, resuming from the persisted
10
+ * conversation exactly like restart recovery does.
11
+ *
12
+ * Retries are deliberately evidence-gated: only failures whose recorded detail classifies as
13
+ * transient (network, 5xx, rate limits) are retried. A missing detail or an unknown
14
+ * classification never retries — an undiagnosed failure repeated is spend without evidence.
15
+ */
16
+ export declare const DEFAULT_WORKER_TASK_ATTEMPTS = 2;
17
+ export declare const WORKER_TASK_RETRY_POLICY: RetryPolicy;
18
+ export type WorkerRetryDecision = {
19
+ retry: true;
20
+ reason: string;
21
+ delayMs: number;
22
+ } | {
23
+ retry: false;
24
+ reason: string;
25
+ };
26
+ export declare function evaluateWorkerRetry(args: {
27
+ laneStatus: string;
28
+ reasonCode: string;
29
+ reasonDetail?: string;
30
+ provider: string;
31
+ /** Restart-durable ladder retries already consumed for this attempt. */
32
+ retriesUsed: number;
33
+ /** Grant budget ceiling for total attempts; defaults to {@link DEFAULT_WORKER_TASK_ATTEMPTS}. */
34
+ maxAttempts?: number;
35
+ }): WorkerRetryDecision;
36
+ //# sourceMappingURL=worker-retry-policy.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"worker-retry-policy.d.ts","sourceRoot":"","sources":["../../../src/core/delegation/worker-retry-policy.ts"],"names":[],"mappings":"AAAA,OAAO,EAAwC,KAAK,WAAW,EAAE,MAAM,uCAAuC,CAAC;AAE/G;;;;;;;;;;;;;GAaG;AACH,eAAO,MAAM,4BAA4B,IAAI,CAAC;AAE9C,eAAO,MAAM,wBAAwB,EAAE,WAKtC,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAAE,KAAK,EAAE,IAAI,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,GAAG;IAAE,KAAK,EAAE,KAAK,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAEtH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE;IACzC,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,wEAAwE;IACxE,WAAW,EAAE,MAAM,CAAC;IACpB,iGAAiG;IACjG,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB,GAAG,mBAAmB,CAYtB"}