@agent-native/core 0.84.1 → 0.84.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. package/corpus/README.md +1 -1
  2. package/corpus/core/CHANGELOG.md +15 -0
  3. package/corpus/core/docs/design/durable-agent-runs.md +34 -35
  4. package/corpus/core/package.json +1 -1
  5. package/corpus/core/src/agent/engine/builder-engine.ts +81 -9
  6. package/corpus/core/src/agent/production-agent.ts +9 -2
  7. package/corpus/core/src/agent/run-loop-with-resume.ts +38 -8
  8. package/corpus/core/src/agent/run-manager.ts +8 -7
  9. package/corpus/core/src/agent/thread-data-builder.ts +6 -0
  10. package/corpus/core/src/cli/design-connect.ts +175 -0
  11. package/corpus/core/src/cli/skills.ts +5 -7
  12. package/corpus/core/src/client/AgentPanel.tsx +8 -0
  13. package/corpus/core/src/client/MultiTabAssistantChat.tsx +4 -1
  14. package/corpus/core/src/client/agent-chat-adapter.ts +253 -24
  15. package/corpus/core/src/client/sharing/ShareButton.tsx +70 -21
  16. package/corpus/core/src/client/sse-event-processor.ts +46 -4
  17. package/corpus/core/src/templates/workspace-core/.agents/skills/reliable-mutations/SKILL.md +15 -17
  18. package/corpus/templates/analytics/AGENTS.md +1 -1
  19. package/corpus/templates/design/actions/apply-motion-edit.ts +50 -18
  20. package/corpus/templates/design/actions/get-motion-timeline.ts +4 -14
  21. package/corpus/templates/design/app/components/design/DesignCanvas.tsx +156 -12
  22. package/corpus/templates/design/app/components/design/MotionDock.tsx +2 -3
  23. package/corpus/templates/design/app/components/design/MultiScreenCanvas.tsx +217 -75
  24. package/corpus/templates/design/app/components/design/QuestionFlow.tsx +39 -39
  25. package/corpus/templates/design/app/components/design/bridge/editor-chrome.bridge.ts +558 -37
  26. package/corpus/templates/design/app/components/design/bridge/hit-test.bridge.ts +75 -3
  27. package/corpus/templates/design/app/components/design/types.ts +13 -0
  28. package/corpus/templates/design/app/components/layout/Layout.tsx +1 -0
  29. package/corpus/templates/design/app/i18n/zh-TW.ts +17 -0
  30. package/corpus/templates/design/app/i18n-data.ts +245 -0
  31. package/corpus/templates/design/app/pages/DesignEditor.tsx +890 -252
  32. package/corpus/templates/design/changelog/2026-06-30-agent-chat-now-reports-saved-design-generations-clearly-when.md +6 -0
  33. package/corpus/templates/design/changelog/2026-06-30-apply-styles-now-appears-only-for-localhost-visual-edit-scre.md +6 -0
  34. package/corpus/templates/design/changelog/2026-06-30-copying-or-dragging-screen-elements-onto-the-infinite-canvas.md +6 -0
  35. package/corpus/templates/design/changelog/2026-06-30-design-questions-now-use-tighter-editor-typography-and-contr.md +6 -0
  36. package/corpus/templates/design/changelog/2026-06-30-element-drags-can-be-cancelled-with-escape-before-they-commi.md +6 -0
  37. package/corpus/templates/design/changelog/2026-06-30-pending-visual-style-edits-now-warn-before-you-leave-the-edi.md +6 -0
  38. package/corpus/templates/design/changelog/2026-06-30-share-options-now-make-export-and-agent-handoff-easier-to-no.md +6 -0
  39. package/corpus/templates/design/changelog/2026-06-30-visual-style-drags-stay-live-while-pending-edits-can-be-appl.md +6 -0
  40. package/corpus/templates/design/changelog/2026-07-01-design-chat-no-longer-shows-a-redundant-context-tab-above-the-composer.md +6 -0
  41. package/corpus/templates/design/changelog/2026-07-01-motion-track-creation-no-longer-fails-in-local-editors.md +6 -0
  42. package/corpus/templates/design/changelog/2026-07-01-share-general-access-menu-stays-open-when-choosing-organization.md +6 -0
  43. package/corpus/templates/design/server/plugins/db.ts +10 -0
  44. package/dist/agent/engine/builder-engine.d.ts.map +1 -1
  45. package/dist/agent/engine/builder-engine.js +61 -10
  46. package/dist/agent/engine/builder-engine.js.map +1 -1
  47. package/dist/agent/production-agent.d.ts.map +1 -1
  48. package/dist/agent/production-agent.js +9 -2
  49. package/dist/agent/production-agent.js.map +1 -1
  50. package/dist/agent/run-loop-with-resume.d.ts +2 -2
  51. package/dist/agent/run-loop-with-resume.d.ts.map +1 -1
  52. package/dist/agent/run-loop-with-resume.js +31 -8
  53. package/dist/agent/run-loop-with-resume.js.map +1 -1
  54. package/dist/agent/run-manager.d.ts +8 -7
  55. package/dist/agent/run-manager.d.ts.map +1 -1
  56. package/dist/agent/run-manager.js +8 -7
  57. package/dist/agent/run-manager.js.map +1 -1
  58. package/dist/agent/thread-data-builder.d.ts +2 -0
  59. package/dist/agent/thread-data-builder.d.ts.map +1 -1
  60. package/dist/agent/thread-data-builder.js +5 -0
  61. package/dist/agent/thread-data-builder.js.map +1 -1
  62. package/dist/cli/design-connect.d.ts +2 -0
  63. package/dist/cli/design-connect.d.ts.map +1 -1
  64. package/dist/cli/design-connect.js +140 -0
  65. package/dist/cli/design-connect.js.map +1 -1
  66. package/dist/cli/skills.d.ts.map +1 -1
  67. package/dist/cli/skills.js +5 -7
  68. package/dist/cli/skills.js.map +1 -1
  69. package/dist/client/AgentPanel.d.ts +5 -1
  70. package/dist/client/AgentPanel.d.ts.map +1 -1
  71. package/dist/client/AgentPanel.js +4 -4
  72. package/dist/client/AgentPanel.js.map +1 -1
  73. package/dist/client/MultiTabAssistantChat.d.ts +3 -1
  74. package/dist/client/MultiTabAssistantChat.d.ts.map +1 -1
  75. package/dist/client/MultiTabAssistantChat.js +2 -2
  76. package/dist/client/MultiTabAssistantChat.js.map +1 -1
  77. package/dist/client/agent-chat-adapter.d.ts.map +1 -1
  78. package/dist/client/agent-chat-adapter.js +216 -22
  79. package/dist/client/agent-chat-adapter.js.map +1 -1
  80. package/dist/client/sharing/ShareButton.d.ts +6 -0
  81. package/dist/client/sharing/ShareButton.d.ts.map +1 -1
  82. package/dist/client/sharing/ShareButton.js +25 -11
  83. package/dist/client/sharing/ShareButton.js.map +1 -1
  84. package/dist/client/sse-event-processor.d.ts +18 -1
  85. package/dist/client/sse-event-processor.d.ts.map +1 -1
  86. package/dist/client/sse-event-processor.js +34 -5
  87. package/dist/client/sse-event-processor.js.map +1 -1
  88. package/dist/collab/awareness.d.ts +2 -2
  89. package/dist/collab/awareness.d.ts.map +1 -1
  90. package/dist/collab/routes.d.ts +2 -2
  91. package/dist/file-upload/actions/upload-image.d.ts +2 -2
  92. package/dist/notifications/routes.d.ts +3 -3
  93. package/dist/observability/routes.d.ts +7 -7
  94. package/dist/resources/handlers.d.ts +2 -2
  95. package/dist/server/agent-engine-api-key-route.d.ts +1 -1
  96. package/dist/server/transcribe-voice.d.ts +1 -1
  97. package/dist/templates/workspace-core/.agents/skills/reliable-mutations/SKILL.md +15 -17
  98. package/docs/design/durable-agent-runs.md +34 -35
  99. package/package.json +1 -1
  100. package/src/templates/workspace-core/.agents/skills/reliable-mutations/SKILL.md +15 -17
@@ -43,8 +43,8 @@ export declare const RUN_BUDGET_EXHAUSTED_ERROR_CODE = "run_budget_exhausted";
43
43
  * exhausts its in-invocation continuation budget without finishing. Generic and
44
44
  * framework-level (not app-specific). Mirrors the `reliable-mutations` skill's
45
45
  * "fail loud, retry as a single bulk action" guidance so the user understands
46
- * the turn stopped before finishing and was not partially saved by the run
47
- * itself. */
46
+ * the turn stopped before finishing without implying that earlier completed
47
+ * tool calls did not persist. */
48
48
  export declare const RUN_BUDGET_EXHAUSTED_MESSAGE: string;
49
49
  /**
50
50
  * Internal entry point used by the agent-chat plugin's run handler. Wraps
@@ -1 +1 @@
1
- {"version":3,"file":"run-loop-with-resume.d.ts","sourceRoot":"","sources":["../../src/agent/run-loop-with-resume.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAGH,OAAO,EACL,YAAY,EAIb,MAAM,uBAAuB,CAAC;AAE/B,OAAO,KAAK,EAAE,4BAA4B,EAAE,MAAM,kBAAkB,CAAC;AAkDrE;;;;;;;;;GASG;AACH,eAAO,MAAM,0BAA0B,IAAI,CAAC;AAE5C;;;;wDAIwD;AACxD,eAAO,MAAM,+BAA+B,yBAAyB,CAAC;AAEtE;;;;;aAKa;AACb,eAAO,MAAM,4BAA4B,QAGS,CAAC;AAEnD;;;;;;;;GAQG;AACH,wBAAsB,iCAAiC,CACrD,IAAI,EAAE,UAAU,CAAC,OAAO,YAAY,CAAC,CAAC,CAAC,CAAC,EACxC,aAAa,CAAC,EAAE,MAAM,EACtB,cAAc,CAAC,EAAE,4BAA4B,GAC5C,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,OAAO,YAAY,CAAC,CAAC,CAAC,CA0HnD"}
1
+ {"version":3,"file":"run-loop-with-resume.d.ts","sourceRoot":"","sources":["../../src/agent/run-loop-with-resume.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAGH,OAAO,EACL,YAAY,EAIb,MAAM,uBAAuB,CAAC;AAE/B,OAAO,KAAK,EAAE,4BAA4B,EAAE,MAAM,kBAAkB,CAAC;AAoErE;;;;;;;;;GASG;AACH,eAAO,MAAM,0BAA0B,IAAI,CAAC;AAE5C;;;;wDAIwD;AACxD,eAAO,MAAM,+BAA+B,yBAAyB,CAAC;AAEtE;;;;;iCAKiC;AACjC,eAAO,MAAM,4BAA4B,QAGkD,CAAC;AAE5F;;;;;;;;GAQG;AACH,wBAAsB,iCAAiC,CACrD,IAAI,EAAE,UAAU,CAAC,OAAO,YAAY,CAAC,CAAC,CAAC,CAAC,EACxC,aAAa,CAAC,EAAE,MAAM,EACtB,cAAc,CAAC,EAAE,4BAA4B,GAC5C,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,OAAO,YAAY,CAAC,CAAC,CAAC,CAsInD"}
@@ -67,6 +67,21 @@ async function appendToolCallJournalNote(messages, threadId) {
67
67
  // up — the model still continues, just without the structured journal.
68
68
  }
69
69
  }
70
+ async function hasCompletedSideEffectToolCallInCurrentTurn(threadId) {
71
+ if (!threadId)
72
+ return false;
73
+ try {
74
+ const events = await getCurrentTurnEventsForThread(threadId);
75
+ if (events.length === 0)
76
+ return false;
77
+ return events.some((event) => event.type === "tool_done" &&
78
+ event.completedSideEffect === true &&
79
+ event.isError !== true);
80
+ }
81
+ catch {
82
+ return false;
83
+ }
84
+ }
70
85
  /**
71
86
  * Cap on continuation iterations inside a single
72
87
  * `runAgentLoopDirectWithSoftTimeout` invocation. The host's hard function
@@ -88,11 +103,11 @@ export const RUN_BUDGET_EXHAUSTED_ERROR_CODE = "run_budget_exhausted";
88
103
  * exhausts its in-invocation continuation budget without finishing. Generic and
89
104
  * framework-level (not app-specific). Mirrors the `reliable-mutations` skill's
90
105
  * "fail loud, retry as a single bulk action" guidance so the user understands
91
- * the turn stopped before finishing and was not partially saved by the run
92
- * itself. */
93
- export const RUN_BUDGET_EXHAUSTED_MESSAGE = "I ran out of time before finishing this step (hosted runs have a ~40s budget). " +
94
- "I stopped rather than leave things half-done — nothing was partially saved by me here. " +
95
- "Please retry, ideally as a single bulk action.";
106
+ * the turn stopped before finishing without implying that earlier completed
107
+ * tool calls did not persist. */
108
+ export const RUN_BUDGET_EXHAUSTED_MESSAGE = "I ran out of time before finishing this step. " +
109
+ "I stopped rather than keep retrying silently. " +
110
+ "Check any completed tool cards above before retrying, ideally as one smaller follow-up.";
96
111
  /**
97
112
  * Internal entry point used by the agent-chat plugin's run handler. Wraps
98
113
  * `runAgentLoop` with soft-timeout + resumable-error continuation recovery.
@@ -168,7 +183,9 @@ export async function runAgentLoopDirectWithSoftTimeout(opts, softTimeoutMs, tim
168
183
  // Clear partial text the client received before the abort so the
169
184
  // resumed model doesn't re-emit it and produce duplicated output.
170
185
  lastAttemptWasUnfinishedContinuation = true;
171
- opts.send({ type: "clear" });
186
+ if (!(await hasCompletedSideEffectToolCallInCurrentTurn(opts.threadId))) {
187
+ opts.send({ type: "clear" });
188
+ }
172
189
  appendAgentLoopContinuation(opts.messages, "run_timeout");
173
190
  await appendToolCallJournalNote(opts.messages, opts.threadId);
174
191
  continue;
@@ -188,7 +205,9 @@ export async function runAgentLoopDirectWithSoftTimeout(opts, softTimeoutMs, tim
188
205
  // the in-memory messages array, so the next attempt re-emits it).
189
206
  if (!upstreamSignal.aborted && isResumableEngineError(err)) {
190
207
  lastAttemptWasUnfinishedContinuation = true;
191
- opts.send({ type: "clear" });
208
+ if (!(await hasCompletedSideEffectToolCallInCurrentTurn(opts.threadId))) {
209
+ opts.send({ type: "clear" });
210
+ }
192
211
  appendAgentLoopContinuation(opts.messages, continuationReasonForResumableError(err));
193
212
  await appendToolCallJournalNote(opts.messages, opts.threadId);
194
213
  continue;
@@ -210,7 +229,11 @@ export async function runAgentLoopDirectWithSoftTimeout(opts, softTimeoutMs, tim
210
229
  if (!upstreamSignal.aborted && lastAttemptWasUnfinishedContinuation) {
211
230
  // Discard any partial text already streamed for the unfinished attempt so
212
231
  // the terminal message stands alone instead of trailing a half sentence.
213
- opts.send({ type: "clear" });
232
+ // Preserve completed tool cards: they are the user's only durable proof
233
+ // that a side effect landed before the final assistant note timed out.
234
+ if (!(await hasCompletedSideEffectToolCallInCurrentTurn(opts.threadId))) {
235
+ opts.send({ type: "clear" });
236
+ }
214
237
  opts.send({
215
238
  type: "error",
216
239
  error: RUN_BUDGET_EXHAUSTED_MESSAGE,
@@ -1 +1 @@
1
- {"version":3,"file":"run-loop-with-resume.js","sourceRoot":"","sources":["../../src/agent/run-loop-with-resume.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAGH,OAAO,EACL,YAAY,EACZ,2BAA2B,EAC3B,sBAAsB,EACtB,mCAAmC,GACpC,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,uBAAuB,EAAE,MAAM,kBAAkB,CAAC;AAE3D,OAAO,EAAE,6BAA6B,EAAE,MAAM,gBAAgB,CAAC;AAC/D,OAAO,EACL,uBAAuB,EACvB,sBAAsB,GACvB,MAAM,wBAAwB,CAAC;AAEhC;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,KAAK,UAAU,yBAAyB,CACtC,QAAyB,EACzB,QAA4B;IAE5B,IAAI,CAAC,QAAQ;QAAE,OAAO;IACtB,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,6BAA6B,CAAC,QAAQ,CAAC,CAAC;QAC7D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAChC,MAAM,OAAO,GAAG,uBAAuB,CAAC,MAAM,CAAC,CAAC;QAChD,MAAM,IAAI,GAAG,sBAAsB,CAAC,OAAO,CAAC,CAAC;QAC7C,IAAI,CAAC,IAAI;YAAE,OAAO;QAClB,QAAQ,CAAC,IAAI,CAAC;YACZ,IAAI,EAAE,MAAM;YACZ,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;SACxC,CAAC,CAAC;IACL,CAAC;IAAC,MAAM,CAAC;QACP,0EAA0E;QAC1E,0EAA0E;QAC1E,uEAAuE;IACzE,CAAC;AACH,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,0BAA0B,GAAG,CAAC,CAAC;AAE5C;;;;wDAIwD;AACxD,MAAM,CAAC,MAAM,+BAA+B,GAAG,sBAAsB,CAAC;AAEtE;;;;;aAKa;AACb,MAAM,CAAC,MAAM,4BAA4B,GACvC,iFAAiF;IACjF,yFAAyF;IACzF,gDAAgD,CAAC;AAEnD;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,iCAAiC,CACrD,IAAwC,EACxC,aAAsB,EACtB,cAA6C;IAE7C,MAAM,SAAS,GAAG,uBAAuB,CAAC,aAAa,EAAE,cAAc,CAAC,CAAC;IACzE,IAAI,SAAS,IAAI,CAAC;QAAE,OAAO,YAAY,CAAC,IAAI,CAAC,CAAC;IAE9C,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC;IACnC,MAAM,KAAK,GAA6C;QACtD,WAAW,EAAE,CAAC;QACd,YAAY,EAAE,CAAC;QACf,eAAe,EAAE,CAAC;QAClB,gBAAgB,EAAE,CAAC;QACnB,KAAK,EAAE,IAAI,CAAC,KAAK;KAClB,CAAC;IAEF,MAAM,QAAQ,GAAG,CAAC,IAA8C,EAAE,EAAE;QAClE,KAAK,CAAC,WAAW,IAAI,IAAI,CAAC,WAAW,CAAC;QACtC,KAAK,CAAC,YAAY,IAAI,IAAI,CAAC,YAAY,CAAC;QACxC,KAAK,CAAC,eAAe,IAAI,IAAI,CAAC,eAAe,CAAC;QAC9C,KAAK,CAAC,gBAAgB,IAAI,IAAI,CAAC,gBAAgB,CAAC;QAChD,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;IAC3B,CAAC,CAAC;IAEF,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,qEAAqE;IACrE,0EAA0E;IAC1E,4EAA4E;IAC5E,8EAA8E;IAC9E,4EAA4E;IAC5E,qEAAqE;IACrE,IAAI,oCAAoC,GAAG,KAAK,CAAC;IACjD,OAAO,CAAC,cAAc,CAAC,OAAO,IAAI,QAAQ,GAAG,0BAA0B,EAAE,CAAC;QACxE,QAAQ,EAAE,CAAC;QACX,oCAAoC,GAAG,KAAK,CAAC;QAC7C,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,iBAAiB,GAAG,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;QACnD,IAAI,cAAc,CAAC,OAAO,EAAE,CAAC;YAC3B,UAAU,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC;aAAM,CAAC;YACN,cAAc,CAAC,gBAAgB,CAAC,OAAO,EAAE,iBAAiB,EAAE;gBAC1D,IAAI,EAAE,IAAI;aACX,CAAC,CAAC;QACL,CAAC;QAED,IAAI,YAAY,GAAG,KAAK,CAAC;QACzB,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,IAAI,UAAU,CAAC,MAAM,CAAC,OAAO;gBAAE,OAAO;YACtC,YAAY,GAAG,IAAI,CAAC;YACpB,UAAU,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC,EAAE,SAAS,CAAC,CAAC;QAEd,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC;gBACnC,GAAG,IAAI;gBACP,MAAM,EAAE,UAAU,CAAC,MAAM;aAC1B,CAAC,CAAC;YACH,QAAQ,CAAC,SAAS,CAAC,CAAC;YACpB,IAAI,YAAY,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,CAAC;gBAC5C,oCAAoC,GAAG,IAAI,CAAC;gBAC5C,2BAA2B,CAAC,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;gBAC1D,MAAM,yBAAyB,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAC9D,SAAS;YACX,CAAC;YACD,OAAO,KAAK,CAAC;QACf,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,YAAY,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,CAAC;gBAC5C,iEAAiE;gBACjE,kEAAkE;gBAClE,oCAAoC,GAAG,IAAI,CAAC;gBAC5C,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;gBAC7B,2BAA2B,CAAC,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;gBAC1D,MAAM,yBAAyB,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAC9D,SAAS;YACX,CAAC;YACD,wEAAwE;YACxE,kEAAkE;YAClE,wEAAwE;YACxE,wEAAwE;YACxE,uEAAuE;YACvE,oEAAoE;YACpE,2CAA2C;YAC3C,EAAE;YACF,uEAAuE;YACvE,yEAAyE;YACzE,oEAAoE;YACpE,qEAAqE;YACrE,kEAAkE;YAClE,IAAI,CAAC,cAAc,CAAC,OAAO,IAAI,sBAAsB,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC3D,oCAAoC,GAAG,IAAI,CAAC;gBAC5C,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;gBAC7B,2BAA2B,CACzB,IAAI,CAAC,QAAQ,EACb,mCAAmC,CAAC,GAAG,CAAC,CACzC,CAAC;gBACF,MAAM,yBAAyB,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAC9D,SAAS;YACX,CAAC;YACD,MAAM,GAAG,CAAC;QACZ,CAAC;gBAAS,CAAC;YACT,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,cAAc,CAAC,mBAAmB,CAAC,OAAO,EAAE,iBAAiB,CAAC,CAAC;QACjE,CAAC;IACH,CAAC;IAED,+EAA+E;IAC/E,wDAAwD;IACxD,wEAAwE;IACxE,0EAA0E;IAC1E,2EAA2E;IAC3E,wEAAwE;IACxE,sEAAsE;IACtE,IAAI,CAAC,cAAc,CAAC,OAAO,IAAI,oCAAoC,EAAE,CAAC;QACpE,0EAA0E;QAC1E,yEAAyE;QACzE,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;QAC7B,IAAI,CAAC,IAAI,CAAC;YACR,IAAI,EAAE,OAAO;YACb,KAAK,EAAE,4BAA4B;YACnC,SAAS,EAAE,+BAA+B;YAC1C,WAAW,EAAE,IAAI;SAClB,CAAC,CAAC;IACL,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC","sourcesContent":["/**\n * Wraps `runAgentLoop` with two layered recovery mechanisms so a single hosted\n * invocation can survive interruptions without showing the user a dead chat:\n *\n * 1. **Soft timeout** — an inner timer that aborts the LLM call before the\n * hosting function's hard limit (Lambda 75s, Vercel 60s, etc.) so we have a\n * chance to gracefully wind down and append a continuation nudge. Without\n * this the function gets killed mid-stream and the user sees a frozen\n * spinner.\n *\n * 2. **Resumable-error continuation** — when the LLM call errors with a\n * transport- or gateway-level interruption (Builder gateway 45s timeout,\n * socket hang up, ECONNRESET, upstream 5xx that survived engine retries),\n * we save the conversation prefix, append a \"continue from where you left\n * off\" message, and run another LLM call. Anthropic's prompt cache makes\n * the resume call dramatically faster than the cold first attempt, and the\n * agent gets explicit context that it was cut off so it doesn't re-do\n * completed work.\n *\n * Both paths route through `appendAgentLoopContinuation` so the agent sees a\n * uniform \"continue\" instruction regardless of which recovery fired.\n */\n\nimport type { EngineMessage } from \"./engine/types.js\";\nimport {\n runAgentLoop,\n appendAgentLoopContinuation,\n isResumableEngineError,\n continuationReasonForResumableError,\n} from \"./production-agent.js\";\nimport { resolveRunSoftTimeoutMs } from \"./run-manager.js\";\nimport type { ResolveRunSoftTimeoutOptions } from \"./run-manager.js\";\nimport { getCurrentTurnEventsForThread } from \"./run-store.js\";\nimport {\n classifyToolCallJournal,\n buildResumeJournalNote,\n} from \"./tool-call-journal.js\";\n\n/**\n * Derive the per-turn tool-call journal from the durable run-event ledger and,\n * when there is anything to report, append a STRUCTURED note to the message\n * prefix so the resumed model:\n * - does NOT re-execute tool calls that already completed (avoiding duplicate\n * side effects like re-sending an email or re-creating a ticket), and\n * - is explicitly told about any tool call that started but whose outcome was\n * never recorded (\"interrupted, unknown outcome\") so it can decide.\n *\n * This is additive to the existing \"continue from where you left off\" nudge —\n * it is appended right after it. When the journal is empty (no completed or\n * interrupted tool calls — e.g. a turn with no tool activity, or a clean\n * continuation), nothing extra is appended and resume behavior is byte-for-byte\n * what it was before. Best-effort: any ledger read/parse failure is swallowed so\n * a journal hiccup can never block a recovery that would otherwise succeed.\n *\n * This prompt-level journal is paired with tool-layer enforcement in\n * production-agent.ts/runToolCall, which refuses to re-execute a journaled-\n * complete write tool (returning the journaled result instead). See\n * `tool-call-journal.ts` (`findCompletedJournalEntry`) for the keying used.\n */\nasync function appendToolCallJournalNote(\n messages: EngineMessage[],\n threadId: string | undefined,\n): Promise<void> {\n if (!threadId) return;\n try {\n const events = await getCurrentTurnEventsForThread(threadId);\n if (events.length === 0) return;\n const journal = classifyToolCallJournal(events);\n const note = buildResumeJournalNote(journal);\n if (!note) return;\n messages.push({\n role: \"user\",\n content: [{ type: \"text\", text: note }],\n });\n } catch {\n // The journal is a hardening layer, never a gate. A failed ledger read or\n // parse must not break the resume that the continuation nudge already set\n // up — the model still continues, just without the structured journal.\n }\n}\n\n/**\n * Cap on continuation iterations inside a single\n * `runAgentLoopDirectWithSoftTimeout` invocation. The host's hard function\n * timeout usually bounds this naturally — but a defensive cap prevents an\n * instant-error spiral from looping forever inside hosting environments with a\n * generous budget.\n *\n * 6 leaves room for: 1 normal completion + a few resume rounds for design\n * generation (prompt + 3 variants ≈ 4 LLM calls), with a small safety margin.\n */\nexport const MAX_RUN_LOOP_CONTINUATIONS = 6;\n\n/** Machine-readable code carried on the give-up terminal `error` event so the\n * client renders a loud \"stopped before finishing\" terminal instead of an\n * ambiguous silent stall. Deliberately NOT in the client's auto-recoverable\n * allow-list (`isAutoRecoverableError`) so it terminates the chain rather than\n * looping another POST that would hit the same wall. */\nexport const RUN_BUDGET_EXHAUSTED_ERROR_CODE = \"run_budget_exhausted\";\n\n/** User-facing terminal message when a hosted run is cut off mid-step and\n * exhausts its in-invocation continuation budget without finishing. Generic and\n * framework-level (not app-specific). Mirrors the `reliable-mutations` skill's\n * \"fail loud, retry as a single bulk action\" guidance so the user understands\n * the turn stopped before finishing and was not partially saved by the run\n * itself. */\nexport const RUN_BUDGET_EXHAUSTED_MESSAGE =\n \"I ran out of time before finishing this step (hosted runs have a ~40s budget). \" +\n \"I stopped rather than leave things half-done — nothing was partially saved by me here. \" +\n \"Please retry, ideally as a single bulk action.\";\n\n/**\n * Internal entry point used by the agent-chat plugin's run handler. Wraps\n * `runAgentLoop` with soft-timeout + resumable-error continuation recovery.\n *\n * The `softTimeoutMs` argument falls back to `resolveRunSoftTimeoutMs(...)` so\n * different hosting environments (Lambda, Vercel, Cloudflare, local dev) get\n * an appropriate inner budget. Setting it to <= 0 disables both layers — the\n * call goes straight to `runAgentLoop` with no wrapping.\n */\nexport async function runAgentLoopDirectWithSoftTimeout(\n opts: Parameters<typeof runAgentLoop>[0],\n softTimeoutMs?: number,\n timeoutOptions?: ResolveRunSoftTimeoutOptions,\n): Promise<Awaited<ReturnType<typeof runAgentLoop>>> {\n const timeoutMs = resolveRunSoftTimeoutMs(softTimeoutMs, timeoutOptions);\n if (timeoutMs <= 0) return runAgentLoop(opts);\n\n const upstreamSignal = opts.signal;\n const usage: Awaited<ReturnType<typeof runAgentLoop>> = {\n inputTokens: 0,\n outputTokens: 0,\n cacheReadTokens: 0,\n cacheWriteTokens: 0,\n model: opts.model,\n };\n\n const addUsage = (next: Awaited<ReturnType<typeof runAgentLoop>>) => {\n usage.inputTokens += next.inputTokens;\n usage.outputTokens += next.outputTokens;\n usage.cacheReadTokens += next.cacheReadTokens;\n usage.cacheWriteTokens += next.cacheWriteTokens;\n usage.model = next.model;\n };\n\n let attempts = 0;\n // Tracks whether the most recent attempt ended by scheduling another\n // continuation (soft-timeout or resumable error → `continue`) rather than\n // returning a finished turn. When the loop then exits because the budget is\n // exhausted (NOT because the user aborted and NOT because the turn finished),\n // this is the silent give-up case: emit a loud terminal so the user sees an\n // unambiguous \"stopped before finishing\" instead of a bare done/\"…\".\n let lastAttemptWasUnfinishedContinuation = false;\n while (!upstreamSignal.aborted && attempts < MAX_RUN_LOOP_CONTINUATIONS) {\n attempts++;\n lastAttemptWasUnfinishedContinuation = false;\n const controller = new AbortController();\n const abortFromUpstream = () => controller.abort();\n if (upstreamSignal.aborted) {\n controller.abort();\n } else {\n upstreamSignal.addEventListener(\"abort\", abortFromUpstream, {\n once: true,\n });\n }\n\n let softTimedOut = false;\n const timer = setTimeout(() => {\n if (controller.signal.aborted) return;\n softTimedOut = true;\n controller.abort();\n }, timeoutMs);\n\n try {\n const nextUsage = await runAgentLoop({\n ...opts,\n signal: controller.signal,\n });\n addUsage(nextUsage);\n if (softTimedOut && !upstreamSignal.aborted) {\n lastAttemptWasUnfinishedContinuation = true;\n appendAgentLoopContinuation(opts.messages, \"run_timeout\");\n await appendToolCallJournalNote(opts.messages, opts.threadId);\n continue;\n }\n return usage;\n } catch (err) {\n if (softTimedOut && !upstreamSignal.aborted) {\n // Clear partial text the client received before the abort so the\n // resumed model doesn't re-emit it and produce duplicated output.\n lastAttemptWasUnfinishedContinuation = true;\n opts.send({ type: \"clear\" });\n appendAgentLoopContinuation(opts.messages, \"run_timeout\");\n await appendToolCallJournalNote(opts.messages, opts.threadId);\n continue;\n }\n // Resumable transport / gateway interruptions: the LLM call was cut off\n // mid-stream (gateway 45s timeout, socket hang up, function-level\n // timeout that didn't trip our soft timer first). Treat it the same way\n // as a soft timeout — append a \"continue from where you left off\" nudge\n // and let the loop run another LLM call. The conversation prefix up to\n // the cut-off is preserved in opts.messages, and Anthropic's prompt\n // cache makes the resume call much faster.\n //\n // Emit 'clear' so any partial streamed text is discarded on the client\n // before the model resumes. Without this the model restarts its sentence\n // from scratch and the fold produces duplicated text in one message\n // (the partial text was already sent to the client but never entered\n // the in-memory messages array, so the next attempt re-emits it).\n if (!upstreamSignal.aborted && isResumableEngineError(err)) {\n lastAttemptWasUnfinishedContinuation = true;\n opts.send({ type: \"clear\" });\n appendAgentLoopContinuation(\n opts.messages,\n continuationReasonForResumableError(err),\n );\n await appendToolCallJournalNote(opts.messages, opts.threadId);\n continue;\n }\n throw err;\n } finally {\n clearTimeout(timer);\n upstreamSignal.removeEventListener(\"abort\", abortFromUpstream);\n }\n }\n\n // The loop exited without a clean return. If the user aborted, that's a Stop —\n // stay silent. Otherwise we only get here by exhausting\n // MAX_RUN_LOOP_CONTINUATIONS while the last attempt was still trying to\n // continue (soft-timeout / resumable error). That is the genuinely-silent\n // give-up the run-manager would otherwise report as a clean `done`: emit a\n // loud, non-auto-continuing terminal so the user knows the turn stopped\n // before finishing and nothing was partially saved by the run itself.\n if (!upstreamSignal.aborted && lastAttemptWasUnfinishedContinuation) {\n // Discard any partial text already streamed for the unfinished attempt so\n // the terminal message stands alone instead of trailing a half sentence.\n opts.send({ type: \"clear\" });\n opts.send({\n type: \"error\",\n error: RUN_BUDGET_EXHAUSTED_MESSAGE,\n errorCode: RUN_BUDGET_EXHAUSTED_ERROR_CODE,\n recoverable: true,\n });\n }\n\n return usage;\n}\n"]}
1
+ {"version":3,"file":"run-loop-with-resume.js","sourceRoot":"","sources":["../../src/agent/run-loop-with-resume.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAGH,OAAO,EACL,YAAY,EACZ,2BAA2B,EAC3B,sBAAsB,EACtB,mCAAmC,GACpC,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,uBAAuB,EAAE,MAAM,kBAAkB,CAAC;AAE3D,OAAO,EAAE,6BAA6B,EAAE,MAAM,gBAAgB,CAAC;AAC/D,OAAO,EACL,uBAAuB,EACvB,sBAAsB,GACvB,MAAM,wBAAwB,CAAC;AAEhC;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,KAAK,UAAU,yBAAyB,CACtC,QAAyB,EACzB,QAA4B;IAE5B,IAAI,CAAC,QAAQ;QAAE,OAAO;IACtB,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,6BAA6B,CAAC,QAAQ,CAAC,CAAC;QAC7D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAChC,MAAM,OAAO,GAAG,uBAAuB,CAAC,MAAM,CAAC,CAAC;QAChD,MAAM,IAAI,GAAG,sBAAsB,CAAC,OAAO,CAAC,CAAC;QAC7C,IAAI,CAAC,IAAI;YAAE,OAAO;QAClB,QAAQ,CAAC,IAAI,CAAC;YACZ,IAAI,EAAE,MAAM;YACZ,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;SACxC,CAAC,CAAC;IACL,CAAC;IAAC,MAAM,CAAC;QACP,0EAA0E;QAC1E,0EAA0E;QAC1E,uEAAuE;IACzE,CAAC;AACH,CAAC;AAED,KAAK,UAAU,2CAA2C,CACxD,QAA4B;IAE5B,IAAI,CAAC,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC5B,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,6BAA6B,CAAC,QAAQ,CAAC,CAAC;QAC7D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC;QACtC,OAAO,MAAM,CAAC,IAAI,CAChB,CAAC,KAAK,EAAE,EAAE,CACR,KAAK,CAAC,IAAI,KAAK,WAAW;YAC1B,KAAK,CAAC,mBAAmB,KAAK,IAAI;YAClC,KAAK,CAAC,OAAO,KAAK,IAAI,CACzB,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,0BAA0B,GAAG,CAAC,CAAC;AAE5C;;;;wDAIwD;AACxD,MAAM,CAAC,MAAM,+BAA+B,GAAG,sBAAsB,CAAC;AAEtE;;;;;iCAKiC;AACjC,MAAM,CAAC,MAAM,4BAA4B,GACvC,gDAAgD;IAChD,gDAAgD;IAChD,yFAAyF,CAAC;AAE5F;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,iCAAiC,CACrD,IAAwC,EACxC,aAAsB,EACtB,cAA6C;IAE7C,MAAM,SAAS,GAAG,uBAAuB,CAAC,aAAa,EAAE,cAAc,CAAC,CAAC;IACzE,IAAI,SAAS,IAAI,CAAC;QAAE,OAAO,YAAY,CAAC,IAAI,CAAC,CAAC;IAE9C,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC;IACnC,MAAM,KAAK,GAA6C;QACtD,WAAW,EAAE,CAAC;QACd,YAAY,EAAE,CAAC;QACf,eAAe,EAAE,CAAC;QAClB,gBAAgB,EAAE,CAAC;QACnB,KAAK,EAAE,IAAI,CAAC,KAAK;KAClB,CAAC;IAEF,MAAM,QAAQ,GAAG,CAAC,IAA8C,EAAE,EAAE;QAClE,KAAK,CAAC,WAAW,IAAI,IAAI,CAAC,WAAW,CAAC;QACtC,KAAK,CAAC,YAAY,IAAI,IAAI,CAAC,YAAY,CAAC;QACxC,KAAK,CAAC,eAAe,IAAI,IAAI,CAAC,eAAe,CAAC;QAC9C,KAAK,CAAC,gBAAgB,IAAI,IAAI,CAAC,gBAAgB,CAAC;QAChD,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;IAC3B,CAAC,CAAC;IAEF,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,qEAAqE;IACrE,0EAA0E;IAC1E,4EAA4E;IAC5E,8EAA8E;IAC9E,4EAA4E;IAC5E,qEAAqE;IACrE,IAAI,oCAAoC,GAAG,KAAK,CAAC;IACjD,OAAO,CAAC,cAAc,CAAC,OAAO,IAAI,QAAQ,GAAG,0BAA0B,EAAE,CAAC;QACxE,QAAQ,EAAE,CAAC;QACX,oCAAoC,GAAG,KAAK,CAAC;QAC7C,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,iBAAiB,GAAG,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;QACnD,IAAI,cAAc,CAAC,OAAO,EAAE,CAAC;YAC3B,UAAU,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC;aAAM,CAAC;YACN,cAAc,CAAC,gBAAgB,CAAC,OAAO,EAAE,iBAAiB,EAAE;gBAC1D,IAAI,EAAE,IAAI;aACX,CAAC,CAAC;QACL,CAAC;QAED,IAAI,YAAY,GAAG,KAAK,CAAC;QACzB,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,IAAI,UAAU,CAAC,MAAM,CAAC,OAAO;gBAAE,OAAO;YACtC,YAAY,GAAG,IAAI,CAAC;YACpB,UAAU,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC,EAAE,SAAS,CAAC,CAAC;QAEd,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC;gBACnC,GAAG,IAAI;gBACP,MAAM,EAAE,UAAU,CAAC,MAAM;aAC1B,CAAC,CAAC;YACH,QAAQ,CAAC,SAAS,CAAC,CAAC;YACpB,IAAI,YAAY,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,CAAC;gBAC5C,oCAAoC,GAAG,IAAI,CAAC;gBAC5C,2BAA2B,CAAC,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;gBAC1D,MAAM,yBAAyB,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAC9D,SAAS;YACX,CAAC;YACD,OAAO,KAAK,CAAC;QACf,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,YAAY,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,CAAC;gBAC5C,iEAAiE;gBACjE,kEAAkE;gBAClE,oCAAoC,GAAG,IAAI,CAAC;gBAC5C,IACE,CAAC,CAAC,MAAM,2CAA2C,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,EACnE,CAAC;oBACD,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;gBAC/B,CAAC;gBACD,2BAA2B,CAAC,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;gBAC1D,MAAM,yBAAyB,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAC9D,SAAS;YACX,CAAC;YACD,wEAAwE;YACxE,kEAAkE;YAClE,wEAAwE;YACxE,wEAAwE;YACxE,uEAAuE;YACvE,oEAAoE;YACpE,2CAA2C;YAC3C,EAAE;YACF,uEAAuE;YACvE,yEAAyE;YACzE,oEAAoE;YACpE,qEAAqE;YACrE,kEAAkE;YAClE,IAAI,CAAC,cAAc,CAAC,OAAO,IAAI,sBAAsB,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC3D,oCAAoC,GAAG,IAAI,CAAC;gBAC5C,IACE,CAAC,CAAC,MAAM,2CAA2C,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,EACnE,CAAC;oBACD,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;gBAC/B,CAAC;gBACD,2BAA2B,CACzB,IAAI,CAAC,QAAQ,EACb,mCAAmC,CAAC,GAAG,CAAC,CACzC,CAAC;gBACF,MAAM,yBAAyB,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAC9D,SAAS;YACX,CAAC;YACD,MAAM,GAAG,CAAC;QACZ,CAAC;gBAAS,CAAC;YACT,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,cAAc,CAAC,mBAAmB,CAAC,OAAO,EAAE,iBAAiB,CAAC,CAAC;QACjE,CAAC;IACH,CAAC;IAED,+EAA+E;IAC/E,wDAAwD;IACxD,wEAAwE;IACxE,0EAA0E;IAC1E,2EAA2E;IAC3E,wEAAwE;IACxE,sEAAsE;IACtE,IAAI,CAAC,cAAc,CAAC,OAAO,IAAI,oCAAoC,EAAE,CAAC;QACpE,0EAA0E;QAC1E,yEAAyE;QACzE,wEAAwE;QACxE,uEAAuE;QACvE,IAAI,CAAC,CAAC,MAAM,2CAA2C,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;YACxE,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;QAC/B,CAAC;QACD,IAAI,CAAC,IAAI,CAAC;YACR,IAAI,EAAE,OAAO;YACb,KAAK,EAAE,4BAA4B;YACnC,SAAS,EAAE,+BAA+B;YAC1C,WAAW,EAAE,IAAI;SAClB,CAAC,CAAC;IACL,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC","sourcesContent":["/**\n * Wraps `runAgentLoop` with two layered recovery mechanisms so a single hosted\n * invocation can survive interruptions without showing the user a dead chat:\n *\n * 1. **Soft timeout** — an inner timer that aborts the LLM call before the\n * hosting function's hard limit (Lambda 75s, Vercel 60s, etc.) so we have a\n * chance to gracefully wind down and append a continuation nudge. Without\n * this the function gets killed mid-stream and the user sees a frozen\n * spinner.\n *\n * 2. **Resumable-error continuation** — when the LLM call errors with a\n * transport- or gateway-level interruption (Builder gateway 45s timeout,\n * socket hang up, ECONNRESET, upstream 5xx that survived engine retries),\n * we save the conversation prefix, append a \"continue from where you left\n * off\" message, and run another LLM call. Anthropic's prompt cache makes\n * the resume call dramatically faster than the cold first attempt, and the\n * agent gets explicit context that it was cut off so it doesn't re-do\n * completed work.\n *\n * Both paths route through `appendAgentLoopContinuation` so the agent sees a\n * uniform \"continue\" instruction regardless of which recovery fired.\n */\n\nimport type { EngineMessage } from \"./engine/types.js\";\nimport {\n runAgentLoop,\n appendAgentLoopContinuation,\n isResumableEngineError,\n continuationReasonForResumableError,\n} from \"./production-agent.js\";\nimport { resolveRunSoftTimeoutMs } from \"./run-manager.js\";\nimport type { ResolveRunSoftTimeoutOptions } from \"./run-manager.js\";\nimport { getCurrentTurnEventsForThread } from \"./run-store.js\";\nimport {\n classifyToolCallJournal,\n buildResumeJournalNote,\n} from \"./tool-call-journal.js\";\n\n/**\n * Derive the per-turn tool-call journal from the durable run-event ledger and,\n * when there is anything to report, append a STRUCTURED note to the message\n * prefix so the resumed model:\n * - does NOT re-execute tool calls that already completed (avoiding duplicate\n * side effects like re-sending an email or re-creating a ticket), and\n * - is explicitly told about any tool call that started but whose outcome was\n * never recorded (\"interrupted, unknown outcome\") so it can decide.\n *\n * This is additive to the existing \"continue from where you left off\" nudge —\n * it is appended right after it. When the journal is empty (no completed or\n * interrupted tool calls — e.g. a turn with no tool activity, or a clean\n * continuation), nothing extra is appended and resume behavior is byte-for-byte\n * what it was before. Best-effort: any ledger read/parse failure is swallowed so\n * a journal hiccup can never block a recovery that would otherwise succeed.\n *\n * This prompt-level journal is paired with tool-layer enforcement in\n * production-agent.ts/runToolCall, which refuses to re-execute a journaled-\n * complete write tool (returning the journaled result instead). See\n * `tool-call-journal.ts` (`findCompletedJournalEntry`) for the keying used.\n */\nasync function appendToolCallJournalNote(\n messages: EngineMessage[],\n threadId: string | undefined,\n): Promise<void> {\n if (!threadId) return;\n try {\n const events = await getCurrentTurnEventsForThread(threadId);\n if (events.length === 0) return;\n const journal = classifyToolCallJournal(events);\n const note = buildResumeJournalNote(journal);\n if (!note) return;\n messages.push({\n role: \"user\",\n content: [{ type: \"text\", text: note }],\n });\n } catch {\n // The journal is a hardening layer, never a gate. A failed ledger read or\n // parse must not break the resume that the continuation nudge already set\n // up — the model still continues, just without the structured journal.\n }\n}\n\nasync function hasCompletedSideEffectToolCallInCurrentTurn(\n threadId: string | undefined,\n): Promise<boolean> {\n if (!threadId) return false;\n try {\n const events = await getCurrentTurnEventsForThread(threadId);\n if (events.length === 0) return false;\n return events.some(\n (event) =>\n event.type === \"tool_done\" &&\n event.completedSideEffect === true &&\n event.isError !== true,\n );\n } catch {\n return false;\n }\n}\n\n/**\n * Cap on continuation iterations inside a single\n * `runAgentLoopDirectWithSoftTimeout` invocation. The host's hard function\n * timeout usually bounds this naturally — but a defensive cap prevents an\n * instant-error spiral from looping forever inside hosting environments with a\n * generous budget.\n *\n * 6 leaves room for: 1 normal completion + a few resume rounds for design\n * generation (prompt + 3 variants ≈ 4 LLM calls), with a small safety margin.\n */\nexport const MAX_RUN_LOOP_CONTINUATIONS = 6;\n\n/** Machine-readable code carried on the give-up terminal `error` event so the\n * client renders a loud \"stopped before finishing\" terminal instead of an\n * ambiguous silent stall. Deliberately NOT in the client's auto-recoverable\n * allow-list (`isAutoRecoverableError`) so it terminates the chain rather than\n * looping another POST that would hit the same wall. */\nexport const RUN_BUDGET_EXHAUSTED_ERROR_CODE = \"run_budget_exhausted\";\n\n/** User-facing terminal message when a hosted run is cut off mid-step and\n * exhausts its in-invocation continuation budget without finishing. Generic and\n * framework-level (not app-specific). Mirrors the `reliable-mutations` skill's\n * \"fail loud, retry as a single bulk action\" guidance so the user understands\n * the turn stopped before finishing without implying that earlier completed\n * tool calls did not persist. */\nexport const RUN_BUDGET_EXHAUSTED_MESSAGE =\n \"I ran out of time before finishing this step. \" +\n \"I stopped rather than keep retrying silently. \" +\n \"Check any completed tool cards above before retrying, ideally as one smaller follow-up.\";\n\n/**\n * Internal entry point used by the agent-chat plugin's run handler. Wraps\n * `runAgentLoop` with soft-timeout + resumable-error continuation recovery.\n *\n * The `softTimeoutMs` argument falls back to `resolveRunSoftTimeoutMs(...)` so\n * different hosting environments (Lambda, Vercel, Cloudflare, local dev) get\n * an appropriate inner budget. Setting it to <= 0 disables both layers — the\n * call goes straight to `runAgentLoop` with no wrapping.\n */\nexport async function runAgentLoopDirectWithSoftTimeout(\n opts: Parameters<typeof runAgentLoop>[0],\n softTimeoutMs?: number,\n timeoutOptions?: ResolveRunSoftTimeoutOptions,\n): Promise<Awaited<ReturnType<typeof runAgentLoop>>> {\n const timeoutMs = resolveRunSoftTimeoutMs(softTimeoutMs, timeoutOptions);\n if (timeoutMs <= 0) return runAgentLoop(opts);\n\n const upstreamSignal = opts.signal;\n const usage: Awaited<ReturnType<typeof runAgentLoop>> = {\n inputTokens: 0,\n outputTokens: 0,\n cacheReadTokens: 0,\n cacheWriteTokens: 0,\n model: opts.model,\n };\n\n const addUsage = (next: Awaited<ReturnType<typeof runAgentLoop>>) => {\n usage.inputTokens += next.inputTokens;\n usage.outputTokens += next.outputTokens;\n usage.cacheReadTokens += next.cacheReadTokens;\n usage.cacheWriteTokens += next.cacheWriteTokens;\n usage.model = next.model;\n };\n\n let attempts = 0;\n // Tracks whether the most recent attempt ended by scheduling another\n // continuation (soft-timeout or resumable error → `continue`) rather than\n // returning a finished turn. When the loop then exits because the budget is\n // exhausted (NOT because the user aborted and NOT because the turn finished),\n // this is the silent give-up case: emit a loud terminal so the user sees an\n // unambiguous \"stopped before finishing\" instead of a bare done/\"…\".\n let lastAttemptWasUnfinishedContinuation = false;\n while (!upstreamSignal.aborted && attempts < MAX_RUN_LOOP_CONTINUATIONS) {\n attempts++;\n lastAttemptWasUnfinishedContinuation = false;\n const controller = new AbortController();\n const abortFromUpstream = () => controller.abort();\n if (upstreamSignal.aborted) {\n controller.abort();\n } else {\n upstreamSignal.addEventListener(\"abort\", abortFromUpstream, {\n once: true,\n });\n }\n\n let softTimedOut = false;\n const timer = setTimeout(() => {\n if (controller.signal.aborted) return;\n softTimedOut = true;\n controller.abort();\n }, timeoutMs);\n\n try {\n const nextUsage = await runAgentLoop({\n ...opts,\n signal: controller.signal,\n });\n addUsage(nextUsage);\n if (softTimedOut && !upstreamSignal.aborted) {\n lastAttemptWasUnfinishedContinuation = true;\n appendAgentLoopContinuation(opts.messages, \"run_timeout\");\n await appendToolCallJournalNote(opts.messages, opts.threadId);\n continue;\n }\n return usage;\n } catch (err) {\n if (softTimedOut && !upstreamSignal.aborted) {\n // Clear partial text the client received before the abort so the\n // resumed model doesn't re-emit it and produce duplicated output.\n lastAttemptWasUnfinishedContinuation = true;\n if (\n !(await hasCompletedSideEffectToolCallInCurrentTurn(opts.threadId))\n ) {\n opts.send({ type: \"clear\" });\n }\n appendAgentLoopContinuation(opts.messages, \"run_timeout\");\n await appendToolCallJournalNote(opts.messages, opts.threadId);\n continue;\n }\n // Resumable transport / gateway interruptions: the LLM call was cut off\n // mid-stream (gateway 45s timeout, socket hang up, function-level\n // timeout that didn't trip our soft timer first). Treat it the same way\n // as a soft timeout — append a \"continue from where you left off\" nudge\n // and let the loop run another LLM call. The conversation prefix up to\n // the cut-off is preserved in opts.messages, and Anthropic's prompt\n // cache makes the resume call much faster.\n //\n // Emit 'clear' so any partial streamed text is discarded on the client\n // before the model resumes. Without this the model restarts its sentence\n // from scratch and the fold produces duplicated text in one message\n // (the partial text was already sent to the client but never entered\n // the in-memory messages array, so the next attempt re-emits it).\n if (!upstreamSignal.aborted && isResumableEngineError(err)) {\n lastAttemptWasUnfinishedContinuation = true;\n if (\n !(await hasCompletedSideEffectToolCallInCurrentTurn(opts.threadId))\n ) {\n opts.send({ type: \"clear\" });\n }\n appendAgentLoopContinuation(\n opts.messages,\n continuationReasonForResumableError(err),\n );\n await appendToolCallJournalNote(opts.messages, opts.threadId);\n continue;\n }\n throw err;\n } finally {\n clearTimeout(timer);\n upstreamSignal.removeEventListener(\"abort\", abortFromUpstream);\n }\n }\n\n // The loop exited without a clean return. If the user aborted, that's a Stop —\n // stay silent. Otherwise we only get here by exhausting\n // MAX_RUN_LOOP_CONTINUATIONS while the last attempt was still trying to\n // continue (soft-timeout / resumable error). That is the genuinely-silent\n // give-up the run-manager would otherwise report as a clean `done`: emit a\n // loud, non-auto-continuing terminal so the user knows the turn stopped\n // before finishing and nothing was partially saved by the run itself.\n if (!upstreamSignal.aborted && lastAttemptWasUnfinishedContinuation) {\n // Discard any partial text already streamed for the unfinished attempt so\n // the terminal message stands alone instead of trailing a half sentence.\n // Preserve completed tool cards: they are the user's only durable proof\n // that a side effect landed before the final assistant note timed out.\n if (!(await hasCompletedSideEffectToolCallInCurrentTurn(opts.threadId))) {\n opts.send({ type: \"clear\" });\n }\n opts.send({\n type: \"error\",\n error: RUN_BUDGET_EXHAUSTED_MESSAGE,\n errorCode: RUN_BUDGET_EXHAUSTED_ERROR_CODE,\n recoverable: true,\n });\n }\n\n return usage;\n}\n"]}
@@ -16,8 +16,8 @@ export interface ActiveRun {
16
16
  *
17
17
  * This MUST fire before the two upstream hard walls that otherwise kill a run
18
18
  * mid-turn with no chance to hand off:
19
- * 1. The Builder model gateway hard-caps a single model call at 45s
20
- * (builder-engine.ts MAX_BUILDER_GATEWAY_TIMEOUT_MS) not raisable.
19
+ * 1. The Builder model gateway keeps a 45s cap only for hosted foreground
20
+ * runs; local and proven background-function runs use longer caps.
21
21
  * 2. Serverless functions are hard-killed around 60-65s (the heartbeat then
22
22
  * reaps the row as a stale_run).
23
23
  * Production data showed every cutoff landing in the 44-70s window with ZERO
@@ -30,11 +30,12 @@ export interface ActiveRun {
30
30
  export declare const DEFAULT_HOSTED_RUN_SOFT_TIMEOUT_MS = 40000;
31
31
  /**
32
32
  * Hard ceiling for the hosted soft timeout. On a hosted runtime the
33
- * auto_continue soft timeout can never usefully exceed this — the gateway
34
- * (45s) / function (~60s) walls kill the run first, so a larger configured or
35
- * env value just guarantees the cutoff is a hard error instead of a graceful
36
- * hand-off. Any resolved value above this is clamped down when hosted. Local
37
- * dev (non-hosted) is left alone so long-running local turns aren't chunked.
33
+ * foreground auto_continue soft timeout can never usefully exceed this — the
34
+ * synchronous function (~60s) wall kills the run first, so a larger configured
35
+ * or env value just guarantees the cutoff is a hard error instead of a graceful
36
+ * hand-off. Any resolved value above this is clamped down for hosted foreground
37
+ * runs. Local dev (non-hosted) is left alone so long-running local turns aren't
38
+ * chunked.
38
39
  *
39
40
  * IMPORTANT: this clamp is for the INTERACTIVE / foreground path and must NOT
40
41
  * be raised. The foreground POST still rides a synchronous serverless function
@@ -1 +1 @@
1
- {"version":3,"file":"run-manager.d.ts","sourceRoot":"","sources":["../../src/agent/run-manager.ts"],"names":[],"mappings":"AAsBA,OAAO,KAAK,EAAE,cAAc,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAEtE,MAAM,WAAW,SAAS;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,6EAA6E;IAC7E,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,QAAQ,EAAE,CAAC;IACnB,MAAM,EAAE,SAAS,CAAC;IAClB,WAAW,EAAE,GAAG,CAAC,CAAC,KAAK,EAAE,QAAQ,KAAK,IAAI,CAAC,CAAC;IAC5C,KAAK,EAAE,eAAe,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;CACnB;AAQD;;;;;;;;;;;;;;;GAeG;AACH,eAAO,MAAM,kCAAkC,QAAS,CAAC;AAEzD;;;;;;;;;;;;;;GAcG;AACH,eAAO,MAAM,8BAA8B,QAAS,CAAC;AAErD;;;;;;;;;;;;;;GAcG;AACH,eAAO,MAAM,kCAAkC,QAAc,CAAC;AAE9D;;;;;GAKG;AACH,eAAO,MAAM,sCAAsC,QACf,CAAC;AAErC,qEAAqE;AACrE,eAAO,MAAM,kCAAkC,QAAsB,CAAC;AAEtE;;;;;GAKG;AACH,eAAO,MAAM,gCAAgC,QAA0B,CAAC;AAExE;;;;;GAKG;AACH,eAAO,MAAM,gCAAgC,QAAiB,CAAC;AA6C/D,MAAM,WAAW,eAAe;IAC9B;;2CAEuC;IACvC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;4DACwD;IACxD,2BAA2B,CAAC,EAAE,OAAO,CAAC;IACtC;;;+EAG2E;IAC3E,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;;;;;OAQG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAC9B;AAED,MAAM,WAAW,4BAA4B;IAC3C,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B;;;;;;OAMG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAC9B;AA0BD,wBAAgB,uBAAuB,CACrC,UAAU,CAAC,EAAE,MAAM,EACnB,OAAO,CAAC,EAAE,4BAA4B,GACrC,MAAM,CAkCR;AAED,wBAAgB,8BAA8B,IAAI,MAAM,CAOvD;AAED,wBAAgB,4BAA4B,IAAI,MAAM,CAOrD;AA6BD;;;;;;GAMG;AACH,wBAAgB,QAAQ,CACtB,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,CACL,IAAI,EAAE,CAAC,KAAK,EAAE,cAAc,KAAK,IAAI,EACrC,MAAM,EAAE,WAAW,KAChB,OAAO,CAAC,IAAI,CAAC,EAClB,UAAU,CAAC,EAAE,CAAC,GAAG,EAAE,SAAS,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EACrD,OAAO,CAAC,EAAE,eAAe,GACxB,SAAS,CAgZX;AAED;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAC5B,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,MAAM,GACd,cAAc,CAAC,UAAU,CAAC,GAAG,IAAI,CAOnC;AAmQD,wEAAwE;AACxE,wBAAgB,qBAAqB,CAAC,QAAQ,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI,CAOxE;AAED;;;;;;;;;GASG;AACH,wBAAsB,0BAA0B,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;IAC1E,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,uFAAuF;IACvF,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC3B,GAAG,IAAI,CAAC,CA8FR;AAkBD,sBAAsB;AACtB,wBAAgB,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI,CAEtD;AAED,gDAAgD;AAChD,wBAAgB,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,GAAE,MAAe,GAAG,OAAO,CAQxE;AAGD,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC"}
1
+ {"version":3,"file":"run-manager.d.ts","sourceRoot":"","sources":["../../src/agent/run-manager.ts"],"names":[],"mappings":"AAsBA,OAAO,KAAK,EAAE,cAAc,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAEtE,MAAM,WAAW,SAAS;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,6EAA6E;IAC7E,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,QAAQ,EAAE,CAAC;IACnB,MAAM,EAAE,SAAS,CAAC;IAClB,WAAW,EAAE,GAAG,CAAC,CAAC,KAAK,EAAE,QAAQ,KAAK,IAAI,CAAC,CAAC;IAC5C,KAAK,EAAE,eAAe,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;CACnB;AAQD;;;;;;;;;;;;;;;GAeG;AACH,eAAO,MAAM,kCAAkC,QAAS,CAAC;AAEzD;;;;;;;;;;;;;;;GAeG;AACH,eAAO,MAAM,8BAA8B,QAAS,CAAC;AAErD;;;;;;;;;;;;;;GAcG;AACH,eAAO,MAAM,kCAAkC,QAAc,CAAC;AAE9D;;;;;GAKG;AACH,eAAO,MAAM,sCAAsC,QACf,CAAC;AAErC,qEAAqE;AACrE,eAAO,MAAM,kCAAkC,QAAsB,CAAC;AAEtE;;;;;GAKG;AACH,eAAO,MAAM,gCAAgC,QAA0B,CAAC;AAExE;;;;;GAKG;AACH,eAAO,MAAM,gCAAgC,QAAiB,CAAC;AA6C/D,MAAM,WAAW,eAAe;IAC9B;;2CAEuC;IACvC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;4DACwD;IACxD,2BAA2B,CAAC,EAAE,OAAO,CAAC;IACtC;;;+EAG2E;IAC3E,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;;;;;OAQG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAC9B;AAED,MAAM,WAAW,4BAA4B;IAC3C,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B;;;;;;OAMG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAC9B;AA0BD,wBAAgB,uBAAuB,CACrC,UAAU,CAAC,EAAE,MAAM,EACnB,OAAO,CAAC,EAAE,4BAA4B,GACrC,MAAM,CAkCR;AAED,wBAAgB,8BAA8B,IAAI,MAAM,CAOvD;AAED,wBAAgB,4BAA4B,IAAI,MAAM,CAOrD;AA6BD;;;;;;GAMG;AACH,wBAAgB,QAAQ,CACtB,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,CACL,IAAI,EAAE,CAAC,KAAK,EAAE,cAAc,KAAK,IAAI,EACrC,MAAM,EAAE,WAAW,KAChB,OAAO,CAAC,IAAI,CAAC,EAClB,UAAU,CAAC,EAAE,CAAC,GAAG,EAAE,SAAS,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EACrD,OAAO,CAAC,EAAE,eAAe,GACxB,SAAS,CAgZX;AAED;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAC5B,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,MAAM,GACd,cAAc,CAAC,UAAU,CAAC,GAAG,IAAI,CAOnC;AAmQD,wEAAwE;AACxE,wBAAgB,qBAAqB,CAAC,QAAQ,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI,CAOxE;AAED;;;;;;;;;GASG;AACH,wBAAsB,0BAA0B,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;IAC1E,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,uFAAuF;IACvF,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC3B,GAAG,IAAI,CAAC,CA8FR;AAkBD,sBAAsB;AACtB,wBAAgB,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI,CAEtD;AAED,gDAAgD;AAChD,wBAAgB,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,GAAE,MAAe,GAAG,OAAO,CAQxE;AAGD,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC"}
@@ -11,8 +11,8 @@ const CLEANUP_DELAY_MS = 5 * 60 * 1000;
11
11
  *
12
12
  * This MUST fire before the two upstream hard walls that otherwise kill a run
13
13
  * mid-turn with no chance to hand off:
14
- * 1. The Builder model gateway hard-caps a single model call at 45s
15
- * (builder-engine.ts MAX_BUILDER_GATEWAY_TIMEOUT_MS) not raisable.
14
+ * 1. The Builder model gateway keeps a 45s cap only for hosted foreground
15
+ * runs; local and proven background-function runs use longer caps.
16
16
  * 2. Serverless functions are hard-killed around 60-65s (the heartbeat then
17
17
  * reaps the row as a stale_run).
18
18
  * Production data showed every cutoff landing in the 44-70s window with ZERO
@@ -25,11 +25,12 @@ const CLEANUP_DELAY_MS = 5 * 60 * 1000;
25
25
  export const DEFAULT_HOSTED_RUN_SOFT_TIMEOUT_MS = 40_000;
26
26
  /**
27
27
  * Hard ceiling for the hosted soft timeout. On a hosted runtime the
28
- * auto_continue soft timeout can never usefully exceed this — the gateway
29
- * (45s) / function (~60s) walls kill the run first, so a larger configured or
30
- * env value just guarantees the cutoff is a hard error instead of a graceful
31
- * hand-off. Any resolved value above this is clamped down when hosted. Local
32
- * dev (non-hosted) is left alone so long-running local turns aren't chunked.
28
+ * foreground auto_continue soft timeout can never usefully exceed this — the
29
+ * synchronous function (~60s) wall kills the run first, so a larger configured
30
+ * or env value just guarantees the cutoff is a hard error instead of a graceful
31
+ * hand-off. Any resolved value above this is clamped down for hosted foreground
32
+ * runs. Local dev (non-hosted) is left alone so long-running local turns aren't
33
+ * chunked.
33
34
  *
34
35
  * IMPORTANT: this clamp is for the INTERACTIVE / foreground path and must NOT
35
36
  * be raised. The foreground POST still rides a synchronous serverless function
@@ -1 +1 @@
1
- {"version":3,"file":"run-manager.js","sourceRoot":"","sources":["../../src/agent/run-manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,oBAAoB,EAAE,MAAM,+BAA+B,CAAC;AACrE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EACL,SAAS,EACT,cAAc,EACd,wBAAwB,EACxB,cAAc,EACd,gBAAgB,EAChB,YAAY,EACZ,iBAAiB,EACjB,UAAU,EACV,cAAc,EACd,cAAc,EACd,kBAAkB,EAClB,eAAe,EACf,WAAW,EACX,0BAA0B,EAC1B,sBAAsB,EACtB,WAAW,EACX,qBAAqB,GACtB,MAAM,gBAAgB,CAAC;AAgBxB,MAAM,UAAU,GAAG,IAAI,GAAG,EAAqB,CAAC;AAChD,MAAM,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAC;AAE9C,uEAAuE;AACvE,MAAM,gBAAgB,GAAG,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC;AAEvC;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,MAAM,kCAAkC,GAAG,MAAM,CAAC;AAEzD;;;;;;;;;;;;;;GAcG;AACH,MAAM,CAAC,MAAM,8BAA8B,GAAG,MAAM,CAAC;AAErD;;;;;;;;;;;;;;GAcG;AACH,MAAM,CAAC,MAAM,kCAAkC,GAAG,EAAE,GAAG,MAAM,CAAC,CAAC,UAAU;AAEzE;;;;;GAKG;AACH,MAAM,CAAC,MAAM,sCAAsC,GACjD,kCAAkC,CAAC;AAErC,qEAAqE;AACrE,MAAM,CAAC,MAAM,kCAAkC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAEtE;;;;;GAKG;AACH,MAAM,CAAC,MAAM,gCAAgC,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAExE;;;;;GAKG;AACH,MAAM,CAAC,MAAM,gCAAgC,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAE/D,MAAM,gCAAgC,GAAG,uBAAuB,CAAC;AAEjE,SAAS,kBAAkB,CAAC,GAAY;IACtC,IACE,OAAO,GAAG,KAAK,QAAQ;QACvB,GAAG,KAAK,IAAI;QACZ,SAAS,IAAI,GAAG;QAChB,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ;QAC/B,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAC7B,CAAC;QACD,OAAO,GAAG,CAAC,OAAO,CAAC;IACrB,CAAC;IACD,OAAO,eAAe,CAAC;AACzB,CAAC;AAED,SAAS,qBAAqB,CAAC,GAAgB;IAC7C,IAAI,GAAG,CAAC,SAAS;QAAE,OAAO,GAAG,CAAC,SAAS,CAAC;IACxC,IAAI,GAAG,CAAC,UAAU,KAAK,GAAG;QAAE,OAAO,gCAAgC,CAAC;IACpE,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,wBAAwB,CAAC,GAAgB;IAChD,IAAI,GAAG,CAAC,UAAU,KAAK,GAAG;QAAE,OAAO,GAAG,CAAC,OAAO,CAAC;IAC/C,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,qBAAqB,CAAC,GAAY;IACzC,IAAI,CAAC,CAAC,GAAG,YAAY,WAAW,CAAC;QAAE,OAAO,IAAI,CAAC;IAC/C,MAAM,SAAS,GAAG,qBAAqB,CAAC,GAAG,CAAC,CAAC;IAC7C,IAAI,oBAAoB,CAAC,GAAG,EAAE,SAAS,CAAC;QAAE,OAAO,KAAK,CAAC;IACvD,IAAI,GAAG,CAAC,UAAU,KAAK,GAAG,IAAI,GAAG,CAAC,UAAU,KAAK,GAAG;QAAE,OAAO,KAAK,CAAC;IACnE,IAAI,wBAAwB,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC;IAC7D,IAAI,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,KAAK,mBAAmB;QAAE,OAAO,KAAK,CAAC;IAC3E,IAAI,CAAC,SAAS;QAAE,OAAO,IAAI,CAAC;IAC5B,MAAM,cAAc,GAAG,SAAS,CAAC,WAAW,EAAE,CAAC;IAC/C,OAAO,CACL,CAAC,cAAc,CAAC,UAAU,CAAC,eAAe,CAAC;QAC3C,cAAc,KAAK,+BAA+B;QAClD,cAAc,KAAK,uBAAuB;QAC1C,cAAc,KAAK,qBAAqB,CACzC,CAAC;AACJ,CAAC;AAuCD,SAAS,eAAe;IACtB,IACE,OAAO,CAAC,GAAG,CAAC,OAAO;QACnB,OAAO,CAAC,GAAG,CAAC,OAAO,KAAK,OAAO;QAC/B,OAAO,CAAC,GAAG,CAAC,aAAa,KAAK,MAAM,EACpC,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IACE,OAAO,CAAC,GAAG,CAAC,wBAAwB;QACpC,OAAO,CAAC,GAAG,CAAC,aAAa,KAAK,MAAM,EACpC,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,OAAO,CACZ,OAAO,CAAC,GAAG,CAAC,QAAQ;QACpB,OAAO,CAAC,GAAG,CAAC,MAAM;QAClB,OAAO,CAAC,GAAG,CAAC,UAAU;QACtB,OAAO,CAAC,GAAG,CAAC,MAAM;QAClB,OAAO,CAAC,GAAG,CAAC,YAAY;QACxB,OAAO,CAAC,GAAG,CAAC,SAAS,CACtB,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,uBAAuB,CACrC,UAAmB,EACnB,OAAsC;IAEtC,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;IACjC,MAAM,UAAU,GAAG,OAAO,EAAE,kBAAkB,KAAK,IAAI,CAAC;IACxD,yEAAyE;IACzE,8EAA8E;IAC9E,4EAA4E;IAC5E,sEAAsE;IACtE,MAAM,OAAO,GAAG,UAAU;QACxB,CAAC,CAAC,kCAAkC;QACpC,CAAC,CAAC,8BAA8B,CAAC;IACnC,0EAA0E;IAC1E,6EAA6E;IAC7E,wEAAwE;IACxE,6EAA6E;IAC7E,sCAAsC;IACtC,MAAM,WAAW,GAAG,CAAC,EAAU,EAAU,EAAE,CACzC,MAAM,IAAI,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;IAExC,IAAI,OAAO,UAAU,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;QAClE,OAAO,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,CAAC;IAC9C,CAAC;IACD,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC;IACvD,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC3B,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC7B,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC;YAAE,OAAO,WAAW,CAAC,GAAG,CAAC,CAAC;IAChE,CAAC;IACD,4EAA4E;IAC5E,yCAAyC;IACzC,IAAI,UAAU,EAAE,CAAC;QACf,OAAO,MAAM,CAAC,CAAC,CAAC,sCAAsC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7D,CAAC;IACD,OAAO,OAAO,EAAE,gBAAgB,IAAI,MAAM;QACxC,CAAC,CAAC,kCAAkC;QACpC,CAAC,CAAC,CAAC,CAAC;AACR,CAAC;AAED,MAAM,UAAU,8BAA8B;IAC5C,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC;IACpD,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC3B,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC7B,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC;YAAE,OAAO,GAAG,CAAC;IACnD,CAAC;IACD,OAAO,kCAAkC,CAAC;AAC5C,CAAC;AAED,MAAM,UAAU,4BAA4B;IAC1C,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,8BAA8B,CAAC;IAC5D,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC3B,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC7B,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC;YAAE,OAAO,GAAG,CAAC;IACnD,CAAC;IACD,OAAO,gCAAgC,CAAC;AAC1C,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAqB;IAC/C,OAAO,CACL,KAAK,CAAC,IAAI,KAAK,MAAM;QACrB,KAAK,CAAC,IAAI,KAAK,OAAO;QACtB,KAAK,CAAC,IAAI,KAAK,iBAAiB;QAChC,KAAK,CAAC,IAAI,KAAK,YAAY;QAC3B,KAAK,CAAC,IAAI,KAAK,eAAe,CAC/B,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CAAC,GAAc,EAAE,MAAM,GAAW,MAAM;IAC/D,GAAG,CAAC,WAAW,GAAG,MAAM,CAAC;IACzB,GAAG,CAAC,MAAM,GAAG,SAAS,CAAC;IACvB,IAAI,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,GAAG,CAAC,KAAK,EAAE,CAAC;QAChD,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IACnC,CAAC;IACD,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IACxB,KAAK,MAAM,UAAU,IAAI,GAAG,CAAC,WAAW,EAAE,CAAC;QACzC,IAAI,CAAC;YACH,UAAU,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC;QAClE,CAAC;QAAC,MAAM,CAAC;YACP,6CAA6C;QAC/C,CAAC;IACH,CAAC;IACD,GAAG,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;AAC1B,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,QAAQ,CACtB,KAAa,EACb,QAAgB,EAChB,KAGkB,EAClB,UAAqD,EACrD,OAAyB;IAEzB,qDAAqD;IACrD,MAAM,aAAa,GAAG,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAChD,IAAI,aAAa,EAAE,CAAC;QAClB,QAAQ,CAAC,aAAa,CAAC,CAAC;IAC1B,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,eAAe,EAAE,CAAC;IACpC,IAAI,YAAY,GAAG,KAAK,CAAC;IACzB,MAAM,GAAG,GAAc;QACrB,KAAK;QACL,QAAQ;QACR,MAAM,EAAE,OAAO,EAAE,MAAM,IAAI,KAAK;QAChC,MAAM,EAAE,EAAE;QACV,MAAM,EAAE,SAAS;QACjB,WAAW,EAAE,IAAI,GAAG,EAAE;QACtB,KAAK;QACL,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;KACtB,CAAC;IAEF,UAAU,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IAC3B,WAAW,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;IAEjC,wEAAwE;IACxE,uEAAuE;IACvE,uDAAuD;IACvD,MAAM,gBAAgB,GAAG,SAAS,CAAC,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,KAAK,CACxE,GAAG,EAAE,GAAE,CAAC,CACT,CAAC;IAEF,4EAA4E;IAC5E,4EAA4E;IAC5E,wEAAwE;IACxE,0EAA0E;IAC1E,sEAAsE;IACtE,IAAI,gBAAgB,GAAkB,OAAO,CAAC,OAAO,EAAE,CAAC;IAExD,wEAAwE;IACxE,yEAAyE;IACzE,0EAA0E;IAC1E,8BAA8B;IAC9B,IAAI,kBAAkB,GAAG,CAAC,CAAC;IAC3B,MAAM,iBAAiB,GAAG,GAAG,EAAE;QAC7B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,IAAI,GAAG,GAAG,kBAAkB,GAAG,IAAI;YAAE,OAAO;QAC5C,kBAAkB,GAAG,GAAG,CAAC;QACzB,eAAe,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACzC,CAAC,CAAC;IAEF,0EAA0E;IAC1E,4EAA4E;IAC5E,0EAA0E;IAC1E,wEAAwE;IACxE,IAAI,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC;IACvC,MAAM,aAAa,GAAG,GAAG,EAAE;QACzB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,IAAI,GAAG,GAAG,cAAc,GAAG,IAAI;YAAE,OAAO;QACxC,cAAc,GAAG,GAAG,CAAC;QACrB,gBAAgB,CAAC,KAAK,CAAC;aACpB,IAAI,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;YACpB,IAAI,KAAK,CAAC,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;gBAC3C,gBAAgB,CAAC,GAAG,EAAE,KAAK,CAAC,MAAM,IAAI,MAAM,CAAC,CAAC;gBAC9C,OAAO;YACT,CAAC;YACD,kEAAkE;YAClE,uEAAuE;YACvE,2CAA2C;YAC3C,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;gBAC1B,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,KAAK,CAAC,CAAC;gBACzC,IAAI,MAAM,KAAK,IAAI,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;oBAC5C,gBAAgB,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;gBACrC,CAAC;YACH,CAAC;QACH,CAAC,CAAC;aACD,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACrB,CAAC,CAAC;IAEF,wEAAwE;IACxE,sEAAsE;IACtE,mEAAmE;IACnE,0DAA0D;IAC1D,MAAM,cAAc,GAAmC,WAAW,CAAC,GAAG,EAAE;QACtE,kBAAkB,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAC1C,aAAa,EAAE,CAAC;IAClB,CAAC,EAAE,IAAI,CAAC,CAAC;IACT,MAAM,aAAa,GAAG,uBAAuB,CAAC,OAAO,EAAE,aAAa,EAAE;QACpE,gBAAgB,EAAE,OAAO,EAAE,2BAA2B,KAAK,IAAI;QAC/D,kBAAkB,EAAE,OAAO,EAAE,kBAAkB,KAAK,IAAI;KACzD,CAAC,CAAC;IACH,MAAM,gBAAgB,GACpB,aAAa,GAAG,CAAC;QACf,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE;YACd,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,IAAI,KAAK,CAAC,MAAM,CAAC,OAAO;gBAAE,OAAO;YAC7D,YAAY,GAAG,IAAI,CAAC;YACpB,IAAI,CAAC;gBACH,IAAI,EAAE,eAAe;gBACrB,MAAM,EAAE,aAAa;aACtB,CAAC,CAAC;YACH,KAAK,CAAC,KAAK,EAAE,CAAC;QAChB,CAAC,EAAE,aAAa,CAAC;QACnB,CAAC,CAAC,IAAI,CAAC;IACX,IAAI,oBAAoB,GAAoB,IAAI,CAAC;IAEjD,MAAM,eAAe,GAAG,CAAC,KAAc,EAAE,KAA2B,EAAE,EAAE;QACtE,MAAM,SAAS,GACb,KAAK,YAAY,WAAW,CAAC,CAAC,CAAC,qBAAqB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAC1E,YAAY,CAAC,KAAK,EAAE;YAClB,KAAK,EAAE,2BAA2B;YAClC,IAAI,EAAE;gBACJ,MAAM,EAAE,mBAAmB;gBAC3B,KAAK;gBACL,SAAS,EAAE,GAAG,CAAC,MAAM;gBACrB,YAAY,EAAE,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO;gBAC7C,WAAW,EAAE,GAAG,CAAC,WAAW;gBAC5B,SAAS;aACV;YACD,KAAK,EAAE;gBACL,KAAK;gBACL,QAAQ;gBACR,UAAU,EAAE,GAAG,CAAC,MAAM,CAAC,MAAM;gBAC7B,SAAS,EAAE,GAAG,CAAC,SAAS;gBACxB,aAAa;aACd;YACD,QAAQ,EAAE;gBACR,QAAQ,EAAE;oBACR,KAAK;oBACL,QAAQ;oBACR,MAAM,EAAE,GAAG,CAAC,MAAM;oBAClB,KAAK;oBACL,UAAU,EAAE,GAAG,CAAC,MAAM,CAAC,MAAM;oBAC7B,SAAS,EAAE,GAAG,CAAC,SAAS;oBACxB,aAAa;oBACb,YAAY;oBACZ,WAAW,EAAE,GAAG,CAAC,WAAW;iBAC7B;aACF;SACF,CAAC,CAAC;IACL,CAAC,CAAC;IAEF,MAAM,YAAY,GAAG,CACnB,QAAkB,EAClB,OAA+C,EAChC,EAAE;QACjB,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAE1B,yDAAyD;QACzD,KAAK,MAAM,UAAU,IAAI,GAAG,CAAC,WAAW,EAAE,CAAC;YACzC,IAAI,CAAC;gBACH,UAAU,CAAC,QAAQ,CAAC,CAAC;YACvB,CAAC;YAAC,MAAM,CAAC;gBACP,GAAG,CAAC,WAAW,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;YACrC,CAAC;QACH,CAAC;QAED,oEAAoE;QACpE,wEAAwE;QACxE,wEAAwE;QACxE,iCAAiC;QACjC,iBAAiB,EAAE,CAAC;QAEpB,uEAAuE;QACvE,yEAAyE;QACzE,qEAAqE;QACrE,0EAA0E;QAC1E,wDAAwD;QACxD,MAAM,UAAU,GAAG,gBAAgB,CAAC,IAAI,CAAC,GAAG,EAAE,CAC5C,cAAc,CAAC,KAAK,EAAE,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CACpE,CAAC;QACF,gBAAgB,GAAG,UAAU,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAC9C,MAAM,WAAW,GAAG,UAAU,CAAC;QAC/B,IAAI,CAAC,OAAO,EAAE,uBAAuB,EAAE,CAAC;YACtC,WAAW,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAC9B,CAAC;QAED,aAAa,EAAE,CAAC;QAChB,OAAO,WAAW,CAAC;IACrB,CAAC,CAAC;IAEF,MAAM,IAAI,GAAG,CAAC,KAAqB,EAAE,EAAE;QACrC,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,IAAI,KAAK,CAAC,MAAM,CAAC,OAAO;YAAE,OAAO;QAE7D,MAAM,QAAQ,GAAa,EAAE,GAAG,EAAE,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC;QAC7D,IAAI,kBAAkB,CAAC,KAAK,CAAC,EAAE,CAAC;YAC9B,oBAAoB,GAAG,QAAQ,CAAC;YAChC,OAAO;QACT,CAAC;QAED,YAAY,CAAC,QAAQ,CAAC,CAAC;IACzB,CAAC,CAAC;IAEF,sEAAsE;IACtE,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC;SACzC,IAAI,CAAC,GAAG,EAAE;QACT,IAAI,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACzB,GAAG,CAAC,MAAM,GAAG,YAAY,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC;YACpD,OAAO;QACT,CAAC;QACD,GAAG,CAAC,MAAM,GAAG,WAAW,CAAC;IAC3B,CAAC,CAAC;SACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;QACb,iEAAiE;QACjE,IAAI,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACzB,GAAG,CAAC,MAAM,GAAG,YAAY,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC;YACpD,OAAO;QACT,CAAC;QACD,GAAG,CAAC,MAAM,GAAG,SAAS,CAAC;QACvB,IAAI,qBAAqB,CAAC,GAAG,CAAC,EAAE,CAAC;YAC/B,eAAe,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC9B,CAAC;QACD,MAAM,YAAY,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC;QAC7C,MAAM,SAAS,GACb,GAAG,YAAY,WAAW,CAAC,CAAC,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACtE,MAAM,OAAO,GACX,GAAG,YAAY,WAAW,CAAC,CAAC,CAAC,wBAAwB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACzE,IAAI,CAAC;YACH,IAAI,EAAE,OAAO;YACb,KAAK,EAAE,YAAY;YACnB,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACnC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/B,GAAG,CAAC,GAAG,YAAY,WAAW,IAAI,GAAG,CAAC,UAAU;gBAC9C,CAAC,CAAC,EAAE,UAAU,EAAE,GAAG,CAAC,UAAU,EAAE;gBAChC,CAAC,CAAC,EAAE,CAAC;SACR,CAAC,CAAC;IACL,CAAC,CAAC;SACD,OAAO,CAAC,KAAK,IAAI,EAAE;QAClB,gEAAgE;QAChE,mEAAmE;QACnE,kEAAkE;QAClE,kEAAkE;QAClE,gEAAgE;QAChE,sCAAsC;QAEtC,oEAAoE;QACpE,mEAAmE;QACnE,8DAA8D;QAC9D,IAAI,eAAe,GAAY,IAAI,CAAC;QACpC,IAAI,wBAAwB,GAAY,IAAI,CAAC;QAC7C,IACE,UAAU;YACV,CAAC,CAAC,GAAG,CAAC,MAAM,KAAK,SAAS,IAAI,GAAG,CAAC,WAAW,KAAK,aAAa,CAAC,EAChE,CAAC;YACD,IAAI,CAAC;gBACH,MAAM,aAAa,GAAc,oBAAoB;oBACnD,CAAC,CAAC,EAAE,GAAG,GAAG,EAAE,MAAM,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,oBAAoB,CAAC,EAAE;oBAC3D,CAAC,CAAC,GAAG,CAAC;gBACR,MAAM,UAAU,CAAC,aAAa,CAAC,CAAC;YAClC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,eAAe,GAAG,GAAG,CAAC;gBACtB,eAAe,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;gBACnC,OAAO,CAAC,KAAK,CACX,0CAA0C,EAC1C,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CACzC,CAAC;YACJ,CAAC;QACH,CAAC;QAED,kEAAkE;QAClE,oEAAoE;QACpE,kBAAkB;QAClB,MAAM,WAAW,GACf,GAAG,CAAC,MAAM,KAAK,SAAS;YACtB,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,GAAG,CAAC,MAAM,KAAK,SAAS,IAAI,eAAe;gBAC3C,CAAC,CAAC,SAAS;gBACX,CAAC,CAAC,WAAW,CAAC;QAEpB,qEAAqE;QACrE,mEAAmE;QACnE,qEAAqE;QACrE,0CAA0C;QAC1C,IAAI,WAAW,KAAK,WAAW,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;YAC7D,mEAAmE;YACnE,mDAAmD;YACnD,mEAAmE;YACnE,sEAAsE;YACtE,kEAAkE;YAClE,oEAAoE;YACpE,0DAA0D;YAC1D,gEAAgE;YAChE,kEAAkE;YAClE,qEAAqE;YACrE,wDAAwD;YACxD,MAAM,aAAa,GACjB,WAAW,KAAK,WAAW;gBACzB,CAAC,CAAC,CAAC,oBAAoB,EAAE,KAAK,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;gBACnD,CAAC,CAAC,oBAAoB,EAAE,KAAK,CAAC,IAAI,KAAK,OAAO;oBAC5C,CAAC,CAAC,oBAAoB,CAAC,KAAK;oBAC5B,CAAC,CAAC,oBAAoB,EAAE,KAAK,CAAC,IAAI,KAAK,eAAe;wBACpD,CAAC,CAAC,+DAA+D;4BAC/D,8DAA8D;4BAC9D,6DAA6D;4BAC7D,4DAA4D;4BAC5D,+DAA+D;4BAC/D,oDAAoD;4BACpD,oBAAoB,CAAC,KAAK;wBAC5B,CAAC,CAAC;4BACE,IAAI,EAAE,OAAO;4BACb,KAAK,EAAE,eAAe;gCACpB,CAAC,CAAC,oCAAoC;gCACtC,CAAC,CAAC,8BAA8B;yBACnC,CAAC;YACZ,MAAM,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAC/C,IAAI,CAAC,IAAI,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC7C,oEAAoE;gBACpE,kEAAkE;gBAClE,oEAAoE;gBACpE,8BAA8B;gBAC9B,MAAM,QAAQ,GAAa;oBACzB,GAAG,EAAE,GAAG,CAAC,MAAM,CAAC,MAAM;oBACtB,KAAK,EAAE,aAAa;iBACrB,CAAC;gBACF,IAAI,CAAC;oBACH,MAAM,YAAY,CAAC,QAAQ,EAAE,EAAE,uBAAuB,EAAE,IAAI,EAAE,CAAC,CAAC;gBAClE,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,wBAAwB,GAAG,GAAG,CAAC;oBAC/B,eAAe,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;oBACnC,OAAO,CAAC,KAAK,CACX,iDAAiD,EACjD,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CACzC,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;QACD,KAAK,MAAM,UAAU,IAAI,GAAG,CAAC,WAAW,EAAE,CAAC;YACzC,GAAG,CAAC,WAAW,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QACrC,CAAC;QAED,wDAAwD;QACxD,aAAa,CAAC,cAAc,CAAC,CAAC;QAC9B,IAAI,gBAAgB;YAAE,YAAY,CAAC,gBAAgB,CAAC,CAAC;QAErD,wEAAwE;QACxE,wEAAwE;QACxE,wDAAwD;QACxD,IAAI,CAAC;YACH,MAAM,gBAAgB,CAAC;YACvB,IAAI,CAAC,wBAAwB,EAAE,CAAC;gBAC9B,MAAM,wBAAwB,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;YACrD,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,8DAA8D;YAC9D,4BAA4B;QAC9B,CAAC;QAED,iEAAiE;QACjE,sEAAsE;QACtE,sEAAsE;QACtE,mEAAmE;QACnE,gEAAgE;QAChE,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;YAC9B,IAAI,SAA6B,CAAC;YAClC,IAAI,WAA+B,CAAC;YACpC,KAAK,IAAI,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAChD,MAAM,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAKxB,CAAC;gBACF,IAAI,EAAE,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;oBACxB,SAAS,GAAG,EAAE,CAAC,SAAS,CAAC;oBACzB,WAAW,GAAG,EAAE,CAAC,KAAK,IAAI,EAAE,CAAC,OAAO,CAAC;oBACrC,MAAM;gBACR,CAAC;YACH,CAAC;YACD,IAAI,eAAe,IAAI,CAAC,SAAS,EAAE,CAAC;gBAClC,SAAS,GAAG,kBAAkB,CAAC;gBAC/B,WAAW;oBACT,WAAW;wBACX,CAAC,eAAe,YAAY,KAAK;4BAC/B,CAAC,CAAC,eAAe,CAAC,OAAO;4BACzB,CAAC,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC;YACjC,CAAC;YACD,MAAM,WAAW,CAAC,KAAK,EAAE,SAAS,IAAI,SAAS,EAAE,WAAW,CAAC,CAAC;QAChE,CAAC;QAED,iEAAiE;QACjE,UAAU,CAAC,GAAG,EAAE;YACd,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACzB,IAAI,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,KAAK,EAAE,CAAC;gBACxC,WAAW,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC,EAAE,gBAAgB,CAAC,CAAC;QACrB,cAAc,CACZ,8BAA8B,EAAE,EAChC,4BAA4B,EAAE,CAC/B,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACpB,CAAC,CAAC,CAAC;IAEL,6DAA6D;IAC7D,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,UAAU,CAAC,QAAQ,CAAC;QAClC,IAAI,KAAK,EAAE,SAAS,EAAE,CAAC;YACrB,KAAK,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;QAC9B,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,0BAA0B;IAC5B,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,cAAc,CAC5B,KAAa,EACb,OAAe;IAEf,MAAM,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAClC,IAAI,GAAG,EAAE,CAAC;QACR,OAAO,iBAAiB,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IACzC,CAAC;IACD,qDAAqD;IACrD,OAAO,gBAAgB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;AAC1C,CAAC;AAED,uDAAuD;AACvD,SAAS,iBAAiB,CACxB,GAAc,EACd,OAAe;IAEf,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;IAClC,IAAI,aAAa,GAAuC,IAAI,CAAC;IAC7D,IAAI,SAAS,GAA0C,IAAI,CAAC;IAE5D,OAAO,IAAI,cAAc,CAAC;QACxB,KAAK,CAAC,UAAU;YACd,MAAM,IAAI,GAAG,GAAG,EAAE;gBAChB,IAAI,CAAC;oBACH,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC;gBACjE,CAAC;gBAAC,MAAM,CAAC;oBACP,IAAI,aAAa;wBAAE,GAAG,CAAC,WAAW,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;oBACzD,IAAI,SAAS;wBAAE,aAAa,CAAC,SAAS,CAAC,CAAC;gBAC1C,CAAC;YACH,CAAC,CAAC;YACF,IAAI,EAAE,CAAC;YACP,SAAS,GAAG,WAAW,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAEtC,sCAAsC;YACtC,KAAK,IAAI,CAAC,GAAG,OAAO,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACjD,IAAI,CAAC;oBACH,UAAU,CAAC,OAAO,CAChB,OAAO,CAAC,MAAM,CACZ,SAAS,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,CAClF,CACF,CAAC;gBACJ,CAAC;gBAAC,MAAM,CAAC;oBACP,OAAO;gBACT,CAAC;YACH,CAAC;YAED,4CAA4C;YAC5C,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;gBAC7B,IAAI,SAAS;oBAAE,aAAa,CAAC,SAAS,CAAC,CAAC;gBACxC,UAAU,CAAC,KAAK,EAAE,CAAC;gBACnB,OAAO;YACT,CAAC;YAED,2BAA2B;YAC3B,aAAa,GAAG,CAAC,KAAe,EAAE,EAAE;gBAClC,IAAI,CAAC;oBACH,UAAU,CAAC,OAAO,CAChB,OAAO,CAAC,MAAM,CACZ,SAAS,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,KAAK,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,CAAC,MAAM,CAClE,CACF,CAAC;oBACF,qCAAqC;oBACrC,IAAI,kBAAkB,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;wBACpC,GAAG,CAAC,WAAW,CAAC,MAAM,CAAC,aAAc,CAAC,CAAC;wBACvC,IAAI,SAAS;4BAAE,aAAa,CAAC,SAAS,CAAC,CAAC;wBACxC,UAAU,CAAC,KAAK,EAAE,CAAC;oBACrB,CAAC;gBACH,CAAC;gBAAC,MAAM,CAAC;oBACP,GAAG,CAAC,WAAW,CAAC,MAAM,CAAC,aAAc,CAAC,CAAC;gBACzC,CAAC;YACH,CAAC,CAAC;YAEF,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;QACrC,CAAC;QACD,MAAM;YACJ,gDAAgD;YAChD,IAAI,aAAa;gBAAE,GAAG,CAAC,WAAW,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;YACzD,IAAI,SAAS;gBAAE,aAAa,CAAC,SAAS,CAAC,CAAC;QAC1C,CAAC;KACF,CAAC,CAAC;AACL,CAAC;AAED,sDAAsD;AACtD,SAAS,gBAAgB,CACvB,KAAa,EACb,OAAe;IAEf,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;IAClC,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,IAAI,SAAS,GAAyC,IAAI,CAAC;IAC3D,IAAI,SAAS,GAA0C,IAAI,CAAC;IAE5D,OAAO,IAAI,cAAc,CAAC;QACxB,KAAK,CAAC,KAAK,CAAC,UAAU;YACpB,IAAI,OAAO,GAAG,OAAO,CAAC;YACtB,MAAM,IAAI,GAAG,GAAG,EAAE;gBAChB,IAAI,CAAC;oBACH,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC;gBACjE,CAAC;gBAAC,MAAM,CAAC;oBACP,SAAS,GAAG,IAAI,CAAC;oBACjB,IAAI,SAAS;wBAAE,aAAa,CAAC,SAAS,CAAC,CAAC;gBAC1C,CAAC;YACH,CAAC,CAAC;YACF,IAAI,EAAE,CAAC;YACP,SAAS,GAAG,WAAW,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAEtC,MAAM,IAAI,GAAG,KAAK,IAAI,EAAE;gBACtB,IAAI,SAAS;oBAAE,OAAO;gBACtB,IAAI,CAAC;oBACH,2BAA2B;oBAC3B,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;oBACvD,KAAK,MAAM,EAAE,GAAG,EAAE,SAAS,EAAE,IAAI,MAAM,EAAE,CAAC;wBACxC,gEAAgE;wBAChE,gEAAgE;wBAChE,kEAAkE;wBAClE,yDAAyD;wBACzD,OAAO,GAAG,GAAG,GAAG,CAAC,CAAC;wBAClB,IAAI,MAAW,CAAC;wBAChB,IAAI,CAAC;4BACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;wBACjC,CAAC;wBAAC,MAAM,CAAC;4BACP,SAAS;wBACX,CAAC;wBACD,IAAI,CAAC;4BACH,UAAU,CAAC,OAAO,CAChB,OAAO,CAAC,MAAM,CACZ,SAAS,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM,CAClD,CACF,CAAC;wBACJ,CAAC;wBAAC,MAAM,CAAC;4BACP,SAAS,GAAG,IAAI,CAAC;4BACjB,OAAO;wBACT,CAAC;wBAED,2BAA2B;wBAC3B,IAAI,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC;4BAC/B,IAAI,SAAS;gCAAE,aAAa,CAAC,SAAS,CAAC,CAAC;4BACxC,UAAU,CAAC,KAAK,EAAE,CAAC;4BACnB,OAAO;wBACT,CAAC;oBACH,CAAC;oBAED,gEAAgE;oBAChE,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;wBACxB,gEAAgE;wBAChE,kEAAkE;wBAClE,WAAW;wBACX,MAAM,WAAW,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;wBACzC,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,CAAC;wBACpC,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;4BACrC,kDAAkD;4BAClD,MAAM,WAAW,GAAG,MAAM,iBAAiB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;4BAC5D,KAAK,MAAM,EAAE,GAAG,EAAE,SAAS,EAAE,IAAI,WAAW,EAAE,CAAC;gCAC7C,wDAAwD;gCACxD,OAAO,GAAG,GAAG,GAAG,CAAC,CAAC;gCAClB,IAAI,MAAW,CAAC;gCAChB,IAAI,CAAC;oCACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;gCACjC,CAAC;gCAAC,MAAM,CAAC;oCACP,SAAS;gCACX,CAAC;gCACD,IAAI,CAAC;oCACH,UAAU,CAAC,OAAO,CAChB,OAAO,CAAC,MAAM,CACZ,SAAS,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM,CAClD,CACF,CAAC;gCACJ,CAAC;gCAAC,MAAM,CAAC;oCACP,SAAS,GAAG,IAAI,CAAC;oCACjB,OAAO;gCACT,CAAC;gCACD,IAAI,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC;oCAC/B,IAAI,SAAS;wCAAE,aAAa,CAAC,SAAS,CAAC,CAAC;oCACxC,UAAU,CAAC,KAAK,EAAE,CAAC;oCACnB,OAAO;gCACT,CAAC;4BACH,CAAC;4BACD,IAAI,GAAG,EAAE,MAAM,KAAK,SAAS,EAAE,CAAC;gCAC9B,IAAI,CAAC;oCACH,UAAU,CAAC,OAAO,CAChB,OAAO,CAAC,MAAM,CACZ,SAAS,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,MAAM,CAC9D,CACF,CAAC;gCACJ,CAAC;gCAAC,MAAM,CAAC;oCACP,SAAS,GAAG,IAAI,CAAC;oCACjB,OAAO;gCACT,CAAC;4BACH,CAAC;iCAAM,IAAI,GAAG,EAAE,MAAM,KAAK,WAAW,EAAE,CAAC;gCACvC,IAAI,CAAC;oCACH,UAAU,CAAC,OAAO,CAChB,OAAO,CAAC,MAAM,CACZ,SAAS,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,MAAM,CAC9D,CACF,CAAC;gCACJ,CAAC;gCAAC,MAAM,CAAC;oCACP,SAAS,GAAG,IAAI,CAAC;oCACjB,OAAO;gCACT,CAAC;4BACH,CAAC;iCAAM,IAAI,GAAG,EAAE,MAAM,KAAK,SAAS,EAAE,CAAC;gCACrC,6DAA6D;gCAC7D,6DAA6D;gCAC7D,4DAA4D;gCAC5D,0DAA0D;gCAC1D,4DAA4D;gCAC5D,wDAAwD;gCACxD,2DAA2D;gCAC3D,mDAAmD;gCACnD,MAAM,sBAAsB,CAC1B,KAAK,EACL,qBAAqB,CACtB,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;gCAClB,IAAI,CAAC;oCACH,UAAU,CAAC,OAAO,CAChB,OAAO,CAAC,MAAM,CACZ,SAAS,IAAI,CAAC,SAAS,CAAC;wCACtB,GAAG,qBAAqB;wCACxB,GAAG,EAAE,OAAO;qCACb,CAAC,MAAM,CACT,CACF,CAAC;gCACJ,CAAC;gCAAC,MAAM,CAAC;oCACP,SAAS,GAAG,IAAI,CAAC;oCACjB,OAAO;gCACT,CAAC;4BACH,CAAC;4BACD,IAAI,SAAS;gCAAE,aAAa,CAAC,SAAS,CAAC,CAAC;4BACxC,UAAU,CAAC,KAAK,EAAE,CAAC;4BACnB,OAAO;wBACT,CAAC;oBACH,CAAC;oBAED,qBAAqB;oBACrB,IAAI,CAAC,SAAS,EAAE,CAAC;wBACf,SAAS,GAAG,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;oBACpC,CAAC;gBACH,CAAC;gBAAC,MAAM,CAAC;oBACP,2BAA2B;oBAC3B,IAAI,CAAC;wBACH,IAAI,SAAS;4BAAE,aAAa,CAAC,SAAS,CAAC,CAAC;wBACxC,UAAU,CAAC,KAAK,EAAE,CAAC;oBACrB,CAAC;oBAAC,MAAM,CAAC,CAAA,CAAC;gBACZ,CAAC;YACH,CAAC,CAAC;YAEF,yCAAyC;YACzC,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,CAAC;gBACpC,IAAI,CAAC,GAAG,EAAE,CAAC;oBACT,IAAI,SAAS;wBAAE,aAAa,CAAC,SAAS,CAAC,CAAC;oBACxC,UAAU,CAAC,KAAK,EAAE,CAAC;oBACnB,OAAO;gBACT,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,UAAU,CAAC,KAAK,EAAE,CAAC;gBACnB,OAAO;YACT,CAAC;YAED,MAAM,IAAI,EAAE,CAAC;QACf,CAAC;QACD,MAAM;YACJ,SAAS,GAAG,IAAI,CAAC;YACjB,IAAI,SAAS;gBAAE,YAAY,CAAC,SAAS,CAAC,CAAC;YACvC,IAAI,SAAS;gBAAE,aAAa,CAAC,SAAS,CAAC,CAAC;QAC1C,CAAC;KACF,CAAC,CAAC;AACL,CAAC;AAED,wEAAwE;AACxE,MAAM,UAAU,qBAAqB,CAAC,QAAgB;IACpD,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IACxC,IAAI,KAAK,EAAE,CAAC;QACV,MAAM,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAClC,IAAI,GAAG;YAAE,OAAO,GAAG,CAAC;IACtB,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAAC,QAAgB;IAgB/D,uEAAuE;IACvE,yEAAyE;IACzE,qDAAqD;IACrD,MAAM,MAAM,GAAG,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IAC/C,IAAI,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;QACxE,OAAO;YACL,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,mEAAmE;YACnE,8DAA8D;YAC9D,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE;YACvB,kEAAkE;YAClE,kEAAkE;YAClE,mEAAmE;YACnE,kEAAkE;YAClE,gCAAgC;YAChC,cAAc,EAAE,MAAM,mBAAmB,CAAC,MAAM,CAAC,KAAK,CAAC;SACxD,CAAC;IACJ,CAAC;IACD,yEAAyE;IACzE,oEAAoE;IACpE,wEAAwE;IACxE,sEAAsE;IACtE,sEAAsE;IACtE,qEAAqE;IACrE,2DAA2D;IAC3D,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,QAAQ,EAAE,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC,CAAC;QACzE,IAAI,CAAC,MAAM;YAAE,OAAO,IAAI,CAAC;QACzB,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAChC,0EAA0E;YAC1E,6EAA6E;YAC7E,uEAAuE;YACvE,2EAA2E;YAC3E,wEAAwE;YACxE,0EAA0E;YAC1E,wEAAwE;YACxE,2EAA2E;YAC3E,IAAI,MAAM,CAAC,YAAY,KAAK,YAAY,EAAE,CAAC;gBACzC,MAAM,SAAS,GAAG,MAAM,0BAA0B,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,KAAK,CACjE,GAAG,EAAE,CAAC,KAAK,CACZ,CAAC;gBACF,IAAI,SAAS;oBAAE,OAAO,IAAI,CAAC;YAC7B,CAAC;YACD,iEAAiE;YACjE,gEAAgE;YAChE,0BAA0B;YAC1B,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;YAC/D,IAAI,MAAM;gBAAE,OAAO,IAAI,CAAC;YACxB,OAAO;gBACL,KAAK,EAAE,MAAM,CAAC,EAAE;gBAChB,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,EAAE;gBAClC,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC,SAAS;gBACnD,cAAc,EAAE,MAAM,CAAC,cAAc;gBACrC,YAAY,EAAE,MAAM,CAAC,YAAY;gBACjC,SAAS,EAAE,MAAM,CAAC,SAAS;aAC5B,CAAC;QACJ,CAAC;QACD,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YACjE,qEAAqE;YACrE,mEAAmE;YACnE,+DAA+D;YAC/D,EAAE;YACF,kEAAkE;YAClE,mEAAmE;YACnE,mEAAmE;YACnE,gEAAgE;YAChE,oEAAoE;YACpE,gEAAgE;YAChE,yCAAyC;YACzC,MAAM,WAAW,GACf,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC,SAAS,CAAC;YAC/D,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW,CAAC;YAC7C,IAAI,WAAW,GAAG,gCAAgC;gBAAE,OAAO,IAAI,CAAC;YAChE,OAAO;gBACL,KAAK,EAAE,MAAM,CAAC,EAAE;gBAChB,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,EAAE;gBAClC,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC,SAAS;gBACnD,cAAc,EAAE,MAAM,CAAC,cAAc;gBACrC,YAAY,EAAE,MAAM,CAAC,YAAY;gBACjC,SAAS,EAAE,MAAM,CAAC,SAAS;aAC5B,CAAC;QACJ,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,2BAA2B;IAC7B,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,KAAK,UAAU,mBAAmB,CAAC,KAAa;IAC9C,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,CAAC;QACpC,IAAI,CAAC,GAAG;YAAE,OAAO,IAAI,CAAC;QACtB,sEAAsE;QACtE,oDAAoD;QACpD,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,GAAG,CAAC,QAAQ,EAAE;YAClD,eAAe,EAAE,IAAI;SACtB,CAAC,CAAC;QACH,IAAI,QAAQ,IAAI,QAAQ,CAAC,EAAE,KAAK,KAAK;YAAE,OAAO,QAAQ,CAAC,cAAc,CAAC;QACtE,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,sBAAsB;AACtB,MAAM,UAAU,MAAM,CAAC,KAAa;IAClC,OAAO,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC;AACvC,CAAC;AAED,gDAAgD;AAChD,MAAM,UAAU,QAAQ,CAAC,KAAa,EAAE,MAAM,GAAW,MAAM;IAC7D,MAAM,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAClC,IAAI,GAAG,EAAE,CAAC;QACR,gBAAgB,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IAChC,CAAC;IACD,mEAAmE;IACnE,cAAc,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IAC9C,OAAO,CAAC,CAAC,GAAG,CAAC;AACf,CAAC;AAED,oEAAoE;AACpE,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC","sourcesContent":["import { captureError } from \"../server/capture-error.js\";\nimport { isLlmCredentialError } from \"./engine/credential-errors.js\";\nimport { EngineError } from \"./engine/types.js\";\nimport {\n insertRun,\n insertRunEvent,\n updateRunStatusIfRunning,\n markRunAborted,\n getRunAbortState,\n getRunStatus,\n getRunEventsSince,\n getRunById,\n getRunByThread,\n cleanupOldRuns,\n updateRunHeartbeat,\n bumpRunProgress,\n reapIfStale,\n reapUnclaimedBackgroundRun,\n ensureTerminalRunEvent,\n setRunError,\n STALE_RUN_ERROR_EVENT,\n} from \"./run-store.js\";\nimport type { AgentChatEvent, RunEvent, RunStatus } from \"./types.js\";\n\nexport interface ActiveRun {\n runId: string;\n threadId: string;\n /** Logical-turn identity (see StartRunOptions.turnId). Defaults to runId. */\n turnId: string;\n events: RunEvent[];\n status: RunStatus;\n subscribers: Set<(event: RunEvent) => void>;\n abort: AbortController;\n abortReason?: string;\n startedAt: number;\n}\n\nconst activeRuns = new Map<string, ActiveRun>();\nconst threadToRun = new Map<string, string>();\n\n/** How long to keep completed runs in memory before cleanup (5 min) */\nconst CLEANUP_DELAY_MS = 5 * 60 * 1000;\n\n/**\n * Default run chunk budget for hosted/serverless deploys.\n *\n * This MUST fire before the two upstream hard walls that otherwise kill a run\n * mid-turn with no chance to hand off:\n * 1. The Builder model gateway hard-caps a single model call at 45s\n * (builder-engine.ts MAX_BUILDER_GATEWAY_TIMEOUT_MS) — not raisable.\n * 2. Serverless functions are hard-killed around 60-65s (the heartbeat then\n * reaps the row as a stale_run).\n * Production data showed every cutoff landing in the 44-70s window with ZERO\n * auto_continue events ever emitted — i.e. the old 45s default raced the 45s\n * gateway and lost, and per-template overrides (e.g. 240_000) pushed it past\n * BOTH walls so it could never fire. 40s leaves ~5s of headroom under the\n * gateway wall to abort, persist the partial turn, write the terminal event,\n * and emit a clean auto_continue so the client resumes seamlessly.\n */\nexport const DEFAULT_HOSTED_RUN_SOFT_TIMEOUT_MS = 40_000;\n\n/**\n * Hard ceiling for the hosted soft timeout. On a hosted runtime the\n * auto_continue soft timeout can never usefully exceed this — the gateway\n * (45s) / function (~60s) walls kill the run first, so a larger configured or\n * env value just guarantees the cutoff is a hard error instead of a graceful\n * hand-off. Any resolved value above this is clamped down when hosted. Local\n * dev (non-hosted) is left alone so long-running local turns aren't chunked.\n *\n * IMPORTANT: this clamp is for the INTERACTIVE / foreground path and must NOT\n * be raised. The foreground POST still rides a synchronous serverless function\n * (~60-65s wall), so 40s remains correct there. The only sanctioned exception\n * is the opt-in `backgroundFunction` mode (see\n * `BACKGROUND_SOFT_TIMEOUT_CEILING_MS`), which runs inside a Netlify background\n * function (no ~60s wall, 15-min budget) and therefore can safely outlast 40s.\n */\nexport const HOSTED_SOFT_TIMEOUT_CEILING_MS = 40_000;\n\n/**\n * Hard ceiling for the soft timeout when a run executes inside a Netlify\n * background function (any deployed function whose name ends in `-background`).\n * Background functions return 202 immediately and run detached for up to 15\n * minutes, so the ~60s synchronous function wall that 40s defends against does\n * NOT apply. 13 minutes leaves ~2 min of headroom under Netlify's 15-min hard\n * kill to abort, persist the partial turn, write the terminal event, and (for\n * the rare >13-min turn) self-fire another background continuation.\n *\n * This ceiling is used ONLY when a caller explicitly opts in with\n * `backgroundFunction: true`. It does not change the foreground/interactive\n * ceiling and does not fire unless the durable-background path dispatched the\n * run into a background function. Per the design doc Guardrail, the 40s\n * interactive clamp stays correct for every non-background run.\n */\nexport const BACKGROUND_SOFT_TIMEOUT_CEILING_MS = 13 * 60_000; // 780_000\n\n/**\n * Default soft-timeout budget for a background-function run when the caller\n * does not pass an explicit `softTimeoutMs`. Same value as the ceiling — we\n * want a background turn to use nearly its whole 15-min budget before handing\n * off to a chained background continuation.\n */\nexport const DEFAULT_BACKGROUND_RUN_SOFT_TIMEOUT_MS =\n BACKGROUND_SOFT_TIMEOUT_CEILING_MS;\n\n/** Default SQL retention for completed run event logs (24 hours). */\nexport const DEFAULT_COMPLETED_RUN_RETENTION_MS = 24 * 60 * 60 * 1000;\n\n/**\n * Default SQL retention for errored/aborted run event logs (7 days). Kept\n * longer than completed runs so cut-off / failed chats survive for pattern\n * analysis (listErroredRuns) — these are rare and small, and they are exactly\n * the runs we need to study to keep hardening reliability.\n */\nexport const DEFAULT_ERRORED_RUN_RETENTION_MS = 7 * 24 * 60 * 60 * 1000;\n\n/**\n * How recently a terminal run must have started for `/runs/active` to surface\n * it. Reconnect after this window won't replay the run — typical real-world\n * disconnects resolve in seconds, so 10 minutes is generous while keeping us\n * from resurrecting ancient turns when the user reopens an old thread.\n */\nexport const TERMINAL_RUN_RECONNECT_WINDOW_MS = 10 * 60 * 1000;\n\nconst PROVIDER_RATE_LIMITED_ERROR_CODE = \"provider_rate_limited\";\n\nfunction getRunErrorMessage(err: unknown): string {\n if (\n typeof err === \"object\" &&\n err !== null &&\n \"message\" in err &&\n typeof err.message === \"string\" &&\n err.message.trim().length > 0\n ) {\n return err.message;\n }\n return \"Unknown error\";\n}\n\nfunction getEngineRunErrorCode(err: EngineError): string | undefined {\n if (err.errorCode) return err.errorCode;\n if (err.statusCode === 429) return PROVIDER_RATE_LIMITED_ERROR_CODE;\n return undefined;\n}\n\nfunction getEngineRunErrorDetails(err: EngineError): string | undefined {\n if (err.statusCode === 429) return err.message;\n return undefined;\n}\n\nfunction shouldCaptureRunError(err: unknown): boolean {\n if (!(err instanceof EngineError)) return true;\n const errorCode = getEngineRunErrorCode(err);\n if (isLlmCredentialError(err, errorCode)) return false;\n if (err.statusCode === 401 || err.statusCode === 403) return false;\n if (/^40[13] status code\\b/i.test(err.message)) return false;\n if (err.message.trim().toLowerCase() === \"connection error.\") return false;\n if (!errorCode) return true;\n const normalizedCode = errorCode.toLowerCase();\n return (\n !normalizedCode.startsWith(\"credits-limit\") &&\n normalizedCode !== \"builder_gateway_network_error\" &&\n normalizedCode !== \"provider_rate_limited\" &&\n normalizedCode !== \"rate_limit_exceeded\"\n );\n}\n\nexport interface StartRunOptions {\n /** Optional internal run chunk budget. When reached, the framework emits an\n * auto-continuation signal instead of a user-facing timeout. Leave unset for\n * no framework-imposed run timeout. */\n softTimeoutMs?: number;\n /** Opt into the hosted/serverless default chunk budget. Only callers with\n * automatic continuation support should enable this. */\n useHostedSoftTimeoutDefault?: boolean;\n /** Stable identity for the logical assistant turn this run belongs to. A\n * turn may span several continuation runs (each chunk is its own run); they\n * share one `turnId` so the durable assistant message can be folded across\n * them instead of dropped per-run. Defaults to the runId (turn == run). */\n turnId?: string;\n /**\n * Opt into the durable-background-function soft-timeout regime for THIS run\n * only. When true, `resolveRunSoftTimeoutMs` lifts the hosted ceiling from\n * 40s to ~13min (`BACKGROUND_SOFT_TIMEOUT_CEILING_MS`) because the run is\n * executing inside a Netlify background function (no ~60s wall). Off by\n * default — the foreground/interactive path never sets this, so its 40s\n * clamp is unchanged. See the design doc + the durable-background dispatch\n * decision in production-agent.ts.\n */\n backgroundFunction?: boolean;\n}\n\nexport interface ResolveRunSoftTimeoutOptions {\n useHostedDefault?: boolean;\n /**\n * Resolve the soft timeout for a run executing inside a Netlify background\n * function. Lifts the hosted clamp to `BACKGROUND_SOFT_TIMEOUT_CEILING_MS`\n * (~13min) for this invocation only and, when no override/env is supplied,\n * defaults to `DEFAULT_BACKGROUND_RUN_SOFT_TIMEOUT_MS`. Does NOT change the\n * foreground ceiling. Off by default.\n */\n backgroundFunction?: boolean;\n}\n\nfunction isHostedRuntime(): boolean {\n if (\n process.env.NETLIFY &&\n process.env.NETLIFY !== \"false\" &&\n process.env.NETLIFY_LOCAL !== \"true\"\n ) {\n return true;\n }\n if (\n process.env.AWS_LAMBDA_FUNCTION_NAME &&\n process.env.NETLIFY_LOCAL !== \"true\"\n ) {\n return true;\n }\n return Boolean(\n process.env.CF_PAGES ||\n process.env.VERCEL ||\n process.env.VERCEL_ENV ||\n process.env.RENDER ||\n process.env.FLY_APP_NAME ||\n process.env.K_SERVICE,\n );\n}\n\nexport function resolveRunSoftTimeoutMs(\n overrideMs?: number,\n options?: ResolveRunSoftTimeoutOptions,\n): number {\n const hosted = isHostedRuntime();\n const background = options?.backgroundFunction === true;\n // The interactive/foreground ceiling is 40s — the synchronous serverless\n // function wall. A background-function run (opt-in only) has no ~60s wall, so\n // it is allowed to outlast that and is clamped to the larger 13-min ceiling\n // instead. The 40s clamp for non-background hosted runs is unchanged.\n const ceiling = background\n ? BACKGROUND_SOFT_TIMEOUT_CEILING_MS\n : HOSTED_SOFT_TIMEOUT_CEILING_MS;\n // A configured/env soft timeout that exceeds the upstream walls can never\n // actually fire (the gateway/function kills the run first), so clamp it down\n // on hosted runtimes. This is what makes auto_continue reach the client\n // instead of the run dying as builder_gateway_timeout / stale_run. `0` means\n // \"disabled\" and is never clamped up.\n const clampHosted = (ms: number): number =>\n hosted && ms > ceiling ? ceiling : ms;\n\n if (typeof overrideMs === \"number\" && Number.isFinite(overrideMs)) {\n return clampHosted(Math.max(0, overrideMs));\n }\n const envValue = process.env.AGENT_RUN_SOFT_TIMEOUT_MS;\n if (envValue !== undefined) {\n const raw = Number(envValue);\n if (Number.isFinite(raw) && raw >= 0) return clampHosted(raw);\n }\n // A background-function run uses the full background budget by default; the\n // foreground default (40s) is unchanged.\n if (background) {\n return hosted ? DEFAULT_BACKGROUND_RUN_SOFT_TIMEOUT_MS : 0;\n }\n return options?.useHostedDefault && hosted\n ? DEFAULT_HOSTED_RUN_SOFT_TIMEOUT_MS\n : 0;\n}\n\nexport function resolveCompletedRunRetentionMs(): number {\n const envValue = process.env.AGENT_RUN_RETENTION_MS;\n if (envValue !== undefined) {\n const raw = Number(envValue);\n if (Number.isFinite(raw) && raw >= 0) return raw;\n }\n return DEFAULT_COMPLETED_RUN_RETENTION_MS;\n}\n\nexport function resolveErroredRunRetentionMs(): number {\n const envValue = process.env.AGENT_ERRORED_RUN_RETENTION_MS;\n if (envValue !== undefined) {\n const raw = Number(envValue);\n if (Number.isFinite(raw) && raw >= 0) return raw;\n }\n return DEFAULT_ERRORED_RUN_RETENTION_MS;\n}\n\nfunction isTerminalRunEvent(event: AgentChatEvent): boolean {\n return (\n event.type === \"done\" ||\n event.type === \"error\" ||\n event.type === \"missing_api_key\" ||\n event.type === \"loop_limit\" ||\n event.type === \"auto_continue\"\n );\n}\n\nfunction abortInMemoryRun(run: ActiveRun, reason: string = \"user\") {\n run.abortReason = reason;\n run.status = \"aborted\";\n if (threadToRun.get(run.threadId) === run.runId) {\n threadToRun.delete(run.threadId);\n }\n run.abort.abort(reason);\n for (const subscriber of run.subscribers) {\n try {\n subscriber({ seq: run.events.length, event: { type: \"done\" } });\n } catch {\n // ignore — subscriber is being removed below\n }\n }\n run.subscribers.clear();\n}\n\n/**\n * Start a new agent run in the background.\n * `runFn` receives a `send` callback and an `AbortSignal`.\n * The run continues even if all SSE subscribers disconnect.\n *\n * Events are persisted to SQL for cross-isolate access (Cloudflare Workers).\n */\nexport function startRun(\n runId: string,\n threadId: string,\n runFn: (\n send: (event: AgentChatEvent) => void,\n signal: AbortSignal,\n ) => Promise<void>,\n onComplete?: (run: ActiveRun) => void | Promise<void>,\n options?: StartRunOptions,\n): ActiveRun {\n // If there's already a run for this thread, abort it\n const existingRunId = threadToRun.get(threadId);\n if (existingRunId) {\n abortRun(existingRunId);\n }\n\n const abort = new AbortController();\n let softTimedOut = false;\n const run: ActiveRun = {\n runId,\n threadId,\n turnId: options?.turnId ?? runId,\n events: [],\n status: \"running\",\n subscribers: new Set(),\n abort,\n startedAt: Date.now(),\n };\n\n activeRuns.set(runId, run);\n threadToRun.set(threadId, runId);\n\n // Persist run to SQL without blocking the response. Keep the promise so\n // final status cannot race ahead of a slow initial INSERT and then get\n // overwritten by a late row stuck at status='running'.\n const insertRunPromise = insertRun(runId, threadId, options?.turnId).catch(\n () => {},\n );\n\n // Per-run event persistence chain: events are chained so SQL inserts commit\n // in seq order. Without this, a fast seq=5 commit before a slow seq=4 means\n // the SQL poller advances its cursor past seq=4 (lastSeq = 5+1 = 6) and\n // reconnecting clients permanently miss that event (silent gap). Chaining\n // per run ensures order without blocking the fast in-memory SSE path.\n let persistenceChain: Promise<void> = Promise.resolve();\n\n // Throttle the durable progress timestamp to at most once per second so\n // a chatty token-by-token stream doesn't translate into one DB write per\n // chunk. The stuck-detector threshold is on the order of tens of seconds,\n // so 1s resolution is plenty.\n let lastProgressBumpAt = 0;\n const bumpProgressIfDue = () => {\n const now = Date.now();\n if (now - lastProgressBumpAt < 1000) return;\n lastProgressBumpAt = now;\n bumpRunProgress(runId).catch(() => {});\n };\n\n // Periodic SQL abort check interval (for cross-isolate abort on Workers).\n // Also self-aborts when our row is no longer status='running' — catches the\n // false-stale-reap zombie scenario where the reaper flipped the row while\n // this isolate was briefly unable to heartbeat (DB latency / GC pause).\n let lastAbortCheck = Date.now() - 3000;\n const checkSqlAbort = () => {\n const now = Date.now();\n if (now - lastAbortCheck < 3000) return;\n lastAbortCheck = now;\n getRunAbortState(runId)\n .then(async (state) => {\n if (state.aborted && !abort.signal.aborted) {\n abortInMemoryRun(run, state.reason ?? \"user\");\n return;\n }\n // If the row is no longer 'running' (reaped / replaced) and we're\n // still executing, self-abort so we stop executing and don't overwrite\n // the newer state with our terminal write.\n if (!abort.signal.aborted) {\n const status = await getRunStatus(runId);\n if (status !== null && status !== \"running\") {\n abortInMemoryRun(run, \"displaced\");\n }\n }\n })\n .catch(() => {});\n };\n\n // Heartbeat: bump heartbeat_at every 1.5s so watchers can detect a dead\n // producer (process crash, HMR restart, isolate eviction) quickly and\n // reap the row. Paired with RUN_STALE_MS (6s) — 4x the interval to\n // tolerate transient DB slowness without false positives.\n const heartbeatTimer: ReturnType<typeof setInterval> = setInterval(() => {\n updateRunHeartbeat(runId).catch(() => {});\n checkSqlAbort();\n }, 1500);\n const softTimeoutMs = resolveRunSoftTimeoutMs(options?.softTimeoutMs, {\n useHostedDefault: options?.useHostedSoftTimeoutDefault === true,\n backgroundFunction: options?.backgroundFunction === true,\n });\n const softTimeoutTimer =\n softTimeoutMs > 0\n ? setTimeout(() => {\n if (run.status !== \"running\" || abort.signal.aborted) return;\n softTimedOut = true;\n send({\n type: \"auto_continue\",\n reason: \"run_timeout\",\n });\n abort.abort();\n }, softTimeoutMs)\n : null;\n let pendingTerminalEvent: RunEvent | null = null;\n\n const captureRunError = (error: unknown, phase: \"run\" | \"completion\") => {\n const errorCode =\n error instanceof EngineError ? getEngineRunErrorCode(error) : undefined;\n captureError(error, {\n route: \"/_agent-native/agent-chat\",\n tags: {\n source: \"agent-run-manager\",\n phase,\n runStatus: run.status,\n softTimedOut: softTimedOut ? \"true\" : \"false\",\n abortReason: run.abortReason,\n errorCode,\n },\n extra: {\n runId,\n threadId,\n eventCount: run.events.length,\n startedAt: run.startedAt,\n softTimeoutMs,\n },\n contexts: {\n agentRun: {\n runId,\n threadId,\n status: run.status,\n phase,\n eventCount: run.events.length,\n startedAt: run.startedAt,\n softTimeoutMs,\n softTimedOut,\n abortReason: run.abortReason,\n },\n },\n });\n };\n\n const emitRunEvent = (\n runEvent: RunEvent,\n options?: { surfacePersistenceError?: boolean },\n ): Promise<void> => {\n run.events.push(runEvent);\n\n // Notify in-memory subscribers (same isolate, fast path)\n for (const subscriber of run.subscribers) {\n try {\n subscriber(runEvent);\n } catch {\n run.subscribers.delete(subscriber);\n }\n }\n\n // Bump the durable progress timestamp. Distinct from the heartbeat:\n // heartbeat = \"process is up\", progress = \"real work is happening.\" The\n // gap between them is what the client-side stuck-detector reads to tell\n // a hung run from a healthy one.\n bumpProgressIfDue();\n\n // Persist event to SQL. Events are chained through persistenceChain so\n // inserts commit in seq order — an out-of-order commit would advance the\n // SQL poller's cursor past the slow row, permanently dropping it for\n // reconnecting clients. Terminal events surface persistence errors so the\n // caller can decide how to handle a failed final write.\n const thisInsert = persistenceChain.then(() =>\n insertRunEvent(runId, runEvent.seq, JSON.stringify(runEvent.event)),\n );\n persistenceChain = thisInsert.catch(() => {});\n const persistence = thisInsert;\n if (!options?.surfacePersistenceError) {\n persistence.catch(() => {});\n }\n\n checkSqlAbort();\n return persistence;\n };\n\n const send = (event: AgentChatEvent) => {\n if (run.status === \"aborted\" && abort.signal.aborted) return;\n\n const runEvent: RunEvent = { seq: run.events.length, event };\n if (isTerminalRunEvent(event)) {\n pendingTerminalEvent = runEvent;\n return;\n }\n\n emitRunEvent(runEvent);\n };\n\n // Run in background — intentionally detached from any HTTP connection\n const runPromise = runFn(send, abort.signal)\n .then(() => {\n if (abort.signal.aborted) {\n run.status = softTimedOut ? \"completed\" : \"aborted\";\n return;\n }\n run.status = \"completed\";\n })\n .catch((err) => {\n // Don't surface abort errors — the run was intentionally stopped\n if (abort.signal.aborted) {\n run.status = softTimedOut ? \"completed\" : \"aborted\";\n return;\n }\n run.status = \"errored\";\n if (shouldCaptureRunError(err)) {\n captureRunError(err, \"run\");\n }\n const errorMessage = getRunErrorMessage(err);\n const errorCode =\n err instanceof EngineError ? getEngineRunErrorCode(err) : undefined;\n const details =\n err instanceof EngineError ? getEngineRunErrorDetails(err) : undefined;\n send({\n type: \"error\",\n error: errorMessage,\n ...(errorCode ? { errorCode } : {}),\n ...(details ? { details } : {}),\n ...(err instanceof EngineError && err.upgradeUrl\n ? { upgradeUrl: err.upgradeUrl }\n : {}),\n });\n })\n .finally(async () => {\n // Ordering matters here — this is the atomic-complete boundary.\n // Invariant: once agent_runs.status flips to \"completed\"/\"errored\"\n // in SQL, thread_data for this turn is already durable. This lets\n // reconnecting clients trust the simple rule \"status != running →\n // fetch thread_data\" without polling/retrying for a race window\n // where onComplete was still pending.\n\n // 1. Await the completion callback (thread_data save). Heartbeat is\n // still ticking so the run doesn't look stale to any concurrent\n // /runs/active check while we wait for SQL writes to land.\n let completionError: unknown = null;\n let terminalPersistenceError: unknown = null;\n if (\n onComplete &&\n !(run.status === \"aborted\" && run.abortReason === \"no_progress\")\n ) {\n try {\n const completionRun: ActiveRun = pendingTerminalEvent\n ? { ...run, events: [...run.events, pendingTerminalEvent] }\n : run;\n await onComplete(completionRun);\n } catch (err) {\n completionError = err;\n captureRunError(err, \"completion\");\n console.error(\n \"[run-manager] onComplete callback error:\",\n err instanceof Error ? err.message : err,\n );\n }\n }\n\n // 2. Compute final status. If the completion callback threw, we'd\n // rather mark the run errored than claim success with incomplete\n // thread_data.\n const finalStatus =\n run.status === \"aborted\"\n ? \"aborted\"\n : run.status === \"errored\" || completionError\n ? \"errored\"\n : \"completed\";\n\n // 3. Emit the terminal event only after thread_data is durable. Live\n // SSE clients close on this event and usually fetch thread_data\n // immediately, so emitting it earlier recreates the final-message\n // race this manager is meant to avoid.\n if (finalStatus === \"completed\" || finalStatus === \"errored\") {\n // Choose the terminal event payload (done / the stashed terminal /\n // a synthesized error). NOTE: the `seq` carried by\n // `pendingTerminalEvent` was captured by `send()` at stash time as\n // `run.events.length` and is NOT authoritative — if the runFn emitted\n // any more events before it actually stopped on the abort signal,\n // those events were pushed and reused that same seq. Persisting the\n // terminal event with the stale seq would collide with an\n // already-persisted streaming event and get silently dropped by\n // insertRunEvent's `ON CONFLICT (run_id, seq) DO NOTHING`, so the\n // client would never see the terminal/continuation signal. We always\n // re-stamp the seq at emit time (max-seq+1) just below.\n const terminalEvent: AgentChatEvent =\n finalStatus === \"completed\"\n ? (pendingTerminalEvent?.event ?? { type: \"done\" })\n : pendingTerminalEvent?.event.type === \"error\"\n ? pendingTerminalEvent.event\n : pendingTerminalEvent?.event.type === \"auto_continue\"\n ? // The run was checkpointed at a soft-timeout/loop boundary and\n // is recoverable: the partial turn is in agent_run_events and\n // the continuation run will re-attempt the thread_data save.\n // Even though the completion save failed (finalStatus stays\n // \"errored\" for SQL/diagnostics), re-emit the auto_continue so\n // the client resumes instead of seeing a dead chat.\n pendingTerminalEvent.event\n : {\n type: \"error\",\n error: completionError\n ? \"Agent response could not be saved.\"\n : \"Agent run ended unexpectedly\",\n };\n const last = run.events[run.events.length - 1];\n if (!last || !isTerminalRunEvent(last.event)) {\n // Assign the seq at EMIT time, not at stash time. `run.events` is a\n // contiguous 0-based log, so `run.events.length` is the next free\n // seq and can never collide with an event that was pushed after the\n // terminal event was stashed.\n const terminal: RunEvent = {\n seq: run.events.length,\n event: terminalEvent,\n };\n try {\n await emitRunEvent(terminal, { surfacePersistenceError: true });\n } catch (err) {\n terminalPersistenceError = err;\n captureRunError(err, \"completion\");\n console.error(\n \"[run-manager] terminal event persistence error:\",\n err instanceof Error ? err.message : err,\n );\n }\n }\n }\n for (const subscriber of run.subscribers) {\n run.subscribers.delete(subscriber);\n }\n\n // 4. Stop the heartbeat — all liveness writes are done.\n clearInterval(heartbeatTimer);\n if (softTimeoutTimer) clearTimeout(softTimeoutTimer);\n\n // 5. Persist final status to SQL. Use the conditional write so a zombie\n // run (reaped or displaced while executing) cannot clobber the newer\n // status written by the reaper or a replacement run.\n try {\n await insertRunPromise;\n if (!terminalPersistenceError) {\n await updateRunStatusIfRunning(runId, finalStatus);\n }\n } catch {\n // Best-effort — reapIfStale will eventually clean this up via\n // the heartbeat-stale path.\n }\n\n // 5b. Record terminal failure classification for errored runs so\n // cut-off / failed chats are queryable for pattern analysis. Read\n // the actual error event the run emitted (errorCode + message) so\n // diagnostics reflect the real cause (builder_gateway_timeout,\n // stale_run, context_length_exceeded, completion_error, …).\n if (finalStatus === \"errored\") {\n let errorCode: string | undefined;\n let errorDetail: string | undefined;\n for (let i = run.events.length - 1; i >= 0; i--) {\n const ev = run.events[i].event as {\n type: string;\n error?: string;\n errorCode?: string;\n details?: string;\n };\n if (ev.type === \"error\") {\n errorCode = ev.errorCode;\n errorDetail = ev.error ?? ev.details;\n break;\n }\n }\n if (completionError && !errorCode) {\n errorCode = \"completion_error\";\n errorDetail =\n errorDetail ??\n (completionError instanceof Error\n ? completionError.message\n : String(completionError));\n }\n await setRunError(runId, errorCode ?? \"unknown\", errorDetail);\n }\n\n // 6. Schedule in-memory cleanup + opportunistic old-run pruning.\n setTimeout(() => {\n activeRuns.delete(runId);\n if (threadToRun.get(threadId) === runId) {\n threadToRun.delete(threadId);\n }\n }, CLEANUP_DELAY_MS);\n cleanupOldRuns(\n resolveCompletedRunRetentionMs(),\n resolveErroredRunRetentionMs(),\n ).catch(() => {});\n });\n\n // On Cloudflare Workers, keep the isolate alive for this run\n try {\n const cfCtx = globalThis.__cf_ctx;\n if (cfCtx?.waitUntil) {\n cfCtx.waitUntil(runPromise);\n }\n } catch {\n // Not on Workers — ignore\n }\n\n return run;\n}\n\n/**\n * Subscribe to a run's events starting from `fromSeq`.\n * Returns a ReadableStream that replays buffered events then live-tails.\n * Cancelling the stream only unsubscribes — does NOT abort the agent.\n *\n * Falls back to SQL polling when the run is not in local memory\n * (cross-isolate reconnection on Workers).\n */\nexport function subscribeToRun(\n runId: string,\n fromSeq: number,\n): ReadableStream<Uint8Array> | null {\n const run = activeRuns.get(runId);\n if (run) {\n return subscribeInMemory(run, fromSeq);\n }\n // Not in local memory — try SQL (cross-isolate path)\n return subscribeFromSQL(runId, fromSeq);\n}\n\n/** In-memory subscription (same isolate, fast path) */\nfunction subscribeInMemory(\n run: ActiveRun,\n fromSeq: number,\n): ReadableStream<Uint8Array> {\n const encoder = new TextEncoder();\n let subscriberRef: ((event: RunEvent) => void) | null = null;\n let pingTimer: ReturnType<typeof setInterval> | null = null;\n\n return new ReadableStream({\n start(controller) {\n const ping = () => {\n try {\n controller.enqueue(encoder.encode(`: ping ${Date.now()}\\n\\n`));\n } catch {\n if (subscriberRef) run.subscribers.delete(subscriberRef);\n if (pingTimer) clearInterval(pingTimer);\n }\n };\n ping();\n pingTimer = setInterval(ping, 10_000);\n\n // Replay buffered events from fromSeq\n for (let i = fromSeq; i < run.events.length; i++) {\n try {\n controller.enqueue(\n encoder.encode(\n `data: ${JSON.stringify({ ...run.events[i].event, seq: run.events[i].seq })}\\n\\n`,\n ),\n );\n } catch {\n return;\n }\n }\n\n // If run is already done, close immediately\n if (run.status !== \"running\") {\n if (pingTimer) clearInterval(pingTimer);\n controller.close();\n return;\n }\n\n // Subscribe to live events\n subscriberRef = (event: RunEvent) => {\n try {\n controller.enqueue(\n encoder.encode(\n `data: ${JSON.stringify({ ...event.event, seq: event.seq })}\\n\\n`,\n ),\n );\n // Close stream after terminal events\n if (isTerminalRunEvent(event.event)) {\n run.subscribers.delete(subscriberRef!);\n if (pingTimer) clearInterval(pingTimer);\n controller.close();\n }\n } catch {\n run.subscribers.delete(subscriberRef!);\n }\n };\n\n run.subscribers.add(subscriberRef);\n },\n cancel() {\n // Only unsubscribe — do NOT abort the agent run\n if (subscriberRef) run.subscribers.delete(subscriberRef);\n if (pingTimer) clearInterval(pingTimer);\n },\n });\n}\n\n/** SQL-based subscription (cross-isolate, polling) */\nfunction subscribeFromSQL(\n runId: string,\n fromSeq: number,\n): ReadableStream<Uint8Array> | null {\n const encoder = new TextEncoder();\n let cancelled = false;\n let pollTimer: ReturnType<typeof setTimeout> | null = null;\n let pingTimer: ReturnType<typeof setInterval> | null = null;\n\n return new ReadableStream({\n async start(controller) {\n let lastSeq = fromSeq;\n const ping = () => {\n try {\n controller.enqueue(encoder.encode(`: ping ${Date.now()}\\n\\n`));\n } catch {\n cancelled = true;\n if (pingTimer) clearInterval(pingTimer);\n }\n };\n ping();\n pingTimer = setInterval(ping, 10_000);\n\n const poll = async () => {\n if (cancelled) return;\n try {\n // Read new events from SQL\n const events = await getRunEventsSince(runId, lastSeq);\n for (const { seq, eventData } of events) {\n // Advance the cursor first, before any parse/enqueue branch can\n // `continue`/`return`. Otherwise a single corrupt (unparseable)\n // event row is re-fetched on every poll tick forever, wedging the\n // SSE stream open and never delivering a terminal event.\n lastSeq = seq + 1;\n let parsed: any;\n try {\n parsed = JSON.parse(eventData);\n } catch {\n continue;\n }\n try {\n controller.enqueue(\n encoder.encode(\n `data: ${JSON.stringify({ ...parsed, seq })}\\n\\n`,\n ),\n );\n } catch {\n cancelled = true;\n return;\n }\n\n // Close on terminal events\n if (isTerminalRunEvent(parsed)) {\n if (pingTimer) clearInterval(pingTimer);\n controller.close();\n return;\n }\n }\n\n // Check if run completed (no terminal event but status changed)\n if (events.length === 0) {\n // Opportunistically reap a stale producer before trusting SQL's\n // \"running\" status — otherwise a crashed server leaves us polling\n // forever.\n await reapIfStale(runId).catch(() => {});\n const run = await getRunById(runId);\n if (!run || run.status !== \"running\") {\n // Run ended — do one final event read, then close\n const finalEvents = await getRunEventsSince(runId, lastSeq);\n for (const { seq, eventData } of finalEvents) {\n // Advance first — see the main poll loop above for why.\n lastSeq = seq + 1;\n let parsed: any;\n try {\n parsed = JSON.parse(eventData);\n } catch {\n continue;\n }\n try {\n controller.enqueue(\n encoder.encode(\n `data: ${JSON.stringify({ ...parsed, seq })}\\n\\n`,\n ),\n );\n } catch {\n cancelled = true;\n return;\n }\n if (isTerminalRunEvent(parsed)) {\n if (pingTimer) clearInterval(pingTimer);\n controller.close();\n return;\n }\n }\n if (run?.status === \"aborted\") {\n try {\n controller.enqueue(\n encoder.encode(\n `data: ${JSON.stringify({ type: \"done\", seq: lastSeq })}\\n\\n`,\n ),\n );\n } catch {\n cancelled = true;\n return;\n }\n } else if (run?.status === \"completed\") {\n try {\n controller.enqueue(\n encoder.encode(\n `data: ${JSON.stringify({ type: \"done\", seq: lastSeq })}\\n\\n`,\n ),\n );\n } catch {\n cancelled = true;\n return;\n }\n } else if (run?.status === \"errored\") {\n // The run row was flipped to `errored` but no terminal event\n // was ever persisted — almost always means a reaper's silent\n // `appendTerminalRunEvent(...).catch(() => {})` swallowed a\n // transient DB error, so the user-facing situation is the\n // same as a stale-run reap. Send the friendly event AND try\n // to persist it so future reconnects replay it from SQL\n // rather than regenerating it (the user used to see a bare\n // \"run_terminal_event_missing\" debug string here).\n await ensureTerminalRunEvent(\n runId,\n STALE_RUN_ERROR_EVENT,\n ).catch(() => {});\n try {\n controller.enqueue(\n encoder.encode(\n `data: ${JSON.stringify({\n ...STALE_RUN_ERROR_EVENT,\n seq: lastSeq,\n })}\\n\\n`,\n ),\n );\n } catch {\n cancelled = true;\n return;\n }\n }\n if (pingTimer) clearInterval(pingTimer);\n controller.close();\n return;\n }\n }\n\n // Schedule next poll\n if (!cancelled) {\n pollTimer = setTimeout(poll, 500);\n }\n } catch {\n // SQL error — close stream\n try {\n if (pingTimer) clearInterval(pingTimer);\n controller.close();\n } catch {}\n }\n };\n\n // Verify run exists before starting poll\n try {\n const run = await getRunById(runId);\n if (!run) {\n if (pingTimer) clearInterval(pingTimer);\n controller.close();\n return;\n }\n } catch {\n controller.close();\n return;\n }\n\n await poll();\n },\n cancel() {\n cancelled = true;\n if (pollTimer) clearTimeout(pollTimer);\n if (pingTimer) clearInterval(pingTimer);\n },\n });\n}\n\n/** Get the active run for a thread (if any) — checks memory then SQL */\nexport function getActiveRunForThread(threadId: string): ActiveRun | null {\n const runId = threadToRun.get(threadId);\n if (runId) {\n const run = activeRuns.get(runId);\n if (run) return run;\n }\n return null;\n}\n\n/**\n * Async version that also checks SQL — for cross-isolate access.\n * Used by the /runs/active endpoint.\n *\n * Returns `heartbeatAt` so the client can independently decide a run is\n * dead even before the server-side stale reap has fired. Returns\n * `lastProgressAt` so the client-side stuck-detector can show a\n * user-visible \"this chat looks stuck\" affordance when a run is alive\n * (heartbeating) but not actually emitting events.\n */\nexport async function getActiveRunForThreadAsync(threadId: string): Promise<{\n runId: string;\n threadId: string;\n turnId: string;\n status: string;\n heartbeatAt: number;\n lastProgressAt: number | null;\n /** How the run was dispatched (NULL/foreground, background, background-processing). */\n dispatchMode?: string | null;\n /**\n * Last reached `_process-run` worker stage as a JSON string\n * `{stage,detail?,at}`. Surfaced so a silent background-worker death is\n * diagnosable from the client WITHOUT the unreadable bg-fn logs.\n */\n diagStage?: string | null;\n} | null> {\n // Check memory first — return both running AND recently-completed runs\n // that still have events in memory. This allows sub-agent tabs to replay\n // the full conversation from completed runs via SSE.\n const memRun = getActiveRunForThread(threadId);\n if (memRun && (memRun.status === \"running\" || memRun.events.length > 0)) {\n return {\n runId: memRun.runId,\n threadId: memRun.threadId,\n turnId: memRun.turnId,\n status: memRun.status,\n // In-memory means this isolate is the producer. By definition, the\n // heartbeat is fresh as of \"now\" — the client can trust this.\n heartbeatAt: Date.now(),\n // For an in-memory run we don't have a separate \"last event emit\"\n // timestamp tracked in JS — the SQL bump is throttled per-second.\n // Read it back from SQL on demand. For the common case the SQL row\n // is well under 1s old; if it isn't, the stuck-detector will pick\n // it up on the next poll cycle.\n lastProgressAt: await fetchLastProgressAt(memRun.runId),\n };\n }\n // Fall back to SQL — also surface recently terminated runs so the client\n // can reconnect and replay synthesized done/error events instead of\n // retrying the original POST. Without this, a POST that fails after the\n // server already accepted (and finished) the run would re-execute the\n // turn and double-apply mutations: the in-memory branch above already\n // returns terminal runs whose events are still buffered, but the SQL\n // path is the only authority once memory has been evicted.\n try {\n const sqlRun = await getRunByThread(threadId, { includeTerminal: true });\n if (!sqlRun) return null;\n if (sqlRun.status === \"running\") {\n // FALLBACK HARDENING: a background-dispatched run that is still UNCLAIMED\n // (dispatch_mode === 'background', never flipped to 'background-processing')\n // past the tight grace means the bg-fn worker never started — a silent\n // async-worker death that the 202-ack inline fallback can't catch. Reap it\n // early and recoverably (background_worker_never_started) so the run no\n // longer hangs for the full 90s window and the client's recoverable-error\n // path can re-drive the turn. Only fires when there is provably no live\n // worker; a claimed/heartbeating run is left alone by the conditional SQL.\n if (sqlRun.dispatchMode === \"background\") {\n const recovered = await reapUnclaimedBackgroundRun(sqlRun.id).catch(\n () => false,\n );\n if (recovered) return null;\n }\n // If the producer is dead (no recent heartbeat), reap before the\n // client can see a stale \"running\" status and enter a reconnect\n // loop it can never exit.\n const reaped = await reapIfStale(sqlRun.id).catch(() => false);\n if (reaped) return null;\n return {\n runId: sqlRun.id,\n threadId: sqlRun.threadId,\n turnId: sqlRun.turnId ?? sqlRun.id,\n status: sqlRun.status,\n heartbeatAt: sqlRun.heartbeatAt ?? sqlRun.startedAt,\n lastProgressAt: sqlRun.lastProgressAt,\n dispatchMode: sqlRun.dispatchMode,\n diagStage: sqlRun.diagStage,\n };\n }\n if (sqlRun.status === \"completed\" || sqlRun.status === \"errored\") {\n // Cap how far back we'll surface terminal runs as \"active\". The goal\n // is to catch the recently-completed-but-reconnecting case, not to\n // resurrect ancient turns when the user reopens an old thread.\n //\n // Measure age from the run's terminal timestamp, not its start. A\n // long-running task that ran 11 minutes and completed five seconds\n // ago should still be reachable — the client's disconnect happened\n // around completion, so completion time is what matters for the\n // \"is the user still here waiting?\" question. Fall back to the last\n // heartbeat (older deployments may have unset completed_at) and\n // finally to startedAt for ancient rows.\n const referenceAt =\n sqlRun.completedAt ?? sqlRun.heartbeatAt ?? sqlRun.startedAt;\n const terminalAge = Date.now() - referenceAt;\n if (terminalAge > TERMINAL_RUN_RECONNECT_WINDOW_MS) return null;\n return {\n runId: sqlRun.id,\n threadId: sqlRun.threadId,\n turnId: sqlRun.turnId ?? sqlRun.id,\n status: sqlRun.status,\n heartbeatAt: sqlRun.heartbeatAt ?? sqlRun.startedAt,\n lastProgressAt: sqlRun.lastProgressAt,\n dispatchMode: sqlRun.dispatchMode,\n diagStage: sqlRun.diagStage,\n };\n }\n } catch {\n // SQL error — fall through\n }\n return null;\n}\n\nasync function fetchLastProgressAt(runId: string): Promise<number | null> {\n try {\n const run = await getRunById(runId);\n if (!run) return null;\n // `getRunById` returns a narrow projection today; ask for the row via\n // the thread lookup which carries last_progress_at.\n const byThread = await getRunByThread(run.threadId, {\n includeTerminal: true,\n });\n if (byThread && byThread.id === runId) return byThread.lastProgressAt;\n return null;\n } catch {\n return null;\n }\n}\n\n/** Get a run by ID */\nexport function getRun(runId: string): ActiveRun | null {\n return activeRuns.get(runId) ?? null;\n}\n\n/** Explicitly abort a run (e.g. Stop button) */\nexport function abortRun(runId: string, reason: string = \"user\"): boolean {\n const run = activeRuns.get(runId);\n if (run) {\n abortInMemoryRun(run, reason);\n }\n // Also mark as aborted in SQL (for cross-isolate abort on Workers)\n markRunAborted(runId, reason).catch(() => {});\n return !!run;\n}\n\n// Re-export so callers can avoid importing from run-store directly.\nexport { tryClaimRunSlot } from \"./run-store.js\";\n"]}
1
+ {"version":3,"file":"run-manager.js","sourceRoot":"","sources":["../../src/agent/run-manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,oBAAoB,EAAE,MAAM,+BAA+B,CAAC;AACrE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EACL,SAAS,EACT,cAAc,EACd,wBAAwB,EACxB,cAAc,EACd,gBAAgB,EAChB,YAAY,EACZ,iBAAiB,EACjB,UAAU,EACV,cAAc,EACd,cAAc,EACd,kBAAkB,EAClB,eAAe,EACf,WAAW,EACX,0BAA0B,EAC1B,sBAAsB,EACtB,WAAW,EACX,qBAAqB,GACtB,MAAM,gBAAgB,CAAC;AAgBxB,MAAM,UAAU,GAAG,IAAI,GAAG,EAAqB,CAAC;AAChD,MAAM,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAC;AAE9C,uEAAuE;AACvE,MAAM,gBAAgB,GAAG,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC;AAEvC;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,MAAM,kCAAkC,GAAG,MAAM,CAAC;AAEzD;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,MAAM,8BAA8B,GAAG,MAAM,CAAC;AAErD;;;;;;;;;;;;;;GAcG;AACH,MAAM,CAAC,MAAM,kCAAkC,GAAG,EAAE,GAAG,MAAM,CAAC,CAAC,UAAU;AAEzE;;;;;GAKG;AACH,MAAM,CAAC,MAAM,sCAAsC,GACjD,kCAAkC,CAAC;AAErC,qEAAqE;AACrE,MAAM,CAAC,MAAM,kCAAkC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAEtE;;;;;GAKG;AACH,MAAM,CAAC,MAAM,gCAAgC,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAExE;;;;;GAKG;AACH,MAAM,CAAC,MAAM,gCAAgC,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAE/D,MAAM,gCAAgC,GAAG,uBAAuB,CAAC;AAEjE,SAAS,kBAAkB,CAAC,GAAY;IACtC,IACE,OAAO,GAAG,KAAK,QAAQ;QACvB,GAAG,KAAK,IAAI;QACZ,SAAS,IAAI,GAAG;QAChB,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ;QAC/B,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAC7B,CAAC;QACD,OAAO,GAAG,CAAC,OAAO,CAAC;IACrB,CAAC;IACD,OAAO,eAAe,CAAC;AACzB,CAAC;AAED,SAAS,qBAAqB,CAAC,GAAgB;IAC7C,IAAI,GAAG,CAAC,SAAS;QAAE,OAAO,GAAG,CAAC,SAAS,CAAC;IACxC,IAAI,GAAG,CAAC,UAAU,KAAK,GAAG;QAAE,OAAO,gCAAgC,CAAC;IACpE,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,wBAAwB,CAAC,GAAgB;IAChD,IAAI,GAAG,CAAC,UAAU,KAAK,GAAG;QAAE,OAAO,GAAG,CAAC,OAAO,CAAC;IAC/C,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,qBAAqB,CAAC,GAAY;IACzC,IAAI,CAAC,CAAC,GAAG,YAAY,WAAW,CAAC;QAAE,OAAO,IAAI,CAAC;IAC/C,MAAM,SAAS,GAAG,qBAAqB,CAAC,GAAG,CAAC,CAAC;IAC7C,IAAI,oBAAoB,CAAC,GAAG,EAAE,SAAS,CAAC;QAAE,OAAO,KAAK,CAAC;IACvD,IAAI,GAAG,CAAC,UAAU,KAAK,GAAG,IAAI,GAAG,CAAC,UAAU,KAAK,GAAG;QAAE,OAAO,KAAK,CAAC;IACnE,IAAI,wBAAwB,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC;IAC7D,IAAI,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,KAAK,mBAAmB;QAAE,OAAO,KAAK,CAAC;IAC3E,IAAI,CAAC,SAAS;QAAE,OAAO,IAAI,CAAC;IAC5B,MAAM,cAAc,GAAG,SAAS,CAAC,WAAW,EAAE,CAAC;IAC/C,OAAO,CACL,CAAC,cAAc,CAAC,UAAU,CAAC,eAAe,CAAC;QAC3C,cAAc,KAAK,+BAA+B;QAClD,cAAc,KAAK,uBAAuB;QAC1C,cAAc,KAAK,qBAAqB,CACzC,CAAC;AACJ,CAAC;AAuCD,SAAS,eAAe;IACtB,IACE,OAAO,CAAC,GAAG,CAAC,OAAO;QACnB,OAAO,CAAC,GAAG,CAAC,OAAO,KAAK,OAAO;QAC/B,OAAO,CAAC,GAAG,CAAC,aAAa,KAAK,MAAM,EACpC,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IACE,OAAO,CAAC,GAAG,CAAC,wBAAwB;QACpC,OAAO,CAAC,GAAG,CAAC,aAAa,KAAK,MAAM,EACpC,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,OAAO,CACZ,OAAO,CAAC,GAAG,CAAC,QAAQ;QACpB,OAAO,CAAC,GAAG,CAAC,MAAM;QAClB,OAAO,CAAC,GAAG,CAAC,UAAU;QACtB,OAAO,CAAC,GAAG,CAAC,MAAM;QAClB,OAAO,CAAC,GAAG,CAAC,YAAY;QACxB,OAAO,CAAC,GAAG,CAAC,SAAS,CACtB,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,uBAAuB,CACrC,UAAmB,EACnB,OAAsC;IAEtC,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;IACjC,MAAM,UAAU,GAAG,OAAO,EAAE,kBAAkB,KAAK,IAAI,CAAC;IACxD,yEAAyE;IACzE,8EAA8E;IAC9E,4EAA4E;IAC5E,sEAAsE;IACtE,MAAM,OAAO,GAAG,UAAU;QACxB,CAAC,CAAC,kCAAkC;QACpC,CAAC,CAAC,8BAA8B,CAAC;IACnC,0EAA0E;IAC1E,6EAA6E;IAC7E,wEAAwE;IACxE,6EAA6E;IAC7E,sCAAsC;IACtC,MAAM,WAAW,GAAG,CAAC,EAAU,EAAU,EAAE,CACzC,MAAM,IAAI,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;IAExC,IAAI,OAAO,UAAU,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;QAClE,OAAO,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,CAAC;IAC9C,CAAC;IACD,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC;IACvD,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC3B,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC7B,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC;YAAE,OAAO,WAAW,CAAC,GAAG,CAAC,CAAC;IAChE,CAAC;IACD,4EAA4E;IAC5E,yCAAyC;IACzC,IAAI,UAAU,EAAE,CAAC;QACf,OAAO,MAAM,CAAC,CAAC,CAAC,sCAAsC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7D,CAAC;IACD,OAAO,OAAO,EAAE,gBAAgB,IAAI,MAAM;QACxC,CAAC,CAAC,kCAAkC;QACpC,CAAC,CAAC,CAAC,CAAC;AACR,CAAC;AAED,MAAM,UAAU,8BAA8B;IAC5C,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC;IACpD,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC3B,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC7B,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC;YAAE,OAAO,GAAG,CAAC;IACnD,CAAC;IACD,OAAO,kCAAkC,CAAC;AAC5C,CAAC;AAED,MAAM,UAAU,4BAA4B;IAC1C,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,8BAA8B,CAAC;IAC5D,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC3B,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC7B,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC;YAAE,OAAO,GAAG,CAAC;IACnD,CAAC;IACD,OAAO,gCAAgC,CAAC;AAC1C,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAqB;IAC/C,OAAO,CACL,KAAK,CAAC,IAAI,KAAK,MAAM;QACrB,KAAK,CAAC,IAAI,KAAK,OAAO;QACtB,KAAK,CAAC,IAAI,KAAK,iBAAiB;QAChC,KAAK,CAAC,IAAI,KAAK,YAAY;QAC3B,KAAK,CAAC,IAAI,KAAK,eAAe,CAC/B,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CAAC,GAAc,EAAE,MAAM,GAAW,MAAM;IAC/D,GAAG,CAAC,WAAW,GAAG,MAAM,CAAC;IACzB,GAAG,CAAC,MAAM,GAAG,SAAS,CAAC;IACvB,IAAI,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,GAAG,CAAC,KAAK,EAAE,CAAC;QAChD,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IACnC,CAAC;IACD,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IACxB,KAAK,MAAM,UAAU,IAAI,GAAG,CAAC,WAAW,EAAE,CAAC;QACzC,IAAI,CAAC;YACH,UAAU,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC;QAClE,CAAC;QAAC,MAAM,CAAC;YACP,6CAA6C;QAC/C,CAAC;IACH,CAAC;IACD,GAAG,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;AAC1B,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,QAAQ,CACtB,KAAa,EACb,QAAgB,EAChB,KAGkB,EAClB,UAAqD,EACrD,OAAyB;IAEzB,qDAAqD;IACrD,MAAM,aAAa,GAAG,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAChD,IAAI,aAAa,EAAE,CAAC;QAClB,QAAQ,CAAC,aAAa,CAAC,CAAC;IAC1B,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,eAAe,EAAE,CAAC;IACpC,IAAI,YAAY,GAAG,KAAK,CAAC;IACzB,MAAM,GAAG,GAAc;QACrB,KAAK;QACL,QAAQ;QACR,MAAM,EAAE,OAAO,EAAE,MAAM,IAAI,KAAK;QAChC,MAAM,EAAE,EAAE;QACV,MAAM,EAAE,SAAS;QACjB,WAAW,EAAE,IAAI,GAAG,EAAE;QACtB,KAAK;QACL,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;KACtB,CAAC;IAEF,UAAU,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IAC3B,WAAW,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;IAEjC,wEAAwE;IACxE,uEAAuE;IACvE,uDAAuD;IACvD,MAAM,gBAAgB,GAAG,SAAS,CAAC,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,KAAK,CACxE,GAAG,EAAE,GAAE,CAAC,CACT,CAAC;IAEF,4EAA4E;IAC5E,4EAA4E;IAC5E,wEAAwE;IACxE,0EAA0E;IAC1E,sEAAsE;IACtE,IAAI,gBAAgB,GAAkB,OAAO,CAAC,OAAO,EAAE,CAAC;IAExD,wEAAwE;IACxE,yEAAyE;IACzE,0EAA0E;IAC1E,8BAA8B;IAC9B,IAAI,kBAAkB,GAAG,CAAC,CAAC;IAC3B,MAAM,iBAAiB,GAAG,GAAG,EAAE;QAC7B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,IAAI,GAAG,GAAG,kBAAkB,GAAG,IAAI;YAAE,OAAO;QAC5C,kBAAkB,GAAG,GAAG,CAAC;QACzB,eAAe,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACzC,CAAC,CAAC;IAEF,0EAA0E;IAC1E,4EAA4E;IAC5E,0EAA0E;IAC1E,wEAAwE;IACxE,IAAI,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC;IACvC,MAAM,aAAa,GAAG,GAAG,EAAE;QACzB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,IAAI,GAAG,GAAG,cAAc,GAAG,IAAI;YAAE,OAAO;QACxC,cAAc,GAAG,GAAG,CAAC;QACrB,gBAAgB,CAAC,KAAK,CAAC;aACpB,IAAI,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;YACpB,IAAI,KAAK,CAAC,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;gBAC3C,gBAAgB,CAAC,GAAG,EAAE,KAAK,CAAC,MAAM,IAAI,MAAM,CAAC,CAAC;gBAC9C,OAAO;YACT,CAAC;YACD,kEAAkE;YAClE,uEAAuE;YACvE,2CAA2C;YAC3C,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;gBAC1B,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,KAAK,CAAC,CAAC;gBACzC,IAAI,MAAM,KAAK,IAAI,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;oBAC5C,gBAAgB,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;gBACrC,CAAC;YACH,CAAC;QACH,CAAC,CAAC;aACD,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACrB,CAAC,CAAC;IAEF,wEAAwE;IACxE,sEAAsE;IACtE,mEAAmE;IACnE,0DAA0D;IAC1D,MAAM,cAAc,GAAmC,WAAW,CAAC,GAAG,EAAE;QACtE,kBAAkB,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAC1C,aAAa,EAAE,CAAC;IAClB,CAAC,EAAE,IAAI,CAAC,CAAC;IACT,MAAM,aAAa,GAAG,uBAAuB,CAAC,OAAO,EAAE,aAAa,EAAE;QACpE,gBAAgB,EAAE,OAAO,EAAE,2BAA2B,KAAK,IAAI;QAC/D,kBAAkB,EAAE,OAAO,EAAE,kBAAkB,KAAK,IAAI;KACzD,CAAC,CAAC;IACH,MAAM,gBAAgB,GACpB,aAAa,GAAG,CAAC;QACf,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE;YACd,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,IAAI,KAAK,CAAC,MAAM,CAAC,OAAO;gBAAE,OAAO;YAC7D,YAAY,GAAG,IAAI,CAAC;YACpB,IAAI,CAAC;gBACH,IAAI,EAAE,eAAe;gBACrB,MAAM,EAAE,aAAa;aACtB,CAAC,CAAC;YACH,KAAK,CAAC,KAAK,EAAE,CAAC;QAChB,CAAC,EAAE,aAAa,CAAC;QACnB,CAAC,CAAC,IAAI,CAAC;IACX,IAAI,oBAAoB,GAAoB,IAAI,CAAC;IAEjD,MAAM,eAAe,GAAG,CAAC,KAAc,EAAE,KAA2B,EAAE,EAAE;QACtE,MAAM,SAAS,GACb,KAAK,YAAY,WAAW,CAAC,CAAC,CAAC,qBAAqB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAC1E,YAAY,CAAC,KAAK,EAAE;YAClB,KAAK,EAAE,2BAA2B;YAClC,IAAI,EAAE;gBACJ,MAAM,EAAE,mBAAmB;gBAC3B,KAAK;gBACL,SAAS,EAAE,GAAG,CAAC,MAAM;gBACrB,YAAY,EAAE,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO;gBAC7C,WAAW,EAAE,GAAG,CAAC,WAAW;gBAC5B,SAAS;aACV;YACD,KAAK,EAAE;gBACL,KAAK;gBACL,QAAQ;gBACR,UAAU,EAAE,GAAG,CAAC,MAAM,CAAC,MAAM;gBAC7B,SAAS,EAAE,GAAG,CAAC,SAAS;gBACxB,aAAa;aACd;YACD,QAAQ,EAAE;gBACR,QAAQ,EAAE;oBACR,KAAK;oBACL,QAAQ;oBACR,MAAM,EAAE,GAAG,CAAC,MAAM;oBAClB,KAAK;oBACL,UAAU,EAAE,GAAG,CAAC,MAAM,CAAC,MAAM;oBAC7B,SAAS,EAAE,GAAG,CAAC,SAAS;oBACxB,aAAa;oBACb,YAAY;oBACZ,WAAW,EAAE,GAAG,CAAC,WAAW;iBAC7B;aACF;SACF,CAAC,CAAC;IACL,CAAC,CAAC;IAEF,MAAM,YAAY,GAAG,CACnB,QAAkB,EAClB,OAA+C,EAChC,EAAE;QACjB,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAE1B,yDAAyD;QACzD,KAAK,MAAM,UAAU,IAAI,GAAG,CAAC,WAAW,EAAE,CAAC;YACzC,IAAI,CAAC;gBACH,UAAU,CAAC,QAAQ,CAAC,CAAC;YACvB,CAAC;YAAC,MAAM,CAAC;gBACP,GAAG,CAAC,WAAW,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;YACrC,CAAC;QACH,CAAC;QAED,oEAAoE;QACpE,wEAAwE;QACxE,wEAAwE;QACxE,iCAAiC;QACjC,iBAAiB,EAAE,CAAC;QAEpB,uEAAuE;QACvE,yEAAyE;QACzE,qEAAqE;QACrE,0EAA0E;QAC1E,wDAAwD;QACxD,MAAM,UAAU,GAAG,gBAAgB,CAAC,IAAI,CAAC,GAAG,EAAE,CAC5C,cAAc,CAAC,KAAK,EAAE,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CACpE,CAAC;QACF,gBAAgB,GAAG,UAAU,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAC9C,MAAM,WAAW,GAAG,UAAU,CAAC;QAC/B,IAAI,CAAC,OAAO,EAAE,uBAAuB,EAAE,CAAC;YACtC,WAAW,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAC9B,CAAC;QAED,aAAa,EAAE,CAAC;QAChB,OAAO,WAAW,CAAC;IACrB,CAAC,CAAC;IAEF,MAAM,IAAI,GAAG,CAAC,KAAqB,EAAE,EAAE;QACrC,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,IAAI,KAAK,CAAC,MAAM,CAAC,OAAO;YAAE,OAAO;QAE7D,MAAM,QAAQ,GAAa,EAAE,GAAG,EAAE,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC;QAC7D,IAAI,kBAAkB,CAAC,KAAK,CAAC,EAAE,CAAC;YAC9B,oBAAoB,GAAG,QAAQ,CAAC;YAChC,OAAO;QACT,CAAC;QAED,YAAY,CAAC,QAAQ,CAAC,CAAC;IACzB,CAAC,CAAC;IAEF,sEAAsE;IACtE,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC;SACzC,IAAI,CAAC,GAAG,EAAE;QACT,IAAI,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACzB,GAAG,CAAC,MAAM,GAAG,YAAY,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC;YACpD,OAAO;QACT,CAAC;QACD,GAAG,CAAC,MAAM,GAAG,WAAW,CAAC;IAC3B,CAAC,CAAC;SACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;QACb,iEAAiE;QACjE,IAAI,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACzB,GAAG,CAAC,MAAM,GAAG,YAAY,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC;YACpD,OAAO;QACT,CAAC;QACD,GAAG,CAAC,MAAM,GAAG,SAAS,CAAC;QACvB,IAAI,qBAAqB,CAAC,GAAG,CAAC,EAAE,CAAC;YAC/B,eAAe,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC9B,CAAC;QACD,MAAM,YAAY,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC;QAC7C,MAAM,SAAS,GACb,GAAG,YAAY,WAAW,CAAC,CAAC,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACtE,MAAM,OAAO,GACX,GAAG,YAAY,WAAW,CAAC,CAAC,CAAC,wBAAwB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACzE,IAAI,CAAC;YACH,IAAI,EAAE,OAAO;YACb,KAAK,EAAE,YAAY;YACnB,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACnC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/B,GAAG,CAAC,GAAG,YAAY,WAAW,IAAI,GAAG,CAAC,UAAU;gBAC9C,CAAC,CAAC,EAAE,UAAU,EAAE,GAAG,CAAC,UAAU,EAAE;gBAChC,CAAC,CAAC,EAAE,CAAC;SACR,CAAC,CAAC;IACL,CAAC,CAAC;SACD,OAAO,CAAC,KAAK,IAAI,EAAE;QAClB,gEAAgE;QAChE,mEAAmE;QACnE,kEAAkE;QAClE,kEAAkE;QAClE,gEAAgE;QAChE,sCAAsC;QAEtC,oEAAoE;QACpE,mEAAmE;QACnE,8DAA8D;QAC9D,IAAI,eAAe,GAAY,IAAI,CAAC;QACpC,IAAI,wBAAwB,GAAY,IAAI,CAAC;QAC7C,IACE,UAAU;YACV,CAAC,CAAC,GAAG,CAAC,MAAM,KAAK,SAAS,IAAI,GAAG,CAAC,WAAW,KAAK,aAAa,CAAC,EAChE,CAAC;YACD,IAAI,CAAC;gBACH,MAAM,aAAa,GAAc,oBAAoB;oBACnD,CAAC,CAAC,EAAE,GAAG,GAAG,EAAE,MAAM,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,oBAAoB,CAAC,EAAE;oBAC3D,CAAC,CAAC,GAAG,CAAC;gBACR,MAAM,UAAU,CAAC,aAAa,CAAC,CAAC;YAClC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,eAAe,GAAG,GAAG,CAAC;gBACtB,eAAe,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;gBACnC,OAAO,CAAC,KAAK,CACX,0CAA0C,EAC1C,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CACzC,CAAC;YACJ,CAAC;QACH,CAAC;QAED,kEAAkE;QAClE,oEAAoE;QACpE,kBAAkB;QAClB,MAAM,WAAW,GACf,GAAG,CAAC,MAAM,KAAK,SAAS;YACtB,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,GAAG,CAAC,MAAM,KAAK,SAAS,IAAI,eAAe;gBAC3C,CAAC,CAAC,SAAS;gBACX,CAAC,CAAC,WAAW,CAAC;QAEpB,qEAAqE;QACrE,mEAAmE;QACnE,qEAAqE;QACrE,0CAA0C;QAC1C,IAAI,WAAW,KAAK,WAAW,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;YAC7D,mEAAmE;YACnE,mDAAmD;YACnD,mEAAmE;YACnE,sEAAsE;YACtE,kEAAkE;YAClE,oEAAoE;YACpE,0DAA0D;YAC1D,gEAAgE;YAChE,kEAAkE;YAClE,qEAAqE;YACrE,wDAAwD;YACxD,MAAM,aAAa,GACjB,WAAW,KAAK,WAAW;gBACzB,CAAC,CAAC,CAAC,oBAAoB,EAAE,KAAK,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;gBACnD,CAAC,CAAC,oBAAoB,EAAE,KAAK,CAAC,IAAI,KAAK,OAAO;oBAC5C,CAAC,CAAC,oBAAoB,CAAC,KAAK;oBAC5B,CAAC,CAAC,oBAAoB,EAAE,KAAK,CAAC,IAAI,KAAK,eAAe;wBACpD,CAAC,CAAC,+DAA+D;4BAC/D,8DAA8D;4BAC9D,6DAA6D;4BAC7D,4DAA4D;4BAC5D,+DAA+D;4BAC/D,oDAAoD;4BACpD,oBAAoB,CAAC,KAAK;wBAC5B,CAAC,CAAC;4BACE,IAAI,EAAE,OAAO;4BACb,KAAK,EAAE,eAAe;gCACpB,CAAC,CAAC,oCAAoC;gCACtC,CAAC,CAAC,8BAA8B;yBACnC,CAAC;YACZ,MAAM,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAC/C,IAAI,CAAC,IAAI,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC7C,oEAAoE;gBACpE,kEAAkE;gBAClE,oEAAoE;gBACpE,8BAA8B;gBAC9B,MAAM,QAAQ,GAAa;oBACzB,GAAG,EAAE,GAAG,CAAC,MAAM,CAAC,MAAM;oBACtB,KAAK,EAAE,aAAa;iBACrB,CAAC;gBACF,IAAI,CAAC;oBACH,MAAM,YAAY,CAAC,QAAQ,EAAE,EAAE,uBAAuB,EAAE,IAAI,EAAE,CAAC,CAAC;gBAClE,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,wBAAwB,GAAG,GAAG,CAAC;oBAC/B,eAAe,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;oBACnC,OAAO,CAAC,KAAK,CACX,iDAAiD,EACjD,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CACzC,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;QACD,KAAK,MAAM,UAAU,IAAI,GAAG,CAAC,WAAW,EAAE,CAAC;YACzC,GAAG,CAAC,WAAW,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QACrC,CAAC;QAED,wDAAwD;QACxD,aAAa,CAAC,cAAc,CAAC,CAAC;QAC9B,IAAI,gBAAgB;YAAE,YAAY,CAAC,gBAAgB,CAAC,CAAC;QAErD,wEAAwE;QACxE,wEAAwE;QACxE,wDAAwD;QACxD,IAAI,CAAC;YACH,MAAM,gBAAgB,CAAC;YACvB,IAAI,CAAC,wBAAwB,EAAE,CAAC;gBAC9B,MAAM,wBAAwB,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;YACrD,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,8DAA8D;YAC9D,4BAA4B;QAC9B,CAAC;QAED,iEAAiE;QACjE,sEAAsE;QACtE,sEAAsE;QACtE,mEAAmE;QACnE,gEAAgE;QAChE,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;YAC9B,IAAI,SAA6B,CAAC;YAClC,IAAI,WAA+B,CAAC;YACpC,KAAK,IAAI,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAChD,MAAM,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAKxB,CAAC;gBACF,IAAI,EAAE,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;oBACxB,SAAS,GAAG,EAAE,CAAC,SAAS,CAAC;oBACzB,WAAW,GAAG,EAAE,CAAC,KAAK,IAAI,EAAE,CAAC,OAAO,CAAC;oBACrC,MAAM;gBACR,CAAC;YACH,CAAC;YACD,IAAI,eAAe,IAAI,CAAC,SAAS,EAAE,CAAC;gBAClC,SAAS,GAAG,kBAAkB,CAAC;gBAC/B,WAAW;oBACT,WAAW;wBACX,CAAC,eAAe,YAAY,KAAK;4BAC/B,CAAC,CAAC,eAAe,CAAC,OAAO;4BACzB,CAAC,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC;YACjC,CAAC;YACD,MAAM,WAAW,CAAC,KAAK,EAAE,SAAS,IAAI,SAAS,EAAE,WAAW,CAAC,CAAC;QAChE,CAAC;QAED,iEAAiE;QACjE,UAAU,CAAC,GAAG,EAAE;YACd,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACzB,IAAI,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,KAAK,EAAE,CAAC;gBACxC,WAAW,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC,EAAE,gBAAgB,CAAC,CAAC;QACrB,cAAc,CACZ,8BAA8B,EAAE,EAChC,4BAA4B,EAAE,CAC/B,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACpB,CAAC,CAAC,CAAC;IAEL,6DAA6D;IAC7D,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,UAAU,CAAC,QAAQ,CAAC;QAClC,IAAI,KAAK,EAAE,SAAS,EAAE,CAAC;YACrB,KAAK,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;QAC9B,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,0BAA0B;IAC5B,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,cAAc,CAC5B,KAAa,EACb,OAAe;IAEf,MAAM,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAClC,IAAI,GAAG,EAAE,CAAC;QACR,OAAO,iBAAiB,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IACzC,CAAC;IACD,qDAAqD;IACrD,OAAO,gBAAgB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;AAC1C,CAAC;AAED,uDAAuD;AACvD,SAAS,iBAAiB,CACxB,GAAc,EACd,OAAe;IAEf,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;IAClC,IAAI,aAAa,GAAuC,IAAI,CAAC;IAC7D,IAAI,SAAS,GAA0C,IAAI,CAAC;IAE5D,OAAO,IAAI,cAAc,CAAC;QACxB,KAAK,CAAC,UAAU;YACd,MAAM,IAAI,GAAG,GAAG,EAAE;gBAChB,IAAI,CAAC;oBACH,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC;gBACjE,CAAC;gBAAC,MAAM,CAAC;oBACP,IAAI,aAAa;wBAAE,GAAG,CAAC,WAAW,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;oBACzD,IAAI,SAAS;wBAAE,aAAa,CAAC,SAAS,CAAC,CAAC;gBAC1C,CAAC;YACH,CAAC,CAAC;YACF,IAAI,EAAE,CAAC;YACP,SAAS,GAAG,WAAW,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAEtC,sCAAsC;YACtC,KAAK,IAAI,CAAC,GAAG,OAAO,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACjD,IAAI,CAAC;oBACH,UAAU,CAAC,OAAO,CAChB,OAAO,CAAC,MAAM,CACZ,SAAS,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,CAClF,CACF,CAAC;gBACJ,CAAC;gBAAC,MAAM,CAAC;oBACP,OAAO;gBACT,CAAC;YACH,CAAC;YAED,4CAA4C;YAC5C,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;gBAC7B,IAAI,SAAS;oBAAE,aAAa,CAAC,SAAS,CAAC,CAAC;gBACxC,UAAU,CAAC,KAAK,EAAE,CAAC;gBACnB,OAAO;YACT,CAAC;YAED,2BAA2B;YAC3B,aAAa,GAAG,CAAC,KAAe,EAAE,EAAE;gBAClC,IAAI,CAAC;oBACH,UAAU,CAAC,OAAO,CAChB,OAAO,CAAC,MAAM,CACZ,SAAS,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,KAAK,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,CAAC,MAAM,CAClE,CACF,CAAC;oBACF,qCAAqC;oBACrC,IAAI,kBAAkB,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;wBACpC,GAAG,CAAC,WAAW,CAAC,MAAM,CAAC,aAAc,CAAC,CAAC;wBACvC,IAAI,SAAS;4BAAE,aAAa,CAAC,SAAS,CAAC,CAAC;wBACxC,UAAU,CAAC,KAAK,EAAE,CAAC;oBACrB,CAAC;gBACH,CAAC;gBAAC,MAAM,CAAC;oBACP,GAAG,CAAC,WAAW,CAAC,MAAM,CAAC,aAAc,CAAC,CAAC;gBACzC,CAAC;YACH,CAAC,CAAC;YAEF,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;QACrC,CAAC;QACD,MAAM;YACJ,gDAAgD;YAChD,IAAI,aAAa;gBAAE,GAAG,CAAC,WAAW,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;YACzD,IAAI,SAAS;gBAAE,aAAa,CAAC,SAAS,CAAC,CAAC;QAC1C,CAAC;KACF,CAAC,CAAC;AACL,CAAC;AAED,sDAAsD;AACtD,SAAS,gBAAgB,CACvB,KAAa,EACb,OAAe;IAEf,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;IAClC,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,IAAI,SAAS,GAAyC,IAAI,CAAC;IAC3D,IAAI,SAAS,GAA0C,IAAI,CAAC;IAE5D,OAAO,IAAI,cAAc,CAAC;QACxB,KAAK,CAAC,KAAK,CAAC,UAAU;YACpB,IAAI,OAAO,GAAG,OAAO,CAAC;YACtB,MAAM,IAAI,GAAG,GAAG,EAAE;gBAChB,IAAI,CAAC;oBACH,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC;gBACjE,CAAC;gBAAC,MAAM,CAAC;oBACP,SAAS,GAAG,IAAI,CAAC;oBACjB,IAAI,SAAS;wBAAE,aAAa,CAAC,SAAS,CAAC,CAAC;gBAC1C,CAAC;YACH,CAAC,CAAC;YACF,IAAI,EAAE,CAAC;YACP,SAAS,GAAG,WAAW,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAEtC,MAAM,IAAI,GAAG,KAAK,IAAI,EAAE;gBACtB,IAAI,SAAS;oBAAE,OAAO;gBACtB,IAAI,CAAC;oBACH,2BAA2B;oBAC3B,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;oBACvD,KAAK,MAAM,EAAE,GAAG,EAAE,SAAS,EAAE,IAAI,MAAM,EAAE,CAAC;wBACxC,gEAAgE;wBAChE,gEAAgE;wBAChE,kEAAkE;wBAClE,yDAAyD;wBACzD,OAAO,GAAG,GAAG,GAAG,CAAC,CAAC;wBAClB,IAAI,MAAW,CAAC;wBAChB,IAAI,CAAC;4BACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;wBACjC,CAAC;wBAAC,MAAM,CAAC;4BACP,SAAS;wBACX,CAAC;wBACD,IAAI,CAAC;4BACH,UAAU,CAAC,OAAO,CAChB,OAAO,CAAC,MAAM,CACZ,SAAS,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM,CAClD,CACF,CAAC;wBACJ,CAAC;wBAAC,MAAM,CAAC;4BACP,SAAS,GAAG,IAAI,CAAC;4BACjB,OAAO;wBACT,CAAC;wBAED,2BAA2B;wBAC3B,IAAI,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC;4BAC/B,IAAI,SAAS;gCAAE,aAAa,CAAC,SAAS,CAAC,CAAC;4BACxC,UAAU,CAAC,KAAK,EAAE,CAAC;4BACnB,OAAO;wBACT,CAAC;oBACH,CAAC;oBAED,gEAAgE;oBAChE,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;wBACxB,gEAAgE;wBAChE,kEAAkE;wBAClE,WAAW;wBACX,MAAM,WAAW,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;wBACzC,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,CAAC;wBACpC,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;4BACrC,kDAAkD;4BAClD,MAAM,WAAW,GAAG,MAAM,iBAAiB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;4BAC5D,KAAK,MAAM,EAAE,GAAG,EAAE,SAAS,EAAE,IAAI,WAAW,EAAE,CAAC;gCAC7C,wDAAwD;gCACxD,OAAO,GAAG,GAAG,GAAG,CAAC,CAAC;gCAClB,IAAI,MAAW,CAAC;gCAChB,IAAI,CAAC;oCACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;gCACjC,CAAC;gCAAC,MAAM,CAAC;oCACP,SAAS;gCACX,CAAC;gCACD,IAAI,CAAC;oCACH,UAAU,CAAC,OAAO,CAChB,OAAO,CAAC,MAAM,CACZ,SAAS,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM,CAClD,CACF,CAAC;gCACJ,CAAC;gCAAC,MAAM,CAAC;oCACP,SAAS,GAAG,IAAI,CAAC;oCACjB,OAAO;gCACT,CAAC;gCACD,IAAI,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC;oCAC/B,IAAI,SAAS;wCAAE,aAAa,CAAC,SAAS,CAAC,CAAC;oCACxC,UAAU,CAAC,KAAK,EAAE,CAAC;oCACnB,OAAO;gCACT,CAAC;4BACH,CAAC;4BACD,IAAI,GAAG,EAAE,MAAM,KAAK,SAAS,EAAE,CAAC;gCAC9B,IAAI,CAAC;oCACH,UAAU,CAAC,OAAO,CAChB,OAAO,CAAC,MAAM,CACZ,SAAS,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,MAAM,CAC9D,CACF,CAAC;gCACJ,CAAC;gCAAC,MAAM,CAAC;oCACP,SAAS,GAAG,IAAI,CAAC;oCACjB,OAAO;gCACT,CAAC;4BACH,CAAC;iCAAM,IAAI,GAAG,EAAE,MAAM,KAAK,WAAW,EAAE,CAAC;gCACvC,IAAI,CAAC;oCACH,UAAU,CAAC,OAAO,CAChB,OAAO,CAAC,MAAM,CACZ,SAAS,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,MAAM,CAC9D,CACF,CAAC;gCACJ,CAAC;gCAAC,MAAM,CAAC;oCACP,SAAS,GAAG,IAAI,CAAC;oCACjB,OAAO;gCACT,CAAC;4BACH,CAAC;iCAAM,IAAI,GAAG,EAAE,MAAM,KAAK,SAAS,EAAE,CAAC;gCACrC,6DAA6D;gCAC7D,6DAA6D;gCAC7D,4DAA4D;gCAC5D,0DAA0D;gCAC1D,4DAA4D;gCAC5D,wDAAwD;gCACxD,2DAA2D;gCAC3D,mDAAmD;gCACnD,MAAM,sBAAsB,CAC1B,KAAK,EACL,qBAAqB,CACtB,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;gCAClB,IAAI,CAAC;oCACH,UAAU,CAAC,OAAO,CAChB,OAAO,CAAC,MAAM,CACZ,SAAS,IAAI,CAAC,SAAS,CAAC;wCACtB,GAAG,qBAAqB;wCACxB,GAAG,EAAE,OAAO;qCACb,CAAC,MAAM,CACT,CACF,CAAC;gCACJ,CAAC;gCAAC,MAAM,CAAC;oCACP,SAAS,GAAG,IAAI,CAAC;oCACjB,OAAO;gCACT,CAAC;4BACH,CAAC;4BACD,IAAI,SAAS;gCAAE,aAAa,CAAC,SAAS,CAAC,CAAC;4BACxC,UAAU,CAAC,KAAK,EAAE,CAAC;4BACnB,OAAO;wBACT,CAAC;oBACH,CAAC;oBAED,qBAAqB;oBACrB,IAAI,CAAC,SAAS,EAAE,CAAC;wBACf,SAAS,GAAG,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;oBACpC,CAAC;gBACH,CAAC;gBAAC,MAAM,CAAC;oBACP,2BAA2B;oBAC3B,IAAI,CAAC;wBACH,IAAI,SAAS;4BAAE,aAAa,CAAC,SAAS,CAAC,CAAC;wBACxC,UAAU,CAAC,KAAK,EAAE,CAAC;oBACrB,CAAC;oBAAC,MAAM,CAAC,CAAA,CAAC;gBACZ,CAAC;YACH,CAAC,CAAC;YAEF,yCAAyC;YACzC,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,CAAC;gBACpC,IAAI,CAAC,GAAG,EAAE,CAAC;oBACT,IAAI,SAAS;wBAAE,aAAa,CAAC,SAAS,CAAC,CAAC;oBACxC,UAAU,CAAC,KAAK,EAAE,CAAC;oBACnB,OAAO;gBACT,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,UAAU,CAAC,KAAK,EAAE,CAAC;gBACnB,OAAO;YACT,CAAC;YAED,MAAM,IAAI,EAAE,CAAC;QACf,CAAC;QACD,MAAM;YACJ,SAAS,GAAG,IAAI,CAAC;YACjB,IAAI,SAAS;gBAAE,YAAY,CAAC,SAAS,CAAC,CAAC;YACvC,IAAI,SAAS;gBAAE,aAAa,CAAC,SAAS,CAAC,CAAC;QAC1C,CAAC;KACF,CAAC,CAAC;AACL,CAAC;AAED,wEAAwE;AACxE,MAAM,UAAU,qBAAqB,CAAC,QAAgB;IACpD,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IACxC,IAAI,KAAK,EAAE,CAAC;QACV,MAAM,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAClC,IAAI,GAAG;YAAE,OAAO,GAAG,CAAC;IACtB,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAAC,QAAgB;IAgB/D,uEAAuE;IACvE,yEAAyE;IACzE,qDAAqD;IACrD,MAAM,MAAM,GAAG,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IAC/C,IAAI,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;QACxE,OAAO;YACL,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,mEAAmE;YACnE,8DAA8D;YAC9D,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE;YACvB,kEAAkE;YAClE,kEAAkE;YAClE,mEAAmE;YACnE,kEAAkE;YAClE,gCAAgC;YAChC,cAAc,EAAE,MAAM,mBAAmB,CAAC,MAAM,CAAC,KAAK,CAAC;SACxD,CAAC;IACJ,CAAC;IACD,yEAAyE;IACzE,oEAAoE;IACpE,wEAAwE;IACxE,sEAAsE;IACtE,sEAAsE;IACtE,qEAAqE;IACrE,2DAA2D;IAC3D,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,QAAQ,EAAE,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC,CAAC;QACzE,IAAI,CAAC,MAAM;YAAE,OAAO,IAAI,CAAC;QACzB,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAChC,0EAA0E;YAC1E,6EAA6E;YAC7E,uEAAuE;YACvE,2EAA2E;YAC3E,wEAAwE;YACxE,0EAA0E;YAC1E,wEAAwE;YACxE,2EAA2E;YAC3E,IAAI,MAAM,CAAC,YAAY,KAAK,YAAY,EAAE,CAAC;gBACzC,MAAM,SAAS,GAAG,MAAM,0BAA0B,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,KAAK,CACjE,GAAG,EAAE,CAAC,KAAK,CACZ,CAAC;gBACF,IAAI,SAAS;oBAAE,OAAO,IAAI,CAAC;YAC7B,CAAC;YACD,iEAAiE;YACjE,gEAAgE;YAChE,0BAA0B;YAC1B,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;YAC/D,IAAI,MAAM;gBAAE,OAAO,IAAI,CAAC;YACxB,OAAO;gBACL,KAAK,EAAE,MAAM,CAAC,EAAE;gBAChB,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,EAAE;gBAClC,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC,SAAS;gBACnD,cAAc,EAAE,MAAM,CAAC,cAAc;gBACrC,YAAY,EAAE,MAAM,CAAC,YAAY;gBACjC,SAAS,EAAE,MAAM,CAAC,SAAS;aAC5B,CAAC;QACJ,CAAC;QACD,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YACjE,qEAAqE;YACrE,mEAAmE;YACnE,+DAA+D;YAC/D,EAAE;YACF,kEAAkE;YAClE,mEAAmE;YACnE,mEAAmE;YACnE,gEAAgE;YAChE,oEAAoE;YACpE,gEAAgE;YAChE,yCAAyC;YACzC,MAAM,WAAW,GACf,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC,SAAS,CAAC;YAC/D,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW,CAAC;YAC7C,IAAI,WAAW,GAAG,gCAAgC;gBAAE,OAAO,IAAI,CAAC;YAChE,OAAO;gBACL,KAAK,EAAE,MAAM,CAAC,EAAE;gBAChB,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,EAAE;gBAClC,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC,SAAS;gBACnD,cAAc,EAAE,MAAM,CAAC,cAAc;gBACrC,YAAY,EAAE,MAAM,CAAC,YAAY;gBACjC,SAAS,EAAE,MAAM,CAAC,SAAS;aAC5B,CAAC;QACJ,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,2BAA2B;IAC7B,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,KAAK,UAAU,mBAAmB,CAAC,KAAa;IAC9C,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,CAAC;QACpC,IAAI,CAAC,GAAG;YAAE,OAAO,IAAI,CAAC;QACtB,sEAAsE;QACtE,oDAAoD;QACpD,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,GAAG,CAAC,QAAQ,EAAE;YAClD,eAAe,EAAE,IAAI;SACtB,CAAC,CAAC;QACH,IAAI,QAAQ,IAAI,QAAQ,CAAC,EAAE,KAAK,KAAK;YAAE,OAAO,QAAQ,CAAC,cAAc,CAAC;QACtE,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,sBAAsB;AACtB,MAAM,UAAU,MAAM,CAAC,KAAa;IAClC,OAAO,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC;AACvC,CAAC;AAED,gDAAgD;AAChD,MAAM,UAAU,QAAQ,CAAC,KAAa,EAAE,MAAM,GAAW,MAAM;IAC7D,MAAM,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAClC,IAAI,GAAG,EAAE,CAAC;QACR,gBAAgB,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IAChC,CAAC;IACD,mEAAmE;IACnE,cAAc,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IAC9C,OAAO,CAAC,CAAC,GAAG,CAAC;AACf,CAAC;AAED,oEAAoE;AACpE,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC","sourcesContent":["import { captureError } from \"../server/capture-error.js\";\nimport { isLlmCredentialError } from \"./engine/credential-errors.js\";\nimport { EngineError } from \"./engine/types.js\";\nimport {\n insertRun,\n insertRunEvent,\n updateRunStatusIfRunning,\n markRunAborted,\n getRunAbortState,\n getRunStatus,\n getRunEventsSince,\n getRunById,\n getRunByThread,\n cleanupOldRuns,\n updateRunHeartbeat,\n bumpRunProgress,\n reapIfStale,\n reapUnclaimedBackgroundRun,\n ensureTerminalRunEvent,\n setRunError,\n STALE_RUN_ERROR_EVENT,\n} from \"./run-store.js\";\nimport type { AgentChatEvent, RunEvent, RunStatus } from \"./types.js\";\n\nexport interface ActiveRun {\n runId: string;\n threadId: string;\n /** Logical-turn identity (see StartRunOptions.turnId). Defaults to runId. */\n turnId: string;\n events: RunEvent[];\n status: RunStatus;\n subscribers: Set<(event: RunEvent) => void>;\n abort: AbortController;\n abortReason?: string;\n startedAt: number;\n}\n\nconst activeRuns = new Map<string, ActiveRun>();\nconst threadToRun = new Map<string, string>();\n\n/** How long to keep completed runs in memory before cleanup (5 min) */\nconst CLEANUP_DELAY_MS = 5 * 60 * 1000;\n\n/**\n * Default run chunk budget for hosted/serverless deploys.\n *\n * This MUST fire before the two upstream hard walls that otherwise kill a run\n * mid-turn with no chance to hand off:\n * 1. The Builder model gateway keeps a 45s cap only for hosted foreground\n * runs; local and proven background-function runs use longer caps.\n * 2. Serverless functions are hard-killed around 60-65s (the heartbeat then\n * reaps the row as a stale_run).\n * Production data showed every cutoff landing in the 44-70s window with ZERO\n * auto_continue events ever emitted — i.e. the old 45s default raced the 45s\n * gateway and lost, and per-template overrides (e.g. 240_000) pushed it past\n * BOTH walls so it could never fire. 40s leaves ~5s of headroom under the\n * gateway wall to abort, persist the partial turn, write the terminal event,\n * and emit a clean auto_continue so the client resumes seamlessly.\n */\nexport const DEFAULT_HOSTED_RUN_SOFT_TIMEOUT_MS = 40_000;\n\n/**\n * Hard ceiling for the hosted soft timeout. On a hosted runtime the\n * foreground auto_continue soft timeout can never usefully exceed this — the\n * synchronous function (~60s) wall kills the run first, so a larger configured\n * or env value just guarantees the cutoff is a hard error instead of a graceful\n * hand-off. Any resolved value above this is clamped down for hosted foreground\n * runs. Local dev (non-hosted) is left alone so long-running local turns aren't\n * chunked.\n *\n * IMPORTANT: this clamp is for the INTERACTIVE / foreground path and must NOT\n * be raised. The foreground POST still rides a synchronous serverless function\n * (~60-65s wall), so 40s remains correct there. The only sanctioned exception\n * is the opt-in `backgroundFunction` mode (see\n * `BACKGROUND_SOFT_TIMEOUT_CEILING_MS`), which runs inside a Netlify background\n * function (no ~60s wall, 15-min budget) and therefore can safely outlast 40s.\n */\nexport const HOSTED_SOFT_TIMEOUT_CEILING_MS = 40_000;\n\n/**\n * Hard ceiling for the soft timeout when a run executes inside a Netlify\n * background function (any deployed function whose name ends in `-background`).\n * Background functions return 202 immediately and run detached for up to 15\n * minutes, so the ~60s synchronous function wall that 40s defends against does\n * NOT apply. 13 minutes leaves ~2 min of headroom under Netlify's 15-min hard\n * kill to abort, persist the partial turn, write the terminal event, and (for\n * the rare >13-min turn) self-fire another background continuation.\n *\n * This ceiling is used ONLY when a caller explicitly opts in with\n * `backgroundFunction: true`. It does not change the foreground/interactive\n * ceiling and does not fire unless the durable-background path dispatched the\n * run into a background function. Per the design doc Guardrail, the 40s\n * interactive clamp stays correct for every non-background run.\n */\nexport const BACKGROUND_SOFT_TIMEOUT_CEILING_MS = 13 * 60_000; // 780_000\n\n/**\n * Default soft-timeout budget for a background-function run when the caller\n * does not pass an explicit `softTimeoutMs`. Same value as the ceiling — we\n * want a background turn to use nearly its whole 15-min budget before handing\n * off to a chained background continuation.\n */\nexport const DEFAULT_BACKGROUND_RUN_SOFT_TIMEOUT_MS =\n BACKGROUND_SOFT_TIMEOUT_CEILING_MS;\n\n/** Default SQL retention for completed run event logs (24 hours). */\nexport const DEFAULT_COMPLETED_RUN_RETENTION_MS = 24 * 60 * 60 * 1000;\n\n/**\n * Default SQL retention for errored/aborted run event logs (7 days). Kept\n * longer than completed runs so cut-off / failed chats survive for pattern\n * analysis (listErroredRuns) — these are rare and small, and they are exactly\n * the runs we need to study to keep hardening reliability.\n */\nexport const DEFAULT_ERRORED_RUN_RETENTION_MS = 7 * 24 * 60 * 60 * 1000;\n\n/**\n * How recently a terminal run must have started for `/runs/active` to surface\n * it. Reconnect after this window won't replay the run — typical real-world\n * disconnects resolve in seconds, so 10 minutes is generous while keeping us\n * from resurrecting ancient turns when the user reopens an old thread.\n */\nexport const TERMINAL_RUN_RECONNECT_WINDOW_MS = 10 * 60 * 1000;\n\nconst PROVIDER_RATE_LIMITED_ERROR_CODE = \"provider_rate_limited\";\n\nfunction getRunErrorMessage(err: unknown): string {\n if (\n typeof err === \"object\" &&\n err !== null &&\n \"message\" in err &&\n typeof err.message === \"string\" &&\n err.message.trim().length > 0\n ) {\n return err.message;\n }\n return \"Unknown error\";\n}\n\nfunction getEngineRunErrorCode(err: EngineError): string | undefined {\n if (err.errorCode) return err.errorCode;\n if (err.statusCode === 429) return PROVIDER_RATE_LIMITED_ERROR_CODE;\n return undefined;\n}\n\nfunction getEngineRunErrorDetails(err: EngineError): string | undefined {\n if (err.statusCode === 429) return err.message;\n return undefined;\n}\n\nfunction shouldCaptureRunError(err: unknown): boolean {\n if (!(err instanceof EngineError)) return true;\n const errorCode = getEngineRunErrorCode(err);\n if (isLlmCredentialError(err, errorCode)) return false;\n if (err.statusCode === 401 || err.statusCode === 403) return false;\n if (/^40[13] status code\\b/i.test(err.message)) return false;\n if (err.message.trim().toLowerCase() === \"connection error.\") return false;\n if (!errorCode) return true;\n const normalizedCode = errorCode.toLowerCase();\n return (\n !normalizedCode.startsWith(\"credits-limit\") &&\n normalizedCode !== \"builder_gateway_network_error\" &&\n normalizedCode !== \"provider_rate_limited\" &&\n normalizedCode !== \"rate_limit_exceeded\"\n );\n}\n\nexport interface StartRunOptions {\n /** Optional internal run chunk budget. When reached, the framework emits an\n * auto-continuation signal instead of a user-facing timeout. Leave unset for\n * no framework-imposed run timeout. */\n softTimeoutMs?: number;\n /** Opt into the hosted/serverless default chunk budget. Only callers with\n * automatic continuation support should enable this. */\n useHostedSoftTimeoutDefault?: boolean;\n /** Stable identity for the logical assistant turn this run belongs to. A\n * turn may span several continuation runs (each chunk is its own run); they\n * share one `turnId` so the durable assistant message can be folded across\n * them instead of dropped per-run. Defaults to the runId (turn == run). */\n turnId?: string;\n /**\n * Opt into the durable-background-function soft-timeout regime for THIS run\n * only. When true, `resolveRunSoftTimeoutMs` lifts the hosted ceiling from\n * 40s to ~13min (`BACKGROUND_SOFT_TIMEOUT_CEILING_MS`) because the run is\n * executing inside a Netlify background function (no ~60s wall). Off by\n * default — the foreground/interactive path never sets this, so its 40s\n * clamp is unchanged. See the design doc + the durable-background dispatch\n * decision in production-agent.ts.\n */\n backgroundFunction?: boolean;\n}\n\nexport interface ResolveRunSoftTimeoutOptions {\n useHostedDefault?: boolean;\n /**\n * Resolve the soft timeout for a run executing inside a Netlify background\n * function. Lifts the hosted clamp to `BACKGROUND_SOFT_TIMEOUT_CEILING_MS`\n * (~13min) for this invocation only and, when no override/env is supplied,\n * defaults to `DEFAULT_BACKGROUND_RUN_SOFT_TIMEOUT_MS`. Does NOT change the\n * foreground ceiling. Off by default.\n */\n backgroundFunction?: boolean;\n}\n\nfunction isHostedRuntime(): boolean {\n if (\n process.env.NETLIFY &&\n process.env.NETLIFY !== \"false\" &&\n process.env.NETLIFY_LOCAL !== \"true\"\n ) {\n return true;\n }\n if (\n process.env.AWS_LAMBDA_FUNCTION_NAME &&\n process.env.NETLIFY_LOCAL !== \"true\"\n ) {\n return true;\n }\n return Boolean(\n process.env.CF_PAGES ||\n process.env.VERCEL ||\n process.env.VERCEL_ENV ||\n process.env.RENDER ||\n process.env.FLY_APP_NAME ||\n process.env.K_SERVICE,\n );\n}\n\nexport function resolveRunSoftTimeoutMs(\n overrideMs?: number,\n options?: ResolveRunSoftTimeoutOptions,\n): number {\n const hosted = isHostedRuntime();\n const background = options?.backgroundFunction === true;\n // The interactive/foreground ceiling is 40s — the synchronous serverless\n // function wall. A background-function run (opt-in only) has no ~60s wall, so\n // it is allowed to outlast that and is clamped to the larger 13-min ceiling\n // instead. The 40s clamp for non-background hosted runs is unchanged.\n const ceiling = background\n ? BACKGROUND_SOFT_TIMEOUT_CEILING_MS\n : HOSTED_SOFT_TIMEOUT_CEILING_MS;\n // A configured/env soft timeout that exceeds the upstream walls can never\n // actually fire (the gateway/function kills the run first), so clamp it down\n // on hosted runtimes. This is what makes auto_continue reach the client\n // instead of the run dying as builder_gateway_timeout / stale_run. `0` means\n // \"disabled\" and is never clamped up.\n const clampHosted = (ms: number): number =>\n hosted && ms > ceiling ? ceiling : ms;\n\n if (typeof overrideMs === \"number\" && Number.isFinite(overrideMs)) {\n return clampHosted(Math.max(0, overrideMs));\n }\n const envValue = process.env.AGENT_RUN_SOFT_TIMEOUT_MS;\n if (envValue !== undefined) {\n const raw = Number(envValue);\n if (Number.isFinite(raw) && raw >= 0) return clampHosted(raw);\n }\n // A background-function run uses the full background budget by default; the\n // foreground default (40s) is unchanged.\n if (background) {\n return hosted ? DEFAULT_BACKGROUND_RUN_SOFT_TIMEOUT_MS : 0;\n }\n return options?.useHostedDefault && hosted\n ? DEFAULT_HOSTED_RUN_SOFT_TIMEOUT_MS\n : 0;\n}\n\nexport function resolveCompletedRunRetentionMs(): number {\n const envValue = process.env.AGENT_RUN_RETENTION_MS;\n if (envValue !== undefined) {\n const raw = Number(envValue);\n if (Number.isFinite(raw) && raw >= 0) return raw;\n }\n return DEFAULT_COMPLETED_RUN_RETENTION_MS;\n}\n\nexport function resolveErroredRunRetentionMs(): number {\n const envValue = process.env.AGENT_ERRORED_RUN_RETENTION_MS;\n if (envValue !== undefined) {\n const raw = Number(envValue);\n if (Number.isFinite(raw) && raw >= 0) return raw;\n }\n return DEFAULT_ERRORED_RUN_RETENTION_MS;\n}\n\nfunction isTerminalRunEvent(event: AgentChatEvent): boolean {\n return (\n event.type === \"done\" ||\n event.type === \"error\" ||\n event.type === \"missing_api_key\" ||\n event.type === \"loop_limit\" ||\n event.type === \"auto_continue\"\n );\n}\n\nfunction abortInMemoryRun(run: ActiveRun, reason: string = \"user\") {\n run.abortReason = reason;\n run.status = \"aborted\";\n if (threadToRun.get(run.threadId) === run.runId) {\n threadToRun.delete(run.threadId);\n }\n run.abort.abort(reason);\n for (const subscriber of run.subscribers) {\n try {\n subscriber({ seq: run.events.length, event: { type: \"done\" } });\n } catch {\n // ignore — subscriber is being removed below\n }\n }\n run.subscribers.clear();\n}\n\n/**\n * Start a new agent run in the background.\n * `runFn` receives a `send` callback and an `AbortSignal`.\n * The run continues even if all SSE subscribers disconnect.\n *\n * Events are persisted to SQL for cross-isolate access (Cloudflare Workers).\n */\nexport function startRun(\n runId: string,\n threadId: string,\n runFn: (\n send: (event: AgentChatEvent) => void,\n signal: AbortSignal,\n ) => Promise<void>,\n onComplete?: (run: ActiveRun) => void | Promise<void>,\n options?: StartRunOptions,\n): ActiveRun {\n // If there's already a run for this thread, abort it\n const existingRunId = threadToRun.get(threadId);\n if (existingRunId) {\n abortRun(existingRunId);\n }\n\n const abort = new AbortController();\n let softTimedOut = false;\n const run: ActiveRun = {\n runId,\n threadId,\n turnId: options?.turnId ?? runId,\n events: [],\n status: \"running\",\n subscribers: new Set(),\n abort,\n startedAt: Date.now(),\n };\n\n activeRuns.set(runId, run);\n threadToRun.set(threadId, runId);\n\n // Persist run to SQL without blocking the response. Keep the promise so\n // final status cannot race ahead of a slow initial INSERT and then get\n // overwritten by a late row stuck at status='running'.\n const insertRunPromise = insertRun(runId, threadId, options?.turnId).catch(\n () => {},\n );\n\n // Per-run event persistence chain: events are chained so SQL inserts commit\n // in seq order. Without this, a fast seq=5 commit before a slow seq=4 means\n // the SQL poller advances its cursor past seq=4 (lastSeq = 5+1 = 6) and\n // reconnecting clients permanently miss that event (silent gap). Chaining\n // per run ensures order without blocking the fast in-memory SSE path.\n let persistenceChain: Promise<void> = Promise.resolve();\n\n // Throttle the durable progress timestamp to at most once per second so\n // a chatty token-by-token stream doesn't translate into one DB write per\n // chunk. The stuck-detector threshold is on the order of tens of seconds,\n // so 1s resolution is plenty.\n let lastProgressBumpAt = 0;\n const bumpProgressIfDue = () => {\n const now = Date.now();\n if (now - lastProgressBumpAt < 1000) return;\n lastProgressBumpAt = now;\n bumpRunProgress(runId).catch(() => {});\n };\n\n // Periodic SQL abort check interval (for cross-isolate abort on Workers).\n // Also self-aborts when our row is no longer status='running' — catches the\n // false-stale-reap zombie scenario where the reaper flipped the row while\n // this isolate was briefly unable to heartbeat (DB latency / GC pause).\n let lastAbortCheck = Date.now() - 3000;\n const checkSqlAbort = () => {\n const now = Date.now();\n if (now - lastAbortCheck < 3000) return;\n lastAbortCheck = now;\n getRunAbortState(runId)\n .then(async (state) => {\n if (state.aborted && !abort.signal.aborted) {\n abortInMemoryRun(run, state.reason ?? \"user\");\n return;\n }\n // If the row is no longer 'running' (reaped / replaced) and we're\n // still executing, self-abort so we stop executing and don't overwrite\n // the newer state with our terminal write.\n if (!abort.signal.aborted) {\n const status = await getRunStatus(runId);\n if (status !== null && status !== \"running\") {\n abortInMemoryRun(run, \"displaced\");\n }\n }\n })\n .catch(() => {});\n };\n\n // Heartbeat: bump heartbeat_at every 1.5s so watchers can detect a dead\n // producer (process crash, HMR restart, isolate eviction) quickly and\n // reap the row. Paired with RUN_STALE_MS (6s) — 4x the interval to\n // tolerate transient DB slowness without false positives.\n const heartbeatTimer: ReturnType<typeof setInterval> = setInterval(() => {\n updateRunHeartbeat(runId).catch(() => {});\n checkSqlAbort();\n }, 1500);\n const softTimeoutMs = resolveRunSoftTimeoutMs(options?.softTimeoutMs, {\n useHostedDefault: options?.useHostedSoftTimeoutDefault === true,\n backgroundFunction: options?.backgroundFunction === true,\n });\n const softTimeoutTimer =\n softTimeoutMs > 0\n ? setTimeout(() => {\n if (run.status !== \"running\" || abort.signal.aborted) return;\n softTimedOut = true;\n send({\n type: \"auto_continue\",\n reason: \"run_timeout\",\n });\n abort.abort();\n }, softTimeoutMs)\n : null;\n let pendingTerminalEvent: RunEvent | null = null;\n\n const captureRunError = (error: unknown, phase: \"run\" | \"completion\") => {\n const errorCode =\n error instanceof EngineError ? getEngineRunErrorCode(error) : undefined;\n captureError(error, {\n route: \"/_agent-native/agent-chat\",\n tags: {\n source: \"agent-run-manager\",\n phase,\n runStatus: run.status,\n softTimedOut: softTimedOut ? \"true\" : \"false\",\n abortReason: run.abortReason,\n errorCode,\n },\n extra: {\n runId,\n threadId,\n eventCount: run.events.length,\n startedAt: run.startedAt,\n softTimeoutMs,\n },\n contexts: {\n agentRun: {\n runId,\n threadId,\n status: run.status,\n phase,\n eventCount: run.events.length,\n startedAt: run.startedAt,\n softTimeoutMs,\n softTimedOut,\n abortReason: run.abortReason,\n },\n },\n });\n };\n\n const emitRunEvent = (\n runEvent: RunEvent,\n options?: { surfacePersistenceError?: boolean },\n ): Promise<void> => {\n run.events.push(runEvent);\n\n // Notify in-memory subscribers (same isolate, fast path)\n for (const subscriber of run.subscribers) {\n try {\n subscriber(runEvent);\n } catch {\n run.subscribers.delete(subscriber);\n }\n }\n\n // Bump the durable progress timestamp. Distinct from the heartbeat:\n // heartbeat = \"process is up\", progress = \"real work is happening.\" The\n // gap between them is what the client-side stuck-detector reads to tell\n // a hung run from a healthy one.\n bumpProgressIfDue();\n\n // Persist event to SQL. Events are chained through persistenceChain so\n // inserts commit in seq order — an out-of-order commit would advance the\n // SQL poller's cursor past the slow row, permanently dropping it for\n // reconnecting clients. Terminal events surface persistence errors so the\n // caller can decide how to handle a failed final write.\n const thisInsert = persistenceChain.then(() =>\n insertRunEvent(runId, runEvent.seq, JSON.stringify(runEvent.event)),\n );\n persistenceChain = thisInsert.catch(() => {});\n const persistence = thisInsert;\n if (!options?.surfacePersistenceError) {\n persistence.catch(() => {});\n }\n\n checkSqlAbort();\n return persistence;\n };\n\n const send = (event: AgentChatEvent) => {\n if (run.status === \"aborted\" && abort.signal.aborted) return;\n\n const runEvent: RunEvent = { seq: run.events.length, event };\n if (isTerminalRunEvent(event)) {\n pendingTerminalEvent = runEvent;\n return;\n }\n\n emitRunEvent(runEvent);\n };\n\n // Run in background — intentionally detached from any HTTP connection\n const runPromise = runFn(send, abort.signal)\n .then(() => {\n if (abort.signal.aborted) {\n run.status = softTimedOut ? \"completed\" : \"aborted\";\n return;\n }\n run.status = \"completed\";\n })\n .catch((err) => {\n // Don't surface abort errors — the run was intentionally stopped\n if (abort.signal.aborted) {\n run.status = softTimedOut ? \"completed\" : \"aborted\";\n return;\n }\n run.status = \"errored\";\n if (shouldCaptureRunError(err)) {\n captureRunError(err, \"run\");\n }\n const errorMessage = getRunErrorMessage(err);\n const errorCode =\n err instanceof EngineError ? getEngineRunErrorCode(err) : undefined;\n const details =\n err instanceof EngineError ? getEngineRunErrorDetails(err) : undefined;\n send({\n type: \"error\",\n error: errorMessage,\n ...(errorCode ? { errorCode } : {}),\n ...(details ? { details } : {}),\n ...(err instanceof EngineError && err.upgradeUrl\n ? { upgradeUrl: err.upgradeUrl }\n : {}),\n });\n })\n .finally(async () => {\n // Ordering matters here — this is the atomic-complete boundary.\n // Invariant: once agent_runs.status flips to \"completed\"/\"errored\"\n // in SQL, thread_data for this turn is already durable. This lets\n // reconnecting clients trust the simple rule \"status != running →\n // fetch thread_data\" without polling/retrying for a race window\n // where onComplete was still pending.\n\n // 1. Await the completion callback (thread_data save). Heartbeat is\n // still ticking so the run doesn't look stale to any concurrent\n // /runs/active check while we wait for SQL writes to land.\n let completionError: unknown = null;\n let terminalPersistenceError: unknown = null;\n if (\n onComplete &&\n !(run.status === \"aborted\" && run.abortReason === \"no_progress\")\n ) {\n try {\n const completionRun: ActiveRun = pendingTerminalEvent\n ? { ...run, events: [...run.events, pendingTerminalEvent] }\n : run;\n await onComplete(completionRun);\n } catch (err) {\n completionError = err;\n captureRunError(err, \"completion\");\n console.error(\n \"[run-manager] onComplete callback error:\",\n err instanceof Error ? err.message : err,\n );\n }\n }\n\n // 2. Compute final status. If the completion callback threw, we'd\n // rather mark the run errored than claim success with incomplete\n // thread_data.\n const finalStatus =\n run.status === \"aborted\"\n ? \"aborted\"\n : run.status === \"errored\" || completionError\n ? \"errored\"\n : \"completed\";\n\n // 3. Emit the terminal event only after thread_data is durable. Live\n // SSE clients close on this event and usually fetch thread_data\n // immediately, so emitting it earlier recreates the final-message\n // race this manager is meant to avoid.\n if (finalStatus === \"completed\" || finalStatus === \"errored\") {\n // Choose the terminal event payload (done / the stashed terminal /\n // a synthesized error). NOTE: the `seq` carried by\n // `pendingTerminalEvent` was captured by `send()` at stash time as\n // `run.events.length` and is NOT authoritative — if the runFn emitted\n // any more events before it actually stopped on the abort signal,\n // those events were pushed and reused that same seq. Persisting the\n // terminal event with the stale seq would collide with an\n // already-persisted streaming event and get silently dropped by\n // insertRunEvent's `ON CONFLICT (run_id, seq) DO NOTHING`, so the\n // client would never see the terminal/continuation signal. We always\n // re-stamp the seq at emit time (max-seq+1) just below.\n const terminalEvent: AgentChatEvent =\n finalStatus === \"completed\"\n ? (pendingTerminalEvent?.event ?? { type: \"done\" })\n : pendingTerminalEvent?.event.type === \"error\"\n ? pendingTerminalEvent.event\n : pendingTerminalEvent?.event.type === \"auto_continue\"\n ? // The run was checkpointed at a soft-timeout/loop boundary and\n // is recoverable: the partial turn is in agent_run_events and\n // the continuation run will re-attempt the thread_data save.\n // Even though the completion save failed (finalStatus stays\n // \"errored\" for SQL/diagnostics), re-emit the auto_continue so\n // the client resumes instead of seeing a dead chat.\n pendingTerminalEvent.event\n : {\n type: \"error\",\n error: completionError\n ? \"Agent response could not be saved.\"\n : \"Agent run ended unexpectedly\",\n };\n const last = run.events[run.events.length - 1];\n if (!last || !isTerminalRunEvent(last.event)) {\n // Assign the seq at EMIT time, not at stash time. `run.events` is a\n // contiguous 0-based log, so `run.events.length` is the next free\n // seq and can never collide with an event that was pushed after the\n // terminal event was stashed.\n const terminal: RunEvent = {\n seq: run.events.length,\n event: terminalEvent,\n };\n try {\n await emitRunEvent(terminal, { surfacePersistenceError: true });\n } catch (err) {\n terminalPersistenceError = err;\n captureRunError(err, \"completion\");\n console.error(\n \"[run-manager] terminal event persistence error:\",\n err instanceof Error ? err.message : err,\n );\n }\n }\n }\n for (const subscriber of run.subscribers) {\n run.subscribers.delete(subscriber);\n }\n\n // 4. Stop the heartbeat — all liveness writes are done.\n clearInterval(heartbeatTimer);\n if (softTimeoutTimer) clearTimeout(softTimeoutTimer);\n\n // 5. Persist final status to SQL. Use the conditional write so a zombie\n // run (reaped or displaced while executing) cannot clobber the newer\n // status written by the reaper or a replacement run.\n try {\n await insertRunPromise;\n if (!terminalPersistenceError) {\n await updateRunStatusIfRunning(runId, finalStatus);\n }\n } catch {\n // Best-effort — reapIfStale will eventually clean this up via\n // the heartbeat-stale path.\n }\n\n // 5b. Record terminal failure classification for errored runs so\n // cut-off / failed chats are queryable for pattern analysis. Read\n // the actual error event the run emitted (errorCode + message) so\n // diagnostics reflect the real cause (builder_gateway_timeout,\n // stale_run, context_length_exceeded, completion_error, …).\n if (finalStatus === \"errored\") {\n let errorCode: string | undefined;\n let errorDetail: string | undefined;\n for (let i = run.events.length - 1; i >= 0; i--) {\n const ev = run.events[i].event as {\n type: string;\n error?: string;\n errorCode?: string;\n details?: string;\n };\n if (ev.type === \"error\") {\n errorCode = ev.errorCode;\n errorDetail = ev.error ?? ev.details;\n break;\n }\n }\n if (completionError && !errorCode) {\n errorCode = \"completion_error\";\n errorDetail =\n errorDetail ??\n (completionError instanceof Error\n ? completionError.message\n : String(completionError));\n }\n await setRunError(runId, errorCode ?? \"unknown\", errorDetail);\n }\n\n // 6. Schedule in-memory cleanup + opportunistic old-run pruning.\n setTimeout(() => {\n activeRuns.delete(runId);\n if (threadToRun.get(threadId) === runId) {\n threadToRun.delete(threadId);\n }\n }, CLEANUP_DELAY_MS);\n cleanupOldRuns(\n resolveCompletedRunRetentionMs(),\n resolveErroredRunRetentionMs(),\n ).catch(() => {});\n });\n\n // On Cloudflare Workers, keep the isolate alive for this run\n try {\n const cfCtx = globalThis.__cf_ctx;\n if (cfCtx?.waitUntil) {\n cfCtx.waitUntil(runPromise);\n }\n } catch {\n // Not on Workers — ignore\n }\n\n return run;\n}\n\n/**\n * Subscribe to a run's events starting from `fromSeq`.\n * Returns a ReadableStream that replays buffered events then live-tails.\n * Cancelling the stream only unsubscribes — does NOT abort the agent.\n *\n * Falls back to SQL polling when the run is not in local memory\n * (cross-isolate reconnection on Workers).\n */\nexport function subscribeToRun(\n runId: string,\n fromSeq: number,\n): ReadableStream<Uint8Array> | null {\n const run = activeRuns.get(runId);\n if (run) {\n return subscribeInMemory(run, fromSeq);\n }\n // Not in local memory — try SQL (cross-isolate path)\n return subscribeFromSQL(runId, fromSeq);\n}\n\n/** In-memory subscription (same isolate, fast path) */\nfunction subscribeInMemory(\n run: ActiveRun,\n fromSeq: number,\n): ReadableStream<Uint8Array> {\n const encoder = new TextEncoder();\n let subscriberRef: ((event: RunEvent) => void) | null = null;\n let pingTimer: ReturnType<typeof setInterval> | null = null;\n\n return new ReadableStream({\n start(controller) {\n const ping = () => {\n try {\n controller.enqueue(encoder.encode(`: ping ${Date.now()}\\n\\n`));\n } catch {\n if (subscriberRef) run.subscribers.delete(subscriberRef);\n if (pingTimer) clearInterval(pingTimer);\n }\n };\n ping();\n pingTimer = setInterval(ping, 10_000);\n\n // Replay buffered events from fromSeq\n for (let i = fromSeq; i < run.events.length; i++) {\n try {\n controller.enqueue(\n encoder.encode(\n `data: ${JSON.stringify({ ...run.events[i].event, seq: run.events[i].seq })}\\n\\n`,\n ),\n );\n } catch {\n return;\n }\n }\n\n // If run is already done, close immediately\n if (run.status !== \"running\") {\n if (pingTimer) clearInterval(pingTimer);\n controller.close();\n return;\n }\n\n // Subscribe to live events\n subscriberRef = (event: RunEvent) => {\n try {\n controller.enqueue(\n encoder.encode(\n `data: ${JSON.stringify({ ...event.event, seq: event.seq })}\\n\\n`,\n ),\n );\n // Close stream after terminal events\n if (isTerminalRunEvent(event.event)) {\n run.subscribers.delete(subscriberRef!);\n if (pingTimer) clearInterval(pingTimer);\n controller.close();\n }\n } catch {\n run.subscribers.delete(subscriberRef!);\n }\n };\n\n run.subscribers.add(subscriberRef);\n },\n cancel() {\n // Only unsubscribe — do NOT abort the agent run\n if (subscriberRef) run.subscribers.delete(subscriberRef);\n if (pingTimer) clearInterval(pingTimer);\n },\n });\n}\n\n/** SQL-based subscription (cross-isolate, polling) */\nfunction subscribeFromSQL(\n runId: string,\n fromSeq: number,\n): ReadableStream<Uint8Array> | null {\n const encoder = new TextEncoder();\n let cancelled = false;\n let pollTimer: ReturnType<typeof setTimeout> | null = null;\n let pingTimer: ReturnType<typeof setInterval> | null = null;\n\n return new ReadableStream({\n async start(controller) {\n let lastSeq = fromSeq;\n const ping = () => {\n try {\n controller.enqueue(encoder.encode(`: ping ${Date.now()}\\n\\n`));\n } catch {\n cancelled = true;\n if (pingTimer) clearInterval(pingTimer);\n }\n };\n ping();\n pingTimer = setInterval(ping, 10_000);\n\n const poll = async () => {\n if (cancelled) return;\n try {\n // Read new events from SQL\n const events = await getRunEventsSince(runId, lastSeq);\n for (const { seq, eventData } of events) {\n // Advance the cursor first, before any parse/enqueue branch can\n // `continue`/`return`. Otherwise a single corrupt (unparseable)\n // event row is re-fetched on every poll tick forever, wedging the\n // SSE stream open and never delivering a terminal event.\n lastSeq = seq + 1;\n let parsed: any;\n try {\n parsed = JSON.parse(eventData);\n } catch {\n continue;\n }\n try {\n controller.enqueue(\n encoder.encode(\n `data: ${JSON.stringify({ ...parsed, seq })}\\n\\n`,\n ),\n );\n } catch {\n cancelled = true;\n return;\n }\n\n // Close on terminal events\n if (isTerminalRunEvent(parsed)) {\n if (pingTimer) clearInterval(pingTimer);\n controller.close();\n return;\n }\n }\n\n // Check if run completed (no terminal event but status changed)\n if (events.length === 0) {\n // Opportunistically reap a stale producer before trusting SQL's\n // \"running\" status — otherwise a crashed server leaves us polling\n // forever.\n await reapIfStale(runId).catch(() => {});\n const run = await getRunById(runId);\n if (!run || run.status !== \"running\") {\n // Run ended — do one final event read, then close\n const finalEvents = await getRunEventsSince(runId, lastSeq);\n for (const { seq, eventData } of finalEvents) {\n // Advance first — see the main poll loop above for why.\n lastSeq = seq + 1;\n let parsed: any;\n try {\n parsed = JSON.parse(eventData);\n } catch {\n continue;\n }\n try {\n controller.enqueue(\n encoder.encode(\n `data: ${JSON.stringify({ ...parsed, seq })}\\n\\n`,\n ),\n );\n } catch {\n cancelled = true;\n return;\n }\n if (isTerminalRunEvent(parsed)) {\n if (pingTimer) clearInterval(pingTimer);\n controller.close();\n return;\n }\n }\n if (run?.status === \"aborted\") {\n try {\n controller.enqueue(\n encoder.encode(\n `data: ${JSON.stringify({ type: \"done\", seq: lastSeq })}\\n\\n`,\n ),\n );\n } catch {\n cancelled = true;\n return;\n }\n } else if (run?.status === \"completed\") {\n try {\n controller.enqueue(\n encoder.encode(\n `data: ${JSON.stringify({ type: \"done\", seq: lastSeq })}\\n\\n`,\n ),\n );\n } catch {\n cancelled = true;\n return;\n }\n } else if (run?.status === \"errored\") {\n // The run row was flipped to `errored` but no terminal event\n // was ever persisted — almost always means a reaper's silent\n // `appendTerminalRunEvent(...).catch(() => {})` swallowed a\n // transient DB error, so the user-facing situation is the\n // same as a stale-run reap. Send the friendly event AND try\n // to persist it so future reconnects replay it from SQL\n // rather than regenerating it (the user used to see a bare\n // \"run_terminal_event_missing\" debug string here).\n await ensureTerminalRunEvent(\n runId,\n STALE_RUN_ERROR_EVENT,\n ).catch(() => {});\n try {\n controller.enqueue(\n encoder.encode(\n `data: ${JSON.stringify({\n ...STALE_RUN_ERROR_EVENT,\n seq: lastSeq,\n })}\\n\\n`,\n ),\n );\n } catch {\n cancelled = true;\n return;\n }\n }\n if (pingTimer) clearInterval(pingTimer);\n controller.close();\n return;\n }\n }\n\n // Schedule next poll\n if (!cancelled) {\n pollTimer = setTimeout(poll, 500);\n }\n } catch {\n // SQL error — close stream\n try {\n if (pingTimer) clearInterval(pingTimer);\n controller.close();\n } catch {}\n }\n };\n\n // Verify run exists before starting poll\n try {\n const run = await getRunById(runId);\n if (!run) {\n if (pingTimer) clearInterval(pingTimer);\n controller.close();\n return;\n }\n } catch {\n controller.close();\n return;\n }\n\n await poll();\n },\n cancel() {\n cancelled = true;\n if (pollTimer) clearTimeout(pollTimer);\n if (pingTimer) clearInterval(pingTimer);\n },\n });\n}\n\n/** Get the active run for a thread (if any) — checks memory then SQL */\nexport function getActiveRunForThread(threadId: string): ActiveRun | null {\n const runId = threadToRun.get(threadId);\n if (runId) {\n const run = activeRuns.get(runId);\n if (run) return run;\n }\n return null;\n}\n\n/**\n * Async version that also checks SQL — for cross-isolate access.\n * Used by the /runs/active endpoint.\n *\n * Returns `heartbeatAt` so the client can independently decide a run is\n * dead even before the server-side stale reap has fired. Returns\n * `lastProgressAt` so the client-side stuck-detector can show a\n * user-visible \"this chat looks stuck\" affordance when a run is alive\n * (heartbeating) but not actually emitting events.\n */\nexport async function getActiveRunForThreadAsync(threadId: string): Promise<{\n runId: string;\n threadId: string;\n turnId: string;\n status: string;\n heartbeatAt: number;\n lastProgressAt: number | null;\n /** How the run was dispatched (NULL/foreground, background, background-processing). */\n dispatchMode?: string | null;\n /**\n * Last reached `_process-run` worker stage as a JSON string\n * `{stage,detail?,at}`. Surfaced so a silent background-worker death is\n * diagnosable from the client WITHOUT the unreadable bg-fn logs.\n */\n diagStage?: string | null;\n} | null> {\n // Check memory first — return both running AND recently-completed runs\n // that still have events in memory. This allows sub-agent tabs to replay\n // the full conversation from completed runs via SSE.\n const memRun = getActiveRunForThread(threadId);\n if (memRun && (memRun.status === \"running\" || memRun.events.length > 0)) {\n return {\n runId: memRun.runId,\n threadId: memRun.threadId,\n turnId: memRun.turnId,\n status: memRun.status,\n // In-memory means this isolate is the producer. By definition, the\n // heartbeat is fresh as of \"now\" — the client can trust this.\n heartbeatAt: Date.now(),\n // For an in-memory run we don't have a separate \"last event emit\"\n // timestamp tracked in JS — the SQL bump is throttled per-second.\n // Read it back from SQL on demand. For the common case the SQL row\n // is well under 1s old; if it isn't, the stuck-detector will pick\n // it up on the next poll cycle.\n lastProgressAt: await fetchLastProgressAt(memRun.runId),\n };\n }\n // Fall back to SQL — also surface recently terminated runs so the client\n // can reconnect and replay synthesized done/error events instead of\n // retrying the original POST. Without this, a POST that fails after the\n // server already accepted (and finished) the run would re-execute the\n // turn and double-apply mutations: the in-memory branch above already\n // returns terminal runs whose events are still buffered, but the SQL\n // path is the only authority once memory has been evicted.\n try {\n const sqlRun = await getRunByThread(threadId, { includeTerminal: true });\n if (!sqlRun) return null;\n if (sqlRun.status === \"running\") {\n // FALLBACK HARDENING: a background-dispatched run that is still UNCLAIMED\n // (dispatch_mode === 'background', never flipped to 'background-processing')\n // past the tight grace means the bg-fn worker never started — a silent\n // async-worker death that the 202-ack inline fallback can't catch. Reap it\n // early and recoverably (background_worker_never_started) so the run no\n // longer hangs for the full 90s window and the client's recoverable-error\n // path can re-drive the turn. Only fires when there is provably no live\n // worker; a claimed/heartbeating run is left alone by the conditional SQL.\n if (sqlRun.dispatchMode === \"background\") {\n const recovered = await reapUnclaimedBackgroundRun(sqlRun.id).catch(\n () => false,\n );\n if (recovered) return null;\n }\n // If the producer is dead (no recent heartbeat), reap before the\n // client can see a stale \"running\" status and enter a reconnect\n // loop it can never exit.\n const reaped = await reapIfStale(sqlRun.id).catch(() => false);\n if (reaped) return null;\n return {\n runId: sqlRun.id,\n threadId: sqlRun.threadId,\n turnId: sqlRun.turnId ?? sqlRun.id,\n status: sqlRun.status,\n heartbeatAt: sqlRun.heartbeatAt ?? sqlRun.startedAt,\n lastProgressAt: sqlRun.lastProgressAt,\n dispatchMode: sqlRun.dispatchMode,\n diagStage: sqlRun.diagStage,\n };\n }\n if (sqlRun.status === \"completed\" || sqlRun.status === \"errored\") {\n // Cap how far back we'll surface terminal runs as \"active\". The goal\n // is to catch the recently-completed-but-reconnecting case, not to\n // resurrect ancient turns when the user reopens an old thread.\n //\n // Measure age from the run's terminal timestamp, not its start. A\n // long-running task that ran 11 minutes and completed five seconds\n // ago should still be reachable — the client's disconnect happened\n // around completion, so completion time is what matters for the\n // \"is the user still here waiting?\" question. Fall back to the last\n // heartbeat (older deployments may have unset completed_at) and\n // finally to startedAt for ancient rows.\n const referenceAt =\n sqlRun.completedAt ?? sqlRun.heartbeatAt ?? sqlRun.startedAt;\n const terminalAge = Date.now() - referenceAt;\n if (terminalAge > TERMINAL_RUN_RECONNECT_WINDOW_MS) return null;\n return {\n runId: sqlRun.id,\n threadId: sqlRun.threadId,\n turnId: sqlRun.turnId ?? sqlRun.id,\n status: sqlRun.status,\n heartbeatAt: sqlRun.heartbeatAt ?? sqlRun.startedAt,\n lastProgressAt: sqlRun.lastProgressAt,\n dispatchMode: sqlRun.dispatchMode,\n diagStage: sqlRun.diagStage,\n };\n }\n } catch {\n // SQL error — fall through\n }\n return null;\n}\n\nasync function fetchLastProgressAt(runId: string): Promise<number | null> {\n try {\n const run = await getRunById(runId);\n if (!run) return null;\n // `getRunById` returns a narrow projection today; ask for the row via\n // the thread lookup which carries last_progress_at.\n const byThread = await getRunByThread(run.threadId, {\n includeTerminal: true,\n });\n if (byThread && byThread.id === runId) return byThread.lastProgressAt;\n return null;\n } catch {\n return null;\n }\n}\n\n/** Get a run by ID */\nexport function getRun(runId: string): ActiveRun | null {\n return activeRuns.get(runId) ?? null;\n}\n\n/** Explicitly abort a run (e.g. Stop button) */\nexport function abortRun(runId: string, reason: string = \"user\"): boolean {\n const run = activeRuns.get(runId);\n if (run) {\n abortInMemoryRun(run, reason);\n }\n // Also mark as aborted in SQL (for cross-isolate abort on Workers)\n markRunAborted(runId, reason).catch(() => {});\n return !!run;\n}\n\n// Re-export so callers can avoid importing from run-store directly.\nexport { tryClaimRunSlot } from \"./run-store.js\";\n"]}