@agent-native/core 0.98.4 → 0.98.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/corpus/README.md +2 -2
- package/corpus/core/CHANGELOG.md +28 -0
- package/corpus/core/docs/content/durable-background-runs.mdx +37 -0
- package/corpus/core/docs/content/external-agents.mdx +62 -7
- package/corpus/core/docs/content/locales/ar-SA/external-agents.mdx +38 -7
- package/corpus/core/docs/content/locales/ar-SA/tracking.mdx +2 -0
- package/corpus/core/docs/content/locales/de-DE/external-agents.mdx +39 -7
- package/corpus/core/docs/content/locales/de-DE/tracking.mdx +2 -0
- package/corpus/core/docs/content/locales/es-ES/external-agents.mdx +39 -7
- package/corpus/core/docs/content/locales/es-ES/tracking.mdx +2 -0
- package/corpus/core/docs/content/locales/fr-FR/external-agents.mdx +39 -7
- package/corpus/core/docs/content/locales/fr-FR/tracking.mdx +4 -2
- package/corpus/core/docs/content/locales/hi-IN/external-agents.mdx +38 -7
- package/corpus/core/docs/content/locales/hi-IN/tracking.mdx +2 -0
- package/corpus/core/docs/content/locales/ja-JP/external-agents.mdx +38 -7
- package/corpus/core/docs/content/locales/ja-JP/tracking.mdx +2 -0
- package/corpus/core/docs/content/locales/ko-KR/external-agents.mdx +38 -7
- package/corpus/core/docs/content/locales/ko-KR/tracking.mdx +2 -0
- package/corpus/core/docs/content/locales/pt-BR/external-agents.mdx +39 -7
- package/corpus/core/docs/content/locales/pt-BR/tracking.mdx +2 -0
- package/corpus/core/docs/content/locales/zh-CN/external-agents.mdx +37 -7
- package/corpus/core/docs/content/locales/zh-CN/tracking.mdx +2 -0
- package/corpus/core/docs/content/locales/zh-TW/durable-background-runs.mdx +14 -0
- package/corpus/core/docs/content/locales/zh-TW/external-agents.mdx +37 -7
- package/corpus/core/docs/content/locales/zh-TW/tracking.mdx +2 -0
- package/corpus/core/docs/content/tracking.mdx +2 -0
- package/corpus/core/package.json +1 -1
- package/corpus/core/src/a2a/handlers.ts +95 -21
- package/corpus/core/src/a2a/task-store.ts +65 -3
- package/corpus/core/src/agent/durable-background.ts +16 -16
- package/corpus/core/src/agent/engine/ai-sdk-engine.ts +25 -8
- package/corpus/core/src/agent/engine/anthropic-engine.ts +17 -3
- package/corpus/core/src/agent/engine/output-tokens.ts +6 -6
- package/corpus/core/src/agent/production-agent.ts +200 -27
- package/corpus/core/src/agent/run-manager.ts +81 -1
- package/corpus/core/src/agent/run-store.ts +301 -41
- package/corpus/core/src/client/AgentPanel.tsx +6 -2
- package/corpus/core/src/client/AssistantChat.tsx +364 -69
- package/corpus/core/src/client/agent-chat-adapter.ts +169 -1
- package/corpus/core/src/client/chat/message-components.tsx +3 -0
- package/corpus/core/src/client/chat/tool-call-display.tsx +20 -36
- package/corpus/core/src/client/conversation/use-near-bottom-autoscroll.ts +7 -0
- package/corpus/core/src/client/extensions/AgentNativeExtensionFrame.tsx +2 -0
- package/corpus/core/src/client/extensions/EmbeddedExtension.tsx +2 -0
- package/corpus/core/src/client/extensions/ExtensionEditor.tsx +9 -2
- package/corpus/core/src/client/extensions/ExtensionViewer.tsx +2 -0
- package/corpus/core/src/client/extensions/InlineExtensionFrame.tsx +2 -0
- package/corpus/core/src/client/extensions/portable-extension.ts +2 -0
- package/corpus/core/src/client/session-replay.ts +361 -29
- package/corpus/core/src/client/sse-event-processor.ts +43 -1
- package/corpus/core/src/demo/actions/toggle-demo-mode.ts +1 -1
- package/corpus/core/src/demo/config.ts +6 -6
- package/corpus/core/src/demo/fetch-interceptor.ts +13 -4
- package/corpus/core/src/demo/redact.ts +13 -149
- package/corpus/core/src/extensions/html-shell.ts +3 -0
- package/corpus/core/src/extensions/session-replay-iframe.ts +131 -0
- package/corpus/core/src/integrations/adapters/slack.ts +123 -13
- package/corpus/core/src/integrations/identity-links-store.ts +210 -0
- package/corpus/core/src/integrations/identity.ts +132 -0
- package/corpus/core/src/integrations/index.ts +6 -0
- package/corpus/core/src/integrations/plugin.ts +58 -1
- package/corpus/core/src/integrations/types.ts +7 -0
- package/corpus/core/src/mcp/build-server.ts +127 -22
- package/corpus/core/src/mcp/external-agent-policy.ts +18 -0
- package/corpus/core/src/mcp/index.ts +1 -0
- package/corpus/core/src/scripts/agent-engines/list-agent-engines.ts +1 -20
- package/corpus/core/src/server/agent-chat/plugin-options.ts +12 -1
- package/corpus/core/src/server/agent-chat/script-entries.ts +20 -3
- package/corpus/core/src/server/agent-chat-plugin.ts +3 -0
- package/corpus/core/src/session-replay-iframe-protocol.ts +58 -0
- package/corpus/core/src/shared/reasoning-effort.ts +35 -0
- package/corpus/core/src/styles/agent-conversation.css +26 -0
- package/corpus/core/src/templates/workspace-core/.agents/skills/external-agents/SKILL.md +34 -5
- package/corpus/core/src/templates/workspace-core/.agents/skills/observability/SKILL.md +0 -15
- package/corpus/templates/analytics/.agents/skills/session-replay/SKILL.md +30 -21
- package/corpus/templates/analytics/AGENTS.md +27 -0
- package/corpus/templates/analytics/actions/create-session-replay-agent-link.ts +0 -1
- package/corpus/templates/analytics/actions/get-error-issue.ts +1 -0
- package/corpus/templates/analytics/actions/get-session-replay-summary.ts +1 -0
- package/corpus/templates/analytics/actions/get-session-replay-timeline.ts +40 -0
- package/corpus/templates/analytics/actions/list-error-issues.ts +13 -0
- package/corpus/templates/analytics/actions/list-session-recordings.ts +1 -0
- package/corpus/templates/analytics/actions/query-agent-native-analytics.ts +4 -0
- package/corpus/templates/analytics/app/global.css +2 -2
- package/corpus/templates/analytics/app/hooks/use-dashboard-chat-context.ts +41 -4
- package/corpus/templates/analytics/app/pages/sessions/SessionDetailPage.tsx +52 -203
- package/corpus/templates/analytics/app/pages/sessions/SessionsPage.tsx +69 -16
- package/corpus/templates/analytics/changelog/2026-07-12-analytics-chat-retries-before-showing-a-generic-no-data-mess.md +6 -0
- package/corpus/templates/analytics/changelog/2026-07-12-analytics-sessions-search-no-longer-loses-fast-keystrokes-an.md +6 -0
- package/corpus/templates/analytics/changelog/2026-07-12-chart-selection-respects-chat-state.md +6 -0
- package/corpus/templates/analytics/changelog/2026-07-12-connected-external-agents-can-now-look-up-sessions-error-iss.md +6 -0
- package/corpus/templates/analytics/changelog/2026-07-12-demo-mode-anonymizes-error-reporting-emails.md +6 -0
- package/corpus/templates/analytics/changelog/2026-07-12-session-identities-stay-visible-when-demo-mode-is-off.md +6 -0
- package/corpus/templates/analytics/changelog/2026-07-12-session-replays-show-iframe-content.md +6 -0
- package/corpus/templates/analytics/netlify.toml +0 -3
- package/corpus/templates/analytics/server/lib/analytics-connector-catalog.ts +14 -0
- package/corpus/templates/analytics/server/lib/error-capture.ts +101 -2
- package/corpus/templates/analytics/server/lib/real-data-actions.ts +24 -0
- package/corpus/templates/analytics/server/lib/session-replay-agent-context.ts +49 -17
- package/corpus/templates/analytics/server/lib/session-replay.ts +18 -16
- package/corpus/templates/analytics/server/plugins/agent-chat.ts +43 -6
- package/corpus/templates/analytics/server/plugins/db.ts +15 -0
- package/corpus/templates/assets/app/global.css +1 -1
- package/corpus/templates/brain/app/global.css +2 -2
- package/corpus/templates/brain/changelog/2026-07-12-fix-long-response-progress.md +6 -0
- package/corpus/templates/forms/app/components/CloudUpgrade.tsx +1 -1
- package/corpus/templates/forms/app/components/ThemeToggle.tsx +22 -6
- package/corpus/templates/forms/app/components/builder/FieldPropertiesPanel.tsx +5 -5
- package/corpus/templates/forms/app/components/builder/FieldRenderer.tsx +3 -3
- package/corpus/templates/forms/app/components/layout/Sidebar.tsx +16 -16
- package/corpus/templates/forms/app/global.css +164 -13
- package/corpus/templates/forms/app/pages/AskPage.tsx +35 -13
- package/corpus/templates/forms/app/pages/FormBuilderPage.tsx +143 -67
- package/corpus/templates/forms/app/pages/FormFillPage.tsx +44 -11
- package/corpus/templates/forms/app/pages/FormsListPage.tsx +42 -24
- package/corpus/templates/forms/app/pages/ResponseInsightsPage.tsx +28 -15
- package/corpus/templates/forms/app/pages/ResponsesPage.tsx +46 -20
- package/corpus/templates/forms/app/routes/_app.settings.tsx +1 -1
- package/corpus/templates/forms/changelog/2026-07-12-ask-forms-suggestions-now-sit-beneath-the-composer-for-a-tig.md +6 -0
- package/corpus/templates/forms/changelog/2026-07-12-builder-response-tables-now-fill-the-view-without-a-redundan.md +6 -0
- package/corpus/templates/forms/changelog/2026-07-12-form-builder-surfaces-are-calmer-fields-stay-stable-on-hover.md +6 -0
- package/corpus/templates/forms/changelog/2026-07-12-response-tables-now-fill-the-available-pane-with-a-flexible-.md +6 -0
- package/corpus/templates/forms/changelog/2026-07-12-smoother-more-tactile-controls-throughout-animated-icon-and-.md +6 -0
- package/corpus/templates/mail/AGENTS.md +3 -1
- package/corpus/templates/mail/app/components/email/EmailThread.tsx +21 -13
- package/corpus/templates/mail/app/components/email/email-iframe-document.ts +18 -0
- package/corpus/templates/mail/server/lib/mail-integrations.ts +3 -0
- package/dist/a2a/handlers.d.ts.map +1 -1
- package/dist/a2a/handlers.js +84 -15
- package/dist/a2a/handlers.js.map +1 -1
- package/dist/a2a/task-store.d.ts +20 -1
- package/dist/a2a/task-store.d.ts.map +1 -1
- package/dist/a2a/task-store.js +57 -4
- package/dist/a2a/task-store.js.map +1 -1
- package/dist/agent/durable-background.d.ts +10 -10
- package/dist/agent/durable-background.d.ts.map +1 -1
- package/dist/agent/durable-background.js +16 -16
- package/dist/agent/durable-background.js.map +1 -1
- package/dist/agent/engine/ai-sdk-engine.d.ts.map +1 -1
- package/dist/agent/engine/ai-sdk-engine.js +19 -8
- package/dist/agent/engine/ai-sdk-engine.js.map +1 -1
- package/dist/agent/engine/anthropic-engine.d.ts.map +1 -1
- package/dist/agent/engine/anthropic-engine.js +11 -3
- package/dist/agent/engine/anthropic-engine.js.map +1 -1
- package/dist/agent/engine/output-tokens.js +6 -6
- package/dist/agent/engine/output-tokens.js.map +1 -1
- package/dist/agent/production-agent.d.ts +52 -0
- package/dist/agent/production-agent.d.ts.map +1 -1
- package/dist/agent/production-agent.js +154 -26
- package/dist/agent/production-agent.js.map +1 -1
- package/dist/agent/run-manager.d.ts +9 -0
- package/dist/agent/run-manager.d.ts.map +1 -1
- package/dist/agent/run-manager.js +71 -1
- package/dist/agent/run-manager.js.map +1 -1
- package/dist/agent/run-store.d.ts +9 -0
- package/dist/agent/run-store.d.ts.map +1 -1
- package/dist/agent/run-store.js +243 -48
- package/dist/agent/run-store.js.map +1 -1
- package/dist/client/AgentPanel.d.ts.map +1 -1
- package/dist/client/AgentPanel.js +6 -2
- package/dist/client/AgentPanel.js.map +1 -1
- package/dist/client/AssistantChat.d.ts +21 -0
- package/dist/client/AssistantChat.d.ts.map +1 -1
- package/dist/client/AssistantChat.js +298 -67
- package/dist/client/AssistantChat.js.map +1 -1
- package/dist/client/agent-chat-adapter.d.ts.map +1 -1
- package/dist/client/agent-chat-adapter.js +146 -1
- package/dist/client/agent-chat-adapter.js.map +1 -1
- package/dist/client/chat/message-components.d.ts.map +1 -1
- package/dist/client/chat/message-components.js +2 -1
- package/dist/client/chat/message-components.js.map +1 -1
- package/dist/client/chat/tool-call-display.d.ts +3 -1
- package/dist/client/chat/tool-call-display.d.ts.map +1 -1
- package/dist/client/chat/tool-call-display.js +13 -37
- package/dist/client/chat/tool-call-display.js.map +1 -1
- package/dist/client/conversation/use-near-bottom-autoscroll.d.ts +1 -0
- package/dist/client/conversation/use-near-bottom-autoscroll.d.ts.map +1 -1
- package/dist/client/conversation/use-near-bottom-autoscroll.js +6 -0
- package/dist/client/conversation/use-near-bottom-autoscroll.js.map +1 -1
- package/dist/client/extensions/AgentNativeExtensionFrame.d.ts.map +1 -1
- package/dist/client/extensions/AgentNativeExtensionFrame.js +2 -1
- package/dist/client/extensions/AgentNativeExtensionFrame.js.map +1 -1
- package/dist/client/extensions/EmbeddedExtension.d.ts.map +1 -1
- package/dist/client/extensions/EmbeddedExtension.js +2 -1
- package/dist/client/extensions/EmbeddedExtension.js.map +1 -1
- package/dist/client/extensions/ExtensionEditor.d.ts.map +1 -1
- package/dist/client/extensions/ExtensionEditor.js +5 -2
- package/dist/client/extensions/ExtensionEditor.js.map +1 -1
- package/dist/client/extensions/ExtensionViewer.d.ts.map +1 -1
- package/dist/client/extensions/ExtensionViewer.js +2 -1
- package/dist/client/extensions/ExtensionViewer.js.map +1 -1
- package/dist/client/extensions/InlineExtensionFrame.d.ts.map +1 -1
- package/dist/client/extensions/InlineExtensionFrame.js +2 -1
- package/dist/client/extensions/InlineExtensionFrame.js.map +1 -1
- package/dist/client/extensions/portable-extension.d.ts.map +1 -1
- package/dist/client/extensions/portable-extension.js +2 -0
- package/dist/client/extensions/portable-extension.js.map +1 -1
- package/dist/client/session-replay.d.ts.map +1 -1
- package/dist/client/session-replay.js +308 -29
- package/dist/client/session-replay.js.map +1 -1
- package/dist/client/sse-event-processor.d.ts +5 -0
- package/dist/client/sse-event-processor.d.ts.map +1 -1
- package/dist/client/sse-event-processor.js +37 -1
- package/dist/client/sse-event-processor.js.map +1 -1
- package/dist/collab/routes.d.ts +1 -1
- package/dist/demo/actions/toggle-demo-mode.js +1 -1
- package/dist/demo/actions/toggle-demo-mode.js.map +1 -1
- package/dist/demo/config.js +6 -6
- package/dist/demo/config.js.map +1 -1
- package/dist/demo/fetch-interceptor.d.ts.map +1 -1
- package/dist/demo/fetch-interceptor.js +13 -4
- package/dist/demo/fetch-interceptor.js.map +1 -1
- package/dist/demo/redact.d.ts +7 -7
- package/dist/demo/redact.d.ts.map +1 -1
- package/dist/demo/redact.js +8 -145
- package/dist/demo/redact.js.map +1 -1
- package/dist/extensions/html-shell.d.ts.map +1 -1
- package/dist/extensions/html-shell.js +2 -0
- package/dist/extensions/html-shell.js.map +1 -1
- package/dist/extensions/session-replay-iframe.d.ts +10 -0
- package/dist/extensions/session-replay-iframe.d.ts.map +1 -0
- package/dist/extensions/session-replay-iframe.js +121 -0
- package/dist/extensions/session-replay-iframe.js.map +1 -0
- package/dist/file-upload/actions/upload-image.d.ts +1 -1
- package/dist/integrations/adapters/slack.d.ts.map +1 -1
- package/dist/integrations/adapters/slack.js +98 -13
- package/dist/integrations/adapters/slack.js.map +1 -1
- package/dist/integrations/identity-links-store.d.ts +23 -0
- package/dist/integrations/identity-links-store.d.ts.map +1 -0
- package/dist/integrations/identity-links-store.js +158 -0
- package/dist/integrations/identity-links-store.js.map +1 -0
- package/dist/integrations/identity.d.ts +11 -0
- package/dist/integrations/identity.d.ts.map +1 -0
- package/dist/integrations/identity.js +100 -0
- package/dist/integrations/identity.js.map +1 -0
- package/dist/integrations/index.d.ts +2 -0
- package/dist/integrations/index.d.ts.map +1 -1
- package/dist/integrations/index.js +2 -0
- package/dist/integrations/index.js.map +1 -1
- package/dist/integrations/plugin.d.ts.map +1 -1
- package/dist/integrations/plugin.js +43 -1
- package/dist/integrations/plugin.js.map +1 -1
- package/dist/integrations/types.d.ts +6 -0
- package/dist/integrations/types.d.ts.map +1 -1
- package/dist/integrations/types.js.map +1 -1
- package/dist/mcp/build-server.d.ts +9 -2
- package/dist/mcp/build-server.d.ts.map +1 -1
- package/dist/mcp/build-server.js +97 -20
- package/dist/mcp/build-server.js.map +1 -1
- package/dist/mcp/external-agent-policy.d.ts +19 -0
- package/dist/mcp/external-agent-policy.d.ts.map +1 -0
- package/dist/mcp/external-agent-policy.js +2 -0
- package/dist/mcp/external-agent-policy.js.map +1 -0
- package/dist/mcp/index.d.ts +1 -0
- package/dist/mcp/index.d.ts.map +1 -1
- package/dist/mcp/index.js.map +1 -1
- package/dist/notifications/routes.d.ts +1 -1
- package/dist/scripts/agent-engines/list-agent-engines.d.ts.map +1 -1
- package/dist/scripts/agent-engines/list-agent-engines.js +1 -19
- package/dist/scripts/agent-engines/list-agent-engines.js.map +1 -1
- package/dist/server/agent-chat/plugin-options.d.ts +11 -1
- package/dist/server/agent-chat/plugin-options.d.ts.map +1 -1
- package/dist/server/agent-chat/plugin-options.js.map +1 -1
- package/dist/server/agent-chat/script-entries.d.ts.map +1 -1
- package/dist/server/agent-chat/script-entries.js +16 -2
- package/dist/server/agent-chat/script-entries.js.map +1 -1
- package/dist/server/agent-chat-plugin.d.ts.map +1 -1
- package/dist/server/agent-chat-plugin.js +3 -0
- package/dist/server/agent-chat-plugin.js.map +1 -1
- package/dist/server/agent-engine-api-key-route.d.ts +1 -1
- package/dist/server/transcribe-voice.d.ts +1 -1
- package/dist/session-replay-iframe-protocol.d.ts +36 -0
- package/dist/session-replay-iframe-protocol.d.ts.map +1 -0
- package/dist/session-replay-iframe-protocol.js +24 -0
- package/dist/session-replay-iframe-protocol.js.map +1 -0
- package/dist/shared/reasoning-effort.d.ts +11 -0
- package/dist/shared/reasoning-effort.d.ts.map +1 -1
- package/dist/shared/reasoning-effort.js +36 -0
- package/dist/shared/reasoning-effort.js.map +1 -1
- package/dist/styles/agent-conversation.css +26 -0
- package/dist/templates/workspace-core/.agents/skills/external-agents/SKILL.md +34 -5
- package/dist/templates/workspace-core/.agents/skills/observability/SKILL.md +0 -15
- package/docs/content/durable-background-runs.mdx +37 -0
- package/docs/content/external-agents.mdx +62 -7
- package/docs/content/locales/ar-SA/external-agents.mdx +38 -7
- package/docs/content/locales/ar-SA/tracking.mdx +2 -0
- package/docs/content/locales/de-DE/external-agents.mdx +39 -7
- package/docs/content/locales/de-DE/tracking.mdx +2 -0
- package/docs/content/locales/es-ES/external-agents.mdx +39 -7
- package/docs/content/locales/es-ES/tracking.mdx +2 -0
- package/docs/content/locales/fr-FR/external-agents.mdx +39 -7
- package/docs/content/locales/fr-FR/tracking.mdx +4 -2
- package/docs/content/locales/hi-IN/external-agents.mdx +38 -7
- package/docs/content/locales/hi-IN/tracking.mdx +2 -0
- package/docs/content/locales/ja-JP/external-agents.mdx +38 -7
- package/docs/content/locales/ja-JP/tracking.mdx +2 -0
- package/docs/content/locales/ko-KR/external-agents.mdx +38 -7
- package/docs/content/locales/ko-KR/tracking.mdx +2 -0
- package/docs/content/locales/pt-BR/external-agents.mdx +39 -7
- package/docs/content/locales/pt-BR/tracking.mdx +2 -0
- package/docs/content/locales/zh-CN/external-agents.mdx +37 -7
- package/docs/content/locales/zh-CN/tracking.mdx +2 -0
- package/docs/content/locales/zh-TW/durable-background-runs.mdx +14 -0
- package/docs/content/locales/zh-TW/external-agents.mdx +37 -7
- package/docs/content/locales/zh-TW/tracking.mdx +2 -0
- package/docs/content/tracking.mdx +2 -0
- package/package.json +2 -2
- package/src/templates/workspace-core/.agents/skills/external-agents/SKILL.md +34 -5
- package/src/templates/workspace-core/.agents/skills/observability/SKILL.md +0 -15
- package/corpus/core/src/observability/hosted-model-experiment.ts +0 -118
- package/dist/observability/hosted-model-experiment.d.ts +0 -39
- package/dist/observability/hosted-model-experiment.d.ts.map +0 -1
- package/dist/observability/hosted-model-experiment.js +0 -90
- package/dist/observability/hosted-model-experiment.js.map +0 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run-store.d.ts","sourceRoot":"","sources":["../../src/agent/run-store.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"run-store.d.ts","sourceRoot":"","sources":["../../src/agent/run-store.ts"],"names":[],"mappings":"AAcA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAIjD;;;;;;GAMG;AACH,eAAO,MAAM,YAAY,QAAS,CAAC;AAEnC;;;;;;;;;;;;;;GAcG;AACH,eAAO,MAAM,uBAAuB,QAAS,CAAC;AAE9C,eAAO,MAAM,qBAAqB;aAChC,IAAI,EAAE,OAAO;aACb,KAAK,EACH,qHAAqH;aACvH,SAAS,EAAE,WAAW;aACtB,WAAW;aACX,OAAO,EACL,gIAAgI;CAC1H,CAAC;AAEX;;;;;;;;GAQG;AACH,eAAO,MAAM,oCAAoC;aAC/C,IAAI,EAAE,OAAO;aACb,KAAK,EACH,gHAAgH;aAClH,SAAS,EAAE,iCAAiC;aAC5C,WAAW;aACX,OAAO,EACL,oNAAoN;CAC9M,CAAC;AAEX;;;;;;GAMG;AACH,eAAO,MAAM,4CAA4C;aACvD,IAAI,EAAE,OAAO;aACb,KAAK,EACH,oHAAoH;aACtH,SAAS,EAAE,0BAA0B;aACrC,WAAW;aACX,OAAO,EACL,yGAAyG;CACnG,CAAC;AAEX;;;;;;;;;;;;;GAaG;AACH,eAAO,MAAM,iCAAiC,QAAS,CAAC;AAExD;;;;;;;;;;;;GAYG;AACH,eAAO,MAAM,4CAA4C,QAAa,CAAC;AAEvE;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,eAAO,MAAM,sCAAsC,QAAS,CAAC;AAa7D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AACH,eAAO,MAAM,4BAA4B,QAAgB,CAAC;AAgR1D;;;;GAIG;AACH,wBAAsB,gBAAgB,CACpC,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,MAAM,EACf,aAAa,EAAE,MAAM,GACpB,OAAO,CAAC,IAAI,CAAC,CAoBf;AAED;;;GAGG;AACH,wBAAsB,eAAe,CACnC,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAcxB;AAED;;;;GAIG;AACH,wBAAsB,oBAAoB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAW1E;AAED,wBAAsB,SAAS,CAC7B,EAAE,EAAE,MAAM,EACV,QAAQ,EAAE,MAAM,EAChB,MAAM,CAAC,EAAE,MAAM,EACf,OAAO,CAAC,EAAE;IACR,YAAY,CAAC,EAAE,YAAY,GAAG,uBAAuB,GAAG,YAAY,CAAC;IACrE;;;;;OAKG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B,GACA,OAAO,CAAC,IAAI,CAAC,CAiBf;AA6ED;;;;;;;;;;;GAWG;AACH,wBAAsB,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAYxE;AAED;;;;;;;;;GASG;AACH,wBAAsB,sBAAsB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC;IACnE,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;CAC/B,GAAG,IAAI,CAAC,CA2BR;AAED;;;;;;GAMG;AACH,wBAAsB,sBAAsB,CAC1C,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAWxB;AAED;;;;GAIG;AACH,wBAAsB,uBAAuB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAO1E;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,6BAA6B,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC,CAkBvE;AAED,0DAA0D;AAC1D,MAAM,WAAW,yBAAyB;IACxC,EAAE,EAAE,MAAM,CAAC;IACX;;;+BAG2B;IAC3B,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;;;;;;GAOG;AACH,wBAAsB,8BAA8B,IAAI,OAAO,CAC7D,yBAAyB,EAAE,CAC5B,CAyBA;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,sCAAsC,CACpD,GAAG,EAAE;IAAE,SAAS,EAAE,MAAM,CAAA;CAAE,EAC1B,GAAG,GAAE,MAAmB,GACvB,OAAO,CAET;AAED;;;;;;GAMG;AACH,wBAAsB,gBAAgB,CACpC,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,MAAM,CAAC,CAUjB;AAED;;;;;;;;;GASG;AACH,wBAAsB,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAS5E;AAED;;;;;;;;;GASG;AACH,wBAAsB,eAAe,CACnC,QAAQ,EAAE,MAAM,EAChB,UAAU,CAAC,EAAE,MAAM,GAClB,OAAO,CAAC;IAAE,OAAO,EAAE,OAAO,CAAC;IAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,CAAC,CAsC3D;AAED;;;;GAIG;AACH,wBAAsB,WAAW,CAC/B,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,MAAM,GAAG,SAAS,EAC7B,WAAW,EAAE,MAAM,GAAG,SAAS,GAC9B,OAAO,CAAC,IAAI,CAAC,CAgBf;AAED;;;;GAIG;AACH,wBAAsB,oBAAoB,CACxC,KAAK,EAAE,MAAM,EACb,cAAc,EAAE,MAAM,GAAG,SAAS,GACjC,OAAO,CAAC,IAAI,CAAC,CAYf;AAoGD;;;;;;;;GAQG;AACH,wBAAsB,8BAA8B,CAClD,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,OAAO,CAAC,CAkClB;AAED;;;;;GAKG;AACH,eAAO,MAAM,cAAc;IACzB,gFAAgF;aAChF,YAAY,EAAE,eAAe;IAC7B,sEAAsE;aACtE,UAAU,EAAE,aAAa;IACzB,8EAA8E;aAC9E,UAAU,EAAE,aAAa;IACzB,4EAA4E;aAC5E,aAAa,EAAE,gBAAgB;IAC/B,yDAAyD;aACzD,aAAa,EAAE,gBAAgB;IAC/B,6EAA6E;aAC7E,eAAe,EAAE,mBAAmB;IACpC,+CAA+C;aAC/C,aAAa,EAAE,gBAAgB;IAC/B,oFAAoF;aACpF,eAAe,EAAE,mBAAmB;IACpC,qDAAqD;aACrD,YAAY,EAAE,eAAe;IAC7B,kFAAkF;aAClF,WAAW,EAAE,cAAc;IAC3B,oEAAoE;aACpE,UAAU,EAAE,aAAa;IACzB;;;;;OAKG;aACH,wBAAwB,EAAE,4BAA4B;IACtD;;;;;;;OAOG;aACH,yBAAyB,EAAE,8BAA8B;CACjD,CAAC;AAEX,MAAM,MAAM,YAAY,GAAG,CAAC,OAAO,cAAc,CAAC,CAAC,MAAM,OAAO,cAAc,CAAC,CAAC;AAEhF;;;;;;;;;;;;;GAaG;AACH,wBAAsB,mBAAmB,CACvC,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,YAAY,EACnB,MAAM,CAAC,EAAE,MAAM,GACd,OAAO,CAAC,IAAI,CAAC,CAiCf;AAED,+EAA+E;AAC/E,wBAAsB,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAYrE;AAED;;;;;GAKG;AACH,wBAAsB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAYlE;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAsB,oBAAoB,CACxC,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,OAAO,GAChB,OAAO,CAAC,IAAI,CAAC,CAcf;AA0RD;;;;GAIG;AACH,wBAAsB,WAAW,CAC/B,KAAK,EAAE,MAAM,EACb,UAAU,CAAC,EAAE,MAAM,GAClB,OAAO,CAAC,OAAO,CAAC,CAalB;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAsB,0BAA0B,CAC9C,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,OAAO,CAAC,CAuClB;AAED,wBAAsB,eAAe,CACnC,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,WAAW,GAAG,SAAS,GAAG,SAAS,GAC1C,OAAO,CAAC,IAAI,CAAC,CASf;AAED;;;;;;;GAOG;AACH,wBAAsB,wBAAwB,CAC5C,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,WAAW,GAAG,SAAS,GAAG,SAAS,GAC1C,OAAO,CAAC,OAAO,CAAC,CAUlB;AAED,kFAAkF;AAClF,wBAAsB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CASxE;AAED,wBAAsB,cAAc,CAClC,KAAK,EAAE,MAAM,EACb,MAAM,CAAC,EAAE,MAAM,GACd,OAAO,CAAC,IAAI,CAAC,CAUf;AAED,wBAAsB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAElE;AAED,wBAAsB,gBAAgB,CACpC,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC;IAAE,OAAO,EAAE,OAAO,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,CAchD;AAED,wBAAsB,cAAc,CAClC,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,MAAM,EACX,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,IAAI,CAAC,CAaf;AAED,wBAAsB,iBAAiB,CACrC,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC,CAWpD;AAED,wBAAsB,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC;IACvD,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5B,GAAG,IAAI,CAAC,CAwBR;AAED;;;;;GAKG;AACH,wBAAsB,uBAAuB,CAC3C,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAAE,GAAG,IAAI,CAAC,CA0BjE;AAED;;;;GAIG;AACH,wBAAgB,8BAA8B,CAAC,GAAG,EAAE;IAClD,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B,GAAG;IACF,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC/B,aAAa,EAAE,OAAO,CAAC;CACxB,CAmBA;AAED,wBAAsB,cAAc,CAClC,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE;IAAE,eAAe,CAAC,EAAE,OAAO,CAAA;CAAE,GACtC,OAAO,CAAC;IACT,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB;;;;;;;;;OASG;IACH,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;CAC9B,GAAG,IAAI,CAAC,CA6CR;AAED,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,4EAA4E;IAC5E,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B;AAED,wBAAsB,iBAAiB,CACrC,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAA;CAAO,GAC/B,OAAO,CAAC,eAAe,EAAE,CAAC,CAmF5B;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,6BAA6B,CACjD,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,cAAc,EAAE,CAAC,CAoC3B;AAED;;;;;GAKG;AACH,wBAAsB,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC,CAuDxD;AAED;;;;wDAIwD;AACxD,wBAAsB,cAAc,CAClC,WAAW,EAAE,MAAM,EACnB,kBAAkB,CAAC,EAAE,MAAM,GAC1B,OAAO,CAAC,IAAI,CAAC,CA6Gf;AAED;;;;;GAKG;AACH,wBAAsB,eAAe,CAAC,OAAO,CAAC,EAAE;IAC9C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,GAAG,OAAO,CACT,KAAK,CAAC;IACJ,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;CAC3B,CAAC,CACH,CA2CA;AAED;;;;;;;;;GASG;AACH,wBAAsB,sBAAsB,CAC1C,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAC7B,OAAO,CAAC,IAAI,CAAC,CAEf"}
|
package/dist/agent/run-store.js
CHANGED
|
@@ -1,8 +1,3 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* SQL persistence for agent runs and events.
|
|
3
|
-
* Enables cross-isolate access on Cloudflare Workers and
|
|
4
|
-
* reliable reconnection after page refreshes.
|
|
5
|
-
*/
|
|
6
1
|
import { getDbExec, intType, isPostgres } from "../db/client.js";
|
|
7
2
|
import { ensureColumnExists, ensureTableExists } from "../db/ddl-guard.js";
|
|
8
3
|
import { widenIntColumnsToBigInt } from "../db/widen-columns.js";
|
|
@@ -126,6 +121,16 @@ export const UNCLAIMED_BACKGROUND_RUN_REDISPATCH_BOUND_MS = 5 * 60_000;
|
|
|
126
121
|
* existing loud reap (`background_worker_never_started`) still fires.
|
|
127
122
|
*/
|
|
128
123
|
export const UNCLAIMED_BACKGROUND_RUN_FAST_SWEEP_MS = 20_000;
|
|
124
|
+
/**
|
|
125
|
+
* FIX 3 (durable-background incident) per-turn run-count ceiling for
|
|
126
|
+
* stale-run recovery — mirrors `chainServerDrivenContinuation`'s own ledger
|
|
127
|
+
* guard in production-agent.ts (`MAX_BACKGROUND_RUN_CONTINUATIONS + 5` = 25).
|
|
128
|
+
* Duplicated as a literal rather than imported: production-agent.ts already
|
|
129
|
+
* imports run-manager.ts, which imports this file, so a runtime import back
|
|
130
|
+
* from here would be circular. Keep this numerically in sync if that
|
|
131
|
+
* constant ever changes.
|
|
132
|
+
*/
|
|
133
|
+
const STALE_RUN_RECOVERY_MAX_TURN_RUNS = 25;
|
|
129
134
|
/**
|
|
130
135
|
* Maximum time the stale reapers (`reapIfStale`, `reapAllStaleRuns`,
|
|
131
136
|
* `cleanupOldRuns`'s heartbeat-stale pass) will suspend reaping a "running"
|
|
@@ -1052,6 +1057,15 @@ export const RUN_DIAG_STAGE = {
|
|
|
1052
1057
|
* recovered by running the turn inline. The run still completes for the user.
|
|
1053
1058
|
*/
|
|
1054
1059
|
foregroundInlineRecovery: "foreground_inline_recovery",
|
|
1060
|
+
/**
|
|
1061
|
+
* FIX 3 (durable-background incident): a stale-run reaper (`reapIfStale` /
|
|
1062
|
+
* `reapAllStaleRuns`) found this background chat-turn run dead (heartbeat
|
|
1063
|
+
* stale, no terminal event) and attempted server-owned recovery — detail
|
|
1064
|
+
* carries the outcome (a recovered successor's runId, or why recovery was
|
|
1065
|
+
* declined: not eligible, payload missing, a newer run already exists, or
|
|
1066
|
+
* the per-turn budget is exhausted). See `attemptStaleRunRecovery`.
|
|
1067
|
+
*/
|
|
1068
|
+
staleRunRecoveryAttempted: "stale_run_recovery_attempted",
|
|
1055
1069
|
};
|
|
1056
1070
|
/**
|
|
1057
1071
|
* Record the last reached pipeline stage (+ optional short detail) for a run.
|
|
@@ -1174,15 +1188,146 @@ export async function setRunInFlightMarker(runId, inFlight) {
|
|
|
1174
1188
|
}
|
|
1175
1189
|
}
|
|
1176
1190
|
/**
|
|
1177
|
-
*
|
|
1178
|
-
*
|
|
1179
|
-
*
|
|
1191
|
+
* Mirrors `production-agent.ts`'s `generateRunId` — duplicated (not
|
|
1192
|
+
* imported) to avoid a run-store.ts <-> production-agent.ts import cycle
|
|
1193
|
+
* (production-agent.ts already imports run-manager.ts, which imports this
|
|
1194
|
+
* file). Keep the format in sync if that one changes.
|
|
1180
1195
|
*/
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1196
|
+
function generateRecoveryRunId() {
|
|
1197
|
+
return `run-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
|
1198
|
+
}
|
|
1199
|
+
/**
|
|
1200
|
+
* FIX 3 (durable-background incident): when a stale-run reaper is about to
|
|
1201
|
+
* flip a BACKGROUND chat-turn run to errored/stale_run, attempt to keep the
|
|
1202
|
+
* TURN alive instead of leaving it dead — a background worker has no
|
|
1203
|
+
* connected client watching it, unlike a foreground run, so nothing else
|
|
1204
|
+
* would ever recover it. Inserts an UNCLAIMED successor row (same turnId,
|
|
1205
|
+
* `dispatch_payload` carried over from the dying run) that the existing
|
|
1206
|
+
* unclaimed-background-run sweep (`agent-chat-plugin.ts`, ~20s fast sweep)
|
|
1207
|
+
* picks up and redispatches automatically — this function never dispatches
|
|
1208
|
+
* anything itself beyond the best-effort immediate attempt its caller fires.
|
|
1209
|
+
*
|
|
1210
|
+
* `db` is threaded through (rather than calling `getDbExec()` internally)
|
|
1211
|
+
* so the caller can run this INSIDE the same transaction as the reap-to-
|
|
1212
|
+
* errored write — see `reapSingleStaleRun` for why that matters.
|
|
1213
|
+
*
|
|
1214
|
+
* Eligibility (ALL must hold, or this is a documented no-op — the caller's
|
|
1215
|
+
* normal loud stale_run failure proceeds unchanged):
|
|
1216
|
+
* - the row is a background chat-turn dispatch (`dispatch_mode` starting
|
|
1217
|
+
* with "background") — a foreground/foreground-self-chain run has a
|
|
1218
|
+
* connected client to recover it via its own `auto_continue` re-POST.
|
|
1219
|
+
* - its `dispatch_payload` is still present — without it there is nothing
|
|
1220
|
+
* to rehydrate the successor's request body from. Read HERE, before the
|
|
1221
|
+
* caller's terminal write (which NULLs `dispatch_payload` on every other
|
|
1222
|
+
* path), so it survives long enough to carry over.
|
|
1223
|
+
* - no newer run already exists for the same turn — avoids stacking a
|
|
1224
|
+
* second successor onto a turn a previous recovery (or a normal
|
|
1225
|
+
* `chainServerDrivenContinuation`) already continued. Combined with the
|
|
1226
|
+
* caller's own atomic "did I win the reap" gate, this guarantees AT MOST
|
|
1227
|
+
* ONE recovery successor per reaped run even under concurrent reapers.
|
|
1228
|
+
* - the per-turn run ledger (`countRunsForTurn`'s underlying query) has
|
|
1229
|
+
* room (`STALE_RUN_RECOVERY_MAX_TURN_RUNS`) — mirrors
|
|
1230
|
+
* `chainServerDrivenContinuation`'s own budget guard so a pathological
|
|
1231
|
+
* turn can't loop forever through reaper-driven recovery either.
|
|
1232
|
+
*/
|
|
1233
|
+
async function attemptStaleRunRecovery(db, runId) {
|
|
1234
|
+
const { rows } = await db.execute({
|
|
1235
|
+
sql: `SELECT thread_id, turn_id, dispatch_mode, dispatch_payload, started_at FROM agent_runs WHERE id = ? LIMIT 1`,
|
|
1236
|
+
args: [runId],
|
|
1237
|
+
});
|
|
1238
|
+
const row = rows?.[0];
|
|
1239
|
+
const dispatchMode = row?.dispatch_mode ?? "";
|
|
1240
|
+
if (!row?.thread_id || !dispatchMode.startsWith("background")) {
|
|
1241
|
+
return { outcome: "not_background" };
|
|
1242
|
+
}
|
|
1243
|
+
const payload = row.dispatch_payload;
|
|
1244
|
+
if (typeof payload !== "string" || payload.length === 0) {
|
|
1245
|
+
return { outcome: "payload_missing" };
|
|
1246
|
+
}
|
|
1247
|
+
const threadId = row.thread_id;
|
|
1248
|
+
const turnId = row.turn_id ?? runId;
|
|
1249
|
+
const startedAt = Number(row.started_at) || 0;
|
|
1250
|
+
const { rows: newerRows } = await db.execute({
|
|
1251
|
+
sql: `SELECT id FROM agent_runs WHERE turn_id = ? AND id != ? AND started_at > ? LIMIT 1`,
|
|
1252
|
+
args: [turnId, runId, startedAt],
|
|
1253
|
+
});
|
|
1254
|
+
if ((newerRows?.length ?? 0) > 0) {
|
|
1255
|
+
return { outcome: "newer_run_exists" };
|
|
1256
|
+
}
|
|
1257
|
+
// Inline COUNT rather than the exported `countRunsForTurn` (which opens
|
|
1258
|
+
// its own `getDbExec()` connection) — this must read through the SAME `db`
|
|
1259
|
+
// handle as everything else here so it participates in the caller's
|
|
1260
|
+
// transaction when one is active.
|
|
1261
|
+
const { rows: countRows } = await db.execute({
|
|
1262
|
+
sql: `SELECT COUNT(*) AS run_count FROM agent_runs WHERE thread_id = ? AND turn_id = ?`,
|
|
1263
|
+
args: [threadId, turnId],
|
|
1264
|
+
});
|
|
1265
|
+
const turnRunCount = Number(countRows?.[0]?.run_count);
|
|
1266
|
+
if (Number.isFinite(turnRunCount) &&
|
|
1267
|
+
turnRunCount > STALE_RUN_RECOVERY_MAX_TURN_RUNS) {
|
|
1268
|
+
return { outcome: "budget_exhausted" };
|
|
1269
|
+
}
|
|
1270
|
+
const successorRunId = generateRecoveryRunId();
|
|
1271
|
+
const now = Date.now();
|
|
1272
|
+
await db.execute({
|
|
1273
|
+
sql: `INSERT INTO agent_runs (id, thread_id, status, started_at, heartbeat_at, last_progress_at, turn_id, dispatch_mode, dispatch_payload) VALUES (?, ?, 'running', ?, ?, ?, ?, 'background', ?) ON CONFLICT (id) DO NOTHING`,
|
|
1274
|
+
args: [successorRunId, threadId, now, now, now, turnId, payload],
|
|
1275
|
+
});
|
|
1276
|
+
return { outcome: "recovered", successorRunId, threadId, turnId };
|
|
1277
|
+
}
|
|
1278
|
+
/**
|
|
1279
|
+
* FIX 3: best-effort immediate redispatch for a stale-run recovery
|
|
1280
|
+
* successor, mirroring the "Unclaimed background-run sweep" redispatch
|
|
1281
|
+
* marker in `agent-chat-plugin.ts` (deliberately omits `continuationCount`
|
|
1282
|
+
* — see that file's comment — so a reaper-recovered chunk starts a fresh
|
|
1283
|
+
* nested-dispatch segment at depth 0). Fire-and-forget and never awaited by
|
|
1284
|
+
* callers: the successor row is already durably persisted (unclaimed,
|
|
1285
|
+
* `dispatch_payload` set) regardless of whether this dispatch lands, so a
|
|
1286
|
+
* failure here just means the existing fast unclaimed-background-run sweep
|
|
1287
|
+
* (`UNCLAIMED_BACKGROUND_RUN_FAST_SWEEP_MS`, ~20s) picks it up instead — the
|
|
1288
|
+
* row is left claimable, never marked errored, by construction (this
|
|
1289
|
+
* function never writes to `agent_runs`).
|
|
1290
|
+
*/
|
|
1291
|
+
function attemptStaleRunRecoveryDispatch(successorRunId) {
|
|
1292
|
+
void (async () => {
|
|
1293
|
+
try {
|
|
1294
|
+
const [{ AGENT_CHAT_BACKGROUND_RUN_FIELD, resolveAgentChatProcessRunDispatchPath, }, { fireInternalDispatch },] = await Promise.all([
|
|
1295
|
+
import("./durable-background.js"),
|
|
1296
|
+
import("../server/self-dispatch.js"),
|
|
1297
|
+
]);
|
|
1298
|
+
await fireInternalDispatch({
|
|
1299
|
+
path: resolveAgentChatProcessRunDispatchPath(),
|
|
1300
|
+
taskId: successorRunId,
|
|
1301
|
+
body: {
|
|
1302
|
+
internalContinuation: true,
|
|
1303
|
+
[AGENT_CHAT_BACKGROUND_RUN_FIELD]: {
|
|
1304
|
+
runId: successorRunId,
|
|
1305
|
+
payloadRef: true,
|
|
1306
|
+
},
|
|
1307
|
+
},
|
|
1308
|
+
});
|
|
1309
|
+
}
|
|
1310
|
+
catch (err) {
|
|
1311
|
+
console.error("[run-store] stale-run recovery redispatch attempt failed (leaving successor claimable for the sweep):", successorRunId, err instanceof Error ? err.message : err);
|
|
1312
|
+
}
|
|
1313
|
+
})();
|
|
1314
|
+
}
|
|
1315
|
+
/**
|
|
1316
|
+
* Shared reap-to-stale-error implementation for a SINGLE run, used by both
|
|
1317
|
+
* `reapIfStale` (per-row, hot read path) and `reapAllStaleRuns` (per-row loop
|
|
1318
|
+
* over a stale-row snapshot — a row's own staleness is re-checked at UPDATE
|
|
1319
|
+
* time exactly as the prior bulk UPDATE did, so a heartbeat that lands
|
|
1320
|
+
* between the snapshot SELECT and this call naturally excludes the row).
|
|
1321
|
+
*
|
|
1322
|
+
* FIX 3: wraps the reap-to-errored write together with the recovery-
|
|
1323
|
+
* successor insert (`attemptStaleRunRecovery`) in ONE transaction so a
|
|
1324
|
+
* client polling `/runs/active` mid-recovery can never observe "errored, no
|
|
1325
|
+
* successor" — both writes commit together or not at all. A recovery
|
|
1326
|
+
* failure (thrown inside the transaction callback) is caught locally and
|
|
1327
|
+
* never rolls back the reap itself: the reap is the critical path, recovery
|
|
1328
|
+
* is strictly additive.
|
|
1329
|
+
*/
|
|
1330
|
+
async function reapSingleStaleRun(runId, maxStaleMs) {
|
|
1186
1331
|
const completedAt = Date.now();
|
|
1187
1332
|
// Background-dispatched runs get the wider stale window so a slow cold-start
|
|
1188
1333
|
// isn't reaped; foreground runs keep the tight 15s window. An explicit
|
|
@@ -1198,8 +1343,7 @@ export async function reapIfStale(runId, maxStaleMs) {
|
|
|
1198
1343
|
: // First `?` is backgroundAwareStaleCutoffSql's CAST param, second is
|
|
1199
1344
|
// inFlightGraceSql's — both bound to the same "now".
|
|
1200
1345
|
[completedAt, completedAt];
|
|
1201
|
-
const
|
|
1202
|
-
sql: `UPDATE agent_runs
|
|
1346
|
+
const updateSql = `UPDATE agent_runs
|
|
1203
1347
|
SET status = 'errored',
|
|
1204
1348
|
completed_at = ?,
|
|
1205
1349
|
error_code = ?,
|
|
@@ -1208,17 +1352,69 @@ export async function reapIfStale(runId, maxStaleMs) {
|
|
|
1208
1352
|
WHERE id = ?
|
|
1209
1353
|
AND status = 'running'
|
|
1210
1354
|
AND ${terminalRunEventExclusionSql()}
|
|
1211
|
-
AND ${staleClause}
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1355
|
+
AND ${staleClause}`;
|
|
1356
|
+
const updateArgs = [
|
|
1357
|
+
completedAt,
|
|
1358
|
+
STALE_RUN_ERROR_EVENT.errorCode,
|
|
1359
|
+
STALE_RUN_ERROR_EVENT.details,
|
|
1360
|
+
STALE_RUN_ERROR_EVENT.errorCode,
|
|
1361
|
+
runId,
|
|
1362
|
+
...staleArgs,
|
|
1363
|
+
];
|
|
1364
|
+
const client = getDbExec();
|
|
1365
|
+
let reaped = false;
|
|
1366
|
+
let outcome = null;
|
|
1367
|
+
if (client.transaction) {
|
|
1368
|
+
await client.transaction(async (tx) => {
|
|
1369
|
+
const { rowsAffected } = await tx.execute({
|
|
1370
|
+
sql: updateSql,
|
|
1371
|
+
args: updateArgs,
|
|
1372
|
+
});
|
|
1373
|
+
reaped = (rowsAffected ?? 0) > 0;
|
|
1374
|
+
if (reaped) {
|
|
1375
|
+
outcome = await attemptStaleRunRecovery(tx, runId).catch(() => null);
|
|
1376
|
+
}
|
|
1377
|
+
});
|
|
1378
|
+
}
|
|
1379
|
+
else {
|
|
1380
|
+
// No transaction primitive on this DbExec (every current implementation
|
|
1381
|
+
// provides one — see db/client.ts — so this is a defensive fallback,
|
|
1382
|
+
// not an expected path). Ordering the recovery design explicitly
|
|
1383
|
+
// tolerates: insert the successor FIRST, then flip the old row terminal
|
|
1384
|
+
// — a still-"running" old row plus an unclaimed successor is a safe
|
|
1385
|
+
// intermediate state; the reverse (errored with no successor briefly
|
|
1386
|
+
// visible) is not. Narrow, accepted gap versus the transactional path:
|
|
1387
|
+
// two concurrent reapers racing this exact fallback on the exact same
|
|
1388
|
+
// run could each pass the "no newer run" check before either inserts,
|
|
1389
|
+
// producing two successors for one turn.
|
|
1390
|
+
outcome = await attemptStaleRunRecovery(client, runId).catch(() => null);
|
|
1391
|
+
const { rowsAffected } = await client.execute({
|
|
1392
|
+
sql: updateSql,
|
|
1393
|
+
args: updateArgs,
|
|
1394
|
+
});
|
|
1395
|
+
reaped = (rowsAffected ?? 0) > 0;
|
|
1396
|
+
}
|
|
1397
|
+
if (reaped && outcome && outcome.outcome !== "not_background") {
|
|
1398
|
+
const detail = outcome.outcome === "recovered"
|
|
1399
|
+
? `recovered successorRunId=${outcome.successorRunId}`
|
|
1400
|
+
: `declined reason=${outcome.outcome}`;
|
|
1401
|
+
await recordRunDiagnostic(runId, RUN_DIAG_STAGE.staleRunRecoveryAttempted, detail).catch(() => { });
|
|
1402
|
+
if (outcome.outcome === "recovered") {
|
|
1403
|
+
attemptStaleRunRecoveryDispatch(outcome.successorRunId);
|
|
1404
|
+
}
|
|
1405
|
+
}
|
|
1406
|
+
return reaped;
|
|
1407
|
+
}
|
|
1408
|
+
/**
|
|
1409
|
+
* If the given run is marked "running" in SQL but its heartbeat is stale
|
|
1410
|
+
* (producer likely crashed), flip it to "errored" so watchers stop waiting.
|
|
1411
|
+
* Returns true if the row was reaped.
|
|
1412
|
+
*/
|
|
1413
|
+
export async function reapIfStale(runId, maxStaleMs) {
|
|
1414
|
+
await ensureRunTables();
|
|
1415
|
+
if (await reconcileTerminalRunFromEvents(runId))
|
|
1416
|
+
return false;
|
|
1417
|
+
const reaped = await reapSingleStaleRun(runId, maxStaleMs);
|
|
1222
1418
|
if (!reaped && (await reconcileTerminalRunFromEvents(runId)))
|
|
1223
1419
|
return false;
|
|
1224
1420
|
if (reaped) {
|
|
@@ -1631,34 +1827,33 @@ export async function reapAllStaleRuns() {
|
|
|
1631
1827
|
await reconcileTerminalRunFromEvents(id);
|
|
1632
1828
|
}
|
|
1633
1829
|
}
|
|
1634
|
-
|
|
1635
|
-
|
|
1636
|
-
|
|
1637
|
-
|
|
1638
|
-
|
|
1639
|
-
|
|
1640
|
-
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
1651
|
-
|
|
1652
|
-
|
|
1653
|
-
|
|
1654
|
-
});
|
|
1830
|
+
// FIX 3 (durable-background incident): reap each stale row individually
|
|
1831
|
+
// via the shared `reapSingleStaleRun` (rather than one bulk UPDATE) so a
|
|
1832
|
+
// background chat-turn run can be checked for server-owned recovery
|
|
1833
|
+
// (`attemptStaleRunRecovery`) before it goes terminal — a single bulk
|
|
1834
|
+
// statement can't express "insert a successor for row A but not row B".
|
|
1835
|
+
// `reapSingleStaleRun` re-applies the identical staleness clause per row
|
|
1836
|
+
// (same as the bulk UPDATE previously did), so a row whose heartbeat
|
|
1837
|
+
// landed between the SELECT above and this loop is still correctly
|
|
1838
|
+
// excluded. This function only runs once, at process startup (see
|
|
1839
|
+
// agent-chat-plugin.ts) — the extra per-row round trips are
|
|
1840
|
+
// inconsequential.
|
|
1841
|
+
let reapedCount = 0;
|
|
1842
|
+
for (const row of stale.rows) {
|
|
1843
|
+
const id = row.id;
|
|
1844
|
+
if (typeof id !== "string")
|
|
1845
|
+
continue;
|
|
1846
|
+
const reaped = await reapSingleStaleRun(id).catch(() => false);
|
|
1847
|
+
if (reaped)
|
|
1848
|
+
reapedCount += 1;
|
|
1849
|
+
}
|
|
1655
1850
|
for (const row of stale.rows) {
|
|
1656
1851
|
const id = row.id;
|
|
1657
1852
|
if (typeof id === "string") {
|
|
1658
1853
|
await safeAppendTerminalRunEvent(id, STALE_RUN_ERROR_EVENT, "reap-all-stale");
|
|
1659
1854
|
}
|
|
1660
1855
|
}
|
|
1661
|
-
return
|
|
1856
|
+
return reapedCount;
|
|
1662
1857
|
}
|
|
1663
1858
|
/** Delete old runs and expire stale "running" rows that haven't had activity
|
|
1664
1859
|
* (e.g. worker crashed before updating status). Completed runs are pruned at
|