@agent-native/core 0.128.4 → 0.129.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (351) hide show
  1. package/corpus/README.md +2 -2
  2. package/corpus/core/CHANGELOG.md +90 -0
  3. package/corpus/core/docs/content/template-design-collaboration-and-full-apps.mdx +3 -1
  4. package/corpus/core/package.json +6 -3
  5. package/corpus/core/src/a2a/client.ts +18 -4
  6. package/corpus/core/src/a2a/server.ts +19 -3
  7. package/corpus/core/src/a2a/types.ts +7 -0
  8. package/corpus/core/src/action.ts +43 -16
  9. package/corpus/core/src/agent/engine/ai-sdk-engine.ts +14 -10
  10. package/corpus/core/src/agent/production-agent.ts +41 -3
  11. package/corpus/core/src/agent/thread-data-builder.ts +5 -0
  12. package/corpus/core/src/chat-threads/migrations.ts +53 -0
  13. package/corpus/core/src/cli/create.ts +4 -0
  14. package/corpus/core/src/cli/skills-content/design-visual-edit-skill.ts +36 -16
  15. package/corpus/core/src/cli/skills.ts +16 -0
  16. package/corpus/core/src/client/AgentPanel.tsx +9 -5
  17. package/corpus/core/src/client/AssistantChat.tsx +138 -46
  18. package/corpus/core/src/client/MultiTabAssistantChat.tsx +16 -10
  19. package/corpus/core/src/client/agent-chat-adapter.ts +21 -3
  20. package/corpus/core/src/client/analytics.ts +159 -102
  21. package/corpus/core/src/client/chat/message-components.tsx +12 -5
  22. package/corpus/core/src/client/chat/runtime.ts +2 -1
  23. package/corpus/core/src/client/chat/tool-call-display.tsx +25 -2
  24. package/corpus/core/src/client/chat-model-groups.ts +3 -0
  25. package/corpus/core/src/client/client-status-requests.ts +161 -0
  26. package/corpus/core/src/client/org/OrgSwitcher.tsx +4 -4
  27. package/corpus/core/src/client/settings/VoiceTranscriptionSection.tsx +2 -2
  28. package/corpus/core/src/client/sse-event-processor.ts +15 -3
  29. package/corpus/core/src/client/use-agent-engine-configured.ts +54 -50
  30. package/corpus/core/src/client/use-chat-models.ts +16 -11
  31. package/corpus/core/src/client/use-chat-threads.ts +32 -9
  32. package/corpus/core/src/client/use-session.ts +4 -5
  33. package/corpus/core/src/db/client.ts +83 -8
  34. package/corpus/core/src/db/ensure-additive-columns.ts +31 -20
  35. package/corpus/core/src/db/migrations.ts +22 -2
  36. package/corpus/core/src/mcp/approval-store.ts +99 -0
  37. package/corpus/core/src/mcp/build-server.ts +579 -303
  38. package/corpus/core/src/mcp/embed-app.ts +1 -1
  39. package/corpus/core/src/mcp/oauth-client-metadata.ts +380 -0
  40. package/corpus/core/src/mcp/oauth-route.ts +57 -52
  41. package/corpus/core/src/mcp/screen-memory-stdio.ts +400 -378
  42. package/corpus/core/src/mcp/server.ts +29 -145
  43. package/corpus/core/src/mcp/stdio.ts +72 -80
  44. package/corpus/core/src/mcp-client/manager.ts +6 -10
  45. package/corpus/core/src/mcp-client/oauth-client.ts +127 -36
  46. package/corpus/core/src/mcp-client/oauth-routes.ts +16 -6
  47. package/corpus/core/src/mcp-client/routes.ts +6 -5
  48. package/corpus/core/src/server/agent-capabilities.ts +20 -0
  49. package/corpus/core/src/server/agent-chat/action-filters-a2a.ts +27 -0
  50. package/corpus/core/src/server/agent-chat-plugin.ts +18 -0
  51. package/corpus/core/src/server/transcribe-voice.ts +14 -5
  52. package/corpus/core/src/templates/default/.agents/skills/security/SKILL.md +13 -2
  53. package/corpus/core/src/templates/workspace-core/.agents/skills/authentication/SKILL.md +5 -2
  54. package/corpus/core/src/templates/workspace-core/.agents/skills/external-agents/SKILL.md +18 -0
  55. package/corpus/core/src/templates/workspace-core/.agents/skills/security/SKILL.md +13 -2
  56. package/corpus/templates/analytics/.agents/skills/security/SKILL.md +13 -2
  57. package/corpus/templates/analytics/app/components/dashboard/DashboardPanelSkeleton.tsx +1 -10
  58. package/corpus/templates/analytics/app/global.css +0 -31
  59. package/corpus/templates/analytics/app/lib/sql-query.ts +46 -20
  60. package/corpus/templates/analytics/changelog/2026-07-28-analytics-dashboard-loading-states-now-use-a-simpler-standar.md +6 -0
  61. package/corpus/templates/analytics/changelog/2026-07-28-analytics-dashboards-now-load-first-party-charts-reliably-in.md +6 -0
  62. package/corpus/templates/analytics/server/lib/first-party-analytics-cache.ts +42 -16
  63. package/corpus/templates/analytics/server/lib/first-party-analytics.ts +4 -2
  64. package/corpus/templates/analytics/shared/sql-query-limits.ts +1 -0
  65. package/corpus/templates/assets/.agents/skills/security/SKILL.md +13 -2
  66. package/corpus/templates/brain/.agents/skills/security/SKILL.md +13 -2
  67. package/corpus/templates/calendar/.agents/skills/security/SKILL.md +13 -2
  68. package/corpus/templates/chat/.agents/skills/security/SKILL.md +13 -2
  69. package/corpus/templates/chat/README.md +0 -2
  70. package/corpus/templates/clips/.agents/skills/security/SKILL.md +13 -2
  71. package/corpus/templates/clips/app/components/player/player-controls.tsx +3 -6
  72. package/corpus/templates/clips/app/components/player/recording-views-badge.tsx +6 -2
  73. package/corpus/templates/clips/app/components/sharing/share-ui.tsx +25 -10
  74. package/corpus/templates/clips/app/components/sharing/viewed-by-popover.tsx +28 -6
  75. package/corpus/templates/clips/app/lib/use-visible-avatar-url.ts +34 -0
  76. package/corpus/templates/clips/app/routes/r.$recordingId.tsx +6 -2
  77. package/corpus/templates/clips/app/routes/share.$shareId.tsx +3 -1
  78. package/corpus/templates/clips/changelog/2026-07-28-copied-share-links-no-longer-leave-an-unnecessary-confirmati.md +6 -0
  79. package/corpus/templates/clips/changelog/2026-07-28-playback-skip-controls-no-longer-show-overlapping-numbers.md +6 -0
  80. package/corpus/templates/clips/changelog/2026-07-28-profile-avatars-now-appear-in-viewer-and-share-access-lists.md +6 -0
  81. package/corpus/templates/clips/changelog/2026-07-28-public-clips-share-buttons-now-use-a-clean-text-only-label.md +6 -0
  82. package/corpus/templates/clips/desktop/src/app.tsx +14 -17
  83. package/corpus/templates/clips/desktop/src/overlays/live-audio-bars.tsx +3 -1
  84. package/corpus/templates/clips/desktop/src/styles.css +13 -10
  85. package/corpus/templates/clips/desktop/src-tauri/src/recording_indicator.rs +2 -2
  86. package/corpus/templates/clips/desktop/src-tauri/src/whisper_speech.rs +36 -0
  87. package/corpus/templates/content/.agents/skills/security/SKILL.md +13 -2
  88. package/corpus/templates/content/actions/_builder-cms-read-client.ts +155 -34
  89. package/corpus/templates/content/actions/_builder-docs-client.ts +151 -30
  90. package/corpus/templates/content/actions/stage-builder-source-bulk-update.ts +5 -2
  91. package/corpus/templates/content/app/root.tsx +35 -14
  92. package/corpus/templates/content/changelog/2026-07-28-builder-backed-files-media-fields-now-accept-direct-file-url.md +6 -0
  93. package/corpus/templates/content/changelog/2026-07-28-builder-content-reads-now-negotiate-the-newest-mcp-protocol-.md +6 -0
  94. package/corpus/templates/content/changelog/2026-07-28-public-content-pages-load-faster-before-the-optional-ask-sid.md +6 -0
  95. package/corpus/templates/design/.agents/skills/security/SKILL.md +13 -2
  96. package/corpus/templates/design/.agents/skills/visual-edit/SKILL.md +36 -16
  97. package/corpus/templates/design/.generated/bridge/editor-chrome.generated.ts +42 -3
  98. package/corpus/templates/design/actions/open-visual-edit.ts +2 -1
  99. package/corpus/templates/design/app/components/design/CanvasContextMenu.tsx +2 -2
  100. package/corpus/templates/design/app/components/design/DesignCanvas.tsx +14 -15
  101. package/corpus/templates/design/app/components/design/ResponsiveInteractBar.tsx +314 -0
  102. package/corpus/templates/design/app/components/design/bridge/editor-chrome.bridge.ts +88 -3
  103. package/corpus/templates/design/app/components/design/keyboard-shortcuts.ts +1 -1
  104. package/corpus/templates/design/app/hooks/useDesignHotkeys.ts +7 -6
  105. package/corpus/templates/design/app/i18n-data.ts +56 -11
  106. package/corpus/templates/design/app/i18n-responsive-interact.ts +183 -0
  107. package/corpus/templates/design/app/lib/agent-chat.ts +60 -0
  108. package/corpus/templates/design/app/pages/design-editor/responsive-interact.ts +89 -0
  109. package/corpus/templates/design/changelog/2026-07-28-design-edits-can-be-applied-through-the-host-coding-agent-fr.md +6 -0
  110. package/corpus/templates/design/changelog/2026-07-28-interact-mode-can-preview-designs-in-responsive-device-sizes.md +6 -0
  111. package/corpus/templates/design/changelog/2026-07-28-interact-mode-now-opens-a-full-screen-responsive-preview-wit.md +6 -0
  112. package/corpus/templates/design/changelog/2026-07-28-show-or-hide-design-editing-chrome-with-cmd-on-apple-devices.md +2 -2
  113. package/corpus/templates/design/changelog/2026-07-28-visual-edit-now-always-shows-the-live-running-app-in-the-can.md +6 -0
  114. package/corpus/templates/dispatch/.agents/skills/security/SKILL.md +13 -2
  115. package/corpus/templates/forms/.agents/skills/security/SKILL.md +13 -2
  116. package/corpus/templates/macros/.agents/skills/security/SKILL.md +13 -2
  117. package/corpus/templates/mail/.agents/skills/security/SKILL.md +13 -2
  118. package/corpus/templates/plan/.agents/skills/security/SKILL.md +13 -2
  119. package/corpus/templates/slides/.agents/skills/security/SKILL.md +13 -2
  120. package/corpus/templates/tasks/.agents/skills/security/SKILL.md +13 -2
  121. package/corpus/toolkit/CHANGELOG.md +13 -0
  122. package/corpus/toolkit/package.json +1 -1
  123. package/corpus/toolkit/src/chat-history.css +1 -2
  124. package/corpus/toolkit/src/composer/TiptapComposer.tsx +45 -0
  125. package/dist/a2a/client.d.ts +7 -0
  126. package/dist/a2a/client.d.ts.map +1 -1
  127. package/dist/a2a/client.js +8 -3
  128. package/dist/a2a/client.js.map +1 -1
  129. package/dist/a2a/server.d.ts.map +1 -1
  130. package/dist/a2a/server.js +18 -3
  131. package/dist/a2a/server.js.map +1 -1
  132. package/dist/a2a/types.d.ts +7 -0
  133. package/dist/a2a/types.d.ts.map +1 -1
  134. package/dist/a2a/types.js.map +1 -1
  135. package/dist/action.d.ts +15 -1
  136. package/dist/action.d.ts.map +1 -1
  137. package/dist/action.js +39 -16
  138. package/dist/action.js.map +1 -1
  139. package/dist/agent/engine/ai-sdk-engine.d.ts.map +1 -1
  140. package/dist/agent/engine/ai-sdk-engine.js +14 -10
  141. package/dist/agent/engine/ai-sdk-engine.js.map +1 -1
  142. package/dist/agent/production-agent.d.ts.map +1 -1
  143. package/dist/agent/production-agent.js +35 -7
  144. package/dist/agent/production-agent.js.map +1 -1
  145. package/dist/agent/thread-data-builder.d.ts +2 -0
  146. package/dist/agent/thread-data-builder.d.ts.map +1 -1
  147. package/dist/agent/thread-data-builder.js +3 -0
  148. package/dist/agent/thread-data-builder.js.map +1 -1
  149. package/dist/chat-threads/migrations.d.ts +21 -0
  150. package/dist/chat-threads/migrations.d.ts.map +1 -0
  151. package/dist/chat-threads/migrations.js +45 -0
  152. package/dist/chat-threads/migrations.js.map +1 -0
  153. package/dist/cli/create.d.ts.map +1 -1
  154. package/dist/cli/create.js +4 -0
  155. package/dist/cli/create.js.map +1 -1
  156. package/dist/cli/skills-content/design-visual-edit-skill.d.ts +1 -1
  157. package/dist/cli/skills-content/design-visual-edit-skill.d.ts.map +1 -1
  158. package/dist/cli/skills-content/design-visual-edit-skill.js +36 -16
  159. package/dist/cli/skills-content/design-visual-edit-skill.js.map +1 -1
  160. package/dist/cli/skills.d.ts.map +1 -1
  161. package/dist/cli/skills.js +15 -0
  162. package/dist/cli/skills.js.map +1 -1
  163. package/dist/client/AgentPanel.d.ts.map +1 -1
  164. package/dist/client/AgentPanel.js +6 -5
  165. package/dist/client/AgentPanel.js.map +1 -1
  166. package/dist/client/AssistantChat.d.ts +37 -0
  167. package/dist/client/AssistantChat.d.ts.map +1 -1
  168. package/dist/client/AssistantChat.js +41 -8
  169. package/dist/client/AssistantChat.js.map +1 -1
  170. package/dist/client/MultiTabAssistantChat.d.ts.map +1 -1
  171. package/dist/client/MultiTabAssistantChat.js +11 -10
  172. package/dist/client/MultiTabAssistantChat.js.map +1 -1
  173. package/dist/client/agent-chat-adapter.d.ts.map +1 -1
  174. package/dist/client/agent-chat-adapter.js +19 -3
  175. package/dist/client/agent-chat-adapter.js.map +1 -1
  176. package/dist/client/analytics.d.ts.map +1 -1
  177. package/dist/client/analytics.js +150 -101
  178. package/dist/client/analytics.js.map +1 -1
  179. package/dist/client/chat/message-components.d.ts +2 -1
  180. package/dist/client/chat/message-components.d.ts.map +1 -1
  181. package/dist/client/chat/message-components.js +11 -4
  182. package/dist/client/chat/message-components.js.map +1 -1
  183. package/dist/client/chat/runtime.d.ts.map +1 -1
  184. package/dist/client/chat/runtime.js +2 -1
  185. package/dist/client/chat/runtime.js.map +1 -1
  186. package/dist/client/chat/tool-call-display.d.ts +4 -1
  187. package/dist/client/chat/tool-call-display.d.ts.map +1 -1
  188. package/dist/client/chat/tool-call-display.js +14 -8
  189. package/dist/client/chat/tool-call-display.js.map +1 -1
  190. package/dist/client/chat-model-groups.d.ts.map +1 -1
  191. package/dist/client/chat-model-groups.js +3 -0
  192. package/dist/client/chat-model-groups.js.map +1 -1
  193. package/dist/client/client-status-requests.d.ts +14 -0
  194. package/dist/client/client-status-requests.d.ts.map +1 -0
  195. package/dist/client/client-status-requests.js +126 -0
  196. package/dist/client/client-status-requests.js.map +1 -0
  197. package/dist/client/org/OrgSwitcher.js +2 -2
  198. package/dist/client/org/OrgSwitcher.js.map +1 -1
  199. package/dist/client/settings/VoiceTranscriptionSection.js +1 -1
  200. package/dist/client/settings/VoiceTranscriptionSection.js.map +1 -1
  201. package/dist/client/sse-event-processor.d.ts +7 -0
  202. package/dist/client/sse-event-processor.d.ts.map +1 -1
  203. package/dist/client/sse-event-processor.js +8 -3
  204. package/dist/client/sse-event-processor.js.map +1 -1
  205. package/dist/client/use-agent-engine-configured.d.ts.map +1 -1
  206. package/dist/client/use-agent-engine-configured.js +29 -43
  207. package/dist/client/use-agent-engine-configured.js.map +1 -1
  208. package/dist/client/use-chat-models.d.ts.map +1 -1
  209. package/dist/client/use-chat-models.js +11 -11
  210. package/dist/client/use-chat-models.js.map +1 -1
  211. package/dist/client/use-chat-threads.d.ts.map +1 -1
  212. package/dist/client/use-chat-threads.js +23 -7
  213. package/dist/client/use-chat-threads.js.map +1 -1
  214. package/dist/client/use-session.d.ts.map +1 -1
  215. package/dist/client/use-session.js +4 -4
  216. package/dist/client/use-session.js.map +1 -1
  217. package/dist/collab/awareness.d.ts +2 -2
  218. package/dist/collab/awareness.d.ts.map +1 -1
  219. package/dist/db/client.d.ts.map +1 -1
  220. package/dist/db/client.js +48 -4
  221. package/dist/db/client.js.map +1 -1
  222. package/dist/db/ensure-additive-columns.d.ts.map +1 -1
  223. package/dist/db/ensure-additive-columns.js +32 -21
  224. package/dist/db/ensure-additive-columns.js.map +1 -1
  225. package/dist/db/migrations.d.ts +9 -0
  226. package/dist/db/migrations.d.ts.map +1 -1
  227. package/dist/db/migrations.js +14 -3
  228. package/dist/db/migrations.js.map +1 -1
  229. package/dist/file-upload/actions/upload-image.d.ts +1 -1
  230. package/dist/mcp/approval-store.d.ts +16 -0
  231. package/dist/mcp/approval-store.d.ts.map +1 -0
  232. package/dist/mcp/approval-store.js +85 -0
  233. package/dist/mcp/approval-store.js.map +1 -0
  234. package/dist/mcp/build-server.d.ts +1 -34
  235. package/dist/mcp/build-server.d.ts.map +1 -1
  236. package/dist/mcp/build-server.js +220 -23
  237. package/dist/mcp/build-server.js.map +1 -1
  238. package/dist/mcp/embed-app.js +1 -1
  239. package/dist/mcp/embed-app.js.map +1 -1
  240. package/dist/mcp/oauth-client-metadata.d.ts +15 -0
  241. package/dist/mcp/oauth-client-metadata.d.ts.map +1 -0
  242. package/dist/mcp/oauth-client-metadata.js +297 -0
  243. package/dist/mcp/oauth-client-metadata.js.map +1 -0
  244. package/dist/mcp/oauth-route.d.ts.map +1 -1
  245. package/dist/mcp/oauth-route.js +39 -53
  246. package/dist/mcp/oauth-route.js.map +1 -1
  247. package/dist/mcp/screen-memory-stdio.d.ts.map +1 -1
  248. package/dist/mcp/screen-memory-stdio.js +304 -296
  249. package/dist/mcp/screen-memory-stdio.js.map +1 -1
  250. package/dist/mcp/server.d.ts +12 -25
  251. package/dist/mcp/server.d.ts.map +1 -1
  252. package/dist/mcp/server.js +24 -133
  253. package/dist/mcp/server.js.map +1 -1
  254. package/dist/mcp/stdio.d.ts.map +1 -1
  255. package/dist/mcp/stdio.js +36 -35
  256. package/dist/mcp/stdio.js.map +1 -1
  257. package/dist/mcp-client/manager.d.ts.map +1 -1
  258. package/dist/mcp-client/manager.js +6 -6
  259. package/dist/mcp-client/manager.js.map +1 -1
  260. package/dist/mcp-client/oauth-client.d.ts +13 -17
  261. package/dist/mcp-client/oauth-client.d.ts.map +1 -1
  262. package/dist/mcp-client/oauth-client.js +74 -16
  263. package/dist/mcp-client/oauth-client.js.map +1 -1
  264. package/dist/mcp-client/oauth-routes.d.ts +2 -2
  265. package/dist/mcp-client/oauth-routes.d.ts.map +1 -1
  266. package/dist/mcp-client/oauth-routes.js +16 -6
  267. package/dist/mcp-client/oauth-routes.js.map +1 -1
  268. package/dist/mcp-client/routes.js +5 -5
  269. package/dist/mcp-client/routes.js.map +1 -1
  270. package/dist/observability/routes.d.ts +3 -3
  271. package/dist/progress/routes.d.ts +1 -1
  272. package/dist/provider-api/actions/custom-provider-registration.d.ts +14 -14
  273. package/dist/provider-api/actions/provider-api.d.ts +8 -8
  274. package/dist/resources/handlers.d.ts +1 -1
  275. package/dist/secrets/routes.d.ts +9 -9
  276. package/dist/server/agent-capabilities.d.ts.map +1 -1
  277. package/dist/server/agent-capabilities.js +21 -0
  278. package/dist/server/agent-capabilities.js.map +1 -1
  279. package/dist/server/agent-chat/action-filters-a2a.d.ts +14 -0
  280. package/dist/server/agent-chat/action-filters-a2a.d.ts.map +1 -1
  281. package/dist/server/agent-chat/action-filters-a2a.js +16 -0
  282. package/dist/server/agent-chat/action-filters-a2a.js.map +1 -1
  283. package/dist/server/agent-chat-plugin.d.ts.map +1 -1
  284. package/dist/server/agent-chat-plugin.js +11 -1
  285. package/dist/server/agent-chat-plugin.js.map +1 -1
  286. package/dist/server/agent-engine-api-key-route.d.ts +1 -1
  287. package/dist/server/realtime-token.d.ts +1 -1
  288. package/dist/server/transcribe-voice.d.ts +1 -1
  289. package/dist/server/transcribe-voice.js +12 -6
  290. package/dist/server/transcribe-voice.js.map +1 -1
  291. package/dist/templates/chat/.agents/skills/security/SKILL.md +13 -2
  292. package/dist/templates/chat/README.md +0 -2
  293. package/dist/templates/default/.agents/skills/security/SKILL.md +13 -2
  294. package/dist/templates/workspace-core/.agents/skills/authentication/SKILL.md +5 -2
  295. package/dist/templates/workspace-core/.agents/skills/external-agents/SKILL.md +18 -0
  296. package/dist/templates/workspace-core/.agents/skills/security/SKILL.md +13 -2
  297. package/docs/content/template-design-collaboration-and-full-apps.mdx +3 -1
  298. package/package.json +7 -4
  299. package/src/a2a/client.ts +18 -4
  300. package/src/a2a/server.ts +19 -3
  301. package/src/a2a/types.ts +7 -0
  302. package/src/action.ts +43 -16
  303. package/src/agent/engine/ai-sdk-engine.ts +14 -10
  304. package/src/agent/production-agent.ts +41 -3
  305. package/src/agent/thread-data-builder.ts +5 -0
  306. package/src/chat-threads/migrations.ts +53 -0
  307. package/src/cli/create.ts +4 -0
  308. package/src/cli/skills-content/design-visual-edit-skill.ts +36 -16
  309. package/src/cli/skills.ts +16 -0
  310. package/src/client/AgentPanel.tsx +9 -5
  311. package/src/client/AssistantChat.tsx +138 -46
  312. package/src/client/MultiTabAssistantChat.tsx +16 -10
  313. package/src/client/agent-chat-adapter.ts +21 -3
  314. package/src/client/analytics.ts +159 -102
  315. package/src/client/chat/message-components.tsx +12 -5
  316. package/src/client/chat/runtime.ts +2 -1
  317. package/src/client/chat/tool-call-display.tsx +25 -2
  318. package/src/client/chat-model-groups.ts +3 -0
  319. package/src/client/client-status-requests.ts +161 -0
  320. package/src/client/org/OrgSwitcher.tsx +4 -4
  321. package/src/client/settings/VoiceTranscriptionSection.tsx +2 -2
  322. package/src/client/sse-event-processor.ts +15 -3
  323. package/src/client/use-agent-engine-configured.ts +54 -50
  324. package/src/client/use-chat-models.ts +16 -11
  325. package/src/client/use-chat-threads.ts +32 -9
  326. package/src/client/use-session.ts +4 -5
  327. package/src/db/client.ts +83 -8
  328. package/src/db/ensure-additive-columns.ts +31 -20
  329. package/src/db/migrations.ts +22 -2
  330. package/src/mcp/approval-store.ts +99 -0
  331. package/src/mcp/build-server.ts +579 -303
  332. package/src/mcp/embed-app.ts +1 -1
  333. package/src/mcp/oauth-client-metadata.ts +380 -0
  334. package/src/mcp/oauth-route.ts +57 -52
  335. package/src/mcp/screen-memory-stdio.ts +400 -378
  336. package/src/mcp/server.ts +29 -145
  337. package/src/mcp/stdio.ts +72 -80
  338. package/src/mcp-client/manager.ts +6 -10
  339. package/src/mcp-client/oauth-client.ts +127 -36
  340. package/src/mcp-client/oauth-routes.ts +16 -6
  341. package/src/mcp-client/routes.ts +6 -5
  342. package/src/server/agent-capabilities.ts +20 -0
  343. package/src/server/agent-chat/action-filters-a2a.ts +27 -0
  344. package/src/server/agent-chat-plugin.ts +18 -0
  345. package/src/server/transcribe-voice.ts +14 -5
  346. package/src/templates/chat/.agents/skills/security/SKILL.md +13 -2
  347. package/src/templates/chat/README.md +0 -2
  348. package/src/templates/default/.agents/skills/security/SKILL.md +13 -2
  349. package/src/templates/workspace-core/.agents/skills/authentication/SKILL.md +5 -2
  350. package/src/templates/workspace-core/.agents/skills/external-agents/SKILL.md +18 -0
  351. package/src/templates/workspace-core/.agents/skills/security/SKILL.md +13 -2
@@ -1170,396 +1170,418 @@ export async function runScreenMemoryMCPStdio(
1170
1170
  const env = opts.env ?? process.env;
1171
1171
  const storeDir = opts.storeDir ?? defaultStoreDir(env);
1172
1172
 
1173
- const { Server } = await import("@modelcontextprotocol/sdk/server/index.js");
1174
- const { StdioServerTransport } =
1175
- await import("@modelcontextprotocol/sdk/server/stdio.js");
1176
- const { CallToolRequestSchema, ListToolsRequestSchema } =
1177
- await import("@modelcontextprotocol/sdk/types.js");
1173
+ const { Server } = await import("@modelcontextprotocol/server");
1174
+ const { serveStdio } = await import("@modelcontextprotocol/server/stdio");
1178
1175
 
1179
- const server = new Server(
1180
- { name: "clips-screen-memory", version: "0.1.0" },
1181
- { capabilities: { tools: {} } },
1182
- );
1183
-
1184
- server.setRequestHandler(ListToolsRequestSchema, async () => ({
1185
- tools: screenMemoryMcpToolDefinitions(),
1186
- }));
1187
-
1188
- server.setRequestHandler(CallToolRequestSchema, async (request: any) => {
1189
- const name = request.params?.name;
1190
- const args = (request.params?.arguments ?? {}) as Record<string, unknown>;
1191
- if (name === "screen_memory_status") {
1192
- const config = readFeatureConfig(storeDir);
1193
- const segments = recentSegments(storeDir, 24 * 60);
1194
- return textResult({
1195
- enabled: config.enabled === true,
1196
- paused: config.paused === true,
1197
- retentionHours: config.retentionHours ?? 24,
1198
- maxBytes: config.maxBytes ?? 20 * 1024 * 1024 * 1024,
1199
- segmentCount: segments.length,
1200
- totalBytes: segments.reduce(
1201
- (sum, segment) => sum + (segment.bytes || 0),
1202
- 0,
1203
- ),
1204
- note: "The Screen Memory store is connected. Local archive paths are intentionally not exposed to agents.",
1205
- });
1206
- }
1207
- if (name === "screen_memory_recent_context") {
1208
- const result = await queryScreenMemoryContextForStore(
1209
- {
1210
- query: typeof args.query === "string" ? args.query : undefined,
1211
- sinceMinutes:
1212
- typeof args.minutes === "number" ? args.minutes : undefined,
1213
- limit: typeof args.limit === "number" ? args.limit : undefined,
1214
- },
1215
- storeDir,
1216
- env,
1176
+ const stdio = serveStdio(
1177
+ () => {
1178
+ const server = new Server(
1179
+ { name: "clips-screen-memory", version: "0.1.0" },
1180
+ { capabilities: { tools: {} } },
1217
1181
  );
1218
- const sanitizedEvidence = result.evidence.map((item) => ({
1219
- ...item,
1220
- excerpt: redactCredentialText(item.excerpt),
1221
- }));
1222
- const sanitizedItems = result.items.map((item) => ({
1223
- ...item,
1224
- text: redactCredentialText(item.text),
1225
- sourceFile: "local-screen-memory",
1226
- }));
1227
- const requestId = `egress-mcp-${Date.now()}-${process.pid}`;
1228
- const packet = {
1229
- question:
1230
- typeof args.query === "string" && args.query.trim()
1231
- ? redactCredentialText(args.query.trim().slice(0, 4_000))
1232
- : "Recent Screen Memory context",
1233
- evidence: sanitizedEvidence.slice(0, 20).map((item) => ({
1234
- id: item.id,
1235
- momentId: item.momentId,
1236
- sourceType: item.sourceType,
1237
- capturedAt: item.capturedAt,
1238
- excerpt: item.excerpt.slice(0, 1_200),
1239
- })),
1240
- };
1241
- appendEgressReceipt(
1242
- storeDir,
1243
- requestId,
1244
- "recent-context",
1245
- packet.evidence,
1182
+
1183
+ server.setRequestHandler(
1184
+ "tools/list",
1185
+ async () =>
1186
+ ({
1187
+ tools: screenMemoryMcpToolDefinitions(),
1188
+ }) as any,
1246
1189
  );
1247
- return textResult(
1248
- // Keep the old MCP envelope available while adding the typed contract.
1249
- {
1250
- events: sanitizedItems,
1251
- ...result,
1252
- items: sanitizedItems,
1253
- evidence: sanitizedEvidence,
1254
- contextFiles: [],
1255
- egress: {
1190
+
1191
+ server.setRequestHandler("tools/call", async (request: any) => {
1192
+ const name = request.params?.name;
1193
+ const args = (request.params?.arguments ?? {}) as Record<
1194
+ string,
1195
+ unknown
1196
+ >;
1197
+ if (name === "screen_memory_status") {
1198
+ const config = readFeatureConfig(storeDir);
1199
+ const segments = recentSegments(storeDir, 24 * 60);
1200
+ return textResult({
1201
+ enabled: config.enabled === true,
1202
+ paused: config.paused === true,
1203
+ retentionHours: config.retentionHours ?? 24,
1204
+ maxBytes: config.maxBytes ?? 20 * 1024 * 1024 * 1024,
1205
+ segmentCount: segments.length,
1206
+ totalBytes: segments.reduce(
1207
+ (sum, segment) => sum + (segment.bytes || 0),
1208
+ 0,
1209
+ ),
1210
+ note: "The Screen Memory store is connected. Local archive paths are intentionally not exposed to agents.",
1211
+ });
1212
+ }
1213
+ if (name === "screen_memory_recent_context") {
1214
+ const result = await queryScreenMemoryContextForStore(
1215
+ {
1216
+ query: typeof args.query === "string" ? args.query : undefined,
1217
+ sinceMinutes:
1218
+ typeof args.minutes === "number" ? args.minutes : undefined,
1219
+ limit: typeof args.limit === "number" ? args.limit : undefined,
1220
+ },
1221
+ storeDir,
1222
+ env,
1223
+ );
1224
+ const sanitizedEvidence = result.evidence.map((item) => ({
1225
+ ...item,
1226
+ excerpt: redactCredentialText(item.excerpt),
1227
+ }));
1228
+ const sanitizedItems = result.items.map((item) => ({
1229
+ ...item,
1230
+ text: redactCredentialText(item.text),
1231
+ sourceFile: "local-screen-memory",
1232
+ }));
1233
+ const requestId = `egress-mcp-${Date.now()}-${process.pid}`;
1234
+ const packet = {
1235
+ question:
1236
+ typeof args.query === "string" && args.query.trim()
1237
+ ? redactCredentialText(args.query.trim().slice(0, 4_000))
1238
+ : "Recent Screen Memory context",
1239
+ evidence: sanitizedEvidence.slice(0, 20).map((item) => ({
1240
+ id: item.id,
1241
+ momentId: item.momentId,
1242
+ sourceType: item.sourceType,
1243
+ capturedAt: item.capturedAt,
1244
+ excerpt: item.excerpt.slice(0, 1_200),
1245
+ })),
1246
+ };
1247
+ appendEgressReceipt(
1248
+ storeDir,
1256
1249
  requestId,
1257
- packet,
1258
- note: "A content-free local activity receipt was recorded before this bounded text packet was returned.",
1259
- },
1260
- },
1261
- );
1262
- }
1263
- if (name === "screen_memory_recent_segments") {
1264
- const minutes = typeof args.minutes === "number" ? args.minutes : 30;
1265
- const segments = recentSegments(storeDir, minutes);
1266
- const requestId = `egress-mcp-${Date.now()}-${process.pid}`;
1267
- const packet = {
1268
- question: `List recent Rewind segment references from the previous ${Math.max(1, Math.min(minutes, 24 * 60))} minutes`,
1269
- evidence: segments.slice(0, 20).map((segment) => ({
1270
- id: segment.id,
1271
- momentId: `segment:${segment.id}`,
1272
- sourceType: "app-context",
1273
- capturedAt: segment.startedAt,
1274
- excerpt: `Local media segment reference from ${segment.startedAt} to ${segment.endedAt}; media path and bytes were not exposed.`,
1275
- })),
1276
- };
1277
- appendEgressReceipt(
1278
- storeDir,
1279
- requestId,
1280
- "recent-segments",
1281
- packet.evidence,
1282
- );
1283
- return textResult({
1284
- localOnly: true,
1285
- mediaApprovalRequired: true,
1286
- segments: segments.map((segment) => ({
1287
- id: segment.id,
1288
- startedAt: segment.startedAt,
1289
- endedAt: segment.endedAt,
1290
- mimeType: segment.mimeType,
1291
- durationMs: segment.durationMs,
1292
- })),
1293
- egress: {
1294
- requestId,
1295
- packet,
1296
- note: "A content-free local activity receipt was recorded before this bounded metadata packet was returned.",
1297
- },
1298
- });
1299
- }
1300
- if (name === "screen_memory_search_chapters") {
1301
- if (typeof args.query !== "string" || !args.query.trim())
1302
- throw new Error("query is required.");
1303
- const document = readScreenMemoryChapters(storeDir);
1304
- if (!document)
1305
- throw new Error(
1306
- "No valid local Rewind chapters index is available yet.",
1307
- );
1308
- const minutes =
1309
- typeof args.minutes === "number" && Number.isFinite(args.minutes)
1310
- ? Math.max(1, Math.min(Math.trunc(args.minutes), 24 * 60))
1311
- : null;
1312
- const cutoff = minutes ? Date.now() - minutes * 60_000 : null;
1313
- const boundedDocument = cutoff
1314
- ? {
1315
- ...document,
1316
- chapters: document.chapters.filter(
1317
- (chapter) => Date.parse(chapter.endedAt) >= cutoff,
1250
+ "recent-context",
1251
+ packet.evidence,
1252
+ );
1253
+ return textResult(
1254
+ // Keep the old MCP envelope available while adding the typed contract.
1255
+ {
1256
+ events: sanitizedItems,
1257
+ ...result,
1258
+ items: sanitizedItems,
1259
+ evidence: sanitizedEvidence,
1260
+ contextFiles: [],
1261
+ egress: {
1262
+ requestId,
1263
+ packet,
1264
+ note: "A content-free local activity receipt was recorded before this bounded text packet was returned.",
1265
+ },
1266
+ },
1267
+ );
1268
+ }
1269
+ if (name === "screen_memory_recent_segments") {
1270
+ const minutes = typeof args.minutes === "number" ? args.minutes : 30;
1271
+ const segments = recentSegments(storeDir, minutes);
1272
+ const requestId = `egress-mcp-${Date.now()}-${process.pid}`;
1273
+ const packet = {
1274
+ question: `List recent Rewind segment references from the previous ${Math.max(1, Math.min(minutes, 24 * 60))} minutes`,
1275
+ evidence: segments.slice(0, 20).map((segment) => ({
1276
+ id: segment.id,
1277
+ momentId: `segment:${segment.id}`,
1278
+ sourceType: "app-context",
1279
+ capturedAt: segment.startedAt,
1280
+ excerpt: `Local media segment reference from ${segment.startedAt} to ${segment.endedAt}; media path and bytes were not exposed.`,
1281
+ })),
1282
+ };
1283
+ appendEgressReceipt(
1284
+ storeDir,
1285
+ requestId,
1286
+ "recent-segments",
1287
+ packet.evidence,
1288
+ );
1289
+ return textResult({
1290
+ localOnly: true,
1291
+ mediaApprovalRequired: true,
1292
+ segments: segments.map((segment) => ({
1293
+ id: segment.id,
1294
+ startedAt: segment.startedAt,
1295
+ endedAt: segment.endedAt,
1296
+ mimeType: segment.mimeType,
1297
+ durationMs: segment.durationMs,
1298
+ })),
1299
+ egress: {
1300
+ requestId,
1301
+ packet,
1302
+ note: "A content-free local activity receipt was recorded before this bounded metadata packet was returned.",
1303
+ },
1304
+ });
1305
+ }
1306
+ if (name === "screen_memory_search_chapters") {
1307
+ if (typeof args.query !== "string" || !args.query.trim())
1308
+ throw new Error("query is required.");
1309
+ const document = readScreenMemoryChapters(storeDir);
1310
+ if (!document)
1311
+ throw new Error(
1312
+ "No valid local Rewind chapters index is available yet.",
1313
+ );
1314
+ const minutes =
1315
+ typeof args.minutes === "number" && Number.isFinite(args.minutes)
1316
+ ? Math.max(1, Math.min(Math.trunc(args.minutes), 24 * 60))
1317
+ : null;
1318
+ const cutoff = minutes ? Date.now() - minutes * 60_000 : null;
1319
+ const boundedDocument = cutoff
1320
+ ? {
1321
+ ...document,
1322
+ chapters: document.chapters.filter(
1323
+ (chapter) => Date.parse(chapter.endedAt) >= cutoff,
1324
+ ),
1325
+ }
1326
+ : document;
1327
+ const candidates = searchScreenMemoryChapters(
1328
+ boundedDocument,
1329
+ args.query.trim(),
1330
+ typeof args.limit === "number" ? args.limit : 5,
1331
+ typeof args.clientHint === "string" ? args.clientHint : undefined,
1332
+ );
1333
+ const ambiguous =
1334
+ candidates.length > 1 &&
1335
+ Math.abs(candidates[0].score - candidates[1].score) <= 0.05;
1336
+ const packet = {
1337
+ question: redactCredentialText(args.query.trim().slice(0, 4_000)),
1338
+ evidence: candidates.map((chapter) => ({
1339
+ id: chapter.id,
1340
+ momentId: chapter.id,
1341
+ sourceType: "chapter",
1342
+ capturedAt: chapter.startedAt,
1343
+ excerpt: redactCredentialText(
1344
+ `${chapter.label}: ${chapter.summary}`,
1345
+ ).slice(0, 1_200),
1346
+ })),
1347
+ };
1348
+ const requestId = `egress-mcp-${Date.now()}-${process.pid}`;
1349
+ appendEgressReceipt(
1350
+ storeDir,
1351
+ requestId,
1352
+ "search-chapters",
1353
+ packet.evidence,
1354
+ );
1355
+ return textResult({
1356
+ localOnly: true,
1357
+ generatedAt: document.generatedAt,
1358
+ state: document.state,
1359
+ coverage: document.coverage,
1360
+ gaps:
1361
+ isRecord(document.coverage) &&
1362
+ Array.isArray(document.coverage.gaps)
1363
+ ? document.coverage.gaps
1364
+ : [],
1365
+ indexState: candidates.some(
1366
+ (candidate) => candidate.indexState === "pending",
1367
+ )
1368
+ ? "pending"
1369
+ : candidates.some(
1370
+ (candidate) => candidate.indexState === "partial",
1371
+ )
1372
+ ? "partial"
1373
+ : "ready",
1374
+ ambiguous,
1375
+ candidates: candidates.map(projectScreenMemoryChapterCandidate),
1376
+ egress: {
1377
+ requestId,
1378
+ packet,
1379
+ note: "A content-free local activity receipt was recorded before this bounded chapter packet was returned.",
1380
+ },
1381
+ });
1382
+ }
1383
+ if (name === "screen_memory_frame_at") {
1384
+ if (typeof args.timestamp !== "string")
1385
+ throw new Error("timestamp is required.");
1386
+ const frame = readScreenMemoryFrame(
1387
+ storeDir,
1388
+ args.timestamp,
1389
+ opts.decodeFrame ?? decodeFrameWithFfmpeg,
1390
+ );
1391
+ appendLocalEvidenceReceipt(storeDir, "frame-at", [
1392
+ { timestamp: frame.timestamp, segmentId: frame.segmentId },
1393
+ ]);
1394
+ return {
1395
+ content: [
1396
+ {
1397
+ type: "text" as const,
1398
+ text: JSON.stringify({
1399
+ localOnly: true,
1400
+ timestamp: frame.timestamp,
1401
+ segmentId: frame.segmentId,
1402
+ coverage: "retained clean segment",
1403
+ note: "This is a bounded local image read. Request screen_memory_request_clip for motion or audio.",
1404
+ }),
1405
+ },
1406
+ frame.image,
1407
+ ],
1408
+ };
1409
+ }
1410
+ if (name === "screen_memory_contact_sheet") {
1411
+ const document = readScreenMemoryChapters(storeDir);
1412
+ const chapter =
1413
+ typeof args.chapterId === "string"
1414
+ ? document?.chapters.find(
1415
+ (candidate) => candidate.id === args.chapterId,
1416
+ )
1417
+ : undefined;
1418
+ const startAt =
1419
+ chapter?.startedAt ??
1420
+ (typeof args.startAt === "string" ? args.startAt : null);
1421
+ const endAt =
1422
+ chapter?.endedAt ??
1423
+ (typeof args.endAt === "string" ? args.endAt : null);
1424
+ if (
1425
+ !startAt ||
1426
+ !endAt ||
1427
+ !isTimestamp(startAt) ||
1428
+ !isTimestamp(endAt)
1429
+ )
1430
+ throw new Error(
1431
+ "Provide a valid chapterId or RFC3339 startAt and endAt.",
1432
+ );
1433
+ const durationMs = Date.parse(endAt) - Date.parse(startAt);
1434
+ if (!contactSheetRangeIsValid(durationMs, Boolean(chapter)))
1435
+ throw new Error(
1436
+ "A contact sheet range must be between zero and five minutes.",
1437
+ );
1438
+ if (Date.parse(endAt) > Date.now() + 5_000)
1439
+ throw new Error(
1440
+ "A Rewind contact sheet cannot include future time.",
1441
+ );
1442
+ const count = Math.min(
1443
+ Math.max(
1444
+ Math.trunc(typeof args.count === "number" ? args.count : 4),
1445
+ 1,
1318
1446
  ),
1319
- }
1320
- : document;
1321
- const candidates = searchScreenMemoryChapters(
1322
- boundedDocument,
1323
- args.query.trim(),
1324
- typeof args.limit === "number" ? args.limit : 5,
1325
- typeof args.clientHint === "string" ? args.clientHint : undefined,
1326
- );
1327
- const ambiguous =
1328
- candidates.length > 1 &&
1329
- Math.abs(candidates[0].score - candidates[1].score) <= 0.05;
1330
- const packet = {
1331
- question: redactCredentialText(args.query.trim().slice(0, 4_000)),
1332
- evidence: candidates.map((chapter) => ({
1333
- id: chapter.id,
1334
- momentId: chapter.id,
1335
- sourceType: "chapter",
1336
- capturedAt: chapter.startedAt,
1337
- excerpt: redactCredentialText(
1338
- `${chapter.label}: ${chapter.summary}`,
1339
- ).slice(0, 1_200),
1340
- })),
1341
- };
1342
- const requestId = `egress-mcp-${Date.now()}-${process.pid}`;
1343
- appendEgressReceipt(
1344
- storeDir,
1345
- requestId,
1346
- "search-chapters",
1347
- packet.evidence,
1348
- );
1349
- return textResult({
1350
- localOnly: true,
1351
- generatedAt: document.generatedAt,
1352
- state: document.state,
1353
- coverage: document.coverage,
1354
- gaps:
1355
- isRecord(document.coverage) && Array.isArray(document.coverage.gaps)
1356
- ? document.coverage.gaps
1357
- : [],
1358
- indexState: candidates.some(
1359
- (candidate) => candidate.indexState === "pending",
1360
- )
1361
- ? "pending"
1362
- : candidates.some((candidate) => candidate.indexState === "partial")
1363
- ? "partial"
1364
- : "ready",
1365
- ambiguous,
1366
- candidates: candidates.map(projectScreenMemoryChapterCandidate),
1367
- egress: {
1368
- requestId,
1369
- packet,
1370
- note: "A content-free local activity receipt was recorded before this bounded chapter packet was returned.",
1371
- },
1372
- });
1373
- }
1374
- if (name === "screen_memory_frame_at") {
1375
- if (typeof args.timestamp !== "string")
1376
- throw new Error("timestamp is required.");
1377
- const frame = readScreenMemoryFrame(
1378
- storeDir,
1379
- args.timestamp,
1380
- opts.decodeFrame ?? decodeFrameWithFfmpeg,
1381
- );
1382
- appendLocalEvidenceReceipt(storeDir, "frame-at", [
1383
- { timestamp: frame.timestamp, segmentId: frame.segmentId },
1384
- ]);
1385
- return {
1386
- content: [
1387
- {
1388
- type: "text" as const,
1389
- text: JSON.stringify({
1390
- localOnly: true,
1391
- timestamp: frame.timestamp,
1392
- segmentId: frame.segmentId,
1393
- coverage: "retained clean segment",
1394
- note: "This is a bounded local image read. Request screen_memory_request_clip for motion or audio.",
1395
- }),
1396
- },
1397
- frame.image,
1398
- ],
1399
- };
1400
- }
1401
- if (name === "screen_memory_contact_sheet") {
1402
- const document = readScreenMemoryChapters(storeDir);
1403
- const chapter =
1404
- typeof args.chapterId === "string"
1405
- ? document?.chapters.find(
1406
- (candidate) => candidate.id === args.chapterId,
1447
+ 8,
1448
+ );
1449
+ const representative = (chapter?.representativeMoments ?? [])
1450
+ .filter(
1451
+ (moment) =>
1452
+ Date.parse(moment.capturedAt) >= Date.parse(startAt) &&
1453
+ Date.parse(moment.capturedAt) <= Date.parse(endAt),
1407
1454
  )
1408
- : undefined;
1409
- const startAt =
1410
- chapter?.startedAt ??
1411
- (typeof args.startAt === "string" ? args.startAt : null);
1412
- const endAt =
1413
- chapter?.endedAt ??
1414
- (typeof args.endAt === "string" ? args.endAt : null);
1415
- if (!startAt || !endAt || !isTimestamp(startAt) || !isTimestamp(endAt))
1416
- throw new Error(
1417
- "Provide a valid chapterId or RFC3339 startAt and endAt.",
1418
- );
1419
- const durationMs = Date.parse(endAt) - Date.parse(startAt);
1420
- if (!contactSheetRangeIsValid(durationMs, Boolean(chapter)))
1421
- throw new Error(
1422
- "A contact sheet range must be between zero and five minutes.",
1423
- );
1424
- if (Date.parse(endAt) > Date.now() + 5_000)
1425
- throw new Error("A Rewind contact sheet cannot include future time.");
1426
- const count = Math.min(
1427
- Math.max(
1428
- Math.trunc(typeof args.count === "number" ? args.count : 4),
1429
- 1,
1430
- ),
1431
- 8,
1432
- );
1433
- const representative = (chapter?.representativeMoments ?? [])
1434
- .filter(
1435
- (moment) =>
1436
- Date.parse(moment.capturedAt) >= Date.parse(startAt) &&
1437
- Date.parse(moment.capturedAt) <= Date.parse(endAt),
1438
- )
1439
- .map((moment) => moment.capturedAt);
1440
- const timestamps = selectContactSheetTimestamps(
1441
- startAt,
1442
- endAt,
1443
- count,
1444
- representative,
1445
- );
1446
- const decoder = opts.decodeFrame ?? decodeFrameWithFfmpeg;
1447
- const frames = timestamps.flatMap((timestamp) => {
1448
- try {
1449
- return [readScreenMemoryFrame(storeDir, timestamp, decoder)];
1450
- } catch {
1451
- // A chapter may span a brief coverage gap. Keep the clean retained
1452
- // frames instead of failing the entire bounded contact sheet.
1453
- return [];
1455
+ .map((moment) => moment.capturedAt);
1456
+ const timestamps = selectContactSheetTimestamps(
1457
+ startAt,
1458
+ endAt,
1459
+ count,
1460
+ representative,
1461
+ );
1462
+ const decoder = opts.decodeFrame ?? decodeFrameWithFfmpeg;
1463
+ const frames = timestamps.flatMap((timestamp) => {
1464
+ try {
1465
+ return [readScreenMemoryFrame(storeDir, timestamp, decoder)];
1466
+ } catch {
1467
+ // A chapter may span a brief coverage gap. Keep the clean retained
1468
+ // frames instead of failing the entire bounded contact sheet.
1469
+ return [];
1470
+ }
1471
+ });
1472
+ if (frames.length === 0) {
1473
+ throw new Error(
1474
+ "No clean retained Rewind frames cover the requested contact-sheet range.",
1475
+ );
1476
+ }
1477
+ appendLocalEvidenceReceipt(
1478
+ storeDir,
1479
+ "contact-sheet",
1480
+ frames.map(({ timestamp, segmentId }) => ({
1481
+ timestamp,
1482
+ segmentId,
1483
+ })),
1484
+ );
1485
+ return {
1486
+ content: [
1487
+ {
1488
+ type: "text" as const,
1489
+ text: JSON.stringify({
1490
+ localOnly: true,
1491
+ chapterId: chapter?.id ?? null,
1492
+ startAt: new Date(Date.parse(startAt)).toISOString(),
1493
+ endAt: new Date(Date.parse(endAt)).toISOString(),
1494
+ frameCount: frames.length,
1495
+ selection: representative.length
1496
+ ? "representative moments then even coverage"
1497
+ : "even temporal coverage",
1498
+ frames: frames.map(({ timestamp, segmentId }) => ({
1499
+ timestamp,
1500
+ segmentId,
1501
+ })),
1502
+ }),
1503
+ },
1504
+ ...frames.map((frame) => frame.image),
1505
+ ],
1506
+ };
1454
1507
  }
1508
+ if (name === "screen_memory_request_clip") {
1509
+ const range = parseHandoffRange(args);
1510
+ const config = readFeatureConfig(storeDir);
1511
+ const requestId = `handoff-${randomUUID()}`;
1512
+ const reason =
1513
+ typeof args.reason === "string"
1514
+ ? redactCredentialText(args.reason.trim().slice(0, 500))
1515
+ : "Visual or audio context requested by the agent";
1516
+ const request = {
1517
+ requestId,
1518
+ status: "pending",
1519
+ requestedAt: new Date().toISOString(),
1520
+ ...range,
1521
+ reason,
1522
+ includeMicrophone: args.includeMicrophone !== false,
1523
+ includeSystemAudio: args.includeSystemAudio !== false,
1524
+ reviewRequired: config.reviewBeforeSending !== false,
1525
+ agentClipRetention: config.agentClipRetention ?? "forever",
1526
+ };
1527
+ writePrivateJson(handoffPath(storeDir, requestId), request);
1528
+ appendEgressEvent(storeDir, {
1529
+ requestId,
1530
+ occurredAt: request.requestedAt,
1531
+ state: "handoff-requested",
1532
+ operation: "request-clip",
1533
+ receipt: {
1534
+ mediaInterval: { startAt: range.startAt, endAt: range.endAt },
1535
+ reviewRequired: request.reviewRequired,
1536
+ },
1537
+ evidenceCount: 0,
1538
+ packetBytes: 0,
1539
+ error: null,
1540
+ });
1541
+ return textResult({
1542
+ requestId,
1543
+ status: "pending",
1544
+ startAt: range.startAt,
1545
+ endAt: range.endAt,
1546
+ reviewRequired: request.reviewRequired,
1547
+ note: request.reviewRequired
1548
+ ? "Clips is waiting for the user to review this bounded range. Poll screen_memory_handoff_status."
1549
+ : "Clips is processing the explicitly requested bounded range. Poll screen_memory_handoff_status.",
1550
+ });
1551
+ }
1552
+ if (name === "screen_memory_handoff_status") {
1553
+ if (typeof args.requestId !== "string") {
1554
+ throw new Error("requestId is required.");
1555
+ }
1556
+ const parsed = JSON.parse(
1557
+ fs.readFileSync(handoffPath(storeDir, args.requestId), "utf-8"),
1558
+ ) as Record<string, unknown>;
1559
+ return textResult({
1560
+ requestId: parsed.requestId,
1561
+ status: parsed.status,
1562
+ startAt: parsed.startAt,
1563
+ endAt: parsed.endAt,
1564
+ recordingId: parsed.recordingId,
1565
+ agentUrl: parsed.agentUrl,
1566
+ contextUrl: parsed.contextUrl,
1567
+ expiresAt: parsed.expiresAt,
1568
+ autoDeleteAt: parsed.autoDeleteAt,
1569
+ autoDeletedAt: parsed.autoDeletedAt,
1570
+ error: parsed.error,
1571
+ });
1572
+ }
1573
+ throw new Error(`Unknown Screen Memory tool: ${name}`);
1455
1574
  });
1456
- if (frames.length === 0) {
1457
- throw new Error(
1458
- "No clean retained Rewind frames cover the requested contact-sheet range.",
1459
- );
1460
- }
1461
- appendLocalEvidenceReceipt(
1462
- storeDir,
1463
- "contact-sheet",
1464
- frames.map(({ timestamp, segmentId }) => ({ timestamp, segmentId })),
1465
- );
1466
- return {
1467
- content: [
1468
- {
1469
- type: "text" as const,
1470
- text: JSON.stringify({
1471
- localOnly: true,
1472
- chapterId: chapter?.id ?? null,
1473
- startAt: new Date(Date.parse(startAt)).toISOString(),
1474
- endAt: new Date(Date.parse(endAt)).toISOString(),
1475
- frameCount: frames.length,
1476
- selection: representative.length
1477
- ? "representative moments then even coverage"
1478
- : "even temporal coverage",
1479
- frames: frames.map(({ timestamp, segmentId }) => ({
1480
- timestamp,
1481
- segmentId,
1482
- })),
1483
- }),
1484
- },
1485
- ...frames.map((frame) => frame.image),
1486
- ],
1487
- };
1488
- }
1489
- if (name === "screen_memory_request_clip") {
1490
- const range = parseHandoffRange(args);
1491
- const config = readFeatureConfig(storeDir);
1492
- const requestId = `handoff-${randomUUID()}`;
1493
- const reason =
1494
- typeof args.reason === "string"
1495
- ? redactCredentialText(args.reason.trim().slice(0, 500))
1496
- : "Visual or audio context requested by the agent";
1497
- const request = {
1498
- requestId,
1499
- status: "pending",
1500
- requestedAt: new Date().toISOString(),
1501
- ...range,
1502
- reason,
1503
- includeMicrophone: args.includeMicrophone !== false,
1504
- includeSystemAudio: args.includeSystemAudio !== false,
1505
- reviewRequired: config.reviewBeforeSending !== false,
1506
- agentClipRetention: config.agentClipRetention ?? "forever",
1507
- };
1508
- writePrivateJson(handoffPath(storeDir, requestId), request);
1509
- appendEgressEvent(storeDir, {
1510
- requestId,
1511
- occurredAt: request.requestedAt,
1512
- state: "handoff-requested",
1513
- operation: "request-clip",
1514
- receipt: {
1515
- mediaInterval: { startAt: range.startAt, endAt: range.endAt },
1516
- reviewRequired: request.reviewRequired,
1517
- },
1518
- evidenceCount: 0,
1519
- packetBytes: 0,
1520
- error: null,
1521
- });
1522
- return textResult({
1523
- requestId,
1524
- status: "pending",
1525
- startAt: range.startAt,
1526
- endAt: range.endAt,
1527
- reviewRequired: request.reviewRequired,
1528
- note: request.reviewRequired
1529
- ? "Clips is waiting for the user to review this bounded range. Poll screen_memory_handoff_status."
1530
- : "Clips is processing the explicitly requested bounded range. Poll screen_memory_handoff_status.",
1531
- });
1532
- }
1533
- if (name === "screen_memory_handoff_status") {
1534
- if (typeof args.requestId !== "string") {
1535
- throw new Error("requestId is required.");
1536
- }
1537
- const parsed = JSON.parse(
1538
- fs.readFileSync(handoffPath(storeDir, args.requestId), "utf-8"),
1539
- ) as Record<string, unknown>;
1540
- return textResult({
1541
- requestId: parsed.requestId,
1542
- status: parsed.status,
1543
- startAt: parsed.startAt,
1544
- endAt: parsed.endAt,
1545
- recordingId: parsed.recordingId,
1546
- agentUrl: parsed.agentUrl,
1547
- contextUrl: parsed.contextUrl,
1548
- expiresAt: parsed.expiresAt,
1549
- autoDeleteAt: parsed.autoDeleteAt,
1550
- autoDeletedAt: parsed.autoDeletedAt,
1551
- error: parsed.error,
1552
- });
1553
- }
1554
- throw new Error(`Unknown Screen Memory tool: ${name}`);
1555
- });
1575
+ return server;
1576
+ },
1577
+ { legacy: "serve" },
1578
+ );
1556
1579
 
1557
1580
  log(`Serving local Screen Memory from ${storeDir}`);
1558
- const transport = new StdioServerTransport();
1559
- await server.connect(transport);
1560
1581
  await new Promise<void>((resolve) => {
1561
- transport.onclose = resolve;
1582
+ process.stdin.once("end", resolve);
1562
1583
  process.once("SIGINT", resolve);
1563
1584
  process.once("SIGTERM", resolve);
1564
1585
  });
1586
+ await stdio.close();
1565
1587
  }