@agent-native/core 0.128.3 → 0.129.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/corpus/README.md +2 -2
- package/corpus/core/CHANGELOG.md +91 -0
- package/corpus/core/docs/content/template-design-collaboration-and-full-apps.mdx +3 -1
- package/corpus/core/package.json +6 -3
- package/corpus/core/src/a2a/client.ts +18 -4
- package/corpus/core/src/a2a/server.ts +19 -3
- package/corpus/core/src/a2a/types.ts +7 -0
- package/corpus/core/src/action.ts +43 -16
- package/corpus/core/src/agent/engine/ai-sdk-engine.ts +14 -10
- package/corpus/core/src/agent/production-agent.ts +41 -3
- package/corpus/core/src/agent/thread-data-builder.ts +5 -0
- package/corpus/core/src/chat-threads/migrations.ts +53 -0
- package/corpus/core/src/cli/create.ts +4 -0
- package/corpus/core/src/cli/skills-content/design-visual-edit-skill.ts +36 -16
- package/corpus/core/src/cli/skills.ts +16 -0
- package/corpus/core/src/client/AgentPanel.tsx +9 -5
- package/corpus/core/src/client/AssistantChat.tsx +138 -46
- package/corpus/core/src/client/MultiTabAssistantChat.tsx +16 -10
- package/corpus/core/src/client/agent-chat-adapter.ts +21 -3
- package/corpus/core/src/client/analytics.ts +159 -102
- package/corpus/core/src/client/chat/legacy-chart-shorthand.tsx +479 -0
- package/corpus/core/src/client/chat/markdown-renderer.tsx +23 -3
- package/corpus/core/src/client/chat/message-components.tsx +12 -5
- package/corpus/core/src/client/chat/runtime.ts +2 -1
- package/corpus/core/src/client/chat/tool-call-display.tsx +25 -2
- package/corpus/core/src/client/chat-model-groups.ts +3 -0
- package/corpus/core/src/client/client-status-requests.ts +161 -0
- package/corpus/core/src/client/settings/VoiceTranscriptionSection.tsx +2 -2
- package/corpus/core/src/client/sse-event-processor.ts +15 -3
- package/corpus/core/src/client/use-agent-engine-configured.ts +54 -50
- package/corpus/core/src/client/use-chat-models.ts +16 -11
- package/corpus/core/src/client/use-chat-threads.ts +32 -9
- package/corpus/core/src/client/use-session.ts +4 -5
- package/corpus/core/src/db/client.ts +83 -8
- package/corpus/core/src/db/ensure-additive-columns.ts +31 -20
- package/corpus/core/src/db/migrations.ts +22 -2
- package/corpus/core/src/extensions/actions.ts +1 -1
- package/corpus/core/src/mcp/approval-store.ts +99 -0
- package/corpus/core/src/mcp/build-server.ts +579 -303
- package/corpus/core/src/mcp/embed-app.ts +1 -1
- package/corpus/core/src/mcp/oauth-client-metadata.ts +380 -0
- package/corpus/core/src/mcp/oauth-route.ts +57 -52
- package/corpus/core/src/mcp/screen-memory-stdio.ts +400 -378
- package/corpus/core/src/mcp/server.ts +29 -145
- package/corpus/core/src/mcp/stdio.ts +72 -80
- package/corpus/core/src/mcp-client/manager.ts +6 -10
- package/corpus/core/src/mcp-client/oauth-client.ts +127 -36
- package/corpus/core/src/mcp-client/oauth-routes.ts +16 -6
- package/corpus/core/src/mcp-client/routes.ts +6 -5
- package/corpus/core/src/server/agent-capabilities.ts +20 -0
- package/corpus/core/src/server/agent-chat/action-filters-a2a.ts +27 -0
- package/corpus/core/src/server/agent-chat-plugin.ts +18 -0
- package/corpus/core/src/server/transcribe-voice.ts +14 -5
- package/corpus/core/src/templates/default/.agents/skills/security/SKILL.md +13 -2
- package/corpus/core/src/templates/workspace-core/.agents/skills/authentication/SKILL.md +5 -2
- package/corpus/core/src/templates/workspace-core/.agents/skills/external-agents/SKILL.md +18 -0
- package/corpus/core/src/templates/workspace-core/.agents/skills/security/SKILL.md +13 -2
- package/corpus/templates/analytics/.agents/skills/data-querying/SKILL.md +12 -0
- package/corpus/templates/analytics/.agents/skills/security/SKILL.md +13 -2
- package/corpus/templates/analytics/actions/generate-chart.ts +2 -2
- package/corpus/templates/analytics/app/components/dashboard/DashboardPanelSkeleton.tsx +1 -10
- package/corpus/templates/analytics/app/global.css +0 -31
- package/corpus/templates/analytics/app/lib/sql-query.ts +46 -20
- package/corpus/templates/analytics/changelog/2026-07-28-analytics-dashboard-loading-states-now-use-a-simpler-standar.md +6 -0
- package/corpus/templates/analytics/changelog/2026-07-28-analytics-dashboards-now-load-first-party-charts-reliably-in.md +6 -0
- package/corpus/templates/analytics/server/lib/first-party-analytics-cache.ts +42 -16
- package/corpus/templates/analytics/server/lib/first-party-analytics.ts +4 -2
- package/corpus/templates/analytics/shared/sql-query-limits.ts +1 -0
- package/corpus/templates/assets/.agents/skills/security/SKILL.md +13 -2
- package/corpus/templates/brain/.agents/skills/security/SKILL.md +13 -2
- package/corpus/templates/calendar/.agents/skills/security/SKILL.md +13 -2
- package/corpus/templates/chat/.agents/skills/security/SKILL.md +13 -2
- package/corpus/templates/clips/.agents/skills/security/SKILL.md +13 -2
- package/corpus/templates/clips/app/components/player/player-controls.tsx +3 -6
- package/corpus/templates/clips/app/routes/r.$recordingId.tsx +6 -2
- package/corpus/templates/clips/app/routes/share.$shareId.tsx +3 -1
- package/corpus/templates/clips/changelog/2026-07-28-copied-share-links-no-longer-leave-an-unnecessary-confirmati.md +6 -0
- package/corpus/templates/clips/changelog/2026-07-28-playback-skip-controls-no-longer-show-overlapping-numbers.md +6 -0
- package/corpus/templates/clips/changelog/2026-07-28-public-clips-share-buttons-now-use-a-clean-text-only-label.md +6 -0
- package/corpus/templates/clips/desktop/src/app.tsx +14 -17
- package/corpus/templates/clips/desktop/src/overlays/live-audio-bars.tsx +3 -1
- package/corpus/templates/clips/desktop/src/styles.css +13 -10
- package/corpus/templates/clips/desktop/src-tauri/src/recording_indicator.rs +2 -2
- package/corpus/templates/clips/desktop/src-tauri/src/whisper_speech.rs +36 -0
- package/corpus/templates/content/.agents/skills/security/SKILL.md +13 -2
- package/corpus/templates/content/actions/_builder-cms-read-client.ts +155 -34
- package/corpus/templates/content/actions/_builder-docs-client.ts +151 -30
- package/corpus/templates/content/actions/stage-builder-source-bulk-update.ts +5 -2
- package/corpus/templates/content/app/root.tsx +35 -14
- package/corpus/templates/content/changelog/2026-07-28-builder-backed-files-media-fields-now-accept-direct-file-url.md +6 -0
- package/corpus/templates/content/changelog/2026-07-28-builder-content-reads-now-negotiate-the-newest-mcp-protocol-.md +6 -0
- package/corpus/templates/content/changelog/2026-07-28-public-content-pages-load-faster-before-the-optional-ask-sid.md +6 -0
- package/corpus/templates/design/.agents/skills/security/SKILL.md +13 -2
- package/corpus/templates/design/.agents/skills/visual-edit/SKILL.md +36 -16
- package/corpus/templates/design/.generated/bridge/editor-chrome.generated.ts +42 -3
- package/corpus/templates/design/actions/open-visual-edit.ts +2 -1
- package/corpus/templates/design/app/components/design/CanvasContextMenu.tsx +2 -2
- package/corpus/templates/design/app/components/design/DesignCanvas.tsx +14 -15
- package/corpus/templates/design/app/components/design/ResponsiveInteractBar.tsx +314 -0
- package/corpus/templates/design/app/components/design/bridge/editor-chrome.bridge.ts +88 -3
- package/corpus/templates/design/app/components/design/keyboard-shortcuts.ts +1 -1
- package/corpus/templates/design/app/hooks/useDesignHotkeys.ts +7 -6
- package/corpus/templates/design/app/i18n-data.ts +56 -11
- package/corpus/templates/design/app/i18n-responsive-interact.ts +183 -0
- package/corpus/templates/design/app/lib/agent-chat.ts +60 -0
- package/corpus/templates/design/app/pages/design-editor/responsive-interact.ts +89 -0
- package/corpus/templates/design/changelog/2026-07-28-design-edits-can-be-applied-through-the-host-coding-agent-fr.md +6 -0
- package/corpus/templates/design/changelog/2026-07-28-interact-mode-can-preview-designs-in-responsive-device-sizes.md +6 -0
- package/corpus/templates/design/changelog/2026-07-28-interact-mode-now-opens-a-full-screen-responsive-preview-wit.md +6 -0
- package/corpus/templates/design/changelog/2026-07-28-show-or-hide-design-editing-chrome-with-cmd-on-apple-devices.md +2 -2
- package/corpus/templates/design/changelog/2026-07-28-visual-edit-now-always-shows-the-live-running-app-in-the-can.md +6 -0
- package/corpus/templates/dispatch/.agents/skills/security/SKILL.md +13 -2
- package/corpus/templates/forms/.agents/skills/security/SKILL.md +13 -2
- package/corpus/templates/macros/.agents/skills/security/SKILL.md +13 -2
- package/corpus/templates/mail/.agents/skills/security/SKILL.md +13 -2
- package/corpus/templates/plan/.agents/skills/security/SKILL.md +13 -2
- package/corpus/templates/slides/.agents/skills/security/SKILL.md +13 -2
- package/corpus/templates/tasks/.agents/skills/security/SKILL.md +13 -2
- package/corpus/toolkit/CHANGELOG.md +13 -0
- package/corpus/toolkit/package.json +1 -1
- package/corpus/toolkit/src/chat-history.css +1 -2
- package/corpus/toolkit/src/composer/TiptapComposer.tsx +45 -0
- package/dist/a2a/client.d.ts +7 -0
- package/dist/a2a/client.d.ts.map +1 -1
- package/dist/a2a/client.js +8 -3
- package/dist/a2a/client.js.map +1 -1
- package/dist/a2a/server.d.ts.map +1 -1
- package/dist/a2a/server.js +18 -3
- package/dist/a2a/server.js.map +1 -1
- package/dist/a2a/types.d.ts +7 -0
- package/dist/a2a/types.d.ts.map +1 -1
- package/dist/a2a/types.js.map +1 -1
- package/dist/action.d.ts +15 -1
- package/dist/action.d.ts.map +1 -1
- package/dist/action.js +39 -16
- package/dist/action.js.map +1 -1
- package/dist/agent/engine/ai-sdk-engine.d.ts.map +1 -1
- package/dist/agent/engine/ai-sdk-engine.js +14 -10
- package/dist/agent/engine/ai-sdk-engine.js.map +1 -1
- package/dist/agent/production-agent.d.ts.map +1 -1
- package/dist/agent/production-agent.js +35 -7
- package/dist/agent/production-agent.js.map +1 -1
- package/dist/agent/thread-data-builder.d.ts +2 -0
- package/dist/agent/thread-data-builder.d.ts.map +1 -1
- package/dist/agent/thread-data-builder.js +3 -0
- package/dist/agent/thread-data-builder.js.map +1 -1
- package/dist/chat-threads/migrations.d.ts +21 -0
- package/dist/chat-threads/migrations.d.ts.map +1 -0
- package/dist/chat-threads/migrations.js +45 -0
- package/dist/chat-threads/migrations.js.map +1 -0
- package/dist/cli/create.d.ts.map +1 -1
- package/dist/cli/create.js +4 -0
- package/dist/cli/create.js.map +1 -1
- package/dist/cli/skills-content/design-visual-edit-skill.d.ts +1 -1
- package/dist/cli/skills-content/design-visual-edit-skill.d.ts.map +1 -1
- package/dist/cli/skills-content/design-visual-edit-skill.js +36 -16
- package/dist/cli/skills-content/design-visual-edit-skill.js.map +1 -1
- package/dist/cli/skills.d.ts.map +1 -1
- package/dist/cli/skills.js +15 -0
- package/dist/cli/skills.js.map +1 -1
- package/dist/client/AgentPanel.d.ts.map +1 -1
- package/dist/client/AgentPanel.js +6 -5
- package/dist/client/AgentPanel.js.map +1 -1
- package/dist/client/AssistantChat.d.ts +37 -0
- package/dist/client/AssistantChat.d.ts.map +1 -1
- package/dist/client/AssistantChat.js +41 -8
- package/dist/client/AssistantChat.js.map +1 -1
- package/dist/client/MultiTabAssistantChat.d.ts.map +1 -1
- package/dist/client/MultiTabAssistantChat.js +11 -10
- package/dist/client/MultiTabAssistantChat.js.map +1 -1
- package/dist/client/agent-chat-adapter.d.ts.map +1 -1
- package/dist/client/agent-chat-adapter.js +19 -3
- package/dist/client/agent-chat-adapter.js.map +1 -1
- package/dist/client/analytics.d.ts.map +1 -1
- package/dist/client/analytics.js +150 -101
- package/dist/client/analytics.js.map +1 -1
- package/dist/client/chat/legacy-chart-shorthand.d.ts +29 -0
- package/dist/client/chat/legacy-chart-shorthand.d.ts.map +1 -0
- package/dist/client/chat/legacy-chart-shorthand.js +330 -0
- package/dist/client/chat/legacy-chart-shorthand.js.map +1 -0
- package/dist/client/chat/markdown-renderer.d.ts.map +1 -1
- package/dist/client/chat/markdown-renderer.js +8 -2
- package/dist/client/chat/markdown-renderer.js.map +1 -1
- package/dist/client/chat/message-components.d.ts +2 -1
- package/dist/client/chat/message-components.d.ts.map +1 -1
- package/dist/client/chat/message-components.js +11 -4
- package/dist/client/chat/message-components.js.map +1 -1
- package/dist/client/chat/runtime.d.ts.map +1 -1
- package/dist/client/chat/runtime.js +2 -1
- package/dist/client/chat/runtime.js.map +1 -1
- package/dist/client/chat/tool-call-display.d.ts +4 -1
- package/dist/client/chat/tool-call-display.d.ts.map +1 -1
- package/dist/client/chat/tool-call-display.js +14 -8
- package/dist/client/chat/tool-call-display.js.map +1 -1
- package/dist/client/chat-model-groups.d.ts.map +1 -1
- package/dist/client/chat-model-groups.js +3 -0
- package/dist/client/chat-model-groups.js.map +1 -1
- package/dist/client/client-status-requests.d.ts +14 -0
- package/dist/client/client-status-requests.d.ts.map +1 -0
- package/dist/client/client-status-requests.js +126 -0
- package/dist/client/client-status-requests.js.map +1 -0
- package/dist/client/settings/VoiceTranscriptionSection.js +1 -1
- package/dist/client/settings/VoiceTranscriptionSection.js.map +1 -1
- package/dist/client/sse-event-processor.d.ts +7 -0
- package/dist/client/sse-event-processor.d.ts.map +1 -1
- package/dist/client/sse-event-processor.js +8 -3
- package/dist/client/sse-event-processor.js.map +1 -1
- package/dist/client/use-agent-engine-configured.d.ts.map +1 -1
- package/dist/client/use-agent-engine-configured.js +29 -43
- package/dist/client/use-agent-engine-configured.js.map +1 -1
- package/dist/client/use-chat-models.d.ts.map +1 -1
- package/dist/client/use-chat-models.js +11 -11
- package/dist/client/use-chat-models.js.map +1 -1
- package/dist/client/use-chat-threads.d.ts.map +1 -1
- package/dist/client/use-chat-threads.js +23 -7
- package/dist/client/use-chat-threads.js.map +1 -1
- package/dist/client/use-session.d.ts.map +1 -1
- package/dist/client/use-session.js +4 -4
- package/dist/client/use-session.js.map +1 -1
- package/dist/collab/awareness.d.ts +2 -2
- package/dist/collab/awareness.d.ts.map +1 -1
- package/dist/db/client.d.ts.map +1 -1
- package/dist/db/client.js +48 -4
- package/dist/db/client.js.map +1 -1
- package/dist/db/ensure-additive-columns.d.ts.map +1 -1
- package/dist/db/ensure-additive-columns.js +32 -21
- package/dist/db/ensure-additive-columns.js.map +1 -1
- package/dist/db/migrations.d.ts +9 -0
- package/dist/db/migrations.d.ts.map +1 -1
- package/dist/db/migrations.js +14 -3
- package/dist/db/migrations.js.map +1 -1
- package/dist/extensions/actions.js +1 -1
- package/dist/extensions/actions.js.map +1 -1
- package/dist/mcp/approval-store.d.ts +16 -0
- package/dist/mcp/approval-store.d.ts.map +1 -0
- package/dist/mcp/approval-store.js +85 -0
- package/dist/mcp/approval-store.js.map +1 -0
- package/dist/mcp/build-server.d.ts +1 -34
- package/dist/mcp/build-server.d.ts.map +1 -1
- package/dist/mcp/build-server.js +220 -23
- package/dist/mcp/build-server.js.map +1 -1
- package/dist/mcp/embed-app.js +1 -1
- package/dist/mcp/embed-app.js.map +1 -1
- package/dist/mcp/oauth-client-metadata.d.ts +15 -0
- package/dist/mcp/oauth-client-metadata.d.ts.map +1 -0
- package/dist/mcp/oauth-client-metadata.js +297 -0
- package/dist/mcp/oauth-client-metadata.js.map +1 -0
- package/dist/mcp/oauth-route.d.ts.map +1 -1
- package/dist/mcp/oauth-route.js +39 -53
- package/dist/mcp/oauth-route.js.map +1 -1
- package/dist/mcp/screen-memory-stdio.d.ts.map +1 -1
- package/dist/mcp/screen-memory-stdio.js +304 -296
- package/dist/mcp/screen-memory-stdio.js.map +1 -1
- package/dist/mcp/server.d.ts +12 -25
- package/dist/mcp/server.d.ts.map +1 -1
- package/dist/mcp/server.js +24 -133
- package/dist/mcp/server.js.map +1 -1
- package/dist/mcp/stdio.d.ts.map +1 -1
- package/dist/mcp/stdio.js +36 -35
- package/dist/mcp/stdio.js.map +1 -1
- package/dist/mcp-client/manager.d.ts.map +1 -1
- package/dist/mcp-client/manager.js +6 -6
- package/dist/mcp-client/manager.js.map +1 -1
- package/dist/mcp-client/oauth-client.d.ts +13 -17
- package/dist/mcp-client/oauth-client.d.ts.map +1 -1
- package/dist/mcp-client/oauth-client.js +74 -16
- package/dist/mcp-client/oauth-client.js.map +1 -1
- package/dist/mcp-client/oauth-routes.d.ts +2 -2
- package/dist/mcp-client/oauth-routes.d.ts.map +1 -1
- package/dist/mcp-client/oauth-routes.js +16 -6
- package/dist/mcp-client/oauth-routes.js.map +1 -1
- package/dist/mcp-client/routes.js +5 -5
- package/dist/mcp-client/routes.js.map +1 -1
- package/dist/notifications/routes.d.ts +1 -1
- package/dist/progress/routes.d.ts +1 -1
- package/dist/resources/handlers.d.ts +1 -1
- package/dist/secrets/routes.d.ts +9 -9
- package/dist/server/agent-capabilities.d.ts.map +1 -1
- package/dist/server/agent-capabilities.js +21 -0
- package/dist/server/agent-capabilities.js.map +1 -1
- package/dist/server/agent-chat/action-filters-a2a.d.ts +14 -0
- package/dist/server/agent-chat/action-filters-a2a.d.ts.map +1 -1
- package/dist/server/agent-chat/action-filters-a2a.js +16 -0
- package/dist/server/agent-chat/action-filters-a2a.js.map +1 -1
- package/dist/server/agent-chat-plugin.d.ts.map +1 -1
- package/dist/server/agent-chat-plugin.js +11 -1
- package/dist/server/agent-chat-plugin.js.map +1 -1
- package/dist/server/transcribe-voice.js +12 -6
- package/dist/server/transcribe-voice.js.map +1 -1
- package/dist/templates/chat/.agents/skills/security/SKILL.md +13 -2
- package/dist/templates/default/.agents/skills/security/SKILL.md +13 -2
- package/dist/templates/workspace-core/.agents/skills/authentication/SKILL.md +5 -2
- package/dist/templates/workspace-core/.agents/skills/external-agents/SKILL.md +18 -0
- package/dist/templates/workspace-core/.agents/skills/security/SKILL.md +13 -2
- package/docs/content/template-design-collaboration-and-full-apps.mdx +3 -1
- package/package.json +7 -4
- package/src/a2a/client.ts +18 -4
- package/src/a2a/server.ts +19 -3
- package/src/a2a/types.ts +7 -0
- package/src/action.ts +43 -16
- package/src/agent/engine/ai-sdk-engine.ts +14 -10
- package/src/agent/production-agent.ts +41 -3
- package/src/agent/thread-data-builder.ts +5 -0
- package/src/chat-threads/migrations.ts +53 -0
- package/src/cli/create.ts +4 -0
- package/src/cli/skills-content/design-visual-edit-skill.ts +36 -16
- package/src/cli/skills.ts +16 -0
- package/src/client/AgentPanel.tsx +9 -5
- package/src/client/AssistantChat.tsx +138 -46
- package/src/client/MultiTabAssistantChat.tsx +16 -10
- package/src/client/agent-chat-adapter.ts +21 -3
- package/src/client/analytics.ts +159 -102
- package/src/client/chat/legacy-chart-shorthand.tsx +479 -0
- package/src/client/chat/markdown-renderer.tsx +23 -3
- package/src/client/chat/message-components.tsx +12 -5
- package/src/client/chat/runtime.ts +2 -1
- package/src/client/chat/tool-call-display.tsx +25 -2
- package/src/client/chat-model-groups.ts +3 -0
- package/src/client/client-status-requests.ts +161 -0
- package/src/client/settings/VoiceTranscriptionSection.tsx +2 -2
- package/src/client/sse-event-processor.ts +15 -3
- package/src/client/use-agent-engine-configured.ts +54 -50
- package/src/client/use-chat-models.ts +16 -11
- package/src/client/use-chat-threads.ts +32 -9
- package/src/client/use-session.ts +4 -5
- package/src/db/client.ts +83 -8
- package/src/db/ensure-additive-columns.ts +31 -20
- package/src/db/migrations.ts +22 -2
- package/src/extensions/actions.ts +1 -1
- package/src/mcp/approval-store.ts +99 -0
- package/src/mcp/build-server.ts +579 -303
- package/src/mcp/embed-app.ts +1 -1
- package/src/mcp/oauth-client-metadata.ts +380 -0
- package/src/mcp/oauth-route.ts +57 -52
- package/src/mcp/screen-memory-stdio.ts +400 -378
- package/src/mcp/server.ts +29 -145
- package/src/mcp/stdio.ts +72 -80
- package/src/mcp-client/manager.ts +6 -10
- package/src/mcp-client/oauth-client.ts +127 -36
- package/src/mcp-client/oauth-routes.ts +16 -6
- package/src/mcp-client/routes.ts +6 -5
- package/src/server/agent-capabilities.ts +20 -0
- package/src/server/agent-chat/action-filters-a2a.ts +27 -0
- package/src/server/agent-chat-plugin.ts +18 -0
- package/src/server/transcribe-voice.ts +14 -5
- package/src/templates/chat/.agents/skills/security/SKILL.md +13 -2
- package/src/templates/default/.agents/skills/security/SKILL.md +13 -2
- package/src/templates/workspace-core/.agents/skills/authentication/SKILL.md +5 -2
- package/src/templates/workspace-core/.agents/skills/external-agents/SKILL.md +18 -0
- package/src/templates/workspace-core/.agents/skills/security/SKILL.md +13 -2
|
@@ -1170,396 +1170,418 @@ export async function runScreenMemoryMCPStdio(
|
|
|
1170
1170
|
const env = opts.env ?? process.env;
|
|
1171
1171
|
const storeDir = opts.storeDir ?? defaultStoreDir(env);
|
|
1172
1172
|
|
|
1173
|
-
const { Server } = await import("@modelcontextprotocol/
|
|
1174
|
-
const {
|
|
1175
|
-
await import("@modelcontextprotocol/sdk/server/stdio.js");
|
|
1176
|
-
const { CallToolRequestSchema, ListToolsRequestSchema } =
|
|
1177
|
-
await import("@modelcontextprotocol/sdk/types.js");
|
|
1173
|
+
const { Server } = await import("@modelcontextprotocol/server");
|
|
1174
|
+
const { serveStdio } = await import("@modelcontextprotocol/server/stdio");
|
|
1178
1175
|
|
|
1179
|
-
const
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
server.setRequestHandler(ListToolsRequestSchema, async () => ({
|
|
1185
|
-
tools: screenMemoryMcpToolDefinitions(),
|
|
1186
|
-
}));
|
|
1187
|
-
|
|
1188
|
-
server.setRequestHandler(CallToolRequestSchema, async (request: any) => {
|
|
1189
|
-
const name = request.params?.name;
|
|
1190
|
-
const args = (request.params?.arguments ?? {}) as Record<string, unknown>;
|
|
1191
|
-
if (name === "screen_memory_status") {
|
|
1192
|
-
const config = readFeatureConfig(storeDir);
|
|
1193
|
-
const segments = recentSegments(storeDir, 24 * 60);
|
|
1194
|
-
return textResult({
|
|
1195
|
-
enabled: config.enabled === true,
|
|
1196
|
-
paused: config.paused === true,
|
|
1197
|
-
retentionHours: config.retentionHours ?? 24,
|
|
1198
|
-
maxBytes: config.maxBytes ?? 20 * 1024 * 1024 * 1024,
|
|
1199
|
-
segmentCount: segments.length,
|
|
1200
|
-
totalBytes: segments.reduce(
|
|
1201
|
-
(sum, segment) => sum + (segment.bytes || 0),
|
|
1202
|
-
0,
|
|
1203
|
-
),
|
|
1204
|
-
note: "The Screen Memory store is connected. Local archive paths are intentionally not exposed to agents.",
|
|
1205
|
-
});
|
|
1206
|
-
}
|
|
1207
|
-
if (name === "screen_memory_recent_context") {
|
|
1208
|
-
const result = await queryScreenMemoryContextForStore(
|
|
1209
|
-
{
|
|
1210
|
-
query: typeof args.query === "string" ? args.query : undefined,
|
|
1211
|
-
sinceMinutes:
|
|
1212
|
-
typeof args.minutes === "number" ? args.minutes : undefined,
|
|
1213
|
-
limit: typeof args.limit === "number" ? args.limit : undefined,
|
|
1214
|
-
},
|
|
1215
|
-
storeDir,
|
|
1216
|
-
env,
|
|
1176
|
+
const stdio = serveStdio(
|
|
1177
|
+
() => {
|
|
1178
|
+
const server = new Server(
|
|
1179
|
+
{ name: "clips-screen-memory", version: "0.1.0" },
|
|
1180
|
+
{ capabilities: { tools: {} } },
|
|
1217
1181
|
);
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
sourceFile: "local-screen-memory",
|
|
1226
|
-
}));
|
|
1227
|
-
const requestId = `egress-mcp-${Date.now()}-${process.pid}`;
|
|
1228
|
-
const packet = {
|
|
1229
|
-
question:
|
|
1230
|
-
typeof args.query === "string" && args.query.trim()
|
|
1231
|
-
? redactCredentialText(args.query.trim().slice(0, 4_000))
|
|
1232
|
-
: "Recent Screen Memory context",
|
|
1233
|
-
evidence: sanitizedEvidence.slice(0, 20).map((item) => ({
|
|
1234
|
-
id: item.id,
|
|
1235
|
-
momentId: item.momentId,
|
|
1236
|
-
sourceType: item.sourceType,
|
|
1237
|
-
capturedAt: item.capturedAt,
|
|
1238
|
-
excerpt: item.excerpt.slice(0, 1_200),
|
|
1239
|
-
})),
|
|
1240
|
-
};
|
|
1241
|
-
appendEgressReceipt(
|
|
1242
|
-
storeDir,
|
|
1243
|
-
requestId,
|
|
1244
|
-
"recent-context",
|
|
1245
|
-
packet.evidence,
|
|
1182
|
+
|
|
1183
|
+
server.setRequestHandler(
|
|
1184
|
+
"tools/list",
|
|
1185
|
+
async () =>
|
|
1186
|
+
({
|
|
1187
|
+
tools: screenMemoryMcpToolDefinitions(),
|
|
1188
|
+
}) as any,
|
|
1246
1189
|
);
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1190
|
+
|
|
1191
|
+
server.setRequestHandler("tools/call", async (request: any) => {
|
|
1192
|
+
const name = request.params?.name;
|
|
1193
|
+
const args = (request.params?.arguments ?? {}) as Record<
|
|
1194
|
+
string,
|
|
1195
|
+
unknown
|
|
1196
|
+
>;
|
|
1197
|
+
if (name === "screen_memory_status") {
|
|
1198
|
+
const config = readFeatureConfig(storeDir);
|
|
1199
|
+
const segments = recentSegments(storeDir, 24 * 60);
|
|
1200
|
+
return textResult({
|
|
1201
|
+
enabled: config.enabled === true,
|
|
1202
|
+
paused: config.paused === true,
|
|
1203
|
+
retentionHours: config.retentionHours ?? 24,
|
|
1204
|
+
maxBytes: config.maxBytes ?? 20 * 1024 * 1024 * 1024,
|
|
1205
|
+
segmentCount: segments.length,
|
|
1206
|
+
totalBytes: segments.reduce(
|
|
1207
|
+
(sum, segment) => sum + (segment.bytes || 0),
|
|
1208
|
+
0,
|
|
1209
|
+
),
|
|
1210
|
+
note: "The Screen Memory store is connected. Local archive paths are intentionally not exposed to agents.",
|
|
1211
|
+
});
|
|
1212
|
+
}
|
|
1213
|
+
if (name === "screen_memory_recent_context") {
|
|
1214
|
+
const result = await queryScreenMemoryContextForStore(
|
|
1215
|
+
{
|
|
1216
|
+
query: typeof args.query === "string" ? args.query : undefined,
|
|
1217
|
+
sinceMinutes:
|
|
1218
|
+
typeof args.minutes === "number" ? args.minutes : undefined,
|
|
1219
|
+
limit: typeof args.limit === "number" ? args.limit : undefined,
|
|
1220
|
+
},
|
|
1221
|
+
storeDir,
|
|
1222
|
+
env,
|
|
1223
|
+
);
|
|
1224
|
+
const sanitizedEvidence = result.evidence.map((item) => ({
|
|
1225
|
+
...item,
|
|
1226
|
+
excerpt: redactCredentialText(item.excerpt),
|
|
1227
|
+
}));
|
|
1228
|
+
const sanitizedItems = result.items.map((item) => ({
|
|
1229
|
+
...item,
|
|
1230
|
+
text: redactCredentialText(item.text),
|
|
1231
|
+
sourceFile: "local-screen-memory",
|
|
1232
|
+
}));
|
|
1233
|
+
const requestId = `egress-mcp-${Date.now()}-${process.pid}`;
|
|
1234
|
+
const packet = {
|
|
1235
|
+
question:
|
|
1236
|
+
typeof args.query === "string" && args.query.trim()
|
|
1237
|
+
? redactCredentialText(args.query.trim().slice(0, 4_000))
|
|
1238
|
+
: "Recent Screen Memory context",
|
|
1239
|
+
evidence: sanitizedEvidence.slice(0, 20).map((item) => ({
|
|
1240
|
+
id: item.id,
|
|
1241
|
+
momentId: item.momentId,
|
|
1242
|
+
sourceType: item.sourceType,
|
|
1243
|
+
capturedAt: item.capturedAt,
|
|
1244
|
+
excerpt: item.excerpt.slice(0, 1_200),
|
|
1245
|
+
})),
|
|
1246
|
+
};
|
|
1247
|
+
appendEgressReceipt(
|
|
1248
|
+
storeDir,
|
|
1256
1249
|
requestId,
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
}
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1250
|
+
"recent-context",
|
|
1251
|
+
packet.evidence,
|
|
1252
|
+
);
|
|
1253
|
+
return textResult(
|
|
1254
|
+
// Keep the old MCP envelope available while adding the typed contract.
|
|
1255
|
+
{
|
|
1256
|
+
events: sanitizedItems,
|
|
1257
|
+
...result,
|
|
1258
|
+
items: sanitizedItems,
|
|
1259
|
+
evidence: sanitizedEvidence,
|
|
1260
|
+
contextFiles: [],
|
|
1261
|
+
egress: {
|
|
1262
|
+
requestId,
|
|
1263
|
+
packet,
|
|
1264
|
+
note: "A content-free local activity receipt was recorded before this bounded text packet was returned.",
|
|
1265
|
+
},
|
|
1266
|
+
},
|
|
1267
|
+
);
|
|
1268
|
+
}
|
|
1269
|
+
if (name === "screen_memory_recent_segments") {
|
|
1270
|
+
const minutes = typeof args.minutes === "number" ? args.minutes : 30;
|
|
1271
|
+
const segments = recentSegments(storeDir, minutes);
|
|
1272
|
+
const requestId = `egress-mcp-${Date.now()}-${process.pid}`;
|
|
1273
|
+
const packet = {
|
|
1274
|
+
question: `List recent Rewind segment references from the previous ${Math.max(1, Math.min(minutes, 24 * 60))} minutes`,
|
|
1275
|
+
evidence: segments.slice(0, 20).map((segment) => ({
|
|
1276
|
+
id: segment.id,
|
|
1277
|
+
momentId: `segment:${segment.id}`,
|
|
1278
|
+
sourceType: "app-context",
|
|
1279
|
+
capturedAt: segment.startedAt,
|
|
1280
|
+
excerpt: `Local media segment reference from ${segment.startedAt} to ${segment.endedAt}; media path and bytes were not exposed.`,
|
|
1281
|
+
})),
|
|
1282
|
+
};
|
|
1283
|
+
appendEgressReceipt(
|
|
1284
|
+
storeDir,
|
|
1285
|
+
requestId,
|
|
1286
|
+
"recent-segments",
|
|
1287
|
+
packet.evidence,
|
|
1288
|
+
);
|
|
1289
|
+
return textResult({
|
|
1290
|
+
localOnly: true,
|
|
1291
|
+
mediaApprovalRequired: true,
|
|
1292
|
+
segments: segments.map((segment) => ({
|
|
1293
|
+
id: segment.id,
|
|
1294
|
+
startedAt: segment.startedAt,
|
|
1295
|
+
endedAt: segment.endedAt,
|
|
1296
|
+
mimeType: segment.mimeType,
|
|
1297
|
+
durationMs: segment.durationMs,
|
|
1298
|
+
})),
|
|
1299
|
+
egress: {
|
|
1300
|
+
requestId,
|
|
1301
|
+
packet,
|
|
1302
|
+
note: "A content-free local activity receipt was recorded before this bounded metadata packet was returned.",
|
|
1303
|
+
},
|
|
1304
|
+
});
|
|
1305
|
+
}
|
|
1306
|
+
if (name === "screen_memory_search_chapters") {
|
|
1307
|
+
if (typeof args.query !== "string" || !args.query.trim())
|
|
1308
|
+
throw new Error("query is required.");
|
|
1309
|
+
const document = readScreenMemoryChapters(storeDir);
|
|
1310
|
+
if (!document)
|
|
1311
|
+
throw new Error(
|
|
1312
|
+
"No valid local Rewind chapters index is available yet.",
|
|
1313
|
+
);
|
|
1314
|
+
const minutes =
|
|
1315
|
+
typeof args.minutes === "number" && Number.isFinite(args.minutes)
|
|
1316
|
+
? Math.max(1, Math.min(Math.trunc(args.minutes), 24 * 60))
|
|
1317
|
+
: null;
|
|
1318
|
+
const cutoff = minutes ? Date.now() - minutes * 60_000 : null;
|
|
1319
|
+
const boundedDocument = cutoff
|
|
1320
|
+
? {
|
|
1321
|
+
...document,
|
|
1322
|
+
chapters: document.chapters.filter(
|
|
1323
|
+
(chapter) => Date.parse(chapter.endedAt) >= cutoff,
|
|
1324
|
+
),
|
|
1325
|
+
}
|
|
1326
|
+
: document;
|
|
1327
|
+
const candidates = searchScreenMemoryChapters(
|
|
1328
|
+
boundedDocument,
|
|
1329
|
+
args.query.trim(),
|
|
1330
|
+
typeof args.limit === "number" ? args.limit : 5,
|
|
1331
|
+
typeof args.clientHint === "string" ? args.clientHint : undefined,
|
|
1332
|
+
);
|
|
1333
|
+
const ambiguous =
|
|
1334
|
+
candidates.length > 1 &&
|
|
1335
|
+
Math.abs(candidates[0].score - candidates[1].score) <= 0.05;
|
|
1336
|
+
const packet = {
|
|
1337
|
+
question: redactCredentialText(args.query.trim().slice(0, 4_000)),
|
|
1338
|
+
evidence: candidates.map((chapter) => ({
|
|
1339
|
+
id: chapter.id,
|
|
1340
|
+
momentId: chapter.id,
|
|
1341
|
+
sourceType: "chapter",
|
|
1342
|
+
capturedAt: chapter.startedAt,
|
|
1343
|
+
excerpt: redactCredentialText(
|
|
1344
|
+
`${chapter.label}: ${chapter.summary}`,
|
|
1345
|
+
).slice(0, 1_200),
|
|
1346
|
+
})),
|
|
1347
|
+
};
|
|
1348
|
+
const requestId = `egress-mcp-${Date.now()}-${process.pid}`;
|
|
1349
|
+
appendEgressReceipt(
|
|
1350
|
+
storeDir,
|
|
1351
|
+
requestId,
|
|
1352
|
+
"search-chapters",
|
|
1353
|
+
packet.evidence,
|
|
1354
|
+
);
|
|
1355
|
+
return textResult({
|
|
1356
|
+
localOnly: true,
|
|
1357
|
+
generatedAt: document.generatedAt,
|
|
1358
|
+
state: document.state,
|
|
1359
|
+
coverage: document.coverage,
|
|
1360
|
+
gaps:
|
|
1361
|
+
isRecord(document.coverage) &&
|
|
1362
|
+
Array.isArray(document.coverage.gaps)
|
|
1363
|
+
? document.coverage.gaps
|
|
1364
|
+
: [],
|
|
1365
|
+
indexState: candidates.some(
|
|
1366
|
+
(candidate) => candidate.indexState === "pending",
|
|
1367
|
+
)
|
|
1368
|
+
? "pending"
|
|
1369
|
+
: candidates.some(
|
|
1370
|
+
(candidate) => candidate.indexState === "partial",
|
|
1371
|
+
)
|
|
1372
|
+
? "partial"
|
|
1373
|
+
: "ready",
|
|
1374
|
+
ambiguous,
|
|
1375
|
+
candidates: candidates.map(projectScreenMemoryChapterCandidate),
|
|
1376
|
+
egress: {
|
|
1377
|
+
requestId,
|
|
1378
|
+
packet,
|
|
1379
|
+
note: "A content-free local activity receipt was recorded before this bounded chapter packet was returned.",
|
|
1380
|
+
},
|
|
1381
|
+
});
|
|
1382
|
+
}
|
|
1383
|
+
if (name === "screen_memory_frame_at") {
|
|
1384
|
+
if (typeof args.timestamp !== "string")
|
|
1385
|
+
throw new Error("timestamp is required.");
|
|
1386
|
+
const frame = readScreenMemoryFrame(
|
|
1387
|
+
storeDir,
|
|
1388
|
+
args.timestamp,
|
|
1389
|
+
opts.decodeFrame ?? decodeFrameWithFfmpeg,
|
|
1390
|
+
);
|
|
1391
|
+
appendLocalEvidenceReceipt(storeDir, "frame-at", [
|
|
1392
|
+
{ timestamp: frame.timestamp, segmentId: frame.segmentId },
|
|
1393
|
+
]);
|
|
1394
|
+
return {
|
|
1395
|
+
content: [
|
|
1396
|
+
{
|
|
1397
|
+
type: "text" as const,
|
|
1398
|
+
text: JSON.stringify({
|
|
1399
|
+
localOnly: true,
|
|
1400
|
+
timestamp: frame.timestamp,
|
|
1401
|
+
segmentId: frame.segmentId,
|
|
1402
|
+
coverage: "retained clean segment",
|
|
1403
|
+
note: "This is a bounded local image read. Request screen_memory_request_clip for motion or audio.",
|
|
1404
|
+
}),
|
|
1405
|
+
},
|
|
1406
|
+
frame.image,
|
|
1407
|
+
],
|
|
1408
|
+
};
|
|
1409
|
+
}
|
|
1410
|
+
if (name === "screen_memory_contact_sheet") {
|
|
1411
|
+
const document = readScreenMemoryChapters(storeDir);
|
|
1412
|
+
const chapter =
|
|
1413
|
+
typeof args.chapterId === "string"
|
|
1414
|
+
? document?.chapters.find(
|
|
1415
|
+
(candidate) => candidate.id === args.chapterId,
|
|
1416
|
+
)
|
|
1417
|
+
: undefined;
|
|
1418
|
+
const startAt =
|
|
1419
|
+
chapter?.startedAt ??
|
|
1420
|
+
(typeof args.startAt === "string" ? args.startAt : null);
|
|
1421
|
+
const endAt =
|
|
1422
|
+
chapter?.endedAt ??
|
|
1423
|
+
(typeof args.endAt === "string" ? args.endAt : null);
|
|
1424
|
+
if (
|
|
1425
|
+
!startAt ||
|
|
1426
|
+
!endAt ||
|
|
1427
|
+
!isTimestamp(startAt) ||
|
|
1428
|
+
!isTimestamp(endAt)
|
|
1429
|
+
)
|
|
1430
|
+
throw new Error(
|
|
1431
|
+
"Provide a valid chapterId or RFC3339 startAt and endAt.",
|
|
1432
|
+
);
|
|
1433
|
+
const durationMs = Date.parse(endAt) - Date.parse(startAt);
|
|
1434
|
+
if (!contactSheetRangeIsValid(durationMs, Boolean(chapter)))
|
|
1435
|
+
throw new Error(
|
|
1436
|
+
"A contact sheet range must be between zero and five minutes.",
|
|
1437
|
+
);
|
|
1438
|
+
if (Date.parse(endAt) > Date.now() + 5_000)
|
|
1439
|
+
throw new Error(
|
|
1440
|
+
"A Rewind contact sheet cannot include future time.",
|
|
1441
|
+
);
|
|
1442
|
+
const count = Math.min(
|
|
1443
|
+
Math.max(
|
|
1444
|
+
Math.trunc(typeof args.count === "number" ? args.count : 4),
|
|
1445
|
+
1,
|
|
1318
1446
|
),
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
);
|
|
1327
|
-
const ambiguous =
|
|
1328
|
-
candidates.length > 1 &&
|
|
1329
|
-
Math.abs(candidates[0].score - candidates[1].score) <= 0.05;
|
|
1330
|
-
const packet = {
|
|
1331
|
-
question: redactCredentialText(args.query.trim().slice(0, 4_000)),
|
|
1332
|
-
evidence: candidates.map((chapter) => ({
|
|
1333
|
-
id: chapter.id,
|
|
1334
|
-
momentId: chapter.id,
|
|
1335
|
-
sourceType: "chapter",
|
|
1336
|
-
capturedAt: chapter.startedAt,
|
|
1337
|
-
excerpt: redactCredentialText(
|
|
1338
|
-
`${chapter.label}: ${chapter.summary}`,
|
|
1339
|
-
).slice(0, 1_200),
|
|
1340
|
-
})),
|
|
1341
|
-
};
|
|
1342
|
-
const requestId = `egress-mcp-${Date.now()}-${process.pid}`;
|
|
1343
|
-
appendEgressReceipt(
|
|
1344
|
-
storeDir,
|
|
1345
|
-
requestId,
|
|
1346
|
-
"search-chapters",
|
|
1347
|
-
packet.evidence,
|
|
1348
|
-
);
|
|
1349
|
-
return textResult({
|
|
1350
|
-
localOnly: true,
|
|
1351
|
-
generatedAt: document.generatedAt,
|
|
1352
|
-
state: document.state,
|
|
1353
|
-
coverage: document.coverage,
|
|
1354
|
-
gaps:
|
|
1355
|
-
isRecord(document.coverage) && Array.isArray(document.coverage.gaps)
|
|
1356
|
-
? document.coverage.gaps
|
|
1357
|
-
: [],
|
|
1358
|
-
indexState: candidates.some(
|
|
1359
|
-
(candidate) => candidate.indexState === "pending",
|
|
1360
|
-
)
|
|
1361
|
-
? "pending"
|
|
1362
|
-
: candidates.some((candidate) => candidate.indexState === "partial")
|
|
1363
|
-
? "partial"
|
|
1364
|
-
: "ready",
|
|
1365
|
-
ambiguous,
|
|
1366
|
-
candidates: candidates.map(projectScreenMemoryChapterCandidate),
|
|
1367
|
-
egress: {
|
|
1368
|
-
requestId,
|
|
1369
|
-
packet,
|
|
1370
|
-
note: "A content-free local activity receipt was recorded before this bounded chapter packet was returned.",
|
|
1371
|
-
},
|
|
1372
|
-
});
|
|
1373
|
-
}
|
|
1374
|
-
if (name === "screen_memory_frame_at") {
|
|
1375
|
-
if (typeof args.timestamp !== "string")
|
|
1376
|
-
throw new Error("timestamp is required.");
|
|
1377
|
-
const frame = readScreenMemoryFrame(
|
|
1378
|
-
storeDir,
|
|
1379
|
-
args.timestamp,
|
|
1380
|
-
opts.decodeFrame ?? decodeFrameWithFfmpeg,
|
|
1381
|
-
);
|
|
1382
|
-
appendLocalEvidenceReceipt(storeDir, "frame-at", [
|
|
1383
|
-
{ timestamp: frame.timestamp, segmentId: frame.segmentId },
|
|
1384
|
-
]);
|
|
1385
|
-
return {
|
|
1386
|
-
content: [
|
|
1387
|
-
{
|
|
1388
|
-
type: "text" as const,
|
|
1389
|
-
text: JSON.stringify({
|
|
1390
|
-
localOnly: true,
|
|
1391
|
-
timestamp: frame.timestamp,
|
|
1392
|
-
segmentId: frame.segmentId,
|
|
1393
|
-
coverage: "retained clean segment",
|
|
1394
|
-
note: "This is a bounded local image read. Request screen_memory_request_clip for motion or audio.",
|
|
1395
|
-
}),
|
|
1396
|
-
},
|
|
1397
|
-
frame.image,
|
|
1398
|
-
],
|
|
1399
|
-
};
|
|
1400
|
-
}
|
|
1401
|
-
if (name === "screen_memory_contact_sheet") {
|
|
1402
|
-
const document = readScreenMemoryChapters(storeDir);
|
|
1403
|
-
const chapter =
|
|
1404
|
-
typeof args.chapterId === "string"
|
|
1405
|
-
? document?.chapters.find(
|
|
1406
|
-
(candidate) => candidate.id === args.chapterId,
|
|
1447
|
+
8,
|
|
1448
|
+
);
|
|
1449
|
+
const representative = (chapter?.representativeMoments ?? [])
|
|
1450
|
+
.filter(
|
|
1451
|
+
(moment) =>
|
|
1452
|
+
Date.parse(moment.capturedAt) >= Date.parse(startAt) &&
|
|
1453
|
+
Date.parse(moment.capturedAt) <= Date.parse(endAt),
|
|
1407
1454
|
)
|
|
1408
|
-
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
|
|
1418
|
-
|
|
1419
|
-
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
|
|
1452
|
-
|
|
1453
|
-
|
|
1455
|
+
.map((moment) => moment.capturedAt);
|
|
1456
|
+
const timestamps = selectContactSheetTimestamps(
|
|
1457
|
+
startAt,
|
|
1458
|
+
endAt,
|
|
1459
|
+
count,
|
|
1460
|
+
representative,
|
|
1461
|
+
);
|
|
1462
|
+
const decoder = opts.decodeFrame ?? decodeFrameWithFfmpeg;
|
|
1463
|
+
const frames = timestamps.flatMap((timestamp) => {
|
|
1464
|
+
try {
|
|
1465
|
+
return [readScreenMemoryFrame(storeDir, timestamp, decoder)];
|
|
1466
|
+
} catch {
|
|
1467
|
+
// A chapter may span a brief coverage gap. Keep the clean retained
|
|
1468
|
+
// frames instead of failing the entire bounded contact sheet.
|
|
1469
|
+
return [];
|
|
1470
|
+
}
|
|
1471
|
+
});
|
|
1472
|
+
if (frames.length === 0) {
|
|
1473
|
+
throw new Error(
|
|
1474
|
+
"No clean retained Rewind frames cover the requested contact-sheet range.",
|
|
1475
|
+
);
|
|
1476
|
+
}
|
|
1477
|
+
appendLocalEvidenceReceipt(
|
|
1478
|
+
storeDir,
|
|
1479
|
+
"contact-sheet",
|
|
1480
|
+
frames.map(({ timestamp, segmentId }) => ({
|
|
1481
|
+
timestamp,
|
|
1482
|
+
segmentId,
|
|
1483
|
+
})),
|
|
1484
|
+
);
|
|
1485
|
+
return {
|
|
1486
|
+
content: [
|
|
1487
|
+
{
|
|
1488
|
+
type: "text" as const,
|
|
1489
|
+
text: JSON.stringify({
|
|
1490
|
+
localOnly: true,
|
|
1491
|
+
chapterId: chapter?.id ?? null,
|
|
1492
|
+
startAt: new Date(Date.parse(startAt)).toISOString(),
|
|
1493
|
+
endAt: new Date(Date.parse(endAt)).toISOString(),
|
|
1494
|
+
frameCount: frames.length,
|
|
1495
|
+
selection: representative.length
|
|
1496
|
+
? "representative moments then even coverage"
|
|
1497
|
+
: "even temporal coverage",
|
|
1498
|
+
frames: frames.map(({ timestamp, segmentId }) => ({
|
|
1499
|
+
timestamp,
|
|
1500
|
+
segmentId,
|
|
1501
|
+
})),
|
|
1502
|
+
}),
|
|
1503
|
+
},
|
|
1504
|
+
...frames.map((frame) => frame.image),
|
|
1505
|
+
],
|
|
1506
|
+
};
|
|
1454
1507
|
}
|
|
1508
|
+
if (name === "screen_memory_request_clip") {
|
|
1509
|
+
const range = parseHandoffRange(args);
|
|
1510
|
+
const config = readFeatureConfig(storeDir);
|
|
1511
|
+
const requestId = `handoff-${randomUUID()}`;
|
|
1512
|
+
const reason =
|
|
1513
|
+
typeof args.reason === "string"
|
|
1514
|
+
? redactCredentialText(args.reason.trim().slice(0, 500))
|
|
1515
|
+
: "Visual or audio context requested by the agent";
|
|
1516
|
+
const request = {
|
|
1517
|
+
requestId,
|
|
1518
|
+
status: "pending",
|
|
1519
|
+
requestedAt: new Date().toISOString(),
|
|
1520
|
+
...range,
|
|
1521
|
+
reason,
|
|
1522
|
+
includeMicrophone: args.includeMicrophone !== false,
|
|
1523
|
+
includeSystemAudio: args.includeSystemAudio !== false,
|
|
1524
|
+
reviewRequired: config.reviewBeforeSending !== false,
|
|
1525
|
+
agentClipRetention: config.agentClipRetention ?? "forever",
|
|
1526
|
+
};
|
|
1527
|
+
writePrivateJson(handoffPath(storeDir, requestId), request);
|
|
1528
|
+
appendEgressEvent(storeDir, {
|
|
1529
|
+
requestId,
|
|
1530
|
+
occurredAt: request.requestedAt,
|
|
1531
|
+
state: "handoff-requested",
|
|
1532
|
+
operation: "request-clip",
|
|
1533
|
+
receipt: {
|
|
1534
|
+
mediaInterval: { startAt: range.startAt, endAt: range.endAt },
|
|
1535
|
+
reviewRequired: request.reviewRequired,
|
|
1536
|
+
},
|
|
1537
|
+
evidenceCount: 0,
|
|
1538
|
+
packetBytes: 0,
|
|
1539
|
+
error: null,
|
|
1540
|
+
});
|
|
1541
|
+
return textResult({
|
|
1542
|
+
requestId,
|
|
1543
|
+
status: "pending",
|
|
1544
|
+
startAt: range.startAt,
|
|
1545
|
+
endAt: range.endAt,
|
|
1546
|
+
reviewRequired: request.reviewRequired,
|
|
1547
|
+
note: request.reviewRequired
|
|
1548
|
+
? "Clips is waiting for the user to review this bounded range. Poll screen_memory_handoff_status."
|
|
1549
|
+
: "Clips is processing the explicitly requested bounded range. Poll screen_memory_handoff_status.",
|
|
1550
|
+
});
|
|
1551
|
+
}
|
|
1552
|
+
if (name === "screen_memory_handoff_status") {
|
|
1553
|
+
if (typeof args.requestId !== "string") {
|
|
1554
|
+
throw new Error("requestId is required.");
|
|
1555
|
+
}
|
|
1556
|
+
const parsed = JSON.parse(
|
|
1557
|
+
fs.readFileSync(handoffPath(storeDir, args.requestId), "utf-8"),
|
|
1558
|
+
) as Record<string, unknown>;
|
|
1559
|
+
return textResult({
|
|
1560
|
+
requestId: parsed.requestId,
|
|
1561
|
+
status: parsed.status,
|
|
1562
|
+
startAt: parsed.startAt,
|
|
1563
|
+
endAt: parsed.endAt,
|
|
1564
|
+
recordingId: parsed.recordingId,
|
|
1565
|
+
agentUrl: parsed.agentUrl,
|
|
1566
|
+
contextUrl: parsed.contextUrl,
|
|
1567
|
+
expiresAt: parsed.expiresAt,
|
|
1568
|
+
autoDeleteAt: parsed.autoDeleteAt,
|
|
1569
|
+
autoDeletedAt: parsed.autoDeletedAt,
|
|
1570
|
+
error: parsed.error,
|
|
1571
|
+
});
|
|
1572
|
+
}
|
|
1573
|
+
throw new Error(`Unknown Screen Memory tool: ${name}`);
|
|
1455
1574
|
});
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
}
|
|
1461
|
-
appendLocalEvidenceReceipt(
|
|
1462
|
-
storeDir,
|
|
1463
|
-
"contact-sheet",
|
|
1464
|
-
frames.map(({ timestamp, segmentId }) => ({ timestamp, segmentId })),
|
|
1465
|
-
);
|
|
1466
|
-
return {
|
|
1467
|
-
content: [
|
|
1468
|
-
{
|
|
1469
|
-
type: "text" as const,
|
|
1470
|
-
text: JSON.stringify({
|
|
1471
|
-
localOnly: true,
|
|
1472
|
-
chapterId: chapter?.id ?? null,
|
|
1473
|
-
startAt: new Date(Date.parse(startAt)).toISOString(),
|
|
1474
|
-
endAt: new Date(Date.parse(endAt)).toISOString(),
|
|
1475
|
-
frameCount: frames.length,
|
|
1476
|
-
selection: representative.length
|
|
1477
|
-
? "representative moments then even coverage"
|
|
1478
|
-
: "even temporal coverage",
|
|
1479
|
-
frames: frames.map(({ timestamp, segmentId }) => ({
|
|
1480
|
-
timestamp,
|
|
1481
|
-
segmentId,
|
|
1482
|
-
})),
|
|
1483
|
-
}),
|
|
1484
|
-
},
|
|
1485
|
-
...frames.map((frame) => frame.image),
|
|
1486
|
-
],
|
|
1487
|
-
};
|
|
1488
|
-
}
|
|
1489
|
-
if (name === "screen_memory_request_clip") {
|
|
1490
|
-
const range = parseHandoffRange(args);
|
|
1491
|
-
const config = readFeatureConfig(storeDir);
|
|
1492
|
-
const requestId = `handoff-${randomUUID()}`;
|
|
1493
|
-
const reason =
|
|
1494
|
-
typeof args.reason === "string"
|
|
1495
|
-
? redactCredentialText(args.reason.trim().slice(0, 500))
|
|
1496
|
-
: "Visual or audio context requested by the agent";
|
|
1497
|
-
const request = {
|
|
1498
|
-
requestId,
|
|
1499
|
-
status: "pending",
|
|
1500
|
-
requestedAt: new Date().toISOString(),
|
|
1501
|
-
...range,
|
|
1502
|
-
reason,
|
|
1503
|
-
includeMicrophone: args.includeMicrophone !== false,
|
|
1504
|
-
includeSystemAudio: args.includeSystemAudio !== false,
|
|
1505
|
-
reviewRequired: config.reviewBeforeSending !== false,
|
|
1506
|
-
agentClipRetention: config.agentClipRetention ?? "forever",
|
|
1507
|
-
};
|
|
1508
|
-
writePrivateJson(handoffPath(storeDir, requestId), request);
|
|
1509
|
-
appendEgressEvent(storeDir, {
|
|
1510
|
-
requestId,
|
|
1511
|
-
occurredAt: request.requestedAt,
|
|
1512
|
-
state: "handoff-requested",
|
|
1513
|
-
operation: "request-clip",
|
|
1514
|
-
receipt: {
|
|
1515
|
-
mediaInterval: { startAt: range.startAt, endAt: range.endAt },
|
|
1516
|
-
reviewRequired: request.reviewRequired,
|
|
1517
|
-
},
|
|
1518
|
-
evidenceCount: 0,
|
|
1519
|
-
packetBytes: 0,
|
|
1520
|
-
error: null,
|
|
1521
|
-
});
|
|
1522
|
-
return textResult({
|
|
1523
|
-
requestId,
|
|
1524
|
-
status: "pending",
|
|
1525
|
-
startAt: range.startAt,
|
|
1526
|
-
endAt: range.endAt,
|
|
1527
|
-
reviewRequired: request.reviewRequired,
|
|
1528
|
-
note: request.reviewRequired
|
|
1529
|
-
? "Clips is waiting for the user to review this bounded range. Poll screen_memory_handoff_status."
|
|
1530
|
-
: "Clips is processing the explicitly requested bounded range. Poll screen_memory_handoff_status.",
|
|
1531
|
-
});
|
|
1532
|
-
}
|
|
1533
|
-
if (name === "screen_memory_handoff_status") {
|
|
1534
|
-
if (typeof args.requestId !== "string") {
|
|
1535
|
-
throw new Error("requestId is required.");
|
|
1536
|
-
}
|
|
1537
|
-
const parsed = JSON.parse(
|
|
1538
|
-
fs.readFileSync(handoffPath(storeDir, args.requestId), "utf-8"),
|
|
1539
|
-
) as Record<string, unknown>;
|
|
1540
|
-
return textResult({
|
|
1541
|
-
requestId: parsed.requestId,
|
|
1542
|
-
status: parsed.status,
|
|
1543
|
-
startAt: parsed.startAt,
|
|
1544
|
-
endAt: parsed.endAt,
|
|
1545
|
-
recordingId: parsed.recordingId,
|
|
1546
|
-
agentUrl: parsed.agentUrl,
|
|
1547
|
-
contextUrl: parsed.contextUrl,
|
|
1548
|
-
expiresAt: parsed.expiresAt,
|
|
1549
|
-
autoDeleteAt: parsed.autoDeleteAt,
|
|
1550
|
-
autoDeletedAt: parsed.autoDeletedAt,
|
|
1551
|
-
error: parsed.error,
|
|
1552
|
-
});
|
|
1553
|
-
}
|
|
1554
|
-
throw new Error(`Unknown Screen Memory tool: ${name}`);
|
|
1555
|
-
});
|
|
1575
|
+
return server;
|
|
1576
|
+
},
|
|
1577
|
+
{ legacy: "serve" },
|
|
1578
|
+
);
|
|
1556
1579
|
|
|
1557
1580
|
log(`Serving local Screen Memory from ${storeDir}`);
|
|
1558
|
-
const transport = new StdioServerTransport();
|
|
1559
|
-
await server.connect(transport);
|
|
1560
1581
|
await new Promise<void>((resolve) => {
|
|
1561
|
-
|
|
1582
|
+
process.stdin.once("end", resolve);
|
|
1562
1583
|
process.once("SIGINT", resolve);
|
|
1563
1584
|
process.once("SIGTERM", resolve);
|
|
1564
1585
|
});
|
|
1586
|
+
await stdio.close();
|
|
1565
1587
|
}
|