@agent-native/core 0.69.0 → 0.70.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -15
- package/corpus/README.md +2 -2
- package/corpus/core/CHANGELOG.md +109 -0
- package/corpus/core/docs/design/durable-agent-runs.md +217 -0
- package/corpus/core/package.json +1 -1
- package/corpus/core/src/action.ts +89 -0
- package/corpus/core/src/agent/model-config.ts +0 -2
- package/corpus/core/src/agent/run-loop-with-resume.ts +48 -0
- package/corpus/core/src/changelog/parse.ts +245 -0
- package/corpus/core/src/cli/changelog.ts +199 -0
- package/corpus/core/src/cli/index.ts +19 -0
- package/corpus/core/src/cli/recap.ts +24 -1
- package/corpus/core/src/cli/skills.ts +74 -47
- package/corpus/core/src/client/AgentPanel.tsx +5 -3
- package/corpus/core/src/client/AssistantChat.tsx +8 -0
- package/corpus/core/src/client/CommandMenu.tsx +183 -75
- package/corpus/core/src/client/agent-chat-adapter.ts +44 -7
- package/corpus/core/src/client/analytics.ts +181 -0
- package/corpus/core/src/client/changelog/Changelog.tsx +321 -0
- package/corpus/core/src/client/client-surface.ts +41 -0
- package/corpus/core/src/client/composer/TiptapComposer.tsx +123 -12
- package/corpus/core/src/client/feedback-context.ts +4 -0
- package/corpus/core/src/client/index.ts +12 -0
- package/corpus/core/src/client/mcp-apps/McpAppRenderer.tsx +54 -0
- package/corpus/core/src/client/settings/useBuilderStatus.ts +82 -20
- package/corpus/core/src/client/sse-event-processor.ts +7 -1
- package/corpus/core/src/mcp/build-server.ts +57 -1
- package/corpus/core/src/mcp/embed-app.ts +197 -7
- package/corpus/core/src/mcp/oauth-route.ts +109 -13
- package/corpus/core/src/server/attribution.ts +211 -0
- package/corpus/core/src/server/auth.ts +6 -0
- package/corpus/core/src/server/better-auth-instance.ts +49 -5
- package/corpus/core/src/server/builder-browser.ts +27 -1
- package/corpus/core/src/server/core-routes-plugin.ts +136 -0
- package/corpus/core/src/server/google-oauth.ts +19 -6
- package/corpus/core/src/server/onboarding-html.ts +35 -0
- package/corpus/core/src/shared/mcp-embed-headers.ts +1 -1
- package/corpus/core/src/styles/agent-conversation.css +21 -0
- package/corpus/core/src/templates/workspace-core/.agents/skills/changelog/SKILL.md +111 -0
- package/corpus/core/src/templates/workspace-core/.agents/skills/reliable-mutations/SKILL.md +72 -0
- package/corpus/core/src/templates/workspace-core/.agents/skills/tracking/SKILL.md +46 -14
- package/corpus/templates/analytics/.agents/skills/dashboard-management/SKILL.md +22 -0
- package/corpus/templates/analytics/AGENTS.md +8 -1
- package/corpus/templates/analytics/CHANGELOG.md +10 -0
- package/corpus/templates/analytics/actions/install-dashboard-template.ts +99 -4
- package/corpus/templates/analytics/actions/open-traffic-dashboard.ts +1 -1
- package/corpus/templates/analytics/actions/update-dashboard.ts +39 -8
- package/corpus/templates/analytics/app/components/dashboard/SqlChart.tsx +30 -3
- package/corpus/templates/analytics/app/components/layout/CommandPalette.tsx +180 -144
- package/corpus/templates/analytics/app/components/layout/Header.tsx +2 -3
- package/corpus/templates/analytics/app/components/layout/Layout.tsx +7 -5
- package/corpus/templates/analytics/app/components/layout/Sidebar.tsx +221 -215
- package/corpus/templates/analytics/app/hooks/use-navigation-state.ts +6 -6
- package/corpus/templates/analytics/app/lib/demo-dashboard-path.ts +1 -1
- package/corpus/templates/analytics/app/lib/last-opened.ts +2 -1
- package/corpus/templates/analytics/app/pages/Settings.tsx +4 -0
- package/corpus/templates/analytics/app/pages/adhoc/explorer-dashboard/index.tsx +3 -3
- package/corpus/templates/analytics/app/pages/adhoc/sql-dashboard/DashboardFilterBar.tsx +2 -2
- package/corpus/templates/analytics/app/pages/adhoc/sql-dashboard/SqlChartCard.tsx +32 -2
- package/corpus/templates/analytics/app/pages/adhoc/sql-dashboard/index.tsx +101 -99
- package/corpus/templates/analytics/app/pages/overview/OverviewPage.tsx +1 -1
- package/corpus/templates/analytics/app/routes/adhoc.$id.tsx +18 -5
- package/corpus/templates/analytics/app/routes/catalog.tsx +1 -1
- package/corpus/templates/analytics/app/routes/dashboards.$id.tsx +9 -0
- package/corpus/templates/analytics/app/routes/traffic.tsx +1 -1
- package/corpus/templates/analytics/changelog/2026-06-23-long-dashboard-and-analysis-names-in-the-sidebar-now-truncat.md +6 -0
- package/corpus/templates/analytics/changelog/2026-06-23-open-any-dashboard-chart-full-screen-in-a-modal-from-the-pan.md +6 -0
- package/corpus/templates/analytics/changelog/2026-06-23-the-dashboards-and-analyses-sidebar-settings-buttons-now-sta.md +6 -0
- package/corpus/templates/analytics/seeds/dashboards/agent-native-templates-first-party.json +145 -0
- package/corpus/templates/analytics/server/db/index.ts +1 -1
- package/corpus/templates/analytics/server/lib/dashboard-catalog.ts +9 -2
- package/corpus/templates/analytics/server/lib/demo-dashboards.ts +1 -1
- package/corpus/templates/analytics/server/plugins/agent-chat.ts +2 -2
- package/corpus/templates/analytics/server/plugins/core-routes.ts +6 -6
- package/corpus/templates/assets/CHANGELOG.md +10 -0
- package/corpus/templates/assets/actions/generate-image.ts +24 -0
- package/corpus/templates/assets/app/components/layout/Sidebar.tsx +2 -2
- package/corpus/templates/assets/app/global.css +17 -1
- package/corpus/templates/assets/app/root.tsx +7 -1
- package/corpus/templates/assets/app/routes/_index.tsx +94 -14
- package/corpus/templates/assets/app/routes/settings.tsx +6 -0
- package/corpus/templates/assets/changelog/2026-06-23-chat-dates-in-the-sidebar-no-longer-wrap-onto-two-lines.md +6 -0
- package/corpus/templates/assets/changelog/2026-06-23-clicking-image-video-or-refine-on-the-start-screen-now-drops.md +6 -0
- package/corpus/templates/assets/changelog/2026-06-23-pick-your-image-generation-model-right-from-the-chat-model-m.md +6 -0
- package/corpus/templates/brain/CHANGELOG.md +10 -0
- package/corpus/templates/brain/app/root.tsx +7 -1
- package/corpus/templates/brain/app/routes/settings.tsx +8 -1
- package/corpus/templates/calendar/CHANGELOG.md +10 -0
- package/corpus/templates/calendar/app/pages/Settings.tsx +4 -0
- package/corpus/templates/calendar/app/root.tsx +7 -1
- package/corpus/templates/chat/CHANGELOG.md +10 -0
- package/corpus/templates/chat/app/root.tsx +7 -1
- package/corpus/templates/clips/CHANGELOG.md +10 -0
- package/corpus/templates/clips/app/components/library/library-grid.tsx +1 -20
- package/corpus/templates/clips/app/components/player/share-dialog.tsx +33 -7
- package/corpus/templates/clips/app/components/player/sign-in-prompt-dialog.tsx +8 -0
- package/corpus/templates/clips/app/components/player/video-player.tsx +150 -20
- package/corpus/templates/clips/app/components/recorder/countdown-overlay.tsx +72 -48
- package/corpus/templates/clips/app/components/recorder/pre-record-panel.tsx +7 -11
- package/corpus/templates/clips/app/components/recorder/recorder-engine.ts +31 -0
- package/corpus/templates/clips/app/components/sharing/slack-share-hint.tsx +71 -0
- package/corpus/templates/clips/app/lib/countdown-audio-cue.ts +35 -17
- package/corpus/templates/clips/app/root.tsx +7 -1
- package/corpus/templates/clips/app/routes/_app.settings._index.tsx +15 -1
- package/corpus/templates/clips/app/routes/record.tsx +78 -5
- package/corpus/templates/clips/app/routes/share.$shareId.tsx +86 -5
- package/corpus/templates/clips/changelog/2026-06-23-clips-shared-in-slack-no-longer-show-a-could-not-start-playb.md +6 -0
- package/corpus/templates/clips/changelog/2026-06-23-made-the-play-button-on-shared-and-embedded-clips-scale-with.md +6 -0
- package/corpus/templates/clips/changelog/2026-06-23-public-clip-share-links-now-show-whether-they-ll-play-inline.md +6 -0
- package/corpus/templates/clips/changelog/2026-06-23-removed-select-button-from-library-hover-a-clip-to-select-it.md +6 -0
- package/corpus/templates/clips/changelog/2026-06-23-screen-camera-recordings-now-keep-the-presenter-s-face-in-th.md +6 -0
- package/corpus/templates/clips/changelog/2026-06-23-shared-video-links-now-play-for-anyone-without-signing-in.md +6 -0
- package/corpus/templates/clips/changelog/2026-06-23-skip-or-cancel-the-recording-countdown-with-buttons-beside-t.md +6 -0
- package/corpus/templates/clips/changelog/2026-06-23-the-desktop-camera-bubble-now-looks-sharp-the-instant-it-ope.md +6 -0
- package/corpus/templates/clips/changelog/2026-06-23-the-desktop-recording-widget-now-grows-downward-only-when-yo.md +6 -0
- package/corpus/templates/clips/changelog/2026-06-23-the-recording-start-sound-is-now-a-softer-more-modern-chime.md +6 -0
- package/corpus/templates/clips/chrome-extension/public/manifest.json +17 -8
- package/corpus/templates/clips/chrome-extension/src/background.ts +769 -180
- package/corpus/templates/clips/chrome-extension/src/content-script.ts +177 -0
- package/corpus/templates/clips/chrome-extension/src/offscreen.ts +466 -329
- package/corpus/templates/clips/chrome-extension/src/overlay.css +377 -0
- package/corpus/templates/clips/chrome-extension/src/overlay.html +12 -0
- package/corpus/templates/clips/chrome-extension/src/overlay.ts +330 -0
- package/corpus/templates/clips/chrome-extension/src/permission.html +227 -0
- package/corpus/templates/clips/chrome-extension/src/permission.ts +105 -0
- package/corpus/templates/clips/chrome-extension/src/popup.html +3 -0
- package/corpus/templates/clips/chrome-extension/src/popup.ts +50 -2
- package/corpus/templates/clips/chrome-extension/src/styles.css +50 -0
- package/corpus/templates/clips/chrome-extension/vite.config.ts +3 -0
- package/corpus/templates/clips/desktop/src/lib/audio-cue.ts +41 -24
- package/corpus/templates/clips/desktop/src/lib/bubble-webrtc.ts +119 -2
- package/corpus/templates/clips/desktop/src/lib/recorder.ts +20 -0
- package/corpus/templates/clips/desktop/src/main.tsx +3 -0
- package/corpus/templates/clips/desktop/src/overlays/countdown.tsx +21 -2
- package/corpus/templates/clips/desktop/src/overlays/region-record-border.tsx +76 -0
- package/corpus/templates/clips/desktop/src/styles.css +119 -2
- package/corpus/templates/clips/desktop/src-tauri/capabilities/default.json +2 -1
- package/corpus/templates/clips/desktop/src-tauri/src/clips/mod.rs +140 -1
- package/corpus/templates/clips/desktop/src-tauri/src/lib.rs +2 -0
- package/corpus/templates/clips/desktop/src-tauri/src/util.rs +4 -1
- package/corpus/templates/clips/server/routes/api/video/[recordingId].get.ts +85 -6
- package/corpus/templates/clips/shared/share-attribution.ts +79 -0
- package/corpus/templates/content/AGENTS.md +20 -3
- package/corpus/templates/content/CHANGELOG.md +10 -0
- package/corpus/templates/content/actions/_database-utils.ts +15 -0
- package/corpus/templates/content/actions/_property-utils.ts +371 -4
- package/corpus/templates/content/actions/configure-document-property.ts +35 -1
- package/corpus/templates/content/actions/create-content-database.ts +5 -1
- package/corpus/templates/content/actions/delete-document-property.ts +52 -2
- package/corpus/templates/content/actions/duplicate-document-property.ts +34 -17
- package/corpus/templates/content/actions/reorder-document-property.ts +79 -0
- package/corpus/templates/content/actions/set-document-property.ts +39 -0
- package/corpus/templates/content/app/components/editor/DocumentBlockFields.tsx +806 -0
- package/corpus/templates/content/app/components/editor/DocumentDatabase.tsx +226 -68
- package/corpus/templates/content/app/components/editor/DocumentEditor.tsx +59 -30
- package/corpus/templates/content/app/components/editor/DocumentProperties.tsx +25 -1
- package/corpus/templates/content/app/components/editor/blockFieldSaveController.ts +180 -0
- package/corpus/templates/content/app/components/editor/blockFieldSaveRegistry.ts +179 -0
- package/corpus/templates/content/app/components/editor/previewDocumentSaveController.ts +244 -0
- package/corpus/templates/content/app/components/editor/previewDocumentSaveRegistry.ts +132 -0
- package/corpus/templates/content/app/global.css +9 -0
- package/corpus/templates/content/app/hooks/use-document-properties.ts +21 -0
- package/corpus/templates/content/app/root.tsx +7 -1
- package/corpus/templates/content/server/db/schema.ts +30 -0
- package/corpus/templates/content/server/plugins/db.ts +87 -0
- package/corpus/templates/content/shared/api.ts +7 -0
- package/corpus/templates/content/shared/properties.ts +109 -0
- package/corpus/templates/design/CHANGELOG.md +10 -0
- package/corpus/templates/design/actions/generate-design.ts +4 -0
- package/corpus/templates/design/app/root.tsx +7 -1
- package/corpus/templates/design/app/routes/settings.tsx +10 -2
- package/corpus/templates/dispatch/CHANGELOG.md +10 -0
- package/corpus/templates/dispatch/app/root.tsx +7 -1
- package/corpus/templates/forms/.agents/skills/form-responses/SKILL.md +10 -0
- package/corpus/templates/forms/CHANGELOG.md +10 -0
- package/corpus/templates/forms/actions/export-responses.ts +6 -0
- package/corpus/templates/forms/actions/list-responses.ts +2 -0
- package/corpus/templates/forms/actions/response-insights.ts +2 -0
- package/corpus/templates/forms/app/pages/ResponsesPage.tsx +138 -1
- package/corpus/templates/forms/app/root.tsx +7 -1
- package/corpus/templates/forms/changelog/2026-06-23-response-tables-now-show-the-page-each-submission-came-from-.md +6 -0
- package/corpus/templates/forms/changelog/2026-06-23-response-tables-now-show-whether-feedback-came-from-the-web-.md +6 -0
- package/corpus/templates/forms/server/db/schema.ts +7 -0
- package/corpus/templates/forms/server/handlers/submissions.ts +19 -2
- package/corpus/templates/forms/server/lib/integrations.ts +76 -2
- package/corpus/templates/forms/server/plugins/db.ts +20 -0
- package/corpus/templates/forms/shared/types.ts +12 -0
- package/corpus/templates/macros/CHANGELOG.md +10 -0
- package/corpus/templates/macros/app/root.tsx +7 -1
- package/corpus/templates/mail/CHANGELOG.md +10 -0
- package/corpus/templates/mail/app/components/layout/CommandPalette.tsx +3 -0
- package/corpus/templates/mail/app/pages/SettingsPage.tsx +27 -0
- package/corpus/templates/plan/.agents/skills/visual-plan/SKILL.md +31 -17
- package/corpus/templates/plan/.agents/skills/visual-plan/references/wireframe.md +7 -4
- package/corpus/templates/plan/.agents/skills/visual-recap/SKILL.md +34 -24
- package/corpus/templates/plan/.agents/skills/visual-recap/references/wireframe.md +7 -4
- package/corpus/templates/plan/AGENTS.md +4 -0
- package/corpus/templates/plan/CHANGELOG.md +10 -0
- package/corpus/templates/plan/actions/view-screen.ts +5 -0
- package/corpus/templates/plan/app/components/layout/Sidebar.tsx +98 -1
- package/corpus/templates/plan/app/components/plan/wireframe/html-artboard.css +3 -1
- package/corpus/templates/plan/app/pages/PlansPage.tsx +107 -7
- package/corpus/templates/plan/app/root.tsx +7 -1
- package/corpus/templates/plan/changelog/2026-06-23-large-diagrams-in-a-plan-now-scroll-within-their-block-inste.md +6 -0
- package/corpus/templates/plan/shared/share-attribution.ts +72 -0
- package/corpus/templates/slides/CHANGELOG.md +10 -0
- package/corpus/templates/slides/app/root.tsx +7 -1
- package/corpus/templates/videos/CHANGELOG.md +4 -151
- package/corpus/templates/videos/app/root.tsx +7 -1
- package/dist/action.js +87 -0
- package/dist/action.js.map +1 -1
- package/dist/agent/engine/builder-engine.d.ts +1 -1
- package/dist/agent/engine/builder-engine.d.ts.map +1 -1
- package/dist/agent/model-config.d.ts +2 -2
- package/dist/agent/model-config.d.ts.map +1 -1
- package/dist/agent/model-config.js +0 -2
- package/dist/agent/model-config.js.map +1 -1
- package/dist/agent/run-loop-with-resume.d.ts +13 -0
- package/dist/agent/run-loop-with-resume.d.ts.map +1 -1
- package/dist/agent/run-loop-with-resume.js +44 -0
- package/dist/agent/run-loop-with-resume.js.map +1 -1
- package/dist/changelog/parse.d.ts +70 -0
- package/dist/changelog/parse.d.ts.map +1 -0
- package/dist/changelog/parse.js +200 -0
- package/dist/changelog/parse.js.map +1 -0
- package/dist/cli/changelog.d.ts +4 -0
- package/dist/cli/changelog.d.ts.map +1 -0
- package/dist/cli/changelog.js +166 -0
- package/dist/cli/changelog.js.map +1 -0
- package/dist/cli/index.js +18 -0
- package/dist/cli/index.js.map +1 -1
- package/dist/cli/recap.d.ts.map +1 -1
- package/dist/cli/recap.js +22 -1
- package/dist/cli/recap.js.map +1 -1
- package/dist/cli/skills.d.ts +3 -3
- package/dist/cli/skills.d.ts.map +1 -1
- package/dist/cli/skills.js +74 -47
- package/dist/cli/skills.js.map +1 -1
- package/dist/client/AgentPanel.d.ts.map +1 -1
- package/dist/client/AgentPanel.js +5 -3
- package/dist/client/AgentPanel.js.map +1 -1
- package/dist/client/AssistantChat.d.ts +6 -1
- package/dist/client/AssistantChat.d.ts.map +1 -1
- package/dist/client/AssistantChat.js +2 -2
- package/dist/client/AssistantChat.js.map +1 -1
- package/dist/client/CommandMenu.d.ts +17 -1
- package/dist/client/CommandMenu.d.ts.map +1 -1
- package/dist/client/CommandMenu.js +49 -13
- package/dist/client/CommandMenu.js.map +1 -1
- package/dist/client/agent-chat-adapter.d.ts.map +1 -1
- package/dist/client/agent-chat-adapter.js +43 -6
- package/dist/client/agent-chat-adapter.js.map +1 -1
- package/dist/client/analytics.d.ts +18 -0
- package/dist/client/analytics.d.ts.map +1 -1
- package/dist/client/analytics.js +177 -0
- package/dist/client/analytics.js.map +1 -1
- package/dist/client/changelog/Changelog.d.ts +47 -0
- package/dist/client/changelog/Changelog.d.ts.map +1 -0
- package/dist/client/changelog/Changelog.js +138 -0
- package/dist/client/changelog/Changelog.js.map +1 -0
- package/dist/client/client-surface.d.ts +17 -0
- package/dist/client/client-surface.d.ts.map +1 -0
- package/dist/client/client-surface.js +24 -0
- package/dist/client/client-surface.js.map +1 -0
- package/dist/client/composer/TiptapComposer.d.ts +26 -1
- package/dist/client/composer/TiptapComposer.d.ts.map +1 -1
- package/dist/client/composer/TiptapComposer.js +27 -5
- package/dist/client/composer/TiptapComposer.js.map +1 -1
- package/dist/client/feedback-context.d.ts +3 -0
- package/dist/client/feedback-context.d.ts.map +1 -1
- package/dist/client/feedback-context.js +2 -0
- package/dist/client/feedback-context.js.map +1 -1
- package/dist/client/index.d.ts +3 -1
- package/dist/client/index.d.ts.map +1 -1
- package/dist/client/index.js +3 -1
- package/dist/client/index.js.map +1 -1
- package/dist/client/mcp-apps/McpAppRenderer.d.ts +1 -0
- package/dist/client/mcp-apps/McpAppRenderer.d.ts.map +1 -1
- package/dist/client/mcp-apps/McpAppRenderer.js +42 -1
- package/dist/client/mcp-apps/McpAppRenderer.js.map +1 -1
- package/dist/client/settings/useBuilderStatus.d.ts +2 -0
- package/dist/client/settings/useBuilderStatus.d.ts.map +1 -1
- package/dist/client/settings/useBuilderStatus.js +58 -10
- package/dist/client/settings/useBuilderStatus.js.map +1 -1
- package/dist/client/sse-event-processor.d.ts.map +1 -1
- package/dist/client/sse-event-processor.js +7 -1
- package/dist/client/sse-event-processor.js.map +1 -1
- package/dist/mcp/build-server.d.ts +21 -0
- package/dist/mcp/build-server.d.ts.map +1 -1
- package/dist/mcp/build-server.js +43 -1
- package/dist/mcp/build-server.js.map +1 -1
- package/dist/mcp/embed-app.d.ts.map +1 -1
- package/dist/mcp/embed-app.js +197 -7
- package/dist/mcp/embed-app.js.map +1 -1
- package/dist/mcp/oauth-route.d.ts.map +1 -1
- package/dist/mcp/oauth-route.js +95 -12
- package/dist/mcp/oauth-route.js.map +1 -1
- package/dist/server/attribution.d.ts +83 -0
- package/dist/server/attribution.d.ts.map +1 -0
- package/dist/server/attribution.js +178 -0
- package/dist/server/attribution.js.map +1 -0
- package/dist/server/auth.d.ts.map +1 -1
- package/dist/server/auth.js +7 -0
- package/dist/server/auth.js.map +1 -1
- package/dist/server/better-auth-instance.d.ts +9 -1
- package/dist/server/better-auth-instance.d.ts.map +1 -1
- package/dist/server/better-auth-instance.js +32 -2
- package/dist/server/better-auth-instance.js.map +1 -1
- package/dist/server/builder-browser.d.ts +4 -0
- package/dist/server/builder-browser.d.ts.map +1 -1
- package/dist/server/builder-browser.js +20 -1
- package/dist/server/builder-browser.js.map +1 -1
- package/dist/server/core-routes-plugin.d.ts +25 -0
- package/dist/server/core-routes-plugin.d.ts.map +1 -1
- package/dist/server/core-routes-plugin.js +101 -0
- package/dist/server/core-routes-plugin.js.map +1 -1
- package/dist/server/google-oauth.d.ts.map +1 -1
- package/dist/server/google-oauth.js +16 -2
- package/dist/server/google-oauth.js.map +1 -1
- package/dist/server/onboarding-html.d.ts.map +1 -1
- package/dist/server/onboarding-html.js +35 -0
- package/dist/server/onboarding-html.js.map +1 -1
- package/dist/shared/mcp-embed-headers.js +1 -1
- package/dist/shared/mcp-embed-headers.js.map +1 -1
- package/dist/styles/agent-conversation.css +21 -0
- package/dist/templates/workspace-core/.agents/skills/changelog/SKILL.md +111 -0
- package/dist/templates/workspace-core/.agents/skills/reliable-mutations/SKILL.md +72 -0
- package/dist/templates/workspace-core/.agents/skills/tracking/SKILL.md +46 -14
- package/docs/design/durable-agent-runs.md +217 -0
- package/package.json +1 -1
- package/src/templates/workspace-core/.agents/skills/changelog/SKILL.md +111 -0
- package/src/templates/workspace-core/.agents/skills/reliable-mutations/SKILL.md +72 -0
- package/src/templates/workspace-core/.agents/skills/tracking/SKILL.md +46 -14
- package/corpus/templates/analytics/app/pages/About.tsx +0 -108
- package/corpus/templates/analytics/app/routes/about.tsx +0 -9
- /package/corpus/templates/analytics/app/routes/{dashboards.tsx → dashboards._index.tsx} +0 -0
|
@@ -1,28 +1,54 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
1
|
+
// Offscreen recording engine (Loom-style, MV3).
|
|
2
|
+
//
|
|
3
|
+
// This document holds the getDisplayMedia()/getUserMedia() stream and the
|
|
4
|
+
// MediaRecorder. Living in an offscreen document (reason DISPLAY_MEDIA) is what
|
|
5
|
+
// lets a recording survive page navigations — the capture is decoupled from any
|
|
6
|
+
// tab. The camera bubble and controls are rendered as on-page overlays by the
|
|
7
|
+
// content script, so we no longer composite the camera into a canvas here; for
|
|
8
|
+
// screen modes we record the display stream directly (+ mixed mic audio), and
|
|
9
|
+
// the bubble shows up in the recording naturally when a full screen/window/tab
|
|
10
|
+
// surface that contains it is captured.
|
|
11
|
+
//
|
|
12
|
+
// Lifecycle: ACQUIRE (show picker, hold stream) → BEGIN (start recorder after
|
|
13
|
+
// the countdown) → PAUSE/RESUME → STOP/CANCEL, plus RESTART (discard and start
|
|
14
|
+
// over on the same stream).
|
|
15
|
+
|
|
16
|
+
type CaptureMode = "screen" | "camera";
|
|
17
|
+
|
|
18
|
+
type AcquireMessage = {
|
|
19
|
+
type: "CLIPS_OFFSCREEN_ACQUIRE";
|
|
5
20
|
sessionId: string;
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
streamId?: string;
|
|
9
|
-
captureSurface: CaptureSurface;
|
|
10
|
-
includeCamera: boolean;
|
|
21
|
+
mode: CaptureMode;
|
|
22
|
+
surface: "browser" | "window" | "monitor";
|
|
11
23
|
includeMicrophone: boolean;
|
|
12
|
-
title?: string | null;
|
|
13
24
|
};
|
|
14
25
|
|
|
15
|
-
type
|
|
16
|
-
type: "
|
|
26
|
+
type BeginMessage = {
|
|
27
|
+
type: "CLIPS_OFFSCREEN_BEGIN";
|
|
17
28
|
sessionId: string;
|
|
29
|
+
recordingId: string;
|
|
30
|
+
uploadUrl: string;
|
|
31
|
+
hasCamera?: boolean;
|
|
32
|
+
// Pre-roll countdown delay, owned here in the offscreen document (a reliable
|
|
33
|
+
// context) rather than the service worker (which can suspend and drop timers).
|
|
34
|
+
startDelayMs?: number;
|
|
35
|
+
// Bearer token so chunk uploads authenticate the same way create-recording
|
|
36
|
+
// does. The offscreen document has no Clips session cookie of its own.
|
|
37
|
+
authToken?: string;
|
|
18
38
|
};
|
|
19
39
|
|
|
20
|
-
type
|
|
21
|
-
type:
|
|
40
|
+
type SimpleMessage = {
|
|
41
|
+
type:
|
|
42
|
+
| "CLIPS_OFFSCREEN_PAUSE"
|
|
43
|
+
| "CLIPS_OFFSCREEN_RESUME"
|
|
44
|
+
| "CLIPS_OFFSCREEN_STOP"
|
|
45
|
+
| "CLIPS_OFFSCREEN_CANCEL"
|
|
46
|
+
| "CLIPS_OFFSCREEN_RESTART"
|
|
47
|
+
| "CLIPS_OFFSCREEN_START_NOW";
|
|
22
48
|
sessionId: string;
|
|
23
49
|
};
|
|
24
50
|
|
|
25
|
-
type StatusName = "recording" | "uploading" | "complete" | "error";
|
|
51
|
+
type StatusName = "recording" | "paused" | "uploading" | "complete" | "error";
|
|
26
52
|
|
|
27
53
|
type UploadResult = {
|
|
28
54
|
ok?: boolean;
|
|
@@ -35,26 +61,39 @@ type UploadResult = {
|
|
|
35
61
|
error?: string;
|
|
36
62
|
};
|
|
37
63
|
|
|
64
|
+
type PreparedStreams = {
|
|
65
|
+
sessionId: string;
|
|
66
|
+
mode: CaptureMode;
|
|
67
|
+
displayStream: MediaStream | null;
|
|
68
|
+
micStream: MediaStream | null;
|
|
69
|
+
cameraStream: MediaStream | null;
|
|
70
|
+
width: number;
|
|
71
|
+
height: number;
|
|
72
|
+
endedListener: (() => void) | null;
|
|
73
|
+
endedTrack: MediaStreamTrack | null;
|
|
74
|
+
};
|
|
75
|
+
|
|
38
76
|
type ActiveRecording = {
|
|
39
77
|
sessionId: string;
|
|
40
78
|
recordingId: string;
|
|
41
79
|
uploadUrl: string;
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
includeMicrophone: boolean;
|
|
80
|
+
authToken: string | null;
|
|
81
|
+
mode: CaptureMode;
|
|
45
82
|
startedAtMs: number;
|
|
46
83
|
mimeType: string;
|
|
47
84
|
recorder: MediaRecorder;
|
|
48
85
|
outputStream: MediaStream;
|
|
49
86
|
sourceStreams: MediaStream[];
|
|
50
|
-
canvas: HTMLCanvasElement;
|
|
51
|
-
canvasStream: MediaStream;
|
|
52
87
|
audioContext: AudioContext | null;
|
|
53
|
-
frameRequestId: number | null;
|
|
54
88
|
chunkIndex: number;
|
|
55
89
|
uploadPromises: Promise<unknown>[];
|
|
56
90
|
uploadFailure: Error | null;
|
|
57
91
|
cancelled: boolean;
|
|
92
|
+
// Set when the recorder is being torn down to start over on the same source
|
|
93
|
+
// streams, so the stop handler skips the usual track cleanup.
|
|
94
|
+
restarting: boolean;
|
|
95
|
+
// Pending pre-roll timer; non-null means the recorder hasn't started yet.
|
|
96
|
+
startTimer: ReturnType<typeof setTimeout> | null;
|
|
58
97
|
dimensions: { width: number; height: number };
|
|
59
98
|
hasAudio: boolean;
|
|
60
99
|
hasCamera: boolean;
|
|
@@ -63,6 +102,7 @@ type ActiveRecording = {
|
|
|
63
102
|
rejectStopped: (error: Error) => void;
|
|
64
103
|
};
|
|
65
104
|
|
|
105
|
+
let prepared: PreparedStreams | null = null;
|
|
66
106
|
let activeRecording: ActiveRecording | null = null;
|
|
67
107
|
|
|
68
108
|
function reportStatus(
|
|
@@ -114,183 +154,79 @@ function waitForMetadata(video: HTMLVideoElement): Promise<void> {
|
|
|
114
154
|
});
|
|
115
155
|
}
|
|
116
156
|
|
|
117
|
-
async function
|
|
157
|
+
async function streamDimensions(
|
|
158
|
+
stream: MediaStream,
|
|
159
|
+
): Promise<{ width: number; height: number }> {
|
|
160
|
+
const track = stream.getVideoTracks()[0];
|
|
161
|
+
const settings = track?.getSettings?.();
|
|
162
|
+
if (settings?.width && settings?.height) {
|
|
163
|
+
return { width: settings.width, height: settings.height };
|
|
164
|
+
}
|
|
118
165
|
const video = document.createElement("video");
|
|
119
166
|
video.muted = true;
|
|
120
|
-
video.playsInline = true;
|
|
121
167
|
video.srcObject = stream;
|
|
122
|
-
await waitForMetadata(video);
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
function drawCover(
|
|
128
|
-
ctx: CanvasRenderingContext2D,
|
|
129
|
-
video: HTMLVideoElement,
|
|
130
|
-
x: number,
|
|
131
|
-
y: number,
|
|
132
|
-
width: number,
|
|
133
|
-
height: number,
|
|
134
|
-
): void {
|
|
135
|
-
const sourceWidth = video.videoWidth || width;
|
|
136
|
-
const sourceHeight = video.videoHeight || height;
|
|
137
|
-
const scale = Math.max(width / sourceWidth, height / sourceHeight);
|
|
138
|
-
const drawWidth = sourceWidth * scale;
|
|
139
|
-
const drawHeight = sourceHeight * scale;
|
|
140
|
-
ctx.drawImage(
|
|
141
|
-
video,
|
|
142
|
-
x + (width - drawWidth) / 2,
|
|
143
|
-
y + (height - drawHeight) / 2,
|
|
144
|
-
drawWidth,
|
|
145
|
-
drawHeight,
|
|
146
|
-
);
|
|
147
|
-
}
|
|
148
|
-
|
|
149
|
-
function drawCameraBubble(
|
|
150
|
-
ctx: CanvasRenderingContext2D,
|
|
151
|
-
video: HTMLVideoElement,
|
|
152
|
-
canvas: HTMLCanvasElement,
|
|
153
|
-
): void {
|
|
154
|
-
const size = Math.max(
|
|
155
|
-
148,
|
|
156
|
-
Math.round(Math.min(canvas.width, canvas.height) * 0.2),
|
|
157
|
-
);
|
|
158
|
-
const margin = Math.max(
|
|
159
|
-
24,
|
|
160
|
-
Math.round(Math.min(canvas.width, canvas.height) * 0.035),
|
|
161
|
-
);
|
|
162
|
-
const x = margin;
|
|
163
|
-
const y = canvas.height - size - margin;
|
|
164
|
-
const radius = size / 2;
|
|
165
|
-
const centerX = x + radius;
|
|
166
|
-
const centerY = y + radius;
|
|
167
|
-
|
|
168
|
-
ctx.save();
|
|
169
|
-
ctx.shadowColor = "rgba(15, 23, 42, 0.35)";
|
|
170
|
-
ctx.shadowBlur = Math.round(size * 0.08);
|
|
171
|
-
ctx.shadowOffsetY = Math.round(size * 0.035);
|
|
172
|
-
ctx.beginPath();
|
|
173
|
-
ctx.arc(centerX, centerY, radius, 0, Math.PI * 2);
|
|
174
|
-
ctx.fillStyle = "#0f172a";
|
|
175
|
-
ctx.fill();
|
|
176
|
-
ctx.restore();
|
|
177
|
-
|
|
178
|
-
ctx.save();
|
|
179
|
-
ctx.beginPath();
|
|
180
|
-
ctx.arc(centerX, centerY, radius - 4, 0, Math.PI * 2);
|
|
181
|
-
ctx.clip();
|
|
182
|
-
drawCover(ctx, video, x + 4, y + 4, size - 8, size - 8);
|
|
183
|
-
ctx.restore();
|
|
184
|
-
|
|
185
|
-
ctx.save();
|
|
186
|
-
ctx.beginPath();
|
|
187
|
-
ctx.arc(centerX, centerY, radius - 2, 0, Math.PI * 2);
|
|
188
|
-
ctx.strokeStyle = "rgba(255, 255, 255, 0.92)";
|
|
189
|
-
ctx.lineWidth = 4;
|
|
190
|
-
ctx.stroke();
|
|
191
|
-
ctx.restore();
|
|
192
|
-
}
|
|
193
|
-
|
|
194
|
-
function startDrawing(params: {
|
|
195
|
-
canvas: HTMLCanvasElement;
|
|
196
|
-
sourceVideo: HTMLVideoElement | null;
|
|
197
|
-
cameraVideo: HTMLVideoElement | null;
|
|
198
|
-
}): number {
|
|
199
|
-
const ctx = params.canvas.getContext("2d");
|
|
200
|
-
if (!ctx) throw new Error("Canvas rendering is unavailable.");
|
|
201
|
-
|
|
202
|
-
const draw = () => {
|
|
203
|
-
ctx.fillStyle = "#020617";
|
|
204
|
-
ctx.fillRect(0, 0, params.canvas.width, params.canvas.height);
|
|
205
|
-
|
|
206
|
-
if (params.sourceVideo) {
|
|
207
|
-
drawCover(
|
|
208
|
-
ctx,
|
|
209
|
-
params.sourceVideo,
|
|
210
|
-
0,
|
|
211
|
-
0,
|
|
212
|
-
params.canvas.width,
|
|
213
|
-
params.canvas.height,
|
|
214
|
-
);
|
|
215
|
-
} else if (params.cameraVideo) {
|
|
216
|
-
drawCover(
|
|
217
|
-
ctx,
|
|
218
|
-
params.cameraVideo,
|
|
219
|
-
0,
|
|
220
|
-
0,
|
|
221
|
-
params.canvas.width,
|
|
222
|
-
params.canvas.height,
|
|
223
|
-
);
|
|
224
|
-
}
|
|
225
|
-
|
|
226
|
-
if (params.sourceVideo && params.cameraVideo) {
|
|
227
|
-
drawCameraBubble(ctx, params.cameraVideo, params.canvas);
|
|
228
|
-
}
|
|
229
|
-
|
|
230
|
-
const recording = activeRecording;
|
|
231
|
-
if (recording && recording.frameRequestId !== null) {
|
|
232
|
-
recording.frameRequestId = requestAnimationFrame(draw);
|
|
233
|
-
}
|
|
168
|
+
await waitForMetadata(video).catch(() => undefined);
|
|
169
|
+
return {
|
|
170
|
+
width: video.videoWidth || 1280,
|
|
171
|
+
height: video.videoHeight || 720,
|
|
234
172
|
};
|
|
235
|
-
|
|
236
|
-
return requestAnimationFrame(draw);
|
|
237
173
|
}
|
|
238
174
|
|
|
239
|
-
function
|
|
175
|
+
function displayConstraints(
|
|
176
|
+
surface: "browser" | "window" | "monitor",
|
|
177
|
+
): MediaStreamConstraints {
|
|
178
|
+
const displaySurface =
|
|
179
|
+
surface === "browser"
|
|
180
|
+
? "browser"
|
|
181
|
+
: surface === "window"
|
|
182
|
+
? "window"
|
|
183
|
+
: "monitor";
|
|
240
184
|
return {
|
|
241
|
-
audio: {
|
|
242
|
-
mandatory: {
|
|
243
|
-
chromeMediaSource: "tab",
|
|
244
|
-
chromeMediaSourceId: streamId,
|
|
245
|
-
},
|
|
246
|
-
},
|
|
247
185
|
video: {
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
chromeMediaSourceId: streamId,
|
|
251
|
-
},
|
|
186
|
+
frameRate: { ideal: 30, max: 30 },
|
|
187
|
+
...({ displaySurface } as object),
|
|
252
188
|
},
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
189
|
+
audio: {
|
|
190
|
+
echoCancellation: false,
|
|
191
|
+
noiseSuppression: false,
|
|
192
|
+
autoGainControl: false,
|
|
193
|
+
},
|
|
194
|
+
} as MediaStreamConstraints;
|
|
258
195
|
}
|
|
259
196
|
|
|
260
|
-
async function
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
197
|
+
async function getMicStream(): Promise<MediaStream | null> {
|
|
198
|
+
try {
|
|
199
|
+
return await navigator.mediaDevices.getUserMedia({
|
|
200
|
+
audio: {
|
|
201
|
+
echoCancellation: true,
|
|
202
|
+
noiseSuppression: true,
|
|
203
|
+
autoGainControl: true,
|
|
204
|
+
channelCount: 1,
|
|
205
|
+
},
|
|
206
|
+
video: false,
|
|
207
|
+
});
|
|
208
|
+
} catch {
|
|
209
|
+
return null;
|
|
210
|
+
}
|
|
271
211
|
}
|
|
272
212
|
|
|
273
213
|
async function createMixedAudio(
|
|
274
214
|
streams: MediaStream[],
|
|
275
|
-
monitorStreams: MediaStream[],
|
|
276
215
|
): Promise<{ audioContext: AudioContext | null; tracks: MediaStreamTrack[] }> {
|
|
277
216
|
const streamsWithAudio = streams.filter(
|
|
278
217
|
(stream) => stream.getAudioTracks().length,
|
|
279
218
|
);
|
|
280
219
|
if (!streamsWithAudio.length) return { audioContext: null, tracks: [] };
|
|
220
|
+
if (streamsWithAudio.length === 1) {
|
|
221
|
+
return { audioContext: null, tracks: streamsWithAudio[0].getAudioTracks() };
|
|
222
|
+
}
|
|
281
223
|
|
|
282
224
|
const audioContext = new AudioContext();
|
|
283
225
|
await audioContext.resume().catch(() => undefined);
|
|
284
226
|
const destination = audioContext.createMediaStreamDestination();
|
|
285
|
-
|
|
286
227
|
for (const stream of streamsWithAudio) {
|
|
287
|
-
|
|
288
|
-
source.connect(destination);
|
|
289
|
-
if (monitorStreams.includes(stream)) {
|
|
290
|
-
source.connect(audioContext.destination);
|
|
291
|
-
}
|
|
228
|
+
audioContext.createMediaStreamSource(stream).connect(destination);
|
|
292
229
|
}
|
|
293
|
-
|
|
294
230
|
return { audioContext, tracks: destination.stream.getAudioTracks() };
|
|
295
231
|
}
|
|
296
232
|
|
|
@@ -335,11 +271,16 @@ async function uploadChunk(
|
|
|
335
271
|
hasCamera: extra.hasCamera,
|
|
336
272
|
});
|
|
337
273
|
const body = await blob.arrayBuffer();
|
|
274
|
+
const headers: Record<string, string> = {
|
|
275
|
+
"Content-Type": blob.type || recording.mimeType,
|
|
276
|
+
"X-Agent-Native-Frontend": "1",
|
|
277
|
+
};
|
|
278
|
+
if (recording.authToken) {
|
|
279
|
+
headers.Authorization = `Bearer ${recording.authToken}`;
|
|
280
|
+
}
|
|
338
281
|
const res = await fetch(url, {
|
|
339
282
|
method: "POST",
|
|
340
|
-
headers
|
|
341
|
-
"Content-Type": blob.type || recording.mimeType,
|
|
342
|
-
},
|
|
283
|
+
headers,
|
|
343
284
|
credentials: "include",
|
|
344
285
|
body,
|
|
345
286
|
});
|
|
@@ -353,74 +294,132 @@ async function uploadChunk(
|
|
|
353
294
|
return data;
|
|
354
295
|
}
|
|
355
296
|
|
|
356
|
-
function
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
recording.frameRequestId = null;
|
|
360
|
-
}
|
|
361
|
-
for (const stream of [
|
|
362
|
-
recording.outputStream,
|
|
363
|
-
recording.canvasStream,
|
|
364
|
-
...recording.sourceStreams,
|
|
365
|
-
]) {
|
|
297
|
+
function stopStreams(streams: (MediaStream | null)[]): void {
|
|
298
|
+
for (const stream of streams) {
|
|
299
|
+
if (!stream) continue;
|
|
366
300
|
for (const track of stream.getTracks()) track.stop();
|
|
367
301
|
}
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
function disposePrepared(): void {
|
|
305
|
+
if (!prepared) return;
|
|
306
|
+
if (prepared.endedTrack && prepared.endedListener) {
|
|
307
|
+
prepared.endedTrack.removeEventListener("ended", prepared.endedListener);
|
|
308
|
+
}
|
|
309
|
+
prepared = null;
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
function cleanup(recording: ActiveRecording): void {
|
|
313
|
+
stopStreams([recording.outputStream, ...recording.sourceStreams]);
|
|
368
314
|
void recording.audioContext?.close().catch(() => undefined);
|
|
369
315
|
}
|
|
370
316
|
|
|
371
|
-
|
|
317
|
+
/* ---------------------------------------------------------------- acquire --- */
|
|
318
|
+
|
|
319
|
+
async function acquire(message: AcquireMessage): Promise<{
|
|
372
320
|
ok: boolean;
|
|
373
|
-
recordingId: string;
|
|
374
321
|
width: number;
|
|
375
322
|
height: number;
|
|
376
|
-
hasAudio: boolean;
|
|
377
|
-
hasCamera: boolean;
|
|
378
323
|
}> {
|
|
379
|
-
if (activeRecording)
|
|
380
|
-
|
|
324
|
+
if (activeRecording) throw new Error("Clips is already recording.");
|
|
325
|
+
// Discard any half-prepared capture from a cancelled attempt.
|
|
326
|
+
stopPreparedStreams();
|
|
327
|
+
disposePrepared();
|
|
328
|
+
|
|
329
|
+
let displayStream: MediaStream | null = null;
|
|
330
|
+
let micStream: MediaStream | null = null;
|
|
331
|
+
let cameraStream: MediaStream | null = null;
|
|
332
|
+
|
|
333
|
+
if (message.mode === "camera") {
|
|
334
|
+
cameraStream = await navigator.mediaDevices.getUserMedia({
|
|
335
|
+
video: {
|
|
336
|
+
width: { ideal: 1280 },
|
|
337
|
+
height: { ideal: 720 },
|
|
338
|
+
facingMode: "user",
|
|
339
|
+
},
|
|
340
|
+
audio: message.includeMicrophone,
|
|
341
|
+
});
|
|
342
|
+
} else {
|
|
343
|
+
// Native "Choose what to share" picker. This is the screenshot Steve showed.
|
|
344
|
+
displayStream = await navigator.mediaDevices.getDisplayMedia(
|
|
345
|
+
displayConstraints(message.surface),
|
|
346
|
+
);
|
|
347
|
+
if (message.includeMicrophone) micStream = await getMicStream();
|
|
381
348
|
}
|
|
382
349
|
|
|
383
|
-
const
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
const
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
350
|
+
const videoStream = displayStream ?? cameraStream;
|
|
351
|
+
if (!videoStream) throw new Error("No media stream was available to record.");
|
|
352
|
+
const { width, height } = await streamDimensions(videoStream);
|
|
353
|
+
|
|
354
|
+
// If the user stops sharing via Chrome's native control, tell the worker so it
|
|
355
|
+
// can run the normal stop/finalize flow.
|
|
356
|
+
const endedTrack = videoStream.getVideoTracks()[0] ?? null;
|
|
357
|
+
const endedListener = () => {
|
|
358
|
+
chrome.runtime.sendMessage({
|
|
359
|
+
type: "CLIPS_NATIVE_ENDED",
|
|
360
|
+
sessionId: message.sessionId,
|
|
361
|
+
});
|
|
362
|
+
};
|
|
363
|
+
endedTrack?.addEventListener("ended", endedListener);
|
|
397
364
|
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
365
|
+
prepared = {
|
|
366
|
+
sessionId: message.sessionId,
|
|
367
|
+
mode: message.mode,
|
|
368
|
+
displayStream,
|
|
369
|
+
micStream,
|
|
370
|
+
cameraStream,
|
|
371
|
+
width,
|
|
372
|
+
height,
|
|
373
|
+
endedListener,
|
|
374
|
+
endedTrack,
|
|
375
|
+
};
|
|
376
|
+
return { ok: true, width, height };
|
|
377
|
+
}
|
|
402
378
|
|
|
403
|
-
|
|
404
|
-
|
|
379
|
+
function stopPreparedStreams(): void {
|
|
380
|
+
if (!prepared) return;
|
|
381
|
+
stopStreams([
|
|
382
|
+
prepared.displayStream,
|
|
383
|
+
prepared.micStream,
|
|
384
|
+
prepared.cameraStream,
|
|
385
|
+
]);
|
|
386
|
+
}
|
|
405
387
|
|
|
406
|
-
|
|
407
|
-
canvas.width = width;
|
|
408
|
-
canvas.height = height;
|
|
409
|
-
const canvasStream = canvas.captureStream(30);
|
|
388
|
+
/* ------------------------------------------------------------------ begin --- */
|
|
410
389
|
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
390
|
+
async function begin(message: BeginMessage): Promise<{
|
|
391
|
+
ok: boolean;
|
|
392
|
+
width: number;
|
|
393
|
+
height: number;
|
|
394
|
+
hasAudio: boolean;
|
|
395
|
+
hasCamera: boolean;
|
|
396
|
+
}> {
|
|
397
|
+
const ready = prepared;
|
|
398
|
+
if (!ready || ready.sessionId !== message.sessionId) {
|
|
399
|
+
throw new Error("No prepared Clips capture was found.");
|
|
400
|
+
}
|
|
401
|
+
if (activeRecording) throw new Error("Clips is already recording.");
|
|
402
|
+
|
|
403
|
+
const videoStream = ready.displayStream ?? ready.cameraStream;
|
|
404
|
+
const videoTrack = videoStream?.getVideoTracks()[0];
|
|
405
|
+
if (!videoTrack) throw new Error("Capture video track was lost.");
|
|
406
|
+
|
|
407
|
+
const audioInputs = [
|
|
408
|
+
...(ready.displayStream ? [ready.displayStream] : []),
|
|
409
|
+
...(ready.micStream ? [ready.micStream] : []),
|
|
410
|
+
...(ready.mode === "camera" && ready.cameraStream
|
|
411
|
+
? [ready.cameraStream]
|
|
412
|
+
: []),
|
|
414
413
|
];
|
|
415
|
-
const
|
|
416
|
-
const mixedAudio = await createMixedAudio(audioStreams, monitorStreams);
|
|
414
|
+
const mixedAudio = await createMixedAudio(audioInputs);
|
|
417
415
|
|
|
418
|
-
const outputStream = new MediaStream([
|
|
419
|
-
...canvasStream.getVideoTracks(),
|
|
420
|
-
...mixedAudio.tracks,
|
|
421
|
-
]);
|
|
416
|
+
const outputStream = new MediaStream([videoTrack, ...mixedAudio.tracks]);
|
|
422
417
|
const mimeType = chooseMimeType();
|
|
423
|
-
const recorder = new MediaRecorder(outputStream, {
|
|
418
|
+
const recorder = new MediaRecorder(outputStream, {
|
|
419
|
+
mimeType,
|
|
420
|
+
videoBitsPerSecond: 4_000_000,
|
|
421
|
+
audioBitsPerSecond: 128_000,
|
|
422
|
+
});
|
|
424
423
|
|
|
425
424
|
let resolveStopped: (result: UploadResult) => void = () => undefined;
|
|
426
425
|
let rejectStopped: (error: Error) => void = () => undefined;
|
|
@@ -430,37 +429,40 @@ async function startRecording(message: OffscreenStartMessage): Promise<{
|
|
|
430
429
|
});
|
|
431
430
|
|
|
432
431
|
const recording: ActiveRecording = {
|
|
433
|
-
sessionId:
|
|
432
|
+
sessionId: ready.sessionId,
|
|
434
433
|
recordingId: message.recordingId,
|
|
435
434
|
uploadUrl: message.uploadUrl,
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
startedAtMs: Date.now(),
|
|
435
|
+
authToken: message.authToken ?? null,
|
|
436
|
+
mode: ready.mode,
|
|
437
|
+
startedAtMs: 0,
|
|
440
438
|
mimeType,
|
|
441
439
|
recorder,
|
|
442
440
|
outputStream,
|
|
443
441
|
sourceStreams: [
|
|
444
|
-
...(
|
|
445
|
-
...(
|
|
442
|
+
...(ready.displayStream ? [ready.displayStream] : []),
|
|
443
|
+
...(ready.micStream ? [ready.micStream] : []),
|
|
444
|
+
...(ready.cameraStream ? [ready.cameraStream] : []),
|
|
446
445
|
],
|
|
447
|
-
canvas,
|
|
448
|
-
canvasStream,
|
|
449
446
|
audioContext: mixedAudio.audioContext,
|
|
450
|
-
frameRequestId: null,
|
|
451
447
|
chunkIndex: 0,
|
|
452
448
|
uploadPromises: [],
|
|
453
449
|
uploadFailure: null,
|
|
454
450
|
cancelled: false,
|
|
455
|
-
|
|
451
|
+
restarting: false,
|
|
452
|
+
startTimer: null,
|
|
453
|
+
dimensions: { width: ready.width, height: ready.height },
|
|
456
454
|
hasAudio: outputStream.getAudioTracks().length > 0,
|
|
457
|
-
hasCamera:
|
|
455
|
+
hasCamera:
|
|
456
|
+
typeof message.hasCamera === "boolean"
|
|
457
|
+
? message.hasCamera
|
|
458
|
+
: ready.mode === "camera",
|
|
458
459
|
stopped,
|
|
459
460
|
resolveStopped,
|
|
460
461
|
rejectStopped,
|
|
461
462
|
};
|
|
463
|
+
// The prepared streams are now owned by the active recording.
|
|
464
|
+
prepared = null;
|
|
462
465
|
activeRecording = recording;
|
|
463
|
-
recording.frameRequestId = startDrawing({ canvas, sourceVideo, cameraVideo });
|
|
464
466
|
|
|
465
467
|
recorder.addEventListener("dataavailable", (event) => {
|
|
466
468
|
if (
|
|
@@ -485,128 +487,263 @@ async function startRecording(message: OffscreenStartMessage): Promise<{
|
|
|
485
487
|
});
|
|
486
488
|
|
|
487
489
|
recorder.addEventListener("stop", () => {
|
|
488
|
-
void (
|
|
489
|
-
if (recording.cancelled) {
|
|
490
|
-
cleanup(recording);
|
|
491
|
-
activeRecording = null;
|
|
492
|
-
recording.resolveStopped({ ok: true, status: "cancelled" });
|
|
493
|
-
return;
|
|
494
|
-
}
|
|
495
|
-
reportStatus(recording.sessionId, "uploading", {
|
|
496
|
-
recordingId: recording.recordingId,
|
|
497
|
-
});
|
|
498
|
-
try {
|
|
499
|
-
const settled = await Promise.allSettled(recording.uploadPromises);
|
|
500
|
-
const rejected = settled.find(
|
|
501
|
-
(item): item is PromiseRejectedResult => item.status === "rejected",
|
|
502
|
-
);
|
|
503
|
-
if (recording.uploadFailure) throw recording.uploadFailure;
|
|
504
|
-
if (rejected) {
|
|
505
|
-
throw rejected.reason instanceof Error
|
|
506
|
-
? rejected.reason
|
|
507
|
-
: new Error(String(rejected.reason));
|
|
508
|
-
}
|
|
509
|
-
const durationMs = Math.max(0, Date.now() - recording.startedAtMs);
|
|
510
|
-
const result = await uploadChunk(
|
|
511
|
-
recording,
|
|
512
|
-
new Blob([], { type: recording.mimeType }),
|
|
513
|
-
recording.chunkIndex,
|
|
514
|
-
{
|
|
515
|
-
isFinal: true,
|
|
516
|
-
total: recording.chunkIndex,
|
|
517
|
-
durationMs,
|
|
518
|
-
width: recording.dimensions.width,
|
|
519
|
-
height: recording.dimensions.height,
|
|
520
|
-
hasAudio: recording.hasAudio,
|
|
521
|
-
hasCamera: recording.hasCamera,
|
|
522
|
-
},
|
|
523
|
-
);
|
|
524
|
-
cleanup(recording);
|
|
525
|
-
activeRecording = null;
|
|
526
|
-
reportStatus(recording.sessionId, "complete", {
|
|
527
|
-
recordingId: recording.recordingId,
|
|
528
|
-
result,
|
|
529
|
-
});
|
|
530
|
-
recording.resolveStopped(result);
|
|
531
|
-
} catch (err) {
|
|
532
|
-
cleanup(recording);
|
|
533
|
-
activeRecording = null;
|
|
534
|
-
const error = err instanceof Error ? err : new Error(String(err));
|
|
535
|
-
reportStatus(recording.sessionId, "error", {
|
|
536
|
-
recordingId: recording.recordingId,
|
|
537
|
-
error: error.message,
|
|
538
|
-
});
|
|
539
|
-
recording.rejectStopped(error);
|
|
540
|
-
}
|
|
541
|
-
})();
|
|
490
|
+
void finalizeStop(recording);
|
|
542
491
|
});
|
|
543
492
|
|
|
544
|
-
recorder.
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
493
|
+
// Run the pre-roll countdown here (reliable) then start the recorder. The
|
|
494
|
+
// worker is told "recording" via reportStatus once it actually starts.
|
|
495
|
+
const delay = Math.max(0, message.startDelayMs ?? 0);
|
|
496
|
+
if (delay > 0) {
|
|
497
|
+
recording.startTimer = setTimeout(() => {
|
|
498
|
+
recording.startTimer = null;
|
|
499
|
+
startRecorderNow(recording);
|
|
500
|
+
}, delay);
|
|
501
|
+
} else {
|
|
502
|
+
startRecorderNow(recording);
|
|
503
|
+
}
|
|
552
504
|
|
|
553
505
|
return {
|
|
554
506
|
ok: true,
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
height,
|
|
507
|
+
width: recording.dimensions.width,
|
|
508
|
+
height: recording.dimensions.height,
|
|
558
509
|
hasAudio: recording.hasAudio,
|
|
559
510
|
hasCamera: recording.hasCamera,
|
|
560
511
|
};
|
|
561
512
|
}
|
|
562
513
|
|
|
563
|
-
|
|
564
|
-
|
|
514
|
+
function startRecorderNow(recording: ActiveRecording): void {
|
|
515
|
+
if (recording.cancelled) return;
|
|
516
|
+
try {
|
|
517
|
+
recording.recorder.start(2000);
|
|
518
|
+
recording.startedAtMs = Date.now();
|
|
519
|
+
reportStatus(recording.sessionId, "recording", {
|
|
520
|
+
recordingId: recording.recordingId,
|
|
521
|
+
width: recording.dimensions.width,
|
|
522
|
+
height: recording.dimensions.height,
|
|
523
|
+
hasAudio: recording.hasAudio,
|
|
524
|
+
hasCamera: recording.hasCamera,
|
|
525
|
+
});
|
|
526
|
+
} catch (err) {
|
|
527
|
+
reportStatus(recording.sessionId, "error", {
|
|
528
|
+
recordingId: recording.recordingId,
|
|
529
|
+
error: err instanceof Error ? err.message : "Could not start recording.",
|
|
530
|
+
});
|
|
531
|
+
}
|
|
532
|
+
}
|
|
533
|
+
|
|
534
|
+
async function finalizeStop(recording: ActiveRecording): Promise<void> {
|
|
535
|
+
if (recording.restarting) {
|
|
536
|
+
// restart() re-homes the source streams; do not stop or upload anything.
|
|
537
|
+
return;
|
|
538
|
+
}
|
|
539
|
+
if (recording.cancelled) {
|
|
540
|
+
cleanup(recording);
|
|
541
|
+
if (activeRecording === recording) activeRecording = null;
|
|
542
|
+
recording.resolveStopped({ ok: true, status: "cancelled" });
|
|
543
|
+
return;
|
|
544
|
+
}
|
|
545
|
+
reportStatus(recording.sessionId, "uploading", {
|
|
546
|
+
recordingId: recording.recordingId,
|
|
547
|
+
});
|
|
548
|
+
try {
|
|
549
|
+
const settled = await Promise.allSettled(recording.uploadPromises);
|
|
550
|
+
if (recording.uploadFailure) throw recording.uploadFailure;
|
|
551
|
+
const rejected = settled.find(
|
|
552
|
+
(item): item is PromiseRejectedResult => item.status === "rejected",
|
|
553
|
+
);
|
|
554
|
+
if (rejected) {
|
|
555
|
+
throw rejected.reason instanceof Error
|
|
556
|
+
? rejected.reason
|
|
557
|
+
: new Error(String(rejected.reason));
|
|
558
|
+
}
|
|
559
|
+
const durationMs = Math.max(0, Date.now() - recording.startedAtMs);
|
|
560
|
+
const result = await uploadChunk(
|
|
561
|
+
recording,
|
|
562
|
+
new Blob([], { type: recording.mimeType }),
|
|
563
|
+
recording.chunkIndex,
|
|
564
|
+
{
|
|
565
|
+
isFinal: true,
|
|
566
|
+
total: recording.chunkIndex,
|
|
567
|
+
durationMs,
|
|
568
|
+
width: recording.dimensions.width,
|
|
569
|
+
height: recording.dimensions.height,
|
|
570
|
+
hasAudio: recording.hasAudio,
|
|
571
|
+
hasCamera: recording.hasCamera,
|
|
572
|
+
},
|
|
573
|
+
);
|
|
574
|
+
cleanup(recording);
|
|
575
|
+
if (activeRecording === recording) activeRecording = null;
|
|
576
|
+
reportStatus(recording.sessionId, "complete", {
|
|
577
|
+
recordingId: recording.recordingId,
|
|
578
|
+
result,
|
|
579
|
+
});
|
|
580
|
+
recording.resolveStopped(result);
|
|
581
|
+
} catch (err) {
|
|
582
|
+
cleanup(recording);
|
|
583
|
+
if (activeRecording === recording) activeRecording = null;
|
|
584
|
+
const error = err instanceof Error ? err : new Error(String(err));
|
|
585
|
+
reportStatus(recording.sessionId, "error", {
|
|
586
|
+
recordingId: recording.recordingId,
|
|
587
|
+
error: error.message,
|
|
588
|
+
});
|
|
589
|
+
recording.rejectStopped(error);
|
|
590
|
+
}
|
|
591
|
+
}
|
|
592
|
+
|
|
593
|
+
/* ------------------------------------------------------- pause/resume/stop --- */
|
|
594
|
+
|
|
595
|
+
function pause(message: SimpleMessage): { ok: boolean } {
|
|
596
|
+
const recording = activeRecording;
|
|
597
|
+
if (recording && recording.sessionId === message.sessionId) {
|
|
598
|
+
if (recording.recorder.state === "recording") recording.recorder.pause();
|
|
599
|
+
reportStatus(recording.sessionId, "paused", {
|
|
600
|
+
recordingId: recording.recordingId,
|
|
601
|
+
});
|
|
602
|
+
}
|
|
603
|
+
return { ok: true };
|
|
604
|
+
}
|
|
605
|
+
|
|
606
|
+
function resume(message: SimpleMessage): { ok: boolean } {
|
|
607
|
+
const recording = activeRecording;
|
|
608
|
+
if (recording && recording.sessionId === message.sessionId) {
|
|
609
|
+
if (recording.recorder.state === "paused") recording.recorder.resume();
|
|
610
|
+
reportStatus(recording.sessionId, "recording", {
|
|
611
|
+
recordingId: recording.recordingId,
|
|
612
|
+
});
|
|
613
|
+
}
|
|
614
|
+
return { ok: true };
|
|
615
|
+
}
|
|
616
|
+
|
|
617
|
+
async function stop(
|
|
618
|
+
message: SimpleMessage,
|
|
565
619
|
): Promise<{ ok: boolean; result: UploadResult }> {
|
|
566
620
|
const recording = activeRecording;
|
|
567
621
|
if (!recording || recording.sessionId !== message.sessionId) {
|
|
568
622
|
throw new Error("No active Clips recording was found.");
|
|
569
623
|
}
|
|
570
|
-
if (recording.
|
|
571
|
-
|
|
624
|
+
if (recording.startTimer !== null) {
|
|
625
|
+
// Stopped during the pre-roll, before the recorder ever started: there is
|
|
626
|
+
// nothing to save, so discard instead of hanging on `stopped`.
|
|
627
|
+
clearTimeout(recording.startTimer);
|
|
628
|
+
recording.startTimer = null;
|
|
629
|
+
recording.cancelled = true;
|
|
630
|
+
cleanup(recording);
|
|
631
|
+
if (activeRecording === recording) activeRecording = null;
|
|
632
|
+
return { ok: true, result: { ok: true, status: "cancelled" } };
|
|
572
633
|
}
|
|
634
|
+
if (recording.recorder.state !== "inactive") recording.recorder.stop();
|
|
573
635
|
return { ok: true, result: await recording.stopped };
|
|
574
636
|
}
|
|
575
637
|
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
638
|
+
function cancel(message: SimpleMessage): { ok: boolean } {
|
|
639
|
+
const recording = activeRecording;
|
|
640
|
+
if (recording && recording.sessionId === message.sessionId) {
|
|
641
|
+
recording.cancelled = true;
|
|
642
|
+
if (recording.startTimer !== null) {
|
|
643
|
+
clearTimeout(recording.startTimer);
|
|
644
|
+
recording.startTimer = null;
|
|
645
|
+
}
|
|
646
|
+
if (recording.recorder.state !== "inactive") recording.recorder.stop();
|
|
647
|
+
cleanup(recording);
|
|
648
|
+
activeRecording = null;
|
|
649
|
+
} else if (prepared && prepared.sessionId === message.sessionId) {
|
|
650
|
+
stopPreparedStreams();
|
|
651
|
+
disposePrepared();
|
|
652
|
+
}
|
|
653
|
+
return { ok: true };
|
|
654
|
+
}
|
|
655
|
+
|
|
656
|
+
// Skip the remaining pre-roll: start the recorder right now.
|
|
657
|
+
function startNow(message: SimpleMessage): { ok: boolean } {
|
|
658
|
+
const recording = activeRecording;
|
|
659
|
+
if (
|
|
660
|
+
recording &&
|
|
661
|
+
recording.sessionId === message.sessionId &&
|
|
662
|
+
recording.startTimer !== null
|
|
663
|
+
) {
|
|
664
|
+
clearTimeout(recording.startTimer);
|
|
665
|
+
recording.startTimer = null;
|
|
666
|
+
startRecorderNow(recording);
|
|
667
|
+
}
|
|
668
|
+
return { ok: true };
|
|
669
|
+
}
|
|
670
|
+
|
|
671
|
+
// Restart: discard the in-progress recording but keep the same source streams
|
|
672
|
+
// (so the user does not have to re-pick a screen), then re-home them into a
|
|
673
|
+
// prepared slot. A fresh recorder is built on the next BEGIN.
|
|
674
|
+
async function restart(
|
|
675
|
+
message: SimpleMessage,
|
|
676
|
+
): Promise<{ ok: boolean; width: number; height: number }> {
|
|
579
677
|
const recording = activeRecording;
|
|
580
678
|
if (!recording || recording.sessionId !== message.sessionId) {
|
|
581
|
-
|
|
679
|
+
throw new Error("No active Clips recording to restart.");
|
|
582
680
|
}
|
|
681
|
+
// restarting + cancelled => the stop handler returns early and the dataavailable
|
|
682
|
+
// handler ignores the final flush, so the source tracks stay live.
|
|
683
|
+
recording.restarting = true;
|
|
583
684
|
recording.cancelled = true;
|
|
584
|
-
if (recording.recorder.state !== "inactive")
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
cleanup(recording);
|
|
685
|
+
if (recording.recorder.state !== "inactive") recording.recorder.stop();
|
|
686
|
+
// Close only the old mixing context; keep the capture tracks alive.
|
|
687
|
+
void recording.audioContext?.close().catch(() => undefined);
|
|
588
688
|
activeRecording = null;
|
|
589
|
-
|
|
689
|
+
|
|
690
|
+
const videoStream =
|
|
691
|
+
recording.sourceStreams.find((s) => s.getVideoTracks().length) ?? null;
|
|
692
|
+
const isCamera = recording.mode === "camera";
|
|
693
|
+
prepared = {
|
|
694
|
+
sessionId: recording.sessionId,
|
|
695
|
+
mode: recording.mode,
|
|
696
|
+
displayStream: isCamera ? null : videoStream,
|
|
697
|
+
cameraStream: isCamera ? videoStream : null,
|
|
698
|
+
micStream:
|
|
699
|
+
recording.sourceStreams.find(
|
|
700
|
+
(s) => s.getAudioTracks().length && s !== videoStream,
|
|
701
|
+
) ?? null,
|
|
702
|
+
width: recording.dimensions.width,
|
|
703
|
+
height: recording.dimensions.height,
|
|
704
|
+
endedListener: null,
|
|
705
|
+
endedTrack: null,
|
|
706
|
+
};
|
|
707
|
+
return {
|
|
708
|
+
ok: true,
|
|
709
|
+
width: recording.dimensions.width,
|
|
710
|
+
height: recording.dimensions.height,
|
|
711
|
+
};
|
|
590
712
|
}
|
|
591
713
|
|
|
592
714
|
chrome.runtime.onMessage.addListener((message, _sender, sendResponse) => {
|
|
593
715
|
if (!message || typeof message !== "object") return false;
|
|
594
716
|
const type = (message as { type?: unknown }).type;
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
717
|
+
let task: Promise<unknown> | null = null;
|
|
718
|
+
switch (type) {
|
|
719
|
+
case "CLIPS_OFFSCREEN_ACQUIRE":
|
|
720
|
+
task = acquire(message as AcquireMessage);
|
|
721
|
+
break;
|
|
722
|
+
case "CLIPS_OFFSCREEN_BEGIN":
|
|
723
|
+
task = begin(message as BeginMessage);
|
|
724
|
+
break;
|
|
725
|
+
case "CLIPS_OFFSCREEN_PAUSE":
|
|
726
|
+
task = Promise.resolve(pause(message as SimpleMessage));
|
|
727
|
+
break;
|
|
728
|
+
case "CLIPS_OFFSCREEN_RESUME":
|
|
729
|
+
task = Promise.resolve(resume(message as SimpleMessage));
|
|
730
|
+
break;
|
|
731
|
+
case "CLIPS_OFFSCREEN_STOP":
|
|
732
|
+
task = stop(message as SimpleMessage);
|
|
733
|
+
break;
|
|
734
|
+
case "CLIPS_OFFSCREEN_CANCEL":
|
|
735
|
+
task = Promise.resolve(cancel(message as SimpleMessage));
|
|
736
|
+
break;
|
|
737
|
+
case "CLIPS_OFFSCREEN_RESTART":
|
|
738
|
+
task = restart(message as SimpleMessage);
|
|
739
|
+
break;
|
|
740
|
+
case "CLIPS_OFFSCREEN_START_NOW":
|
|
741
|
+
task = Promise.resolve(startNow(message as SimpleMessage));
|
|
742
|
+
break;
|
|
743
|
+
default:
|
|
744
|
+
return false;
|
|
601
745
|
}
|
|
602
746
|
|
|
603
|
-
const task =
|
|
604
|
-
type === "CLIPS_OFFSCREEN_START"
|
|
605
|
-
? startRecording(message as OffscreenStartMessage)
|
|
606
|
-
: type === "CLIPS_OFFSCREEN_STOP"
|
|
607
|
-
? stopRecording(message as OffscreenStopMessage)
|
|
608
|
-
: cancelRecording(message as OffscreenCancelMessage);
|
|
609
|
-
|
|
610
747
|
void task.then(sendResponse).catch((err) =>
|
|
611
748
|
sendResponse({
|
|
612
749
|
ok: false,
|