@agent-native/core 0.100.2 → 0.101.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/corpus/README.md +2 -2
- package/corpus/core/CHANGELOG.md +65 -0
- package/corpus/core/docs/content/locales/ar-SA/pr-visual-recap.mdx +48 -30
- package/corpus/core/docs/content/locales/ar-SA/template-plan.mdx +7 -0
- package/corpus/core/docs/content/locales/de-DE/pr-visual-recap.mdx +48 -30
- package/corpus/core/docs/content/locales/de-DE/template-plan.mdx +7 -0
- package/corpus/core/docs/content/locales/es-ES/pr-visual-recap.mdx +48 -30
- package/corpus/core/docs/content/locales/es-ES/template-plan.mdx +14 -0
- package/corpus/core/docs/content/locales/fr-FR/pr-visual-recap.mdx +48 -30
- package/corpus/core/docs/content/locales/fr-FR/template-plan.mdx +7 -0
- package/corpus/core/docs/content/locales/hi-IN/pr-visual-recap.mdx +48 -30
- package/corpus/core/docs/content/locales/hi-IN/template-plan.mdx +7 -0
- package/corpus/core/docs/content/locales/ja-JP/pr-visual-recap.mdx +48 -30
- package/corpus/core/docs/content/locales/ja-JP/template-plan.mdx +7 -0
- package/corpus/core/docs/content/locales/ko-KR/pr-visual-recap.mdx +48 -30
- package/corpus/core/docs/content/locales/ko-KR/template-plan.mdx +7 -0
- package/corpus/core/docs/content/locales/pt-BR/pr-visual-recap.mdx +48 -30
- package/corpus/core/docs/content/locales/pt-BR/template-plan.mdx +14 -0
- package/corpus/core/docs/content/locales/zh-CN/pr-visual-recap.mdx +48 -30
- package/corpus/core/docs/content/locales/zh-CN/template-plan.mdx +7 -0
- package/corpus/core/docs/content/locales/zh-TW/pr-visual-recap.mdx +48 -30
- package/corpus/core/docs/content/locales/zh-TW/template-plan.mdx +7 -0
- package/corpus/core/docs/content/pr-visual-recap.mdx +50 -32
- package/corpus/core/docs/content/template-plan.mdx +14 -0
- package/corpus/core/docs/content/toolkit-comments-review.mdx +16 -2
- package/corpus/core/package.json +4 -1
- package/corpus/core/src/a2a/index.ts +2 -1
- package/corpus/core/src/a2a/server.ts +34 -9
- package/corpus/core/src/agent/production-agent.ts +56 -10
- package/corpus/core/src/cli/create.ts +47 -19
- package/corpus/core/src/cli/pr-visual-recap-workflow.ts +2 -3
- package/corpus/core/src/cli/recap.ts +3 -4932
- package/corpus/core/src/cli/skills-content/visual-plan-skill.ts +6 -4
- package/corpus/core/src/cli/skills-content/visual-recap-skill.ts +7 -1
- package/corpus/core/src/client/AgentPanel.tsx +19 -17
- package/corpus/core/src/client/analytics.ts +8 -1
- package/corpus/core/src/client/chat/markdown-renderer.tsx +54 -8
- package/corpus/core/src/client/chat/message-components.tsx +8 -2
- package/corpus/core/src/client/clipboard.ts +28 -1
- package/corpus/core/src/client/guided-questions.tsx +4 -0
- package/corpus/core/src/client/i18n.tsx +2 -1
- package/corpus/core/src/client/index.ts +4 -0
- package/corpus/core/src/client/review/ReviewCommentComposer.tsx +108 -0
- package/corpus/core/src/client/review/ReviewThreadPanel.tsx +482 -150
- package/corpus/core/src/client/review/index.ts +8 -0
- package/corpus/core/src/client/review/use-review.ts +35 -1
- package/corpus/core/src/client/use-agent-chat-context.ts +3 -2
- package/corpus/core/src/review/actions/create-review-comment.ts +2 -1
- package/corpus/core/src/review/actions/get-review-feedback.ts +6 -11
- package/corpus/core/src/review/actions/list-review-comments.ts +38 -4
- package/corpus/core/src/review/actions/reply-review-comment.ts +33 -22
- package/corpus/core/src/review/actions/resolve-review-thread.ts +29 -2
- package/corpus/core/src/review/actions/send-review-thread-to-agent.ts +62 -0
- package/corpus/core/src/review/identity.ts +101 -0
- package/corpus/core/src/review/index.ts +14 -0
- package/corpus/core/src/review/store.ts +315 -8
- package/corpus/core/src/review/types.ts +4 -0
- package/corpus/core/src/server/action-discovery.ts +4 -0
- package/corpus/core/src/server/action-routes.ts +125 -4
- package/corpus/core/src/server/agent-chat/plugin-options.ts +16 -0
- package/corpus/core/src/server/agent-chat-plugin.ts +1 -0
- package/corpus/core/src/server/index.ts +8 -0
- package/corpus/core/src/vite/action-types-plugin.ts +4 -0
- package/corpus/templates/analytics/changelog/2026-07-14-daily-dashboard-email-captures-authenticate-with-a-sessi.md +6 -0
- package/corpus/templates/analytics/server/lib/dashboard-report.ts +156 -3
- package/corpus/templates/assets/app/components/generation/GenerationResults.tsx +402 -142
- package/corpus/templates/assets/app/i18n/zh-TW.ts +1 -0
- package/corpus/templates/assets/app/i18n-data.ts +1 -0
- package/corpus/templates/assets/app/routes/library.tsx +187 -48
- package/corpus/templates/assets/changelog/2026-07-13-merged-the-candidates-panel-into-the-library-as-a-drafts-tab.md +6 -0
- package/corpus/templates/chat/app/lib/agent-page.tsx +40 -0
- package/corpus/templates/chat/app/routes/agent.tsx +5 -3
- package/corpus/templates/chat/changelog/2026-07-14-fixed-chat-template-startup-with-older-core-versions.md +6 -0
- package/corpus/templates/clips/.agents/skills/ai-video-tools/SKILL.md +5 -0
- package/corpus/templates/clips/AGENTS.md +4 -0
- package/corpus/templates/clips/actions/get-feature-flags.ts +30 -0
- package/corpus/templates/clips/actions/lib/transcript-preview.ts +48 -0
- package/corpus/templates/clips/actions/request-transcript.ts +41 -1
- package/corpus/templates/clips/actions/view-screen.ts +5 -9
- package/corpus/templates/clips/app/components/library/library-layout.tsx +1 -1
- package/corpus/templates/clips/app/components/player/video-player.tsx +39 -1
- package/corpus/templates/clips/app/hooks/use-desktop-promo.ts +6 -12
- package/corpus/templates/clips/app/hooks/use-mse-video-source.ts +146 -0
- package/corpus/templates/clips/app/lib/capture-install-options.ts +21 -0
- package/corpus/templates/clips/app/lib/fmp4.ts +271 -0
- package/corpus/templates/clips/app/lib/mse-video-loader.ts +585 -0
- package/corpus/templates/clips/app/routes/_app.dictate.tsx +0 -4
- package/corpus/templates/clips/app/routes/_app.meetings.$meetingId.tsx +5 -13
- package/corpus/templates/clips/app/routes/_app.meetings._index.tsx +2 -37
- package/corpus/templates/clips/app/routes/download.tsx +3 -2
- package/corpus/templates/clips/changelog/2026-07-10-shared-clips-now-start-playing-instantly-instead-of-showing-.md +6 -0
- package/corpus/templates/clips/changelog/2026-07-14-clips-now-distinguishes-transcript-previews-from-incomplete-.md +6 -0
- package/corpus/templates/clips/changelog/2026-07-14-desktop-app-prompts-stay-hidden-after-you-download-the-insta.md +6 -0
- package/corpus/templates/clips/desktop/src/lib/recorder.ts +5 -0
- package/corpus/templates/clips/desktop/src/overlays/recording-pill.tsx +10 -0
- package/corpus/templates/clips/desktop/src-tauri/src/lib.rs +4 -0
- package/corpus/templates/clips/desktop/src-tauri/src/native_screen/custom_capture.rs +2518 -0
- package/corpus/templates/clips/desktop/src-tauri/src/native_screen/live_upload.rs +353 -0
- package/corpus/templates/clips/desktop/src-tauri/src/native_screen.rs +558 -67
- package/corpus/templates/clips/desktop/src-tauri/src/recording_indicator.rs +11 -1
- package/corpus/templates/clips/desktop/src-tauri/src/remote_flags.rs +188 -0
- package/corpus/templates/clips/desktop/src-tauri/src/whisper_speech.rs +1 -1
- package/corpus/templates/clips/server/lib/post-finalize-dispatch.ts +1 -0
- package/corpus/templates/clips/server/plugins/agent-chat.ts +5 -0
- package/corpus/templates/clips/server/routes/api/_agent-native-background/post-finalize-worker.post.ts +5 -1
- package/corpus/templates/clips/shared/feature-flags.ts +41 -0
- package/corpus/templates/design/.agents/skills/design-generation/SKILL.md +3 -0
- package/corpus/templates/design/.agents/skills/design-review-feedback/SKILL.md +42 -0
- package/corpus/templates/design/AGENTS.md +6 -0
- package/corpus/templates/design/README.md +1 -0
- package/corpus/templates/design/actions/navigate.ts +4 -4
- package/corpus/templates/design/actions/view-screen.ts +116 -2
- package/corpus/templates/design/agent-native.app-skill.json +5 -0
- package/corpus/templates/design/app/components/design/DesignCanvas.tsx +52 -59
- package/corpus/templates/design/app/components/design/DesignExtensionsPanel.tsx +13 -9
- package/corpus/templates/design/app/components/design/EditPanel.tsx +79 -11
- package/corpus/templates/design/app/components/design/LayersPanel.tsx +78 -65
- package/corpus/templates/design/app/components/design/ReviewCommentsPanel.tsx +161 -0
- package/corpus/templates/design/app/components/design/ReviewStatusControl.tsx +96 -0
- package/corpus/templates/design/app/components/design/bridge/hit-test.bridge.ts +166 -0
- package/corpus/templates/design/app/components/layout/Layout.tsx +1 -0
- package/corpus/templates/design/app/components/visual-editor/ReviewCanvasPins.tsx +1077 -0
- package/corpus/templates/design/app/components/visual-editor/index.ts +1 -0
- package/corpus/templates/design/app/components/visual-editor/review-canvas-state.ts +42 -0
- package/corpus/templates/design/app/hooks/use-design-systems.ts +2 -2
- package/corpus/templates/design/app/hooks/use-navigation-state.ts +9 -6
- package/corpus/templates/design/app/hooks/use-question-flow.ts +7 -1
- package/corpus/templates/design/app/i18n/ar-SA.ts +56 -1
- package/corpus/templates/design/app/i18n/de-DE.ts +57 -1
- package/corpus/templates/design/app/i18n/en-US.ts +56 -1
- package/corpus/templates/design/app/i18n/es-ES.ts +56 -1
- package/corpus/templates/design/app/i18n/fr-FR.ts +57 -1
- package/corpus/templates/design/app/i18n/hi-IN.ts +56 -1
- package/corpus/templates/design/app/i18n/ja-JP.ts +56 -1
- package/corpus/templates/design/app/i18n/ko-KR.ts +56 -1
- package/corpus/templates/design/app/i18n/pt-BR.ts +57 -1
- package/corpus/templates/design/app/i18n/zh-CN.ts +56 -1
- package/corpus/templates/design/app/i18n/zh-TW.ts +51 -0
- package/corpus/templates/design/app/pages/Present.tsx +172 -14
- package/corpus/templates/design/app/pages/design-editor/overview-camera.ts +23 -0
- package/corpus/templates/design/app/pages/design-editor/tool-state.ts +10 -0
- package/corpus/templates/design/app/pages/present-review-state.ts +18 -0
- package/corpus/templates/design/app/public-routes.ts +8 -0
- package/corpus/templates/design/app/root.tsx +7 -6
- package/corpus/templates/design/changelog/2026-07-13-dragged-design-assets-now-appear-at-the-visible-drop-locatio.md +6 -0
- package/corpus/templates/design/changelog/2026-07-13-reviewers-can-pin-comments-on-shared-designs-and-apply-verif.md +6 -0
- package/corpus/templates/design/server/plugins/agent-chat.ts +10 -0
- package/corpus/templates/design/server/plugins/auth.ts +7 -5
- package/corpus/templates/design/server/plugins/review.ts +9 -0
- package/corpus/templates/design/shared/review-anchor.ts +83 -0
- package/corpus/templates/design/shared/review-summary.ts +25 -0
- package/corpus/templates/forms/.agents/skills/form-publishing/SKILL.md +2 -0
- package/corpus/templates/forms/AGENTS.md +4 -0
- package/corpus/templates/forms/actions/create-form.ts +2 -1
- package/corpus/templates/forms/actions/update-form.ts +5 -2
- package/corpus/templates/forms/app/i18n/ar-SA.ts +3 -0
- package/corpus/templates/forms/app/i18n/de-DE.ts +3 -0
- package/corpus/templates/forms/app/i18n/en-US.ts +3 -0
- package/corpus/templates/forms/app/i18n/es-ES.ts +3 -0
- package/corpus/templates/forms/app/i18n/fr-FR.ts +3 -0
- package/corpus/templates/forms/app/i18n/hi-IN.ts +3 -0
- package/corpus/templates/forms/app/i18n/ja-JP.ts +3 -0
- package/corpus/templates/forms/app/i18n/ko-KR.ts +3 -0
- package/corpus/templates/forms/app/i18n/pt-BR.ts +3 -0
- package/corpus/templates/forms/app/i18n/zh-CN.ts +3 -0
- package/corpus/templates/forms/app/i18n/zh-TW.ts +3 -0
- package/corpus/templates/forms/app/pages/FormBuilderPage.tsx +19 -0
- package/corpus/templates/forms/changelog/2026-07-14-fixed-the-ask-forms-loading-input-appearing-above-the-welcom.md +6 -0
- package/corpus/templates/forms/changelog/2026-07-14-form-social-previews-now-show-the-form-title-description-and.md +6 -0
- package/corpus/templates/forms/changelog/2026-07-14-forms-can-email-the-form-owner-when-new-responses-arrive.md +6 -0
- package/corpus/templates/forms/server/handlers/submissions.ts +17 -0
- package/corpus/templates/forms/server/lib/form-og-image.ts +64 -6
- package/corpus/templates/forms/server/lib/public-form-ssr.ts +6 -2
- package/corpus/templates/forms/server/lib/response-email.ts +65 -0
- package/corpus/templates/forms/server/plugins/agent-chat.ts +1 -0
- package/corpus/templates/forms/server/routes/api/forms/og/[...slug]/og.png.get.ts +184 -0
- package/corpus/templates/forms/shared/types.ts +2 -0
- package/corpus/templates/plan/.agents/skills/visual-plan/SKILL.md +6 -4
- package/corpus/templates/plan/.agents/skills/visual-recap/SKILL.md +7 -1
- package/dist/a2a/index.d.ts +2 -1
- package/dist/a2a/index.d.ts.map +1 -1
- package/dist/a2a/index.js +1 -1
- package/dist/a2a/index.js.map +1 -1
- package/dist/a2a/server.d.ts +25 -0
- package/dist/a2a/server.d.ts.map +1 -1
- package/dist/a2a/server.js +30 -6
- package/dist/a2a/server.js.map +1 -1
- package/dist/agent/production-agent.d.ts +2 -0
- package/dist/agent/production-agent.d.ts.map +1 -1
- package/dist/agent/production-agent.js +37 -12
- package/dist/agent/production-agent.js.map +1 -1
- package/dist/cli/create.d.ts +5 -3
- package/dist/cli/create.d.ts.map +1 -1
- package/dist/cli/create.js +44 -19
- package/dist/cli/create.js.map +1 -1
- package/dist/cli/pr-visual-recap-workflow.d.ts +2 -2
- package/dist/cli/pr-visual-recap-workflow.d.ts.map +1 -1
- package/dist/cli/pr-visual-recap-workflow.js +2 -2
- package/dist/cli/pr-visual-recap-workflow.js.map +1 -1
- package/dist/cli/recap.d.ts +3 -562
- package/dist/cli/recap.d.ts.map +1 -1
- package/dist/cli/recap.js +3 -3875
- package/dist/cli/recap.js.map +1 -1
- package/dist/cli/skills-content/visual-plan-skill.d.ts +1 -1
- package/dist/cli/skills-content/visual-plan-skill.d.ts.map +1 -1
- package/dist/cli/skills-content/visual-plan-skill.js +6 -4
- package/dist/cli/skills-content/visual-plan-skill.js.map +1 -1
- package/dist/cli/skills-content/visual-recap-skill.d.ts +1 -1
- package/dist/cli/skills-content/visual-recap-skill.d.ts.map +1 -1
- package/dist/cli/skills-content/visual-recap-skill.js +7 -1
- package/dist/cli/skills-content/visual-recap-skill.js.map +1 -1
- package/dist/client/AgentPanel.d.ts.map +1 -1
- package/dist/client/AgentPanel.js +3 -3
- package/dist/client/AgentPanel.js.map +1 -1
- package/dist/client/analytics.d.ts +5 -0
- package/dist/client/analytics.d.ts.map +1 -1
- package/dist/client/analytics.js +3 -1
- package/dist/client/analytics.js.map +1 -1
- package/dist/client/chat/markdown-renderer.d.ts +1 -0
- package/dist/client/chat/markdown-renderer.d.ts.map +1 -1
- package/dist/client/chat/markdown-renderer.js +36 -1
- package/dist/client/chat/markdown-renderer.js.map +1 -1
- package/dist/client/chat/message-components.d.ts.map +1 -1
- package/dist/client/chat/message-components.js +5 -2
- package/dist/client/chat/message-components.js.map +1 -1
- package/dist/client/clipboard.d.ts +3 -1
- package/dist/client/clipboard.d.ts.map +1 -1
- package/dist/client/clipboard.js +24 -1
- package/dist/client/clipboard.js.map +1 -1
- package/dist/client/guided-questions.d.ts +3 -1
- package/dist/client/guided-questions.d.ts.map +1 -1
- package/dist/client/guided-questions.js +2 -1
- package/dist/client/guided-questions.js.map +1 -1
- package/dist/client/i18n.d.ts.map +1 -1
- package/dist/client/i18n.js +3 -1
- package/dist/client/i18n.js.map +1 -1
- package/dist/client/index.d.ts +1 -1
- package/dist/client/index.d.ts.map +1 -1
- package/dist/client/index.js +1 -1
- package/dist/client/index.js.map +1 -1
- package/dist/client/review/ReviewCommentComposer.d.ts +18 -0
- package/dist/client/review/ReviewCommentComposer.d.ts.map +1 -0
- package/dist/client/review/ReviewCommentComposer.js +30 -0
- package/dist/client/review/ReviewCommentComposer.js.map +1 -0
- package/dist/client/review/ReviewThreadPanel.d.ts +28 -1
- package/dist/client/review/ReviewThreadPanel.d.ts.map +1 -1
- package/dist/client/review/ReviewThreadPanel.js +151 -39
- package/dist/client/review/ReviewThreadPanel.js.map +1 -1
- package/dist/client/review/index.d.ts +3 -2
- package/dist/client/review/index.d.ts.map +1 -1
- package/dist/client/review/index.js +2 -1
- package/dist/client/review/index.js.map +1 -1
- package/dist/client/review/use-review.d.ts +23 -0
- package/dist/client/review/use-review.d.ts.map +1 -1
- package/dist/client/review/use-review.js +3 -0
- package/dist/client/review/use-review.js.map +1 -1
- package/dist/client/use-agent-chat-context.d.ts +1 -1
- package/dist/client/use-agent-chat-context.d.ts.map +1 -1
- package/dist/client/use-agent-chat-context.js +4 -2
- package/dist/client/use-agent-chat-context.js.map +1 -1
- package/dist/collab/awareness.d.ts +2 -2
- package/dist/collab/awareness.d.ts.map +1 -1
- package/dist/collab/struct-routes.d.ts +1 -1
- package/dist/notifications/routes.d.ts +3 -3
- package/dist/observability/routes.d.ts +5 -5
- package/dist/progress/routes.d.ts +1 -1
- package/dist/provider-api/corpus-jobs.d.ts +2 -2
- package/dist/resources/handlers.d.ts +2 -2
- package/dist/review/actions/create-review-comment.js +2 -1
- package/dist/review/actions/create-review-comment.js.map +1 -1
- package/dist/review/actions/get-review-feedback.js +6 -11
- package/dist/review/actions/get-review-feedback.js.map +1 -1
- package/dist/review/actions/list-review-comments.d.ts +1 -0
- package/dist/review/actions/list-review-comments.js +25 -4
- package/dist/review/actions/list-review-comments.js.map +1 -1
- package/dist/review/actions/reply-review-comment.js +12 -5
- package/dist/review/actions/reply-review-comment.js.map +1 -1
- package/dist/review/actions/resolve-review-thread.d.ts +3 -0
- package/dist/review/actions/resolve-review-thread.js +20 -3
- package/dist/review/actions/resolve-review-thread.js.map +1 -1
- package/dist/review/actions/send-review-thread-to-agent.d.ts +17 -0
- package/dist/review/actions/send-review-thread-to-agent.d.ts.map +1 -0
- package/dist/review/actions/send-review-thread-to-agent.js +49 -0
- package/dist/review/actions/send-review-thread-to-agent.js.map +1 -0
- package/dist/review/identity.d.ts +6 -0
- package/dist/review/identity.d.ts.map +1 -0
- package/dist/review/identity.js +68 -0
- package/dist/review/identity.js.map +1 -0
- package/dist/review/index.d.ts +3 -1
- package/dist/review/index.d.ts.map +1 -1
- package/dist/review/index.js +2 -1
- package/dist/review/index.js.map +1 -1
- package/dist/review/store.d.ts +36 -0
- package/dist/review/store.d.ts.map +1 -1
- package/dist/review/store.js +223 -8
- package/dist/review/store.js.map +1 -1
- package/dist/review/types.d.ts +4 -0
- package/dist/review/types.d.ts.map +1 -1
- package/dist/review/types.js.map +1 -1
- package/dist/secrets/routes.d.ts +9 -9
- package/dist/server/action-discovery.d.ts.map +1 -1
- package/dist/server/action-discovery.js +4 -0
- package/dist/server/action-discovery.js.map +1 -1
- package/dist/server/action-routes.d.ts +45 -0
- package/dist/server/action-routes.d.ts.map +1 -1
- package/dist/server/action-routes.js +71 -5
- package/dist/server/action-routes.js.map +1 -1
- package/dist/server/agent-chat/plugin-options.d.ts +16 -0
- package/dist/server/agent-chat/plugin-options.d.ts.map +1 -1
- package/dist/server/agent-chat/plugin-options.js.map +1 -1
- package/dist/server/agent-chat-plugin.d.ts.map +1 -1
- package/dist/server/agent-chat-plugin.js +1 -0
- package/dist/server/agent-chat-plugin.js.map +1 -1
- package/dist/server/index.d.ts +3 -1
- package/dist/server/index.d.ts.map +1 -1
- package/dist/server/index.js +2 -0
- package/dist/server/index.js.map +1 -1
- package/dist/vite/action-types-plugin.d.ts.map +1 -1
- package/dist/vite/action-types-plugin.js +4 -0
- package/dist/vite/action-types-plugin.js.map +1 -1
- package/docs/content/locales/ar-SA/pr-visual-recap.mdx +48 -30
- package/docs/content/locales/ar-SA/template-plan.mdx +7 -0
- package/docs/content/locales/de-DE/pr-visual-recap.mdx +48 -30
- package/docs/content/locales/de-DE/template-plan.mdx +7 -0
- package/docs/content/locales/es-ES/pr-visual-recap.mdx +48 -30
- package/docs/content/locales/es-ES/template-plan.mdx +14 -0
- package/docs/content/locales/fr-FR/pr-visual-recap.mdx +48 -30
- package/docs/content/locales/fr-FR/template-plan.mdx +7 -0
- package/docs/content/locales/hi-IN/pr-visual-recap.mdx +48 -30
- package/docs/content/locales/hi-IN/template-plan.mdx +7 -0
- package/docs/content/locales/ja-JP/pr-visual-recap.mdx +48 -30
- package/docs/content/locales/ja-JP/template-plan.mdx +7 -0
- package/docs/content/locales/ko-KR/pr-visual-recap.mdx +48 -30
- package/docs/content/locales/ko-KR/template-plan.mdx +7 -0
- package/docs/content/locales/pt-BR/pr-visual-recap.mdx +48 -30
- package/docs/content/locales/pt-BR/template-plan.mdx +14 -0
- package/docs/content/locales/zh-CN/pr-visual-recap.mdx +48 -30
- package/docs/content/locales/zh-CN/template-plan.mdx +7 -0
- package/docs/content/locales/zh-TW/pr-visual-recap.mdx +48 -30
- package/docs/content/locales/zh-TW/template-plan.mdx +7 -0
- package/docs/content/pr-visual-recap.mdx +50 -32
- package/docs/content/template-plan.mdx +14 -0
- package/docs/content/toolkit-comments-review.mdx +16 -2
- package/package.json +4 -1
- package/corpus/templates/forms/server/routes/api/forms/og/[slug]/og.png.get.ts +0 -45
|
@@ -0,0 +1,2518 @@
|
|
|
1
|
+
//! Custom ScreenCaptureKit capture engine.
|
|
2
|
+
//!
|
|
3
|
+
//! Owns everything between "SCStream hands us a sample buffer" and "a
|
|
4
|
+
//! fragmented MP4 grows on disk": the AVAssetWriter wrapper
|
|
5
|
+
//! (`CustomScreenCaptureWriter`), the realtime `LiveAudioMixer` that combines
|
|
6
|
+
//! mic + system audio into one track, the PCM decode/resample helpers, the
|
|
7
|
+
//! AVFoundation/CoreMedia FFI glue, and the capture watchdog that rebuilds a
|
|
8
|
+
//! stopped/stalled SCStream in place. Session management and live upload stay
|
|
9
|
+
//! in the parent `native_screen` module and the `live_upload` sibling.
|
|
10
|
+
|
|
11
|
+
use super::*;
|
|
12
|
+
use screencapturekit::error::SCError;
|
|
13
|
+
use screencapturekit::stream::delegate_trait::SCStreamDelegateTrait;
|
|
14
|
+
|
|
15
|
+
/// `AVAssetWriter.status` raw value for `.completed`.
|
|
16
|
+
const AV_WRITER_STATUS_COMPLETED: i64 = 2;
|
|
17
|
+
/// `kAudioFormatMPEG4AAC` FourCC ('aac ') for the writer's audio output.
|
|
18
|
+
const AUDIO_FORMAT_AAC: i64 = 0x6161_6320;
|
|
19
|
+
/// Cadence of delegate-produced output segments in segmented mode (seconds of
|
|
20
|
+
/// media time). Smaller = data becomes uploadable sooner, at a small
|
|
21
|
+
/// container-overhead cost.
|
|
22
|
+
const OUTPUT_SEGMENT_INTERVAL_SECONDS: i64 = 1;
|
|
23
|
+
/// Capture-time H.264 budget in bits per pixel per frame. 0.15 bpp matches
|
|
24
|
+
/// Cap's "instant" quality tier (~3.3 Mbps at 1280x720@24) and keeps most
|
|
25
|
+
/// recordings small enough to upload without a post-capture transcode.
|
|
26
|
+
const CAPTURE_VIDEO_BPP: f64 = 0.15;
|
|
27
|
+
|
|
28
|
+
// Self-healing capture watchdog. ScreenCaptureKit can silently stop feeding a
|
|
29
|
+
// display stream when the captured display changes Spaces (virtual desktops),
|
|
30
|
+
// a full-screen app takes over, or the display config changes — sometimes with
|
|
31
|
+
// a `did_stop_with_error` callback, sometimes by just going quiet. Either way
|
|
32
|
+
// the recording file stops growing with no app-level signal. The watchdog
|
|
33
|
+
// notices the gap and rebuilds the SCStream in place so recording continues.
|
|
34
|
+
//
|
|
35
|
+
// How long without any delivered sample buffer before we treat the stream as
|
|
36
|
+
// dead and rebuild it. Generous enough not to trip on a momentarily idle
|
|
37
|
+
// screen (SCK still delivers idle frames, which count as activity here).
|
|
38
|
+
const CAPTURE_STALL_TIMEOUT: Duration = Duration::from_secs(4);
|
|
39
|
+
// Watchdog poll cadence.
|
|
40
|
+
const CAPTURE_WATCHDOG_POLL: Duration = Duration::from_millis(1000);
|
|
41
|
+
// Consecutive failed restarts (rebuild or start error, or an immediate
|
|
42
|
+
// re-stall) before giving up and finalizing whatever was captured. A single
|
|
43
|
+
// successful stretch of frames resets the counter.
|
|
44
|
+
const CAPTURE_MAX_RESTARTS: u32 = 5;
|
|
45
|
+
|
|
46
|
+
/// Liveness state shared between the SCK output handler (which records that a
|
|
47
|
+
/// sample arrived), the stream delegate (which records an OS-reported stop),
|
|
48
|
+
/// and the watchdog thread (which reads both to decide when to rebuild the
|
|
49
|
+
/// stream).
|
|
50
|
+
pub(crate) struct CaptureWatch {
|
|
51
|
+
/// Wall-clock time of the most recent delivered sample buffer.
|
|
52
|
+
last_activity: Mutex<Instant>,
|
|
53
|
+
/// Set by the stream delegate when ScreenCaptureKit reports the stream
|
|
54
|
+
/// stopped. Consumed by the watchdog to force an immediate rebuild.
|
|
55
|
+
stream_stopped: Mutex<Option<String>>,
|
|
56
|
+
/// The user stopped capture through macOS itself (menu-bar "Stop
|
|
57
|
+
/// Sharing", SCStreamError code -3817). The watchdog must treat this as
|
|
58
|
+
/// a clean stop request, never as a failure to rebuild from.
|
|
59
|
+
user_stopped: AtomicBool,
|
|
60
|
+
/// True between pause and resume. The capture source (SCStream) is
|
|
61
|
+
/// intentionally stopped while the writer/file/uploader stay alive, so
|
|
62
|
+
/// the watchdog must not read the silence as a stall and rebuild — resume
|
|
63
|
+
/// brings a fresh stream back and clears this.
|
|
64
|
+
paused: AtomicBool,
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
impl CaptureWatch {
|
|
68
|
+
fn new() -> Self {
|
|
69
|
+
Self {
|
|
70
|
+
last_activity: Mutex::new(Instant::now()),
|
|
71
|
+
stream_stopped: Mutex::new(None),
|
|
72
|
+
user_stopped: AtomicBool::new(false),
|
|
73
|
+
paused: AtomicBool::new(false),
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
fn note_activity(&self) {
|
|
78
|
+
if let Ok(mut guard) = self.last_activity.lock() {
|
|
79
|
+
*guard = Instant::now();
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
fn since_activity(&self) -> Duration {
|
|
84
|
+
self.last_activity
|
|
85
|
+
.lock()
|
|
86
|
+
.map(|t| t.elapsed())
|
|
87
|
+
.unwrap_or_default()
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
fn note_stream_stopped(&self, reason: String) {
|
|
91
|
+
if let Ok(mut guard) = self.stream_stopped.lock() {
|
|
92
|
+
if guard.is_none() {
|
|
93
|
+
*guard = Some(reason);
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
fn take_stream_stopped(&self) -> Option<String> {
|
|
99
|
+
self.stream_stopped.lock().ok().and_then(|mut g| g.take())
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
fn note_user_stopped(&self) {
|
|
103
|
+
self.user_stopped.store(true, Ordering::SeqCst);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
fn user_stopped(&self) -> bool {
|
|
107
|
+
self.user_stopped.load(Ordering::SeqCst)
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
fn set_paused(&self, paused: bool) {
|
|
111
|
+
self.paused.store(paused, Ordering::SeqCst);
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
fn is_paused(&self) -> bool {
|
|
115
|
+
self.paused.load(Ordering::SeqCst)
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
// ---------------------------------------------------------------------------
|
|
120
|
+
// Segmented output (live-upload mode)
|
|
121
|
+
//
|
|
122
|
+
// With `movieFragmentInterval` + a writer-owned file, `finishWriting`
|
|
123
|
+
// DEFRAGMENTS the file in place (fragmented layout -> classic mdat+moov), so
|
|
124
|
+
// byte ranges streamed to the server during recording no longer match the
|
|
125
|
+
// final file and the uploaded clip comes back corrupt. The fix is Apple's
|
|
126
|
+
// segment API: the writer is created WITHOUT an output URL, produces discrete
|
|
127
|
+
// fMP4 segments through `AVAssetWriterDelegate`, and WE append them to the
|
|
128
|
+
// local file ourselves. AVFoundation never owns the file, so nothing is ever
|
|
129
|
+
// rewritten — the file is append-only by construction and the live uploader
|
|
130
|
+
// can safely tail it forever.
|
|
131
|
+
// ---------------------------------------------------------------------------
|
|
132
|
+
|
|
133
|
+
/// Owns the local recording file in segmented mode. The ObjC delegate appends
|
|
134
|
+
/// every segment it receives; append order matches delivery order (delegate
|
|
135
|
+
/// callbacks are serial, and the file mutex serializes any stragglers).
|
|
136
|
+
pub(super) struct SegmentSink {
|
|
137
|
+
file: Mutex<std::fs::File>,
|
|
138
|
+
/// First write error, if any; surfaced by `finish()` so a failing disk
|
|
139
|
+
/// turns into a reported error instead of a silently truncated file.
|
|
140
|
+
failed: Mutex<Option<String>>,
|
|
141
|
+
segments: AtomicU64,
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
impl SegmentSink {
|
|
145
|
+
fn create(path: &Path) -> Result<Arc<Self>, String> {
|
|
146
|
+
let file = std::fs::File::create(path)
|
|
147
|
+
.map_err(|e| format!("could not create recording file {}: {e}", path.display()))?;
|
|
148
|
+
Ok(Arc::new(Self {
|
|
149
|
+
file: Mutex::new(file),
|
|
150
|
+
failed: Mutex::new(None),
|
|
151
|
+
segments: AtomicU64::new(0),
|
|
152
|
+
}))
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
fn append(&self, bytes: &[u8]) {
|
|
156
|
+
use std::io::Write;
|
|
157
|
+
let Ok(mut file) = self.file.lock() else {
|
|
158
|
+
return;
|
|
159
|
+
};
|
|
160
|
+
if let Err(err) = file.write_all(bytes) {
|
|
161
|
+
if let Ok(mut failed) = self.failed.lock() {
|
|
162
|
+
if failed.is_none() {
|
|
163
|
+
eprintln!("[mixer] segment write failed: {err}");
|
|
164
|
+
*failed = Some(format!("segment write failed: {err}"));
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
return;
|
|
168
|
+
}
|
|
169
|
+
let n = self.segments.fetch_add(1, Ordering::Relaxed) + 1;
|
|
170
|
+
if n == 1 {
|
|
171
|
+
eprintln!(
|
|
172
|
+
"[mixer] first output segment written ({} bytes)",
|
|
173
|
+
bytes.len()
|
|
174
|
+
);
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
fn failure(&self) -> Option<String> {
|
|
179
|
+
self.failed.lock().ok().and_then(|guard| guard.clone())
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
/// Instance variables for the segment delegate: just the shared sink.
|
|
184
|
+
struct SegmentDelegateIvars {
|
|
185
|
+
sink: Arc<SegmentSink>,
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
objc2::define_class!(
|
|
189
|
+
// SAFETY: NSObject has no subclassing requirements and the type has no
|
|
190
|
+
// Drop impl. Methods are called by AVFoundation on its own serial queue.
|
|
191
|
+
#[unsafe(super(objc2::runtime::NSObject))]
|
|
192
|
+
#[name = "ClipsSegmentWriterDelegate"]
|
|
193
|
+
#[ivars = SegmentDelegateIvars]
|
|
194
|
+
struct SegmentWriterDelegate;
|
|
195
|
+
|
|
196
|
+
impl SegmentWriterDelegate {
|
|
197
|
+
/// `AVAssetWriterDelegate` — receives each fMP4 segment (type 1 =
|
|
198
|
+
/// initialization, 2 = separable media) as it is produced.
|
|
199
|
+
#[unsafe(method(assetWriter:didOutputSegmentData:segmentType:))]
|
|
200
|
+
fn did_output_segment(
|
|
201
|
+
&self,
|
|
202
|
+
_writer: *mut objc2::runtime::AnyObject,
|
|
203
|
+
data: *mut objc2::runtime::AnyObject,
|
|
204
|
+
_segment_type: isize,
|
|
205
|
+
) {
|
|
206
|
+
// Crossing the ObjC boundary: a Rust panic here would abort the
|
|
207
|
+
// whole process, so contain it.
|
|
208
|
+
let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
|
|
209
|
+
if data.is_null() {
|
|
210
|
+
return;
|
|
211
|
+
}
|
|
212
|
+
let (ptr, len): (*const std::ffi::c_void, usize) = unsafe {
|
|
213
|
+
(
|
|
214
|
+
objc2::msg_send![&*data, bytes],
|
|
215
|
+
objc2::msg_send![&*data, length],
|
|
216
|
+
)
|
|
217
|
+
};
|
|
218
|
+
if ptr.is_null() || len == 0 {
|
|
219
|
+
return;
|
|
220
|
+
}
|
|
221
|
+
let bytes = unsafe { std::slice::from_raw_parts(ptr as *const u8, len) };
|
|
222
|
+
use objc2::DefinedClass;
|
|
223
|
+
self.ivars().sink.append(bytes);
|
|
224
|
+
}));
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
);
|
|
228
|
+
|
|
229
|
+
impl SegmentWriterDelegate {
|
|
230
|
+
fn new(sink: Arc<SegmentSink>) -> objc2::rc::Retained<Self> {
|
|
231
|
+
use objc2::AllocAnyThread;
|
|
232
|
+
let this = Self::alloc().set_ivars(SegmentDelegateIvars { sink });
|
|
233
|
+
unsafe { objc2::msg_send![super(this), init] }
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
/// Cheap-to-clone handle around one `AVAssetWriter` producing a fragmented
|
|
238
|
+
/// MP4. SCK callbacks append through it from multiple dispatch queues; the
|
|
239
|
+
/// stop path calls [`Self::finish`]. All clones share the same underlying
|
|
240
|
+
/// state (everything inside is an `Arc`).
|
|
241
|
+
pub(crate) struct CustomScreenCaptureWriter {
|
|
242
|
+
inner: Arc<Mutex<CustomScreenCaptureWriterState>>,
|
|
243
|
+
/// Present only in live-mixing mode; combines the two incoming audio
|
|
244
|
+
/// streams into the one track written to `mixed_audio_input`. Lives on its
|
|
245
|
+
/// own lock so PCM mixing on the audio callbacks never blocks video frame
|
|
246
|
+
/// appends (which only need `inner`). Lock order is always `mixer` →
|
|
247
|
+
/// `inner`; never take `mixer` while holding `inner`.
|
|
248
|
+
mixer: Option<Arc<Mutex<LiveAudioMixer>>>,
|
|
249
|
+
/// The writer session has started (first video frame ran
|
|
250
|
+
/// `startSessionAtSourceTime:`). Written under the `inner` lock, read
|
|
251
|
+
/// lock-free by the audio callbacks.
|
|
252
|
+
started: Arc<AtomicBool>,
|
|
253
|
+
/// Source-time (seconds, as f64 bits) the writer session started at.
|
|
254
|
+
/// `f64::NAN` until known; mixed audio earlier than this must be dropped
|
|
255
|
+
/// or the writer rejects it. Written before `started` flips true.
|
|
256
|
+
session_start_bits: Arc<AtomicU64>,
|
|
257
|
+
/// Once true the writer accepts no more samples — set on stop, cancel, and
|
|
258
|
+
/// any append failure.
|
|
259
|
+
appends_closed: Arc<AtomicBool>,
|
|
260
|
+
/// Samples skipped because an AVAssetWriterInput wasn't ready; logged
|
|
261
|
+
/// periodically so realtime backpressure is visible.
|
|
262
|
+
dropped_samples: Arc<AtomicU64>,
|
|
263
|
+
/// Accumulated pause time (seconds, as f64 bits) to subtract from the
|
|
264
|
+
/// zero-based session timeline so a pause/resume leaves no gap in the
|
|
265
|
+
/// append-only file. Zero until the first resume. Read by the video
|
|
266
|
+
/// retime path and the live audio mixer.
|
|
267
|
+
pause_offset_bits: Arc<AtomicU64>,
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
/// The lock-guarded half of the writer: the retained AVFoundation objects
|
|
271
|
+
/// plus lifecycle flags. Every Objective-C call on these handles happens
|
|
272
|
+
/// while holding this state's mutex (see the SAFETY note below).
|
|
273
|
+
struct CustomScreenCaptureWriterState {
|
|
274
|
+
writer: objc2::rc::Retained<objc2::runtime::AnyObject>,
|
|
275
|
+
video_input: objc2::rc::Retained<objc2::runtime::AnyObject>,
|
|
276
|
+
system_audio_input: Option<objc2::rc::Retained<objc2::runtime::AnyObject>>,
|
|
277
|
+
mic_audio_input: Option<objc2::rc::Retained<objc2::runtime::AnyObject>>,
|
|
278
|
+
/// Single output track used when mic + system audio are mixed live into one
|
|
279
|
+
/// stream. Mutually exclusive with `system_audio_input` / `mic_audio_input`.
|
|
280
|
+
mixed_audio_input: Option<objc2::rc::Retained<objc2::runtime::AnyObject>>,
|
|
281
|
+
/// Segmented (delegate-fed) output mode; see the "Segmented output"
|
|
282
|
+
/// section. When true, `initialSegmentStartTime` must be set before
|
|
283
|
+
/// `startWriting`.
|
|
284
|
+
segmented: bool,
|
|
285
|
+
/// Local-file writer for segmented mode (we own the file, not
|
|
286
|
+
/// AVFoundation). `None` in plain file mode.
|
|
287
|
+
segment_sink: Option<Arc<SegmentSink>>,
|
|
288
|
+
/// Keeps the ObjC delegate alive — `AVAssetWriter.delegate` is weak.
|
|
289
|
+
#[allow(dead_code)]
|
|
290
|
+
segment_delegate: Option<objc2::rc::Retained<SegmentWriterDelegate>>,
|
|
291
|
+
/// First video frame's PTS as (value, timescale). In segmented mode the
|
|
292
|
+
/// writer session starts at ZERO and every sample is rebased against this
|
|
293
|
+
/// — the segment API preserves source timestamps verbatim, so appending
|
|
294
|
+
/// raw host-clock PTS would give the clip a media timeline starting at
|
|
295
|
+
/// "seconds since boot" (browsers then show wall-clock-like times).
|
|
296
|
+
session_start_time: Option<(i64, i32)>,
|
|
297
|
+
finished: bool,
|
|
298
|
+
failed: Option<String>,
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
// SAFETY: `Retained<AnyObject>` is `!Send`/`!Sync` by default because objc2
|
|
302
|
+
// cannot know an arbitrary object's threading contract. Here every
|
|
303
|
+
// Objective-C call on the retained writer/input handles (`startWriting`,
|
|
304
|
+
// `startSessionAtSourceTime:`, `appendSampleBuffer:`, `markAsFinished`,
|
|
305
|
+
// `finishWriting…`, `status`/`error` reads) happens while holding the `inner`
|
|
306
|
+
// mutex, so access is serialized even though callbacks arrive on multiple SCK
|
|
307
|
+
// dispatch queues. The remaining shared fields are lock-free atomics
|
|
308
|
+
// (`started`, `session_start_bits`, `appends_closed`, `dropped_samples`), and
|
|
309
|
+
// the mixer holds no Objective-C state and lives behind its own mutex.
|
|
310
|
+
// Objective-C retain/release itself is atomic, so moving the retained
|
|
311
|
+
// pointers across threads is sound.
|
|
312
|
+
unsafe impl Send for CustomScreenCaptureWriter {}
|
|
313
|
+
unsafe impl Sync for CustomScreenCaptureWriter {}
|
|
314
|
+
unsafe impl Send for CustomScreenCaptureWriterState {}
|
|
315
|
+
|
|
316
|
+
#[derive(Clone)]
|
|
317
|
+
/// The `SCStreamOutputTrait` sink registered for all three output types
|
|
318
|
+
/// (screen / system audio / microphone). One instance is shared across the
|
|
319
|
+
/// registrations and across watchdog stream rebuilds, so the same writer
|
|
320
|
+
/// keeps receiving samples over the whole recording.
|
|
321
|
+
struct CustomScreenCaptureOutputHandler {
|
|
322
|
+
writer: CustomScreenCaptureWriter,
|
|
323
|
+
recording_enabled: Arc<AtomicBool>,
|
|
324
|
+
mic_ready: Option<Arc<AtomicBool>>,
|
|
325
|
+
watch: Arc<CaptureWatch>,
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
impl Clone for CustomScreenCaptureWriter {
|
|
329
|
+
fn clone(&self) -> Self {
|
|
330
|
+
Self {
|
|
331
|
+
inner: Arc::clone(&self.inner),
|
|
332
|
+
mixer: self.mixer.clone(),
|
|
333
|
+
started: Arc::clone(&self.started),
|
|
334
|
+
session_start_bits: Arc::clone(&self.session_start_bits),
|
|
335
|
+
appends_closed: Arc::clone(&self.appends_closed),
|
|
336
|
+
dropped_samples: Arc::clone(&self.dropped_samples),
|
|
337
|
+
pause_offset_bits: Arc::clone(&self.pause_offset_bits),
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
impl SCStreamOutputTrait for CustomScreenCaptureOutputHandler {
|
|
343
|
+
fn did_output_sample_buffer(
|
|
344
|
+
&self,
|
|
345
|
+
sample_buffer: screencapturekit::cm::CMSampleBuffer,
|
|
346
|
+
of_type: SCStreamOutputType,
|
|
347
|
+
) {
|
|
348
|
+
if matches!(of_type, SCStreamOutputType::Microphone) {
|
|
349
|
+
if let Some(mic_ready) = &self.mic_ready {
|
|
350
|
+
mic_ready.store(true, Ordering::Relaxed);
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
if !self.recording_enabled.load(Ordering::SeqCst) {
|
|
354
|
+
return;
|
|
355
|
+
}
|
|
356
|
+
// A delivered buffer (even a content-less idle frame) proves the stream
|
|
357
|
+
// is still alive; record it so the watchdog can tell a genuinely stalled
|
|
358
|
+
// stream apart from a quiet one.
|
|
359
|
+
self.watch.note_activity();
|
|
360
|
+
if self.writer.appends_closed.load(Ordering::SeqCst) {
|
|
361
|
+
return;
|
|
362
|
+
}
|
|
363
|
+
// Append every screen frame that carries an image buffer, INCLUDING
|
|
364
|
+
// idle/blank ones. On a static screen (e.g. after switching to another
|
|
365
|
+
// Space) ScreenCaptureKit delivers Idle frames; dropping them starves
|
|
366
|
+
// the video track, and fragmented-MP4 interleaving then can't complete
|
|
367
|
+
// a fragment — the file stops growing even though audio keeps flowing.
|
|
368
|
+
// Idle frames still reference the current surface, and the encoder
|
|
369
|
+
// turns repeats into tiny P-frames, so appending them is cheap. Only
|
|
370
|
+
// frames with no image buffer are skipped (nothing to encode).
|
|
371
|
+
if matches!(of_type, SCStreamOutputType::Screen) {
|
|
372
|
+
use std::sync::atomic::AtomicU64;
|
|
373
|
+
static SCREEN_SEEN: AtomicU64 = AtomicU64::new(0);
|
|
374
|
+
static SCREEN_BUFFERLESS: AtomicU64 = AtomicU64::new(0);
|
|
375
|
+
let seen = SCREEN_SEEN.fetch_add(1, Ordering::Relaxed) + 1;
|
|
376
|
+
if sample_buffer.image_buffer().is_none() {
|
|
377
|
+
let skipped = SCREEN_BUFFERLESS.fetch_add(1, Ordering::Relaxed) + 1;
|
|
378
|
+
if skipped == 1 || skipped % 100 == 0 {
|
|
379
|
+
eprintln!(
|
|
380
|
+
"[mixer] screen frame without image buffer skipped ({skipped} so far, status={:?})",
|
|
381
|
+
sample_buffer.frame_status()
|
|
382
|
+
);
|
|
383
|
+
}
|
|
384
|
+
return;
|
|
385
|
+
}
|
|
386
|
+
if seen == 1 || seen % 512 == 0 {
|
|
387
|
+
eprintln!(
|
|
388
|
+
"[mixer] screen frames delivered: seen={seen} bufferless={} status_now={:?}",
|
|
389
|
+
SCREEN_BUFFERLESS.load(Ordering::Relaxed),
|
|
390
|
+
sample_buffer.frame_status()
|
|
391
|
+
);
|
|
392
|
+
}
|
|
393
|
+
}
|
|
394
|
+
// ScreenCaptureKit invokes this from its own dispatch queues through an
|
|
395
|
+
// Objective-C boundary. Two failure modes can abort the whole process:
|
|
396
|
+
// - an Objective-C exception (e.g. AVFoundation), which `catch_unwind`
|
|
397
|
+
// CANNOT catch ("Rust cannot catch foreign exceptions"), so we wrap
|
|
398
|
+
// the body in `objc2::exception::catch` first; and
|
|
399
|
+
// - a Rust panic, contained by the outer `catch_unwind`.
|
|
400
|
+
// Either way we log the cause and cancel the capture instead of dying.
|
|
401
|
+
let panic_result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
|
|
402
|
+
let exc_result = objc2::exception::catch(std::panic::AssertUnwindSafe(|| {
|
|
403
|
+
self.writer.append_sample(&sample_buffer, of_type);
|
|
404
|
+
}));
|
|
405
|
+
if let Err(exc) = exc_result {
|
|
406
|
+
let detail = describe_objc_exception(exc);
|
|
407
|
+
eprintln!(
|
|
408
|
+
"[mixer] append_sample raised Objective-C exception for {of_type:?}: {detail}; cancelling capture"
|
|
409
|
+
);
|
|
410
|
+
self.writer.appends_closed.store(true, Ordering::SeqCst);
|
|
411
|
+
}
|
|
412
|
+
}));
|
|
413
|
+
if panic_result.is_err() {
|
|
414
|
+
eprintln!("[mixer] panic while appending {of_type:?} sample; cancelling capture");
|
|
415
|
+
self.writer.appends_closed.store(true, Ordering::SeqCst);
|
|
416
|
+
}
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
impl CustomScreenCaptureWriter {
|
|
421
|
+
/// Build the `AVAssetWriter` + inputs for one recording file. With live
|
|
422
|
+
/// mixing (mic + system) a single mixed audio track is created;
|
|
423
|
+
/// otherwise each captured source gets its own track. In live-upload
|
|
424
|
+
/// mode the writer produces movie fragments (append-only file); without
|
|
425
|
+
/// it, a regular MP4 with faststart.
|
|
426
|
+
fn new(
|
|
427
|
+
output_path: &Path,
|
|
428
|
+
width: u32,
|
|
429
|
+
height: u32,
|
|
430
|
+
capture_system_audio: bool,
|
|
431
|
+
include_audio: bool,
|
|
432
|
+
mix_live: bool,
|
|
433
|
+
) -> Result<Self, String> {
|
|
434
|
+
use objc2::msg_send;
|
|
435
|
+
use objc2::runtime::AnyObject;
|
|
436
|
+
|
|
437
|
+
#[link(name = "AVFoundation", kind = "framework")]
|
|
438
|
+
extern "C" {
|
|
439
|
+
static AVFileTypeMPEG4: *const AnyObject;
|
|
440
|
+
static AVMediaTypeVideo: *const AnyObject;
|
|
441
|
+
static AVFileTypeProfileMPEG4AppleHLS: *const AnyObject;
|
|
442
|
+
}
|
|
443
|
+
// Force-load UniformTypeIdentifiers so the runtime UTType lookup in
|
|
444
|
+
// segmented mode resolves.
|
|
445
|
+
#[link(name = "UniformTypeIdentifiers", kind = "framework")]
|
|
446
|
+
extern "C" {}
|
|
447
|
+
|
|
448
|
+
let segmented = crate::remote_flags::current().custom_sck_pipeline_live_upload_enabled;
|
|
449
|
+
unsafe {
|
|
450
|
+
let writer_cls = av_class_named("AVAssetWriter")
|
|
451
|
+
.ok_or_else(|| "AVAssetWriter missing".to_string())?;
|
|
452
|
+
|
|
453
|
+
let (writer, segment_sink, segment_delegate) = if segmented {
|
|
454
|
+
// Segmented mode: the writer has NO output URL. It produces
|
|
455
|
+
// discrete fMP4 segments through the delegate, and WE append
|
|
456
|
+
// them to the local file — so the file is append-only by
|
|
457
|
+
// construction and `finishWriting` cannot defragment/rewrite
|
|
458
|
+
// it (which would invalidate live-uploaded byte ranges).
|
|
459
|
+
let ident = av_ns_string_from("public.mpeg-4")
|
|
460
|
+
.ok_or_else(|| "NSString for UTType failed".to_string())?;
|
|
461
|
+
let ut_cls =
|
|
462
|
+
av_class_named("UTType").ok_or_else(|| "UTType missing".to_string())?;
|
|
463
|
+
let ut: *mut AnyObject = msg_send![ut_cls, typeWithIdentifier: &*ident];
|
|
464
|
+
if ut.is_null() {
|
|
465
|
+
return Err("UTType public.mpeg-4 unavailable".into());
|
|
466
|
+
}
|
|
467
|
+
let allocated: *mut AnyObject = msg_send![writer_cls, alloc];
|
|
468
|
+
let writer_raw: *mut AnyObject = msg_send![allocated, initWithContentType: ut];
|
|
469
|
+
if writer_raw.is_null() {
|
|
470
|
+
if !allocated.is_null() {
|
|
471
|
+
let _ = objc2::rc::Retained::from_raw(allocated);
|
|
472
|
+
}
|
|
473
|
+
return Err("AVAssetWriter initWithContentType failed".into());
|
|
474
|
+
}
|
|
475
|
+
let writer = objc2::rc::Retained::from_raw(writer_raw)
|
|
476
|
+
.ok_or_else(|| "AVAssetWriter retain failed".to_string())?;
|
|
477
|
+
|
|
478
|
+
let interval = ObjcCMTime {
|
|
479
|
+
value: OUTPUT_SEGMENT_INTERVAL_SECONDS,
|
|
480
|
+
timescale: 1,
|
|
481
|
+
flags: 1,
|
|
482
|
+
epoch: 0,
|
|
483
|
+
};
|
|
484
|
+
let _: () = msg_send![&*writer, setPreferredOutputSegmentInterval: interval];
|
|
485
|
+
let _: () =
|
|
486
|
+
msg_send![&*writer, setOutputFileTypeProfile: AVFileTypeProfileMPEG4AppleHLS];
|
|
487
|
+
|
|
488
|
+
let sink = SegmentSink::create(output_path)?;
|
|
489
|
+
let delegate = SegmentWriterDelegate::new(Arc::clone(&sink));
|
|
490
|
+
// `AVAssetWriter.delegate` is weak — the Retained delegate is
|
|
491
|
+
// stored in the writer state to keep it alive.
|
|
492
|
+
let _: () = msg_send![&*writer, setDelegate: &*delegate];
|
|
493
|
+
(writer, Some(sink), Some(delegate))
|
|
494
|
+
} else {
|
|
495
|
+
// Plain file mode: AVFoundation owns the file; faststart is
|
|
496
|
+
// safe because nothing tails the file during recording.
|
|
497
|
+
let url = av_file_url(output_path).ok_or_else(|| {
|
|
498
|
+
format!("could not build output URL for {}", output_path.display())
|
|
499
|
+
})?;
|
|
500
|
+
let allocated: *mut AnyObject = msg_send![writer_cls, alloc];
|
|
501
|
+
let mut err_ptr: *mut AnyObject = std::ptr::null_mut();
|
|
502
|
+
let writer_raw: *mut AnyObject = msg_send![
|
|
503
|
+
allocated,
|
|
504
|
+
initWithURL: &*url,
|
|
505
|
+
fileType: AVFileTypeMPEG4,
|
|
506
|
+
error: &mut err_ptr
|
|
507
|
+
];
|
|
508
|
+
if writer_raw.is_null() {
|
|
509
|
+
let detail = av_error_suffix(err_ptr);
|
|
510
|
+
if !allocated.is_null() {
|
|
511
|
+
let _ = objc2::rc::Retained::from_raw(allocated);
|
|
512
|
+
}
|
|
513
|
+
return Err(format!("AVAssetWriter init failed{detail}"));
|
|
514
|
+
}
|
|
515
|
+
let writer = objc2::rc::Retained::from_raw(writer_raw)
|
|
516
|
+
.ok_or_else(|| "AVAssetWriter retain failed".to_string())?;
|
|
517
|
+
let _: () = msg_send![&*writer, setShouldOptimizeForNetworkUse: true];
|
|
518
|
+
(writer, None, None)
|
|
519
|
+
};
|
|
520
|
+
let input_cls = av_class_named("AVAssetWriterInput")
|
|
521
|
+
.ok_or_else(|| "AVAssetWriterInput missing".to_string())?;
|
|
522
|
+
let video_settings = av_video_output_settings(width, height)?;
|
|
523
|
+
let video_raw: *mut AnyObject = msg_send![
|
|
524
|
+
input_cls,
|
|
525
|
+
assetWriterInputWithMediaType: AVMediaTypeVideo,
|
|
526
|
+
outputSettings: &*video_settings
|
|
527
|
+
];
|
|
528
|
+
if video_raw.is_null() {
|
|
529
|
+
return Err("AVAssetWriterInput video allocation failed".into());
|
|
530
|
+
}
|
|
531
|
+
let video_input = objc2::rc::Retained::retain(video_raw)
|
|
532
|
+
.ok_or_else(|| "AVAssetWriterInput video retain failed".to_string())?;
|
|
533
|
+
let _: () = msg_send![&*video_input, setExpectsMediaDataInRealTime: true];
|
|
534
|
+
let can_add_video: bool = msg_send![&*writer, canAddInput: &*video_input];
|
|
535
|
+
if !can_add_video {
|
|
536
|
+
return Err("AVAssetWriter cannot add video input".into());
|
|
537
|
+
}
|
|
538
|
+
let _: () = msg_send![&*writer, addInput: &*video_input];
|
|
539
|
+
|
|
540
|
+
// Live-mixing mode writes one combined audio track; otherwise each
|
|
541
|
+
// captured source gets its own track (mixed later by ffmpeg, or
|
|
542
|
+
// left as the single captured source).
|
|
543
|
+
let (system_audio_input, mic_audio_input, mixed_audio_input, mixer) = if mix_live {
|
|
544
|
+
let mixed = av_make_audio_writer_input(input_cls, &writer)?;
|
|
545
|
+
(
|
|
546
|
+
None,
|
|
547
|
+
None,
|
|
548
|
+
Some(mixed),
|
|
549
|
+
Some(LiveAudioMixer::new(segmented)?),
|
|
550
|
+
)
|
|
551
|
+
} else {
|
|
552
|
+
let system_audio_input = if capture_system_audio {
|
|
553
|
+
Some(av_make_audio_writer_input(input_cls, &writer)?)
|
|
554
|
+
} else {
|
|
555
|
+
None
|
|
556
|
+
};
|
|
557
|
+
let mic_audio_input = if include_audio {
|
|
558
|
+
Some(av_make_audio_writer_input(input_cls, &writer)?)
|
|
559
|
+
} else {
|
|
560
|
+
None
|
|
561
|
+
};
|
|
562
|
+
(system_audio_input, mic_audio_input, None, None)
|
|
563
|
+
};
|
|
564
|
+
|
|
565
|
+
Ok(Self {
|
|
566
|
+
inner: Arc::new(Mutex::new(CustomScreenCaptureWriterState {
|
|
567
|
+
writer,
|
|
568
|
+
video_input,
|
|
569
|
+
system_audio_input,
|
|
570
|
+
mic_audio_input,
|
|
571
|
+
mixed_audio_input,
|
|
572
|
+
segmented,
|
|
573
|
+
segment_sink,
|
|
574
|
+
segment_delegate,
|
|
575
|
+
session_start_time: None,
|
|
576
|
+
finished: false,
|
|
577
|
+
failed: None,
|
|
578
|
+
})),
|
|
579
|
+
mixer: mixer.map(|m| Arc::new(Mutex::new(m))),
|
|
580
|
+
started: Arc::new(AtomicBool::new(false)),
|
|
581
|
+
session_start_bits: Arc::new(AtomicU64::new(f64::NAN.to_bits())),
|
|
582
|
+
appends_closed: Arc::new(AtomicBool::new(false)),
|
|
583
|
+
dropped_samples: Arc::new(AtomicU64::new(0)),
|
|
584
|
+
pause_offset_bits: Arc::new(AtomicU64::new(0.0_f64.to_bits())),
|
|
585
|
+
})
|
|
586
|
+
}
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
/// Whether the writer runs the segmented (zero-based, append-only) output
|
|
590
|
+
/// used by live upload. Only that mode rebases sample timestamps, so it's
|
|
591
|
+
/// the only mode where a fresh SCStream can be spliced onto the same file.
|
|
592
|
+
pub(super) fn segmented(&self) -> bool {
|
|
593
|
+
self.inner.lock().map(|g| g.segmented).unwrap_or(false)
|
|
594
|
+
}
|
|
595
|
+
|
|
596
|
+
/// Whether the writer session has begun (first video frame appended).
|
|
597
|
+
pub(super) fn is_started(&self) -> bool {
|
|
598
|
+
self.started.load(Ordering::SeqCst)
|
|
599
|
+
}
|
|
600
|
+
|
|
601
|
+
/// Accumulated pause offset in seconds. Every appended sample skips this
|
|
602
|
+
/// much wall-clock time so a pause/resume leaves no gap in the file.
|
|
603
|
+
pub(super) fn pause_offset(&self) -> f64 {
|
|
604
|
+
f64::from_bits(self.pause_offset_bits.load(Ordering::SeqCst))
|
|
605
|
+
}
|
|
606
|
+
|
|
607
|
+
/// Set the accumulated pause offset (clamped to non-negative). Resume reads
|
|
608
|
+
/// the prior value, applies `prior + paused_for` once the replacement stream
|
|
609
|
+
/// is about to start, and restores the prior value if startup fails — so a
|
|
610
|
+
/// failed-then-retried resume never double-counts the same pause.
|
|
611
|
+
pub(super) fn set_pause_offset(&self, seconds: f64) {
|
|
612
|
+
self.pause_offset_bits
|
|
613
|
+
.store(seconds.max(0.0).to_bits(), Ordering::SeqCst);
|
|
614
|
+
}
|
|
615
|
+
|
|
616
|
+
/// Route one SCK sample buffer to the right writer input. Mixed-mode
|
|
617
|
+
/// audio goes through [`Self::append_mixed_audio`] (no writer lock while
|
|
618
|
+
/// decoding); everything else appends directly under the `inner` lock.
|
|
619
|
+
/// The first video frame starts the writer session.
|
|
620
|
+
fn append_sample(
|
|
621
|
+
&self,
|
|
622
|
+
sample: &screencapturekit::cm::CMSampleBuffer,
|
|
623
|
+
of_type: SCStreamOutputType,
|
|
624
|
+
) {
|
|
625
|
+
// Live-mixing mode: mic + system audio are combined into one track
|
|
626
|
+
// before being written, instead of going to separate inputs. Handled
|
|
627
|
+
// before taking the writer lock — the decode/resample/mix work happens
|
|
628
|
+
// on the audio callbacks and must not delay video frame appends.
|
|
629
|
+
if self.mixer.is_some()
|
|
630
|
+
&& matches!(
|
|
631
|
+
of_type,
|
|
632
|
+
SCStreamOutputType::Audio | SCStreamOutputType::Microphone
|
|
633
|
+
)
|
|
634
|
+
{
|
|
635
|
+
self.append_mixed_audio(sample, of_type);
|
|
636
|
+
return;
|
|
637
|
+
}
|
|
638
|
+
|
|
639
|
+
let Ok(mut guard) = self.inner.lock() else {
|
|
640
|
+
return;
|
|
641
|
+
};
|
|
642
|
+
if guard.finished || guard.failed.is_some() {
|
|
643
|
+
return;
|
|
644
|
+
}
|
|
645
|
+
let input = match of_type {
|
|
646
|
+
SCStreamOutputType::Screen => Some(guard.video_input.clone()),
|
|
647
|
+
SCStreamOutputType::Audio => guard.system_audio_input.clone(),
|
|
648
|
+
SCStreamOutputType::Microphone => guard.mic_audio_input.clone(),
|
|
649
|
+
};
|
|
650
|
+
let Some(input) = input else {
|
|
651
|
+
return;
|
|
652
|
+
};
|
|
653
|
+
let timing = match sample.sample_timing_info(0) {
|
|
654
|
+
Ok(timing) if timing.presentation_time_stamp.is_valid() => timing,
|
|
655
|
+
_ => return,
|
|
656
|
+
};
|
|
657
|
+
|
|
658
|
+
unsafe {
|
|
659
|
+
if !self.ensure_session_started(&mut guard, timing.presentation_time_stamp) {
|
|
660
|
+
return;
|
|
661
|
+
}
|
|
662
|
+
if guard.segmented {
|
|
663
|
+
// Segmented mode runs a ZERO-based session timeline: append a
|
|
664
|
+
// copy with PTS/DTS rebased against the session start.
|
|
665
|
+
// Appending the raw buffer here would put host-clock times in
|
|
666
|
+
// the track — the writer then buffers/starves (frozen video)
|
|
667
|
+
// and browsers show wall-clock timestamps.
|
|
668
|
+
let Some(base) = guard.session_start_time else {
|
|
669
|
+
return;
|
|
670
|
+
};
|
|
671
|
+
let pause_offset = self.pause_offset();
|
|
672
|
+
match retimed_sample_copy(sample, &timing, base, pause_offset) {
|
|
673
|
+
Ok(copy) => {
|
|
674
|
+
self.append_sample_ptr(&mut guard, &input, copy.as_ptr());
|
|
675
|
+
}
|
|
676
|
+
Err(err) => {
|
|
677
|
+
drop(guard);
|
|
678
|
+
self.fail(format!("sample retime failed: {err}"));
|
|
679
|
+
}
|
|
680
|
+
}
|
|
681
|
+
} else {
|
|
682
|
+
self.append_sample_ptr(&mut guard, &input, sample.as_ptr());
|
|
683
|
+
}
|
|
684
|
+
}
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
/// Pull PCM out of an audio sample, push it into the mixer, then append any
|
|
688
|
+
/// mixed buffers the mixer is ready to emit. The session is started by the
|
|
689
|
+
/// video track, so emitted audio is held until that happens.
|
|
690
|
+
///
|
|
691
|
+
/// Locking: PCM decode + resample run with no lock held; timeline placement
|
|
692
|
+
/// and draining run under the mixer lock, which stays held through the
|
|
693
|
+
/// appends below so two concurrently-draining audio callbacks can't
|
|
694
|
+
/// interleave out-of-PTS-order appends; the writer (`inner`) lock is taken
|
|
695
|
+
/// last, only around the actual appends.
|
|
696
|
+
fn append_mixed_audio(
|
|
697
|
+
&self,
|
|
698
|
+
sample: &screencapturekit::cm::CMSampleBuffer,
|
|
699
|
+
of_type: SCStreamOutputType,
|
|
700
|
+
) {
|
|
701
|
+
let (source, label) = match of_type {
|
|
702
|
+
SCStreamOutputType::Audio => (MixSource::System, "system"),
|
|
703
|
+
SCStreamOutputType::Microphone => (MixSource::Mic, "mic"),
|
|
704
|
+
_ => return,
|
|
705
|
+
};
|
|
706
|
+
// Heavy part (decode + resample) — pure function of the sample.
|
|
707
|
+
let Some((interleaved, pts_seconds)) = extract_interleaved_stereo(sample, label) else {
|
|
708
|
+
return;
|
|
709
|
+
};
|
|
710
|
+
|
|
711
|
+
let Some(mixer) = self.mixer.as_ref() else {
|
|
712
|
+
return;
|
|
713
|
+
};
|
|
714
|
+
let Ok(mut mixer_guard) = mixer.lock() else {
|
|
715
|
+
return;
|
|
716
|
+
};
|
|
717
|
+
mixer_guard.set_pause_offset(self.pause_offset());
|
|
718
|
+
mixer_guard.push(source, &interleaved, pts_seconds);
|
|
719
|
+
if !self.started.load(Ordering::SeqCst) {
|
|
720
|
+
return;
|
|
721
|
+
}
|
|
722
|
+
let start_secs = f64::from_bits(self.session_start_bits.load(Ordering::SeqCst));
|
|
723
|
+
if !start_secs.is_nan() {
|
|
724
|
+
mixer_guard.set_min_start(start_secs);
|
|
725
|
+
}
|
|
726
|
+
let emitted = match mixer_guard.drain_ready(false) {
|
|
727
|
+
Ok(buffers) => buffers,
|
|
728
|
+
Err(err) => {
|
|
729
|
+
drop(mixer_guard);
|
|
730
|
+
self.fail(err);
|
|
731
|
+
return;
|
|
732
|
+
}
|
|
733
|
+
};
|
|
734
|
+
if emitted.is_empty() {
|
|
735
|
+
return;
|
|
736
|
+
}
|
|
737
|
+
|
|
738
|
+
let Ok(mut guard) = self.inner.lock() else {
|
|
739
|
+
return;
|
|
740
|
+
};
|
|
741
|
+
if guard.finished || guard.failed.is_some() {
|
|
742
|
+
return;
|
|
743
|
+
}
|
|
744
|
+
let Some(input) = guard.mixed_audio_input.clone() else {
|
|
745
|
+
return;
|
|
746
|
+
};
|
|
747
|
+
for buffer in &emitted {
|
|
748
|
+
unsafe {
|
|
749
|
+
self.append_sample_ptr(&mut guard, &input, buffer.as_ptr());
|
|
750
|
+
}
|
|
751
|
+
if guard.failed.is_some() {
|
|
752
|
+
break;
|
|
753
|
+
}
|
|
754
|
+
}
|
|
755
|
+
}
|
|
756
|
+
|
|
757
|
+
/// Close the writer to further samples and record the failure reason.
|
|
758
|
+
fn fail(&self, err: String) {
|
|
759
|
+
self.appends_closed.store(true, Ordering::SeqCst);
|
|
760
|
+
if let Ok(mut guard) = self.inner.lock() {
|
|
761
|
+
if guard.failed.is_none() {
|
|
762
|
+
guard.failed = Some(err);
|
|
763
|
+
}
|
|
764
|
+
}
|
|
765
|
+
}
|
|
766
|
+
|
|
767
|
+
/// Start the writer session at the first video frame's PTS (once). Also
|
|
768
|
+
/// publishes the session start time to the lock-free atomics the audio
|
|
769
|
+
/// path reads. Returns false (and records the failure) if AVFoundation
|
|
770
|
+
/// refuses to start.
|
|
771
|
+
unsafe fn ensure_session_started(
|
|
772
|
+
&self,
|
|
773
|
+
guard: &mut CustomScreenCaptureWriterState,
|
|
774
|
+
pts: screencapturekit::cm::CMTime,
|
|
775
|
+
) -> bool {
|
|
776
|
+
use objc2::msg_send;
|
|
777
|
+
|
|
778
|
+
if self.started.load(Ordering::SeqCst) {
|
|
779
|
+
return true;
|
|
780
|
+
}
|
|
781
|
+
// `startWriting` / `startSessionAtSourceTime:` can raise Objective-C
|
|
782
|
+
// exceptions (e.g. invalid state). Catch them so they don't abort the
|
|
783
|
+
// process from inside the realtime callback.
|
|
784
|
+
let writer_ptr = &*guard.writer as *const objc2::runtime::AnyObject;
|
|
785
|
+
let segmented = guard.segmented;
|
|
786
|
+
// Segmented mode: the session runs on a ZERO-based timeline and every
|
|
787
|
+
// appended sample is rebased (see `session_start_time`). Plain file
|
|
788
|
+
// mode keeps the source clock — `startSessionAtSourceTime:` writes an
|
|
789
|
+
// implicit edit so playback still starts at zero there.
|
|
790
|
+
let start = if segmented {
|
|
791
|
+
ObjcCMTime {
|
|
792
|
+
value: 0,
|
|
793
|
+
timescale: pts.timescale.max(1),
|
|
794
|
+
flags: 1,
|
|
795
|
+
epoch: 0,
|
|
796
|
+
}
|
|
797
|
+
} else {
|
|
798
|
+
ObjcCMTime::from(pts)
|
|
799
|
+
};
|
|
800
|
+
let outcome = objc2::exception::catch(std::panic::AssertUnwindSafe(|| unsafe {
|
|
801
|
+
// Segmented output requires the initial segment start time to be
|
|
802
|
+
// set before writing starts; it must equal the session start.
|
|
803
|
+
if segmented {
|
|
804
|
+
let _: () = msg_send![&*writer_ptr, setInitialSegmentStartTime: start];
|
|
805
|
+
}
|
|
806
|
+
let ok: bool = msg_send![&*writer_ptr, startWriting];
|
|
807
|
+
if !ok {
|
|
808
|
+
return false;
|
|
809
|
+
}
|
|
810
|
+
let _: () = msg_send![&*writer_ptr, startSessionAtSourceTime: start];
|
|
811
|
+
true
|
|
812
|
+
}));
|
|
813
|
+
match outcome {
|
|
814
|
+
Ok(true) => {
|
|
815
|
+
guard.session_start_time = Some((pts.value, pts.timescale.max(1)));
|
|
816
|
+
// Publish the session start time before flipping `started` so
|
|
817
|
+
// lock-free readers that observe `started == true` always see
|
|
818
|
+
// a valid start time.
|
|
819
|
+
if let Some(secs) = pts.as_seconds() {
|
|
820
|
+
self.session_start_bits
|
|
821
|
+
.store(secs.to_bits(), Ordering::SeqCst);
|
|
822
|
+
}
|
|
823
|
+
self.started.store(true, Ordering::SeqCst);
|
|
824
|
+
true
|
|
825
|
+
}
|
|
826
|
+
Ok(false) => {
|
|
827
|
+
self.appends_closed.store(true, Ordering::SeqCst);
|
|
828
|
+
guard.failed = Some(format!(
|
|
829
|
+
"AVAssetWriter startWriting failed{}",
|
|
830
|
+
av_writer_error_suffix(&guard.writer)
|
|
831
|
+
));
|
|
832
|
+
false
|
|
833
|
+
}
|
|
834
|
+
Err(exc) => {
|
|
835
|
+
self.appends_closed.store(true, Ordering::SeqCst);
|
|
836
|
+
let detail = describe_objc_exception(exc);
|
|
837
|
+
eprintln!("[mixer] startWriting raised Objective-C exception: {detail}");
|
|
838
|
+
guard.failed = Some(format!("AVAssetWriter startWriting raised: {detail}"));
|
|
839
|
+
false
|
|
840
|
+
}
|
|
841
|
+
}
|
|
842
|
+
}
|
|
843
|
+
|
|
844
|
+
/// Append one CMSampleBuffer to a writer input, containing Objective-C
|
|
845
|
+
/// exceptions and recording any failure into `guard.failed`. Skips (and
|
|
846
|
+
/// counts) samples when the input reports not-ready — realtime capture
|
|
847
|
+
/// must never block the SCK callback.
|
|
848
|
+
unsafe fn append_sample_ptr(
|
|
849
|
+
&self,
|
|
850
|
+
guard: &mut CustomScreenCaptureWriterState,
|
|
851
|
+
input: &objc2::rc::Retained<objc2::runtime::AnyObject>,
|
|
852
|
+
sample_ptr: *mut std::ffi::c_void,
|
|
853
|
+
) {
|
|
854
|
+
use objc2::msg_send;
|
|
855
|
+
|
|
856
|
+
let ready: bool = msg_send![&**input, isReadyForMoreMediaData];
|
|
857
|
+
if !ready {
|
|
858
|
+
// Realtime mode: skip rather than block the capture callback, but
|
|
859
|
+
// count and periodically log so sustained backpressure is visible.
|
|
860
|
+
let dropped = self.dropped_samples.fetch_add(1, Ordering::Relaxed) + 1;
|
|
861
|
+
if dropped == 1 || dropped % 100 == 0 {
|
|
862
|
+
eprintln!("[mixer] writer input not ready; dropped {dropped} sample(s) so far");
|
|
863
|
+
}
|
|
864
|
+
return;
|
|
865
|
+
}
|
|
866
|
+
// `appendSampleBuffer:` throws Objective-C exceptions on bad input
|
|
867
|
+
// (format/timestamp/state). Those can't be caught by `catch_unwind`
|
|
868
|
+
// and would abort the app, so contain them here.
|
|
869
|
+
let input_ptr = &**input as *const objc2::runtime::AnyObject;
|
|
870
|
+
let outcome = objc2::exception::catch(std::panic::AssertUnwindSafe(|| unsafe {
|
|
871
|
+
let appended: bool = msg_send![&*input_ptr, appendSampleBuffer: sample_ptr];
|
|
872
|
+
appended
|
|
873
|
+
}));
|
|
874
|
+
match outcome {
|
|
875
|
+
Ok(true) => {}
|
|
876
|
+
Ok(false) => {
|
|
877
|
+
self.appends_closed.store(true, Ordering::SeqCst);
|
|
878
|
+
guard.failed = Some(format!(
|
|
879
|
+
"AVAssetWriter appendSampleBuffer failed{}",
|
|
880
|
+
av_writer_error_suffix(&guard.writer)
|
|
881
|
+
));
|
|
882
|
+
}
|
|
883
|
+
Err(exc) => {
|
|
884
|
+
self.appends_closed.store(true, Ordering::SeqCst);
|
|
885
|
+
let detail = describe_objc_exception(exc);
|
|
886
|
+
eprintln!("[mixer] appendSampleBuffer raised Objective-C exception: {detail}");
|
|
887
|
+
guard.failed = Some(format!("AVAssetWriter appendSampleBuffer raised: {detail}"));
|
|
888
|
+
}
|
|
889
|
+
}
|
|
890
|
+
}
|
|
891
|
+
|
|
892
|
+
/// Stop accepting samples, flush audio still held by the mixer, mark all
|
|
893
|
+
/// inputs finished, and finalize the file. With `wait_for_finalize` the
|
|
894
|
+
/// call blocks (bounded) until AVFoundation completes and verifies the
|
|
895
|
+
/// writer status; without it, finalization completes in the background
|
|
896
|
+
/// (the fragmented file is already playable up to the last fragment).
|
|
897
|
+
pub(super) fn finish(&self, wait_for_finalize: bool) -> Result<(), String> {
|
|
898
|
+
self.appends_closed.store(true, Ordering::SeqCst);
|
|
899
|
+
let dropped = self.dropped_samples.load(Ordering::Relaxed);
|
|
900
|
+
eprintln!(
|
|
901
|
+
"[mixer] writer finish requested (wait={wait_for_finalize}, dropped_samples={dropped})"
|
|
902
|
+
);
|
|
903
|
+
|
|
904
|
+
// Flush any audio still held in the mixer (treating a missing source as
|
|
905
|
+
// silence) before we tear the writer down. Lock order: mixer → inner.
|
|
906
|
+
if let Some(mixer) = self.mixer.as_ref() {
|
|
907
|
+
if self.started.load(Ordering::SeqCst) {
|
|
908
|
+
let mut mixer_guard = mixer.lock().map_err(|e| e.to_string())?;
|
|
909
|
+
let start_secs = f64::from_bits(self.session_start_bits.load(Ordering::SeqCst));
|
|
910
|
+
if !start_secs.is_nan() {
|
|
911
|
+
mixer_guard.set_min_start(start_secs);
|
|
912
|
+
}
|
|
913
|
+
let emitted = mixer_guard.drain_ready(true);
|
|
914
|
+
let mut guard = self.inner.lock().map_err(|e| e.to_string())?;
|
|
915
|
+
if !guard.finished && guard.failed.is_none() {
|
|
916
|
+
match emitted {
|
|
917
|
+
Ok(buffers) => {
|
|
918
|
+
if let Some(input) = guard.mixed_audio_input.clone() {
|
|
919
|
+
for buffer in &buffers {
|
|
920
|
+
unsafe {
|
|
921
|
+
self.append_sample_ptr(&mut guard, &input, buffer.as_ptr());
|
|
922
|
+
}
|
|
923
|
+
if guard.failed.is_some() {
|
|
924
|
+
break;
|
|
925
|
+
}
|
|
926
|
+
}
|
|
927
|
+
}
|
|
928
|
+
}
|
|
929
|
+
Err(err) => {
|
|
930
|
+
guard.failed = Some(err);
|
|
931
|
+
}
|
|
932
|
+
}
|
|
933
|
+
}
|
|
934
|
+
}
|
|
935
|
+
}
|
|
936
|
+
|
|
937
|
+
let (
|
|
938
|
+
writer,
|
|
939
|
+
video_input,
|
|
940
|
+
system_audio_input,
|
|
941
|
+
mic_audio_input,
|
|
942
|
+
mixed_audio_input,
|
|
943
|
+
segment_sink,
|
|
944
|
+
started,
|
|
945
|
+
failed,
|
|
946
|
+
) = {
|
|
947
|
+
let mut guard = self.inner.lock().map_err(|e| e.to_string())?;
|
|
948
|
+
if guard.finished {
|
|
949
|
+
return guard.failed.clone().map_or(Ok(()), Err);
|
|
950
|
+
}
|
|
951
|
+
guard.finished = true;
|
|
952
|
+
(
|
|
953
|
+
guard.writer.clone(),
|
|
954
|
+
guard.video_input.clone(),
|
|
955
|
+
guard.system_audio_input.clone(),
|
|
956
|
+
guard.mic_audio_input.clone(),
|
|
957
|
+
guard.mixed_audio_input.clone(),
|
|
958
|
+
guard.segment_sink.clone(),
|
|
959
|
+
self.started.load(Ordering::SeqCst),
|
|
960
|
+
guard.failed.clone(),
|
|
961
|
+
)
|
|
962
|
+
};
|
|
963
|
+
if let Some(err) = failed {
|
|
964
|
+
return Err(err);
|
|
965
|
+
}
|
|
966
|
+
unsafe {
|
|
967
|
+
use block2::RcBlock;
|
|
968
|
+
use objc2::msg_send;
|
|
969
|
+
use std::sync::mpsc;
|
|
970
|
+
use std::time::Duration as StdDuration;
|
|
971
|
+
|
|
972
|
+
if !started {
|
|
973
|
+
let _: () = msg_send![&*writer, cancelWriting];
|
|
974
|
+
return Err("AVAssetWriter received no samples".into());
|
|
975
|
+
}
|
|
976
|
+
let _: () = msg_send![&*video_input, markAsFinished];
|
|
977
|
+
if let Some(input) = system_audio_input.as_ref() {
|
|
978
|
+
let _: () = msg_send![&**input, markAsFinished];
|
|
979
|
+
}
|
|
980
|
+
if let Some(input) = mic_audio_input.as_ref() {
|
|
981
|
+
let _: () = msg_send![&**input, markAsFinished];
|
|
982
|
+
}
|
|
983
|
+
if let Some(input) = mixed_audio_input.as_ref() {
|
|
984
|
+
let _: () = msg_send![&**input, markAsFinished];
|
|
985
|
+
}
|
|
986
|
+
|
|
987
|
+
let (tx, rx) = mpsc::sync_channel::<()>(1);
|
|
988
|
+
let block = RcBlock::new(move || {
|
|
989
|
+
let _ = tx.send(());
|
|
990
|
+
});
|
|
991
|
+
let _: () = msg_send![&*writer, finishWritingWithCompletionHandler: &*block];
|
|
992
|
+
if wait_for_finalize && rx.recv_timeout(StdDuration::from_secs(15)).is_err() {
|
|
993
|
+
let _: () = msg_send![&*writer, cancelWriting];
|
|
994
|
+
return Err("AVAssetWriter finalize timed out".into());
|
|
995
|
+
}
|
|
996
|
+
if wait_for_finalize {
|
|
997
|
+
let status: i64 = msg_send![&*writer, status];
|
|
998
|
+
if status != AV_WRITER_STATUS_COMPLETED {
|
|
999
|
+
return Err(format!(
|
|
1000
|
+
"AVAssetWriter finalize failed (status={status}{})",
|
|
1001
|
+
av_writer_error_suffix(&writer)
|
|
1002
|
+
));
|
|
1003
|
+
}
|
|
1004
|
+
}
|
|
1005
|
+
}
|
|
1006
|
+
// Segmented mode: the delegate delivers the final segment before the
|
|
1007
|
+
// completion handler fires, so by now the local file is complete —
|
|
1008
|
+
// unless a disk write failed along the way.
|
|
1009
|
+
if let Some(sink) = segment_sink.as_ref() {
|
|
1010
|
+
if let Some(err) = sink.failure() {
|
|
1011
|
+
return Err(err);
|
|
1012
|
+
}
|
|
1013
|
+
}
|
|
1014
|
+
eprintln!("[mixer] writer finish completed (wait={wait_for_finalize})");
|
|
1015
|
+
Ok(())
|
|
1016
|
+
}
|
|
1017
|
+
}
|
|
1018
|
+
|
|
1019
|
+
// ---------------------------------------------------------------------------
|
|
1020
|
+
// Live audio mixer
|
|
1021
|
+
//
|
|
1022
|
+
// Combines the two ScreenCaptureKit audio streams (system + mic) into a single
|
|
1023
|
+
// interleaved-stereo track in real time, so the recorded file already has one
|
|
1024
|
+
// mixed audio track and no post-recording ffmpeg mixing pass is needed.
|
|
1025
|
+
//
|
|
1026
|
+
// The hard part is that the two streams arrive on independent callbacks with
|
|
1027
|
+
// their own timestamps and can start late (mic warmup) or stall. We place each
|
|
1028
|
+
// stream on a shared sample timeline (anchored to the first audio sample),
|
|
1029
|
+
// zero-filling gaps, and only emit output up to the point where we have data we
|
|
1030
|
+
// trust from every still-active source.
|
|
1031
|
+
// ---------------------------------------------------------------------------
|
|
1032
|
+
|
|
1033
|
+
/// Frames per emitted mixed buffer (~85ms at 48kHz).
|
|
1034
|
+
const MIX_CHUNK_FRAMES: i64 = 4096;
|
|
1035
|
+
/// A source with no data for this long stops bounding output (treated as
|
|
1036
|
+
/// silent) so one stalled source can't freeze the mixed track.
|
|
1037
|
+
const MIX_STALL_TIMEOUT: Duration = Duration::from_millis(250);
|
|
1038
|
+
/// How long after mixer creation to keep waiting for a source that hasn't
|
|
1039
|
+
/// produced its first buffer (mic warmup); afterwards it counts as absent.
|
|
1040
|
+
const MIX_SOURCE_GRACE: Duration = Duration::from_millis(2000);
|
|
1041
|
+
const AUDIO_FORMAT_LPCM: u32 = 0x6C70_636D; // 'lpcm'
|
|
1042
|
+
const AUDIO_FORMAT_FLAGS_FLOAT_PACKED: u32 = 1 | 8; // float + packed, interleaved, little-endian
|
|
1043
|
+
|
|
1044
|
+
#[derive(Clone, Copy, PartialEq, Eq)]
|
|
1045
|
+
/// Which capture stream a pushed PCM chunk came from.
|
|
1046
|
+
enum MixSource {
|
|
1047
|
+
System,
|
|
1048
|
+
Mic,
|
|
1049
|
+
}
|
|
1050
|
+
|
|
1051
|
+
/// One source's contiguous PCM ring on the shared output timeline. Gaps
|
|
1052
|
+
/// between pushes are zero-filled; consumed frames are dropped from the
|
|
1053
|
+
/// front as the mixer emits.
|
|
1054
|
+
struct MixerTimeline {
|
|
1055
|
+
/// Absolute frame index (anchor = 0) of `samples[0]`.
|
|
1056
|
+
base_frame: i64,
|
|
1057
|
+
/// Interleaved stereo f32 samples from `base_frame` onward.
|
|
1058
|
+
samples: Vec<f32>,
|
|
1059
|
+
started: bool,
|
|
1060
|
+
last_push: Option<Instant>,
|
|
1061
|
+
}
|
|
1062
|
+
|
|
1063
|
+
impl MixerTimeline {
|
|
1064
|
+
fn new() -> Self {
|
|
1065
|
+
Self {
|
|
1066
|
+
base_frame: 0,
|
|
1067
|
+
samples: Vec::new(),
|
|
1068
|
+
started: false,
|
|
1069
|
+
last_push: None,
|
|
1070
|
+
}
|
|
1071
|
+
}
|
|
1072
|
+
|
|
1073
|
+
/// One past the last frame this source has data for (absolute index).
|
|
1074
|
+
fn end_frame(&self) -> i64 {
|
|
1075
|
+
self.base_frame + (self.samples.len() / 2) as i64
|
|
1076
|
+
}
|
|
1077
|
+
|
|
1078
|
+
/// Stereo sample at an absolute frame index; silence outside the
|
|
1079
|
+
/// buffered range, so callers never need bounds checks.
|
|
1080
|
+
fn sample_at(&self, frame: i64) -> (f32, f32) {
|
|
1081
|
+
if frame < self.base_frame {
|
|
1082
|
+
return (0.0, 0.0);
|
|
1083
|
+
}
|
|
1084
|
+
let offset = ((frame - self.base_frame) as usize) * 2;
|
|
1085
|
+
if offset + 1 < self.samples.len() {
|
|
1086
|
+
(self.samples[offset], self.samples[offset + 1])
|
|
1087
|
+
} else {
|
|
1088
|
+
(0.0, 0.0)
|
|
1089
|
+
}
|
|
1090
|
+
}
|
|
1091
|
+
}
|
|
1092
|
+
|
|
1093
|
+
enum SourceBound {
|
|
1094
|
+
/// Started and recently fed; bounds output to `end_frame`.
|
|
1095
|
+
Active(i64),
|
|
1096
|
+
/// Started but no recent data; treat beyond its data as silence.
|
|
1097
|
+
Stalled,
|
|
1098
|
+
/// Not started yet but still within the warmup grace window; hold output.
|
|
1099
|
+
Pending,
|
|
1100
|
+
/// Not started and past grace; treat as silent.
|
|
1101
|
+
Absent,
|
|
1102
|
+
}
|
|
1103
|
+
|
|
1104
|
+
/// Realtime mic + system mixer. Places each source on a shared 48kHz
|
|
1105
|
+
/// stereo timeline anchored at the first audio PTS, then emits summed
|
|
1106
|
+
/// chunks up to the point every still-active source has data for. See the
|
|
1107
|
+
/// section comment above for the full design rationale.
|
|
1108
|
+
struct LiveAudioMixer {
|
|
1109
|
+
format_desc: screencapturekit::cm::CMFormatDescription,
|
|
1110
|
+
sample_rate: i32,
|
|
1111
|
+
/// Subtract the writer session start from emitted PTS (segmented mode,
|
|
1112
|
+
/// where the session timeline is zero-based). Plain file mode keeps
|
|
1113
|
+
/// absolute source time to match `startSessionAtSourceTime:`.
|
|
1114
|
+
rebase_output: bool,
|
|
1115
|
+
/// Session start in source seconds, recorded by `set_min_start`.
|
|
1116
|
+
session_start_seconds: Option<f64>,
|
|
1117
|
+
/// Accumulated pause time (seconds) subtracted from every incoming source
|
|
1118
|
+
/// PTS so a pause/resume leaves no gap on the audio timeline — mirrors the
|
|
1119
|
+
/// video path's `pause_offset`.
|
|
1120
|
+
pause_offset_seconds: f64,
|
|
1121
|
+
anchor_seconds: Option<f64>,
|
|
1122
|
+
out_pos: i64,
|
|
1123
|
+
system: MixerTimeline,
|
|
1124
|
+
mic: MixerTimeline,
|
|
1125
|
+
created_at: Instant,
|
|
1126
|
+
}
|
|
1127
|
+
|
|
1128
|
+
impl LiveAudioMixer {
|
|
1129
|
+
fn new(rebase_output: bool) -> Result<Self, String> {
|
|
1130
|
+
let sample_rate = AUDIO_OUTPUT_SAMPLE_RATE as i32;
|
|
1131
|
+
let asbd = AudioStreamBasicDescription {
|
|
1132
|
+
sample_rate: AUDIO_OUTPUT_SAMPLE_RATE as f64,
|
|
1133
|
+
format_id: AUDIO_FORMAT_LPCM,
|
|
1134
|
+
format_flags: AUDIO_FORMAT_FLAGS_FLOAT_PACKED,
|
|
1135
|
+
bytes_per_packet: 8,
|
|
1136
|
+
frames_per_packet: 1,
|
|
1137
|
+
bytes_per_frame: 8,
|
|
1138
|
+
channels_per_frame: 2,
|
|
1139
|
+
bits_per_channel: 32,
|
|
1140
|
+
reserved: 0,
|
|
1141
|
+
};
|
|
1142
|
+
let mut desc: *mut std::ffi::c_void = std::ptr::null_mut();
|
|
1143
|
+
let status = unsafe {
|
|
1144
|
+
CMAudioFormatDescriptionCreate(
|
|
1145
|
+
std::ptr::null(),
|
|
1146
|
+
&asbd,
|
|
1147
|
+
0,
|
|
1148
|
+
std::ptr::null(),
|
|
1149
|
+
0,
|
|
1150
|
+
std::ptr::null(),
|
|
1151
|
+
std::ptr::null(),
|
|
1152
|
+
&mut desc,
|
|
1153
|
+
)
|
|
1154
|
+
};
|
|
1155
|
+
if status != 0 || desc.is_null() {
|
|
1156
|
+
return Err(format!(
|
|
1157
|
+
"CMAudioFormatDescriptionCreate failed (status={status})"
|
|
1158
|
+
));
|
|
1159
|
+
}
|
|
1160
|
+
let format_desc = screencapturekit::cm::CMFormatDescription::from_raw(desc)
|
|
1161
|
+
.ok_or_else(|| "CMAudioFormatDescription wrap failed".to_string())?;
|
|
1162
|
+
Ok(Self {
|
|
1163
|
+
format_desc,
|
|
1164
|
+
sample_rate,
|
|
1165
|
+
rebase_output,
|
|
1166
|
+
session_start_seconds: None,
|
|
1167
|
+
pause_offset_seconds: 0.0,
|
|
1168
|
+
anchor_seconds: None,
|
|
1169
|
+
out_pos: 0,
|
|
1170
|
+
system: MixerTimeline::new(),
|
|
1171
|
+
mic: MixerTimeline::new(),
|
|
1172
|
+
created_at: Instant::now(),
|
|
1173
|
+
})
|
|
1174
|
+
}
|
|
1175
|
+
|
|
1176
|
+
/// Place decoded PCM on the source's timeline at its PTS-derived frame
|
|
1177
|
+
/// position, zero-filling any gap since the previous push (capped so a
|
|
1178
|
+
/// glitched timestamp can't allocate gigabytes).
|
|
1179
|
+
fn push(&mut self, source: MixSource, interleaved: &[f32], pts_seconds: f64) {
|
|
1180
|
+
let frames = interleaved.len() / 2;
|
|
1181
|
+
if frames == 0 {
|
|
1182
|
+
return;
|
|
1183
|
+
}
|
|
1184
|
+
// Collapse the paused gap: pull post-resume audio back onto the
|
|
1185
|
+
// continuous timeline so its frame positions abut the pre-pause data
|
|
1186
|
+
// instead of leaving a silence hole (which `push` would otherwise cap
|
|
1187
|
+
// at `max_gap` and desync from the video track).
|
|
1188
|
+
let pts_seconds = pts_seconds - self.pause_offset_seconds;
|
|
1189
|
+
let anchor = *self.anchor_seconds.get_or_insert(pts_seconds);
|
|
1190
|
+
let frame_index =
|
|
1191
|
+
(((pts_seconds - anchor) * self.sample_rate as f64).round() as i64).max(0);
|
|
1192
|
+
let out_pos = self.out_pos;
|
|
1193
|
+
// Cap silence inserted for a timestamp gap. A glitched/discontinuous
|
|
1194
|
+
// PTS could otherwise compute a multi-billion-frame gap and try to
|
|
1195
|
+
// allocate gigabytes of zeros, aborting the process.
|
|
1196
|
+
let max_gap = self.sample_rate as i64 * 2;
|
|
1197
|
+
let timeline = match source {
|
|
1198
|
+
MixSource::System => &mut self.system,
|
|
1199
|
+
MixSource::Mic => &mut self.mic,
|
|
1200
|
+
};
|
|
1201
|
+
if !timeline.started {
|
|
1202
|
+
timeline.started = true;
|
|
1203
|
+
// Never start behind already-emitted output.
|
|
1204
|
+
timeline.base_frame = frame_index.max(out_pos);
|
|
1205
|
+
}
|
|
1206
|
+
let cur_end = timeline.end_frame();
|
|
1207
|
+
if frame_index > cur_end {
|
|
1208
|
+
let gap = (frame_index - cur_end).min(max_gap) as usize;
|
|
1209
|
+
timeline
|
|
1210
|
+
.samples
|
|
1211
|
+
.extend(std::iter::repeat(0.0_f32).take(gap * 2));
|
|
1212
|
+
}
|
|
1213
|
+
timeline.samples.extend_from_slice(interleaved);
|
|
1214
|
+
timeline.last_push = Some(Instant::now());
|
|
1215
|
+
}
|
|
1216
|
+
|
|
1217
|
+
/// Update the accumulated pause offset (seconds) applied to incoming
|
|
1218
|
+
/// source PTS in `push`. Set from the writer's shared offset on resume.
|
|
1219
|
+
fn set_pause_offset(&mut self, seconds: f64) {
|
|
1220
|
+
self.pause_offset_seconds = seconds;
|
|
1221
|
+
}
|
|
1222
|
+
|
|
1223
|
+
/// Advance the output cursor so we never emit audio earlier than the
|
|
1224
|
+
/// writer session start (the writer rejects samples before it).
|
|
1225
|
+
fn set_min_start(&mut self, start_seconds: f64) {
|
|
1226
|
+
self.session_start_seconds = Some(start_seconds);
|
|
1227
|
+
if let Some(anchor) = self.anchor_seconds {
|
|
1228
|
+
// ceil, not round: rounding down would place the first emitted
|
|
1229
|
+
// sample a fraction of a frame BEFORE the session start, which
|
|
1230
|
+
// AVAssetWriter can reject when it writes the fragment.
|
|
1231
|
+
let floor = (((start_seconds - anchor) * self.sample_rate as f64).ceil() as i64).max(0);
|
|
1232
|
+
if floor > self.out_pos {
|
|
1233
|
+
self.out_pos = floor;
|
|
1234
|
+
self.drain_consumed();
|
|
1235
|
+
}
|
|
1236
|
+
}
|
|
1237
|
+
}
|
|
1238
|
+
|
|
1239
|
+
/// How a source currently bounds output: actively feeding (bound to its
|
|
1240
|
+
/// data end), stalled/absent (ignored), or still warming up (holds all
|
|
1241
|
+
/// output back).
|
|
1242
|
+
fn classify(&self, timeline: &MixerTimeline, now: Instant) -> SourceBound {
|
|
1243
|
+
if !timeline.started {
|
|
1244
|
+
if now.duration_since(self.created_at) < MIX_SOURCE_GRACE {
|
|
1245
|
+
SourceBound::Pending
|
|
1246
|
+
} else {
|
|
1247
|
+
SourceBound::Absent
|
|
1248
|
+
}
|
|
1249
|
+
} else if timeline
|
|
1250
|
+
.last_push
|
|
1251
|
+
.map_or(true, |t| now.duration_since(t) >= MIX_STALL_TIMEOUT)
|
|
1252
|
+
{
|
|
1253
|
+
SourceBound::Stalled
|
|
1254
|
+
} else {
|
|
1255
|
+
SourceBound::Active(timeline.end_frame())
|
|
1256
|
+
}
|
|
1257
|
+
}
|
|
1258
|
+
|
|
1259
|
+
/// The frame up to which mixing is safe: the minimum data end across
|
|
1260
|
+
/// active sources (or everything buffered when flushing at stop).
|
|
1261
|
+
fn compute_safe_end(&self, flush: bool) -> i64 {
|
|
1262
|
+
if flush {
|
|
1263
|
+
return self.system.end_frame().max(self.mic.end_frame());
|
|
1264
|
+
}
|
|
1265
|
+
let now = Instant::now();
|
|
1266
|
+
let sys = self.classify(&self.system, now);
|
|
1267
|
+
let mic = self.classify(&self.mic, now);
|
|
1268
|
+
if matches!(sys, SourceBound::Pending) || matches!(mic, SourceBound::Pending) {
|
|
1269
|
+
// Still expecting a source to start; don't run ahead of it.
|
|
1270
|
+
return self.out_pos;
|
|
1271
|
+
}
|
|
1272
|
+
let mut bound = i64::MAX;
|
|
1273
|
+
let mut any_active = false;
|
|
1274
|
+
for b in [&sys, &mic] {
|
|
1275
|
+
if let SourceBound::Active(end) = b {
|
|
1276
|
+
bound = bound.min(*end);
|
|
1277
|
+
any_active = true;
|
|
1278
|
+
}
|
|
1279
|
+
}
|
|
1280
|
+
if any_active {
|
|
1281
|
+
bound
|
|
1282
|
+
} else {
|
|
1283
|
+
// Everything stalled/absent: drain whatever frozen data we have.
|
|
1284
|
+
self.system.end_frame().max(self.mic.end_frame())
|
|
1285
|
+
}
|
|
1286
|
+
}
|
|
1287
|
+
|
|
1288
|
+
/// Emit mixed sample buffers covering `out_pos..safe_end` in
|
|
1289
|
+
/// `MIX_CHUNK_FRAMES` chunks: sum system + mic per frame, clamp, wrap as
|
|
1290
|
+
/// LPCM `CMSampleBuffer`s with contiguous PTS.
|
|
1291
|
+
fn drain_ready(
|
|
1292
|
+
&mut self,
|
|
1293
|
+
flush: bool,
|
|
1294
|
+
) -> Result<Vec<screencapturekit::cm::CMSampleBuffer>, String> {
|
|
1295
|
+
if self.anchor_seconds.is_none() {
|
|
1296
|
+
return Ok(Vec::new());
|
|
1297
|
+
}
|
|
1298
|
+
let safe_end = self.compute_safe_end(flush);
|
|
1299
|
+
if safe_end <= self.out_pos {
|
|
1300
|
+
return Ok(Vec::new());
|
|
1301
|
+
}
|
|
1302
|
+
let mut emitted = Vec::new();
|
|
1303
|
+
let mut a = self.out_pos;
|
|
1304
|
+
while a < safe_end {
|
|
1305
|
+
let b = (a + MIX_CHUNK_FRAMES).min(safe_end);
|
|
1306
|
+
let n = (b - a) as usize;
|
|
1307
|
+
let mut interleaved = vec![0.0_f32; n * 2];
|
|
1308
|
+
for f in 0..n {
|
|
1309
|
+
let frame = a + f as i64;
|
|
1310
|
+
let (sl, sr) = self.system.sample_at(frame);
|
|
1311
|
+
let (ml, mr) = self.mic.sample_at(frame);
|
|
1312
|
+
interleaved[f * 2] = (sl + ml).clamp(-1.0, 1.0);
|
|
1313
|
+
interleaved[f * 2 + 1] = (sr + mr).clamp(-1.0, 1.0);
|
|
1314
|
+
}
|
|
1315
|
+
emitted.push(self.build_sample_buffer(&interleaved, a)?);
|
|
1316
|
+
a = b;
|
|
1317
|
+
}
|
|
1318
|
+
self.out_pos = safe_end;
|
|
1319
|
+
self.drain_consumed();
|
|
1320
|
+
Ok(emitted)
|
|
1321
|
+
}
|
|
1322
|
+
|
|
1323
|
+
/// Free PCM below the output cursor from both timelines (already mixed
|
|
1324
|
+
/// and emitted; `sample_at` treats it as silence if ever re-read).
|
|
1325
|
+
fn drain_consumed(&mut self) {
|
|
1326
|
+
let out_pos = self.out_pos;
|
|
1327
|
+
for timeline in [&mut self.system, &mut self.mic] {
|
|
1328
|
+
if out_pos > timeline.base_frame {
|
|
1329
|
+
let drop_frames = (out_pos - timeline.base_frame) as usize;
|
|
1330
|
+
let drop_samples = (drop_frames * 2).min(timeline.samples.len());
|
|
1331
|
+
timeline.samples.drain(0..drop_samples);
|
|
1332
|
+
timeline.base_frame = out_pos;
|
|
1333
|
+
}
|
|
1334
|
+
}
|
|
1335
|
+
}
|
|
1336
|
+
|
|
1337
|
+
/// Wrap raw interleaved f32 PCM as a ready-to-append LPCM
|
|
1338
|
+
/// `CMSampleBuffer` whose PTS continues the mixer's output timeline.
|
|
1339
|
+
fn build_sample_buffer(
|
|
1340
|
+
&self,
|
|
1341
|
+
interleaved: &[f32],
|
|
1342
|
+
start_frame: i64,
|
|
1343
|
+
) -> Result<screencapturekit::cm::CMSampleBuffer, String> {
|
|
1344
|
+
let frames = interleaved.len() / 2;
|
|
1345
|
+
let bytes = unsafe {
|
|
1346
|
+
std::slice::from_raw_parts(
|
|
1347
|
+
interleaved.as_ptr() as *const u8,
|
|
1348
|
+
std::mem::size_of_val(interleaved),
|
|
1349
|
+
)
|
|
1350
|
+
};
|
|
1351
|
+
let block = screencapturekit::cm::CMBlockBuffer::create(bytes)
|
|
1352
|
+
.ok_or_else(|| "CMBlockBuffer create failed".to_string())?;
|
|
1353
|
+
let anchor = self.anchor_seconds.unwrap_or(0.0);
|
|
1354
|
+
// Zero-based session timeline: emit PTS relative to the session start
|
|
1355
|
+
// (set_min_start guarantees emitted frames are never earlier than it).
|
|
1356
|
+
let base = if self.rebase_output {
|
|
1357
|
+
self.session_start_seconds.unwrap_or(anchor)
|
|
1358
|
+
} else {
|
|
1359
|
+
0.0
|
|
1360
|
+
};
|
|
1361
|
+
let pts_value = ((anchor - base) * self.sample_rate as f64).round() as i64 + start_frame;
|
|
1362
|
+
let pts = ObjcCMTime {
|
|
1363
|
+
value: pts_value,
|
|
1364
|
+
timescale: self.sample_rate,
|
|
1365
|
+
flags: 1,
|
|
1366
|
+
epoch: 0,
|
|
1367
|
+
};
|
|
1368
|
+
let mut out: *mut std::ffi::c_void = std::ptr::null_mut();
|
|
1369
|
+
let status = unsafe {
|
|
1370
|
+
CMAudioSampleBufferCreateReadyWithPacketDescriptions(
|
|
1371
|
+
std::ptr::null(),
|
|
1372
|
+
block.as_ptr(),
|
|
1373
|
+
self.format_desc.as_ptr(),
|
|
1374
|
+
frames as isize,
|
|
1375
|
+
pts,
|
|
1376
|
+
std::ptr::null(),
|
|
1377
|
+
&mut out,
|
|
1378
|
+
)
|
|
1379
|
+
};
|
|
1380
|
+
if status != 0 || out.is_null() {
|
|
1381
|
+
return Err(format!(
|
|
1382
|
+
"CMAudioSampleBufferCreate failed (status={status})"
|
|
1383
|
+
));
|
|
1384
|
+
}
|
|
1385
|
+
screencapturekit::cm::CMSampleBuffer::from_raw(out)
|
|
1386
|
+
.ok_or_else(|| "CMSampleBuffer wrap failed".to_string())
|
|
1387
|
+
}
|
|
1388
|
+
}
|
|
1389
|
+
|
|
1390
|
+
/// Decode little-endian f32 samples from a CoreMedia byte buffer. Copies
|
|
1391
|
+
/// instead of reinterpreting the pointer: CoreAudio makes no alignment
|
|
1392
|
+
/// guarantee, and casting an unaligned `*const u8` to `&[f32]` is undefined
|
|
1393
|
+
/// behavior.
|
|
1394
|
+
fn bytes_to_f32_vec(bytes: &[u8]) -> Vec<f32> {
|
|
1395
|
+
bytes
|
|
1396
|
+
.chunks_exact(4)
|
|
1397
|
+
.map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]]))
|
|
1398
|
+
.collect()
|
|
1399
|
+
}
|
|
1400
|
+
|
|
1401
|
+
// CoreAudio format flags (AudioFormatFlags).
|
|
1402
|
+
const K_AUDIO_FLAG_IS_FLOAT: u32 = 1 << 0;
|
|
1403
|
+
const K_AUDIO_FLAG_IS_SIGNED_INT: u32 = 1 << 2;
|
|
1404
|
+
|
|
1405
|
+
/// Render a caught Objective-C exception (name + reason) for logging.
|
|
1406
|
+
fn describe_objc_exception(
|
|
1407
|
+
exc: Option<objc2::rc::Retained<objc2::exception::Exception>>,
|
|
1408
|
+
) -> String {
|
|
1409
|
+
match exc {
|
|
1410
|
+
Some(e) => format!("{e:?}"),
|
|
1411
|
+
None => "unknown Objective-C exception".to_string(),
|
|
1412
|
+
}
|
|
1413
|
+
}
|
|
1414
|
+
|
|
1415
|
+
/// Read the source stream's `AudioStreamBasicDescription` (rate, channels,
|
|
1416
|
+
/// sample format) off a sample buffer; `None` when unavailable.
|
|
1417
|
+
fn source_asbd(
|
|
1418
|
+
sample: &screencapturekit::cm::CMSampleBuffer,
|
|
1419
|
+
) -> Option<AudioStreamBasicDescription> {
|
|
1420
|
+
let format = sample.format_description()?;
|
|
1421
|
+
let asbd = unsafe { CMAudioFormatDescriptionGetStreamBasicDescription(format.as_ptr()) };
|
|
1422
|
+
if asbd.is_null() {
|
|
1423
|
+
return None;
|
|
1424
|
+
}
|
|
1425
|
+
Some(unsafe { std::ptr::read(asbd) })
|
|
1426
|
+
}
|
|
1427
|
+
|
|
1428
|
+
/// Decode one audio buffer's raw bytes into f32 samples according to the
|
|
1429
|
+
/// stream's sample format (float32/float64 or signed int16/int32).
|
|
1430
|
+
fn decode_samples_to_f32(bytes: &[u8], is_float: bool, bits: u32) -> Vec<f32> {
|
|
1431
|
+
match (is_float, bits) {
|
|
1432
|
+
(true, 32) => bytes_to_f32_vec(bytes),
|
|
1433
|
+
(true, 64) => bytes
|
|
1434
|
+
.chunks_exact(8)
|
|
1435
|
+
.map(|c| f64::from_le_bytes(c.try_into().unwrap()) as f32)
|
|
1436
|
+
.collect(),
|
|
1437
|
+
(false, 16) => bytes
|
|
1438
|
+
.chunks_exact(2)
|
|
1439
|
+
.map(|c| i16::from_le_bytes([c[0], c[1]]) as f32 / 32768.0)
|
|
1440
|
+
.collect(),
|
|
1441
|
+
(false, 32) => bytes
|
|
1442
|
+
.chunks_exact(4)
|
|
1443
|
+
.map(|c| i32::from_le_bytes(c.try_into().unwrap()) as f32 / 2_147_483_648.0)
|
|
1444
|
+
.collect(),
|
|
1445
|
+
// Unknown: best-effort treat as float32.
|
|
1446
|
+
_ => bytes_to_f32_vec(bytes),
|
|
1447
|
+
}
|
|
1448
|
+
}
|
|
1449
|
+
|
|
1450
|
+
/// Resample interleaved-stereo f32 from `src_rate` to `dst_rate` with linear
|
|
1451
|
+
/// interpolation. ScreenCaptureKit delivers the microphone at its native rate
|
|
1452
|
+
/// (often 44.1 kHz / mono-upmixed), which is not the 48 kHz the mixer assumes;
|
|
1453
|
+
/// without this the mic plays back pitch-shifted and unintelligible.
|
|
1454
|
+
fn resample_interleaved_stereo(input: &[f32], src_rate: f64, dst_rate: f64) -> Vec<f32> {
|
|
1455
|
+
let in_frames = input.len() / 2;
|
|
1456
|
+
if in_frames == 0 || src_rate <= 0.0 || dst_rate <= 0.0 || (src_rate - dst_rate).abs() < 1.0 {
|
|
1457
|
+
return input.to_vec();
|
|
1458
|
+
}
|
|
1459
|
+
let ratio = dst_rate / src_rate;
|
|
1460
|
+
let out_frames = ((in_frames as f64) * ratio).round() as usize;
|
|
1461
|
+
if out_frames == 0 {
|
|
1462
|
+
return Vec::new();
|
|
1463
|
+
}
|
|
1464
|
+
let mut out = vec![0.0_f32; out_frames * 2];
|
|
1465
|
+
let step = src_rate / dst_rate;
|
|
1466
|
+
for f in 0..out_frames {
|
|
1467
|
+
let pos = f as f64 * step;
|
|
1468
|
+
let i = pos.floor() as usize;
|
|
1469
|
+
let frac = (pos - i as f64) as f32;
|
|
1470
|
+
let i1 = (i + 1).min(in_frames - 1);
|
|
1471
|
+
for ch in 0..2 {
|
|
1472
|
+
let a = input[i * 2 + ch];
|
|
1473
|
+
let b = input[i1 * 2 + ch];
|
|
1474
|
+
out[f * 2 + ch] = a + (b - a) * frac;
|
|
1475
|
+
}
|
|
1476
|
+
}
|
|
1477
|
+
out
|
|
1478
|
+
}
|
|
1479
|
+
|
|
1480
|
+
fn extract_interleaved_stereo(
|
|
1481
|
+
sample: &screencapturekit::cm::CMSampleBuffer,
|
|
1482
|
+
label: &str,
|
|
1483
|
+
) -> Option<(Vec<f32>, f64)> {
|
|
1484
|
+
let frames = sample.num_samples();
|
|
1485
|
+
if frames == 0 {
|
|
1486
|
+
return None;
|
|
1487
|
+
}
|
|
1488
|
+
let timing = sample.sample_timing_info(0).ok()?;
|
|
1489
|
+
let pts_seconds = timing.presentation_time_stamp.as_seconds()?;
|
|
1490
|
+
let abl = sample.audio_buffer_list()?;
|
|
1491
|
+
let num_buffers = abl.num_buffers();
|
|
1492
|
+
if num_buffers == 0 {
|
|
1493
|
+
return None;
|
|
1494
|
+
}
|
|
1495
|
+
|
|
1496
|
+
let asbd = source_asbd(sample);
|
|
1497
|
+
let (is_float, bits, src_rate) = match asbd {
|
|
1498
|
+
Some(a) => {
|
|
1499
|
+
// If neither float nor signed-int is flagged, assume float (SCK default).
|
|
1500
|
+
let is_float = a.format_flags & K_AUDIO_FLAG_IS_FLOAT != 0
|
|
1501
|
+
|| a.format_flags & K_AUDIO_FLAG_IS_SIGNED_INT == 0;
|
|
1502
|
+
(is_float, a.bits_per_channel, a.sample_rate)
|
|
1503
|
+
}
|
|
1504
|
+
None => (true, 32, AUDIO_OUTPUT_SAMPLE_RATE as f64),
|
|
1505
|
+
};
|
|
1506
|
+
|
|
1507
|
+
log_audio_format_once(label, &asbd, num_buffers, &abl, frames);
|
|
1508
|
+
|
|
1509
|
+
let mut out = vec![0.0_f32; frames * 2];
|
|
1510
|
+
if num_buffers >= 2 {
|
|
1511
|
+
// Non-interleaved (planar): one channel per buffer.
|
|
1512
|
+
let left = decode_samples_to_f32(abl.get(0)?.data(), is_float, bits);
|
|
1513
|
+
let right = decode_samples_to_f32(abl.get(1)?.data(), is_float, bits);
|
|
1514
|
+
let n = frames.min(left.len()).min(right.len());
|
|
1515
|
+
for i in 0..n {
|
|
1516
|
+
out[i * 2] = left[i];
|
|
1517
|
+
out[i * 2 + 1] = right[i];
|
|
1518
|
+
}
|
|
1519
|
+
} else {
|
|
1520
|
+
let buf = abl.get(0)?;
|
|
1521
|
+
let channels = buf.number_channels.max(1) as usize;
|
|
1522
|
+
let decoded = decode_samples_to_f32(buf.data(), is_float, bits);
|
|
1523
|
+
if channels >= 2 {
|
|
1524
|
+
// Interleaved multi-channel: take the first two channels.
|
|
1525
|
+
let n = frames.min(decoded.len() / channels);
|
|
1526
|
+
for i in 0..n {
|
|
1527
|
+
out[i * 2] = decoded[i * channels];
|
|
1528
|
+
out[i * 2 + 1] = decoded[i * channels + 1];
|
|
1529
|
+
}
|
|
1530
|
+
} else {
|
|
1531
|
+
// Mono: duplicate into both channels.
|
|
1532
|
+
let n = frames.min(decoded.len());
|
|
1533
|
+
for i in 0..n {
|
|
1534
|
+
out[i * 2] = decoded[i];
|
|
1535
|
+
out[i * 2 + 1] = decoded[i];
|
|
1536
|
+
}
|
|
1537
|
+
}
|
|
1538
|
+
}
|
|
1539
|
+
|
|
1540
|
+
// Normalize every source to the mixer's output rate so contiguous placement
|
|
1541
|
+
// on the timeline matches real time (otherwise the mic is pitch-shifted).
|
|
1542
|
+
out = resample_interleaved_stereo(&out, src_rate, AUDIO_OUTPUT_SAMPLE_RATE as f64);
|
|
1543
|
+
Some((out, pts_seconds))
|
|
1544
|
+
}
|
|
1545
|
+
|
|
1546
|
+
/// Logs the decoded audio format the first time each source is seen, so format
|
|
1547
|
+
/// mismatches (rate / channels / int-vs-float / interleaving) are diagnosable.
|
|
1548
|
+
fn log_audio_format_once(
|
|
1549
|
+
label: &str,
|
|
1550
|
+
asbd: &Option<AudioStreamBasicDescription>,
|
|
1551
|
+
num_buffers: usize,
|
|
1552
|
+
abl: &screencapturekit::cm::AudioBufferList,
|
|
1553
|
+
frames: usize,
|
|
1554
|
+
) {
|
|
1555
|
+
use std::sync::atomic::AtomicBool;
|
|
1556
|
+
static SYS_LOGGED: AtomicBool = AtomicBool::new(false);
|
|
1557
|
+
static MIC_LOGGED: AtomicBool = AtomicBool::new(false);
|
|
1558
|
+
let flag = if label == "mic" {
|
|
1559
|
+
&MIC_LOGGED
|
|
1560
|
+
} else {
|
|
1561
|
+
&SYS_LOGGED
|
|
1562
|
+
};
|
|
1563
|
+
if flag.swap(true, Ordering::SeqCst) {
|
|
1564
|
+
return;
|
|
1565
|
+
}
|
|
1566
|
+
let ch0 = abl.get(0).map(|b| (b.number_channels, b.data_byte_size()));
|
|
1567
|
+
let ch1 = abl.get(1).map(|b| (b.number_channels, b.data_byte_size()));
|
|
1568
|
+
eprintln!(
|
|
1569
|
+
"[mixer] {label} format: asbd={asbd:?} num_buffers={num_buffers} frames={frames} buf0={ch0:?} buf1={ch1:?}"
|
|
1570
|
+
);
|
|
1571
|
+
}
|
|
1572
|
+
|
|
1573
|
+
/// Mirror of CoreAudio's `AudioStreamBasicDescription` (repr(C) so it can
|
|
1574
|
+
/// cross the FFI boundary by value).
|
|
1575
|
+
#[repr(C)]
|
|
1576
|
+
#[derive(Copy, Clone, Debug)]
|
|
1577
|
+
struct AudioStreamBasicDescription {
|
|
1578
|
+
sample_rate: f64,
|
|
1579
|
+
format_id: u32,
|
|
1580
|
+
format_flags: u32,
|
|
1581
|
+
bytes_per_packet: u32,
|
|
1582
|
+
frames_per_packet: u32,
|
|
1583
|
+
bytes_per_frame: u32,
|
|
1584
|
+
channels_per_frame: u32,
|
|
1585
|
+
bits_per_channel: u32,
|
|
1586
|
+
reserved: u32,
|
|
1587
|
+
}
|
|
1588
|
+
|
|
1589
|
+
/// `CMSampleTimingInfo` mirror for the retiming FFI call.
|
|
1590
|
+
#[repr(C)]
|
|
1591
|
+
#[derive(Copy, Clone)]
|
|
1592
|
+
struct ObjcCMSampleTimingInfo {
|
|
1593
|
+
duration: ObjcCMTime,
|
|
1594
|
+
presentation_time_stamp: ObjcCMTime,
|
|
1595
|
+
decode_time_stamp: ObjcCMTime,
|
|
1596
|
+
}
|
|
1597
|
+
|
|
1598
|
+
/// Rebase a timestamp onto the zero-based session timeline (subtract the
|
|
1599
|
+
/// first video frame's PTS). `pause_offset_seconds` is additionally subtracted
|
|
1600
|
+
/// so time spent paused collapses to nothing — the file stays gapless across a
|
|
1601
|
+
/// pause/resume. Invalid times pass through untouched.
|
|
1602
|
+
fn rebased_time(
|
|
1603
|
+
t: screencapturekit::cm::CMTime,
|
|
1604
|
+
base: (i64, i32),
|
|
1605
|
+
pause_offset_seconds: f64,
|
|
1606
|
+
) -> ObjcCMTime {
|
|
1607
|
+
const K_CMTIME_FLAG_VALID: u32 = 1;
|
|
1608
|
+
if t.flags & K_CMTIME_FLAG_VALID == 0 {
|
|
1609
|
+
return ObjcCMTime::from(t);
|
|
1610
|
+
}
|
|
1611
|
+
let (base_value, base_timescale) = base;
|
|
1612
|
+
let base_in_t = if t.timescale == base_timescale {
|
|
1613
|
+
base_value
|
|
1614
|
+
} else {
|
|
1615
|
+
// Convert the base into this timestamp's timescale before subtracting.
|
|
1616
|
+
(i128::from(base_value) * i128::from(t.timescale) / i128::from(base_timescale.max(1)))
|
|
1617
|
+
as i64
|
|
1618
|
+
};
|
|
1619
|
+
// Convert the accumulated pause time into this timestamp's timescale.
|
|
1620
|
+
let pause_in_t = (pause_offset_seconds * f64::from(t.timescale)).round() as i64;
|
|
1621
|
+
ObjcCMTime {
|
|
1622
|
+
value: t.value - base_in_t - pause_in_t,
|
|
1623
|
+
timescale: t.timescale,
|
|
1624
|
+
flags: t.flags,
|
|
1625
|
+
epoch: t.epoch,
|
|
1626
|
+
}
|
|
1627
|
+
}
|
|
1628
|
+
|
|
1629
|
+
/// Copy a sample buffer with its PTS/DTS rebased onto the session timeline.
|
|
1630
|
+
/// One timing entry applies to every sample in the buffer (SCK video buffers
|
|
1631
|
+
/// hold one frame; audio buffers have uniform per-sample timing).
|
|
1632
|
+
fn retimed_sample_copy(
|
|
1633
|
+
sample: &screencapturekit::cm::CMSampleBuffer,
|
|
1634
|
+
timing: &screencapturekit::cm::CMSampleTimingInfo,
|
|
1635
|
+
base: (i64, i32),
|
|
1636
|
+
pause_offset_seconds: f64,
|
|
1637
|
+
) -> Result<screencapturekit::cm::CMSampleBuffer, String> {
|
|
1638
|
+
let new_timing = ObjcCMSampleTimingInfo {
|
|
1639
|
+
duration: ObjcCMTime::from(timing.duration),
|
|
1640
|
+
presentation_time_stamp: rebased_time(
|
|
1641
|
+
timing.presentation_time_stamp,
|
|
1642
|
+
base,
|
|
1643
|
+
pause_offset_seconds,
|
|
1644
|
+
),
|
|
1645
|
+
decode_time_stamp: rebased_time(timing.decode_time_stamp, base, pause_offset_seconds),
|
|
1646
|
+
};
|
|
1647
|
+
let mut out: *mut std::ffi::c_void = std::ptr::null_mut();
|
|
1648
|
+
let status = unsafe {
|
|
1649
|
+
CMSampleBufferCreateCopyWithNewTiming(
|
|
1650
|
+
std::ptr::null(),
|
|
1651
|
+
sample.as_ptr(),
|
|
1652
|
+
1,
|
|
1653
|
+
&new_timing,
|
|
1654
|
+
&mut out,
|
|
1655
|
+
)
|
|
1656
|
+
};
|
|
1657
|
+
if status != 0 || out.is_null() {
|
|
1658
|
+
return Err(format!(
|
|
1659
|
+
"CMSampleBufferCreateCopyWithNewTiming failed (status={status})"
|
|
1660
|
+
));
|
|
1661
|
+
}
|
|
1662
|
+
screencapturekit::cm::CMSampleBuffer::from_raw(out)
|
|
1663
|
+
.ok_or_else(|| "retimed CMSampleBuffer wrap failed".to_string())
|
|
1664
|
+
}
|
|
1665
|
+
|
|
1666
|
+
// CoreMedia C API used to hand-build the mixer's LPCM output buffers —
|
|
1667
|
+
// the screencapturekit crate has no constructors for these.
|
|
1668
|
+
#[link(name = "CoreMedia", kind = "framework")]
|
|
1669
|
+
extern "C" {
|
|
1670
|
+
fn CMAudioFormatDescriptionCreate(
|
|
1671
|
+
allocator: *const std::ffi::c_void,
|
|
1672
|
+
asbd: *const AudioStreamBasicDescription,
|
|
1673
|
+
layout_size: usize,
|
|
1674
|
+
layout: *const std::ffi::c_void,
|
|
1675
|
+
magic_cookie_size: usize,
|
|
1676
|
+
magic_cookie: *const std::ffi::c_void,
|
|
1677
|
+
extensions: *const std::ffi::c_void,
|
|
1678
|
+
format_description_out: *mut *mut std::ffi::c_void,
|
|
1679
|
+
) -> i32;
|
|
1680
|
+
|
|
1681
|
+
fn CMAudioSampleBufferCreateReadyWithPacketDescriptions(
|
|
1682
|
+
allocator: *const std::ffi::c_void,
|
|
1683
|
+
data_buffer: *mut std::ffi::c_void,
|
|
1684
|
+
format_description: *mut std::ffi::c_void,
|
|
1685
|
+
num_samples: isize,
|
|
1686
|
+
presentation_time_stamp: ObjcCMTime,
|
|
1687
|
+
packet_descriptions: *const std::ffi::c_void,
|
|
1688
|
+
sample_buffer_out: *mut *mut std::ffi::c_void,
|
|
1689
|
+
) -> i32;
|
|
1690
|
+
|
|
1691
|
+
fn CMAudioFormatDescriptionGetStreamBasicDescription(
|
|
1692
|
+
format_description: *mut std::ffi::c_void,
|
|
1693
|
+
) -> *const AudioStreamBasicDescription;
|
|
1694
|
+
|
|
1695
|
+
fn CMSampleBufferCreateCopyWithNewTiming(
|
|
1696
|
+
allocator: *const std::ffi::c_void,
|
|
1697
|
+
original: *mut std::ffi::c_void,
|
|
1698
|
+
num_timing_entries: isize,
|
|
1699
|
+
timing_array: *const ObjcCMSampleTimingInfo,
|
|
1700
|
+
sample_buffer_out: *mut *mut std::ffi::c_void,
|
|
1701
|
+
) -> i32;
|
|
1702
|
+
}
|
|
1703
|
+
|
|
1704
|
+
/// Mirror of CoreMedia's `CMTime`, with objc2 `Encode` impls so it can be
|
|
1705
|
+
/// passed by value through `msg_send!` (e.g. `startSessionAtSourceTime:`,
|
|
1706
|
+
/// `setPreferredOutputSegmentInterval:`).
|
|
1707
|
+
#[repr(C)]
|
|
1708
|
+
#[derive(Copy, Clone)]
|
|
1709
|
+
struct ObjcCMTime {
|
|
1710
|
+
value: i64,
|
|
1711
|
+
timescale: i32,
|
|
1712
|
+
flags: u32,
|
|
1713
|
+
epoch: i64,
|
|
1714
|
+
}
|
|
1715
|
+
|
|
1716
|
+
unsafe impl objc2::encode::RefEncode for ObjcCMTime {
|
|
1717
|
+
const ENCODING_REF: objc2::encode::Encoding =
|
|
1718
|
+
objc2::encode::Encoding::Pointer(&<Self as objc2::encode::Encode>::ENCODING);
|
|
1719
|
+
}
|
|
1720
|
+
|
|
1721
|
+
unsafe impl objc2::encode::Encode for ObjcCMTime {
|
|
1722
|
+
const ENCODING: objc2::encode::Encoding = objc2::encode::Encoding::Struct(
|
|
1723
|
+
"CMTime",
|
|
1724
|
+
&[
|
|
1725
|
+
<i64 as objc2::encode::Encode>::ENCODING,
|
|
1726
|
+
<i32 as objc2::encode::Encode>::ENCODING,
|
|
1727
|
+
<u32 as objc2::encode::Encode>::ENCODING,
|
|
1728
|
+
<i64 as objc2::encode::Encode>::ENCODING,
|
|
1729
|
+
],
|
|
1730
|
+
);
|
|
1731
|
+
}
|
|
1732
|
+
|
|
1733
|
+
impl From<screencapturekit::cm::CMTime> for ObjcCMTime {
|
|
1734
|
+
fn from(value: screencapturekit::cm::CMTime) -> Self {
|
|
1735
|
+
Self {
|
|
1736
|
+
value: value.value,
|
|
1737
|
+
timescale: value.timescale,
|
|
1738
|
+
flags: value.flags,
|
|
1739
|
+
epoch: value.epoch,
|
|
1740
|
+
}
|
|
1741
|
+
}
|
|
1742
|
+
}
|
|
1743
|
+
|
|
1744
|
+
/// Look up an Objective-C class by name at runtime.
|
|
1745
|
+
unsafe fn av_class_named(name: &str) -> Option<&'static objc2::runtime::AnyClass> {
|
|
1746
|
+
let bytes = std::ffi::CString::new(name).ok()?;
|
|
1747
|
+
objc2::runtime::AnyClass::get(&bytes)
|
|
1748
|
+
}
|
|
1749
|
+
|
|
1750
|
+
/// Build a retained `NSString` from a Rust string.
|
|
1751
|
+
unsafe fn av_ns_string_from(s: &str) -> Option<objc2::rc::Retained<objc2::runtime::AnyObject>> {
|
|
1752
|
+
use objc2::runtime::AnyObject;
|
|
1753
|
+
use objc2::{class, msg_send};
|
|
1754
|
+
|
|
1755
|
+
let cstr = std::ffi::CString::new(s).ok()?;
|
|
1756
|
+
let allocated: *mut AnyObject = msg_send![class!(NSString), alloc];
|
|
1757
|
+
if allocated.is_null() {
|
|
1758
|
+
return None;
|
|
1759
|
+
}
|
|
1760
|
+
let inited: *mut AnyObject = msg_send![allocated, initWithUTF8String: cstr.as_ptr()];
|
|
1761
|
+
if inited.is_null() {
|
|
1762
|
+
let _ = objc2::rc::Retained::from_raw(allocated);
|
|
1763
|
+
None
|
|
1764
|
+
} else {
|
|
1765
|
+
objc2::rc::Retained::from_raw(inited)
|
|
1766
|
+
}
|
|
1767
|
+
}
|
|
1768
|
+
|
|
1769
|
+
/// Build a retained `NSURL` file URL for the writer's output path.
|
|
1770
|
+
unsafe fn av_file_url(path: &Path) -> Option<objc2::rc::Retained<objc2::runtime::AnyObject>> {
|
|
1771
|
+
use objc2::runtime::AnyObject;
|
|
1772
|
+
use objc2::{class, msg_send};
|
|
1773
|
+
|
|
1774
|
+
let path_str = path.to_str()?;
|
|
1775
|
+
let nsstr = av_ns_string_from(path_str)?;
|
|
1776
|
+
let url: *mut AnyObject = msg_send![class!(NSURL), fileURLWithPath: &*nsstr];
|
|
1777
|
+
if url.is_null() {
|
|
1778
|
+
None
|
|
1779
|
+
} else {
|
|
1780
|
+
objc2::rc::Retained::retain(url)
|
|
1781
|
+
}
|
|
1782
|
+
}
|
|
1783
|
+
|
|
1784
|
+
/// Boxed `NSNumber` for integer settings-dictionary values.
|
|
1785
|
+
unsafe fn av_number_i64(value: i64) -> Option<objc2::rc::Retained<objc2::runtime::AnyObject>> {
|
|
1786
|
+
use objc2::runtime::AnyObject;
|
|
1787
|
+
use objc2::{class, msg_send};
|
|
1788
|
+
|
|
1789
|
+
let raw: *mut AnyObject = msg_send![class!(NSNumber), numberWithLongLong: value];
|
|
1790
|
+
if raw.is_null() {
|
|
1791
|
+
None
|
|
1792
|
+
} else {
|
|
1793
|
+
objc2::rc::Retained::retain(raw)
|
|
1794
|
+
}
|
|
1795
|
+
}
|
|
1796
|
+
|
|
1797
|
+
/// Boxed boolean `NSNumber` for settings-dictionary flags.
|
|
1798
|
+
unsafe fn av_number_bool(value: bool) -> Option<objc2::rc::Retained<objc2::runtime::AnyObject>> {
|
|
1799
|
+
use objc2::runtime::AnyObject;
|
|
1800
|
+
use objc2::{class, msg_send};
|
|
1801
|
+
|
|
1802
|
+
let raw: *mut AnyObject = msg_send![class!(NSNumber), numberWithBool: value];
|
|
1803
|
+
if raw.is_null() {
|
|
1804
|
+
None
|
|
1805
|
+
} else {
|
|
1806
|
+
objc2::rc::Retained::retain(raw)
|
|
1807
|
+
}
|
|
1808
|
+
}
|
|
1809
|
+
|
|
1810
|
+
/// Fresh `NSMutableDictionary` for AVFoundation output settings.
|
|
1811
|
+
unsafe fn av_dict() -> Result<objc2::rc::Retained<objc2::runtime::AnyObject>, String> {
|
|
1812
|
+
use objc2::runtime::AnyObject;
|
|
1813
|
+
use objc2::{class, msg_send};
|
|
1814
|
+
|
|
1815
|
+
let raw: *mut AnyObject = msg_send![class!(NSMutableDictionary), dictionary];
|
|
1816
|
+
if raw.is_null() {
|
|
1817
|
+
Err("NSMutableDictionary allocation failed".into())
|
|
1818
|
+
} else {
|
|
1819
|
+
objc2::rc::Retained::retain(raw)
|
|
1820
|
+
.ok_or_else(|| "NSMutableDictionary retain failed".to_string())
|
|
1821
|
+
}
|
|
1822
|
+
}
|
|
1823
|
+
|
|
1824
|
+
/// `dict[key] = value` on an `NSMutableDictionary`.
|
|
1825
|
+
unsafe fn av_dict_set(
|
|
1826
|
+
dict: &objc2::runtime::AnyObject,
|
|
1827
|
+
key: *const objc2::runtime::AnyObject,
|
|
1828
|
+
value: &objc2::runtime::AnyObject,
|
|
1829
|
+
) {
|
|
1830
|
+
use objc2::msg_send;
|
|
1831
|
+
|
|
1832
|
+
let _: () = msg_send![dict, setObject: value, forKey: key];
|
|
1833
|
+
}
|
|
1834
|
+
|
|
1835
|
+
/// H.264 output settings for the video writer input, sized to the capture
|
|
1836
|
+
/// dimensions and with frame reordering (B-frames) disabled — required for
|
|
1837
|
+
/// stable fragmented-MP4 writing (see the comment inside).
|
|
1838
|
+
unsafe fn av_video_output_settings(
|
|
1839
|
+
width: u32,
|
|
1840
|
+
height: u32,
|
|
1841
|
+
) -> Result<objc2::rc::Retained<objc2::runtime::AnyObject>, String> {
|
|
1842
|
+
use objc2::runtime::AnyObject;
|
|
1843
|
+
|
|
1844
|
+
#[link(name = "AVFoundation", kind = "framework")]
|
|
1845
|
+
extern "C" {
|
|
1846
|
+
static AVVideoCodecKey: *const AnyObject;
|
|
1847
|
+
static AVVideoCodecTypeH264: *const AnyObject;
|
|
1848
|
+
static AVVideoHeightKey: *const AnyObject;
|
|
1849
|
+
static AVVideoWidthKey: *const AnyObject;
|
|
1850
|
+
static AVVideoCompressionPropertiesKey: *const AnyObject;
|
|
1851
|
+
static AVVideoAllowFrameReorderingKey: *const AnyObject;
|
|
1852
|
+
static AVVideoAverageBitRateKey: *const AnyObject;
|
|
1853
|
+
static AVVideoExpectedSourceFrameRateKey: *const AnyObject;
|
|
1854
|
+
static AVVideoMaxKeyFrameIntervalKey: *const AnyObject;
|
|
1855
|
+
}
|
|
1856
|
+
|
|
1857
|
+
let settings = av_dict()?;
|
|
1858
|
+
let width_value =
|
|
1859
|
+
av_number_i64(width as i64).ok_or_else(|| "NSNumber width failed".to_string())?;
|
|
1860
|
+
let height_value =
|
|
1861
|
+
av_number_i64(height as i64).ok_or_else(|| "NSNumber height failed".to_string())?;
|
|
1862
|
+
av_dict_set(&settings, AVVideoCodecKey, &*AVVideoCodecTypeH264);
|
|
1863
|
+
av_dict_set(&settings, AVVideoWidthKey, &width_value);
|
|
1864
|
+
av_dict_set(&settings, AVVideoHeightKey, &height_value);
|
|
1865
|
+
|
|
1866
|
+
// Segmented output (preferredOutputSegmentInterval) needs monotonic video timing
|
|
1867
|
+
// inside each fragment. The encoder's default B-frames (frame reordering)
|
|
1868
|
+
// intermittently kill the writer with -11800 / OSStatus -16341 when a
|
|
1869
|
+
// reordered frame group straddles a fragment boundary, truncating the
|
|
1870
|
+
// recording at an exact fragment-boundary timestamp. Screen capture gains
|
|
1871
|
+
// almost nothing from B-frames; disable reordering.
|
|
1872
|
+
let compression = av_dict()?;
|
|
1873
|
+
let no_reordering =
|
|
1874
|
+
av_number_bool(false).ok_or_else(|| "NSNumber reordering flag failed".to_string())?;
|
|
1875
|
+
av_dict_set(&compression, AVVideoAllowFrameReorderingKey, &no_reordering);
|
|
1876
|
+
|
|
1877
|
+
// Capture-time rate control (mirrors Cap's AVAssetWriter setup). Without
|
|
1878
|
+
// an explicit budget the encoder default runs several times larger and
|
|
1879
|
+
// every upload needs an ffmpeg transcode pass to shrink it.
|
|
1880
|
+
let average_bit_rate =
|
|
1881
|
+
(CAPTURE_VIDEO_BPP * f64::from(width) * f64::from(height) * f64::from(NATIVE_CAPTURE_FPS))
|
|
1882
|
+
as i64;
|
|
1883
|
+
let bit_rate =
|
|
1884
|
+
av_number_i64(average_bit_rate).ok_or_else(|| "NSNumber bit rate failed".to_string())?;
|
|
1885
|
+
av_dict_set(&compression, AVVideoAverageBitRateKey, &bit_rate);
|
|
1886
|
+
let expected_fps = av_number_i64(i64::from(NATIVE_CAPTURE_FPS))
|
|
1887
|
+
.ok_or_else(|| "NSNumber expected fps failed".to_string())?;
|
|
1888
|
+
av_dict_set(
|
|
1889
|
+
&compression,
|
|
1890
|
+
AVVideoExpectedSourceFrameRateKey,
|
|
1891
|
+
&expected_fps,
|
|
1892
|
+
);
|
|
1893
|
+
// ~0.75s keyframe cadence: guarantees at least one sync sample per movie
|
|
1894
|
+
// fragment (1s interval) so every fragment stays independently seekable.
|
|
1895
|
+
let keyframe_interval = av_number_i64((i64::from(NATIVE_CAPTURE_FPS) * 3 / 4).max(1))
|
|
1896
|
+
.ok_or_else(|| "NSNumber keyframe interval failed".to_string())?;
|
|
1897
|
+
av_dict_set(
|
|
1898
|
+
&compression,
|
|
1899
|
+
AVVideoMaxKeyFrameIntervalKey,
|
|
1900
|
+
&keyframe_interval,
|
|
1901
|
+
);
|
|
1902
|
+
|
|
1903
|
+
av_dict_set(&settings, AVVideoCompressionPropertiesKey, &compression);
|
|
1904
|
+
Ok(settings)
|
|
1905
|
+
}
|
|
1906
|
+
|
|
1907
|
+
/// AAC 48kHz stereo 128kbps output settings for the audio writer inputs
|
|
1908
|
+
/// (both the mixed track and per-source tracks).
|
|
1909
|
+
unsafe fn av_audio_output_settings(
|
|
1910
|
+
) -> Result<objc2::rc::Retained<objc2::runtime::AnyObject>, String> {
|
|
1911
|
+
use objc2::runtime::AnyObject;
|
|
1912
|
+
|
|
1913
|
+
#[link(name = "AVFoundation", kind = "framework")]
|
|
1914
|
+
extern "C" {
|
|
1915
|
+
static AVEncoderBitRateKey: *const AnyObject;
|
|
1916
|
+
static AVFormatIDKey: *const AnyObject;
|
|
1917
|
+
static AVNumberOfChannelsKey: *const AnyObject;
|
|
1918
|
+
static AVSampleRateKey: *const AnyObject;
|
|
1919
|
+
}
|
|
1920
|
+
|
|
1921
|
+
let settings = av_dict()?;
|
|
1922
|
+
let format = av_number_i64(AUDIO_FORMAT_AAC)
|
|
1923
|
+
.ok_or_else(|| "NSNumber audio format failed".to_string())?;
|
|
1924
|
+
let sample_rate =
|
|
1925
|
+
av_number_i64(48_000).ok_or_else(|| "NSNumber sample rate failed".to_string())?;
|
|
1926
|
+
let channels = av_number_i64(2).ok_or_else(|| "NSNumber channels failed".to_string())?;
|
|
1927
|
+
let bitrate = av_number_i64(128_000).ok_or_else(|| "NSNumber bitrate failed".to_string())?;
|
|
1928
|
+
av_dict_set(&settings, AVFormatIDKey, &format);
|
|
1929
|
+
av_dict_set(&settings, AVSampleRateKey, &sample_rate);
|
|
1930
|
+
av_dict_set(&settings, AVNumberOfChannelsKey, &channels);
|
|
1931
|
+
av_dict_set(&settings, AVEncoderBitRateKey, &bitrate);
|
|
1932
|
+
Ok(settings)
|
|
1933
|
+
}
|
|
1934
|
+
|
|
1935
|
+
/// Create an AAC `AVAssetWriterInput` (realtime mode), attach it to the
|
|
1936
|
+
/// writer, and return it retained.
|
|
1937
|
+
unsafe fn av_make_audio_writer_input(
|
|
1938
|
+
input_cls: &objc2::runtime::AnyClass,
|
|
1939
|
+
writer: &objc2::runtime::AnyObject,
|
|
1940
|
+
) -> Result<objc2::rc::Retained<objc2::runtime::AnyObject>, String> {
|
|
1941
|
+
use objc2::msg_send;
|
|
1942
|
+
use objc2::runtime::AnyObject;
|
|
1943
|
+
|
|
1944
|
+
#[link(name = "AVFoundation", kind = "framework")]
|
|
1945
|
+
extern "C" {
|
|
1946
|
+
static AVMediaTypeAudio: *const AnyObject;
|
|
1947
|
+
}
|
|
1948
|
+
|
|
1949
|
+
let settings = av_audio_output_settings()?;
|
|
1950
|
+
let raw: *mut AnyObject = msg_send![
|
|
1951
|
+
input_cls,
|
|
1952
|
+
assetWriterInputWithMediaType: AVMediaTypeAudio,
|
|
1953
|
+
outputSettings: &*settings
|
|
1954
|
+
];
|
|
1955
|
+
if raw.is_null() {
|
|
1956
|
+
return Err("AVAssetWriterInput audio allocation failed".into());
|
|
1957
|
+
}
|
|
1958
|
+
let input = objc2::rc::Retained::retain(raw)
|
|
1959
|
+
.ok_or_else(|| "AVAssetWriterInput audio retain failed".to_string())?;
|
|
1960
|
+
let _: () = msg_send![&*input, setExpectsMediaDataInRealTime: true];
|
|
1961
|
+
let can_add: bool = msg_send![writer, canAddInput: &*input];
|
|
1962
|
+
if !can_add {
|
|
1963
|
+
return Err("AVAssetWriter cannot add audio input".into());
|
|
1964
|
+
}
|
|
1965
|
+
let _: () = msg_send![writer, addInput: &*input];
|
|
1966
|
+
Ok(input)
|
|
1967
|
+
}
|
|
1968
|
+
|
|
1969
|
+
/// Render an `NSError` for logs: description, domain+code, failure reason,
|
|
1970
|
+
/// and the underlying error's domain+code (where the raw OSStatus that
|
|
1971
|
+
/// names the real cause usually hides).
|
|
1972
|
+
unsafe fn av_error_suffix(err_obj: *mut objc2::runtime::AnyObject) -> String {
|
|
1973
|
+
if err_obj.is_null() {
|
|
1974
|
+
return String::new();
|
|
1975
|
+
}
|
|
1976
|
+
let desc_obj: *mut objc2::runtime::AnyObject = objc2::msg_send![err_obj, localizedDescription];
|
|
1977
|
+
let mut out = av_string_suffix(desc_obj);
|
|
1978
|
+
|
|
1979
|
+
// Domain + code identify the error class; the localizedDescription alone
|
|
1980
|
+
// is usually a generic "The operation could not be completed".
|
|
1981
|
+
let domain_obj: *mut objc2::runtime::AnyObject = objc2::msg_send![err_obj, domain];
|
|
1982
|
+
let code: i64 = objc2::msg_send![err_obj, code];
|
|
1983
|
+
out.push_str(&format!(
|
|
1984
|
+
" [domain{} code={code}]",
|
|
1985
|
+
av_string_suffix(domain_obj)
|
|
1986
|
+
));
|
|
1987
|
+
|
|
1988
|
+
let reason_obj: *mut objc2::runtime::AnyObject =
|
|
1989
|
+
objc2::msg_send![err_obj, localizedFailureReason];
|
|
1990
|
+
if !reason_obj.is_null() {
|
|
1991
|
+
out.push_str(&format!(" reason{}", av_string_suffix(reason_obj)));
|
|
1992
|
+
}
|
|
1993
|
+
|
|
1994
|
+
// The underlying error carries the raw OSStatus naming the real cause
|
|
1995
|
+
// (e.g. CoreMedia -12780); surface its domain + code too.
|
|
1996
|
+
if let Some(cls) = av_class_named("NSString") {
|
|
1997
|
+
let key_cstr = b"NSUnderlyingError\0".as_ptr() as *const i8;
|
|
1998
|
+
let key: *mut objc2::runtime::AnyObject =
|
|
1999
|
+
objc2::msg_send![cls, stringWithUTF8String: key_cstr];
|
|
2000
|
+
if !key.is_null() {
|
|
2001
|
+
let user_info: *mut objc2::runtime::AnyObject = objc2::msg_send![err_obj, userInfo];
|
|
2002
|
+
if !user_info.is_null() {
|
|
2003
|
+
let underlying: *mut objc2::runtime::AnyObject =
|
|
2004
|
+
objc2::msg_send![user_info, objectForKey: key];
|
|
2005
|
+
if !underlying.is_null() {
|
|
2006
|
+
let u_domain: *mut objc2::runtime::AnyObject =
|
|
2007
|
+
objc2::msg_send![underlying, domain];
|
|
2008
|
+
let u_code: i64 = objc2::msg_send![underlying, code];
|
|
2009
|
+
out.push_str(&format!(
|
|
2010
|
+
" underlying=[domain{} code={u_code}]",
|
|
2011
|
+
av_string_suffix(u_domain)
|
|
2012
|
+
));
|
|
2013
|
+
}
|
|
2014
|
+
}
|
|
2015
|
+
}
|
|
2016
|
+
}
|
|
2017
|
+
out
|
|
2018
|
+
}
|
|
2019
|
+
|
|
2020
|
+
/// Render the writer's current status + `error` property for logs.
|
|
2021
|
+
unsafe fn av_writer_error_suffix(writer: &objc2::runtime::AnyObject) -> String {
|
|
2022
|
+
let status: i64 = objc2::msg_send![writer, status];
|
|
2023
|
+
let err_obj: *mut objc2::runtime::AnyObject = objc2::msg_send![writer, error];
|
|
2024
|
+
format!(" (writer status={status}){}", av_error_suffix(err_obj))
|
|
2025
|
+
}
|
|
2026
|
+
|
|
2027
|
+
/// `": <string>"` from an `NSString` pointer, or empty when nil.
|
|
2028
|
+
unsafe fn av_string_suffix(obj: *mut objc2::runtime::AnyObject) -> String {
|
|
2029
|
+
if obj.is_null() {
|
|
2030
|
+
return String::new();
|
|
2031
|
+
}
|
|
2032
|
+
let utf8: *const i8 = objc2::msg_send![obj, UTF8String];
|
|
2033
|
+
if utf8.is_null() {
|
|
2034
|
+
return String::new();
|
|
2035
|
+
}
|
|
2036
|
+
let cstr = std::ffi::CStr::from_ptr(utf8);
|
|
2037
|
+
format!(": {}", cstr.to_string_lossy())
|
|
2038
|
+
}
|
|
2039
|
+
|
|
2040
|
+
/// Plain, `Send` capture parameters kept so the watchdog can rebuild the
|
|
2041
|
+
/// SCStream from scratch after an interruption without holding on to any
|
|
2042
|
+
/// non-`Send` ScreenCaptureKit handles. `width`/`height` are fixed to the
|
|
2043
|
+
/// dimensions the writer was created with so a rebuilt stream keeps producing
|
|
2044
|
+
/// frames the existing video input accepts.
|
|
2045
|
+
#[derive(Clone)]
|
|
2046
|
+
struct RestartParams {
|
|
2047
|
+
include_audio: bool,
|
|
2048
|
+
capture_system_audio: bool,
|
|
2049
|
+
mic_device_id: Option<String>,
|
|
2050
|
+
mic_device_label: Option<String>,
|
|
2051
|
+
target_display_id: Option<u32>,
|
|
2052
|
+
capture_region: Option<NativeCaptureRegion>,
|
|
2053
|
+
width: u32,
|
|
2054
|
+
height: u32,
|
|
2055
|
+
}
|
|
2056
|
+
|
|
2057
|
+
/// Everything the pause/resume path needs to stop the capture source without
|
|
2058
|
+
/// tearing down the writer, and to splice a fresh SCStream onto the same
|
|
2059
|
+
/// (append-only) file on resume. Cheap to clone — all handles are `Arc`s or
|
|
2060
|
+
/// plain data — so it lives alongside the backend and shares the watchdog's
|
|
2061
|
+
/// stream/handler/watch.
|
|
2062
|
+
pub(crate) struct CustomCaptureResume {
|
|
2063
|
+
stream: Arc<Mutex<SCStream>>,
|
|
2064
|
+
handler: CustomScreenCaptureOutputHandler,
|
|
2065
|
+
watch: Arc<CaptureWatch>,
|
|
2066
|
+
params: RestartParams,
|
|
2067
|
+
}
|
|
2068
|
+
|
|
2069
|
+
impl CustomCaptureResume {
|
|
2070
|
+
/// Pause: stop only the capture source (SCStream). The writer, file, and
|
|
2071
|
+
/// live uploader stay alive; mic/screen go cold. The watchdog is told to
|
|
2072
|
+
/// hold so it never reads the silence as a stall and rebuilds.
|
|
2073
|
+
pub(crate) fn pause(&self) {
|
|
2074
|
+
self.watch.set_paused(true);
|
|
2075
|
+
if let Ok(guard) = self.stream.lock() {
|
|
2076
|
+
let _ = guard.stop_capture();
|
|
2077
|
+
}
|
|
2078
|
+
eprintln!("[mixer] capture paused; source stream stopped, writer/file kept open");
|
|
2079
|
+
}
|
|
2080
|
+
|
|
2081
|
+
/// Resume: build a fresh SCStream wired to the SAME writer and start it,
|
|
2082
|
+
/// after advancing the writer's pause offset by `paused_for` so the new
|
|
2083
|
+
/// samples rebase past the pause gap. The result is one continuous
|
|
2084
|
+
/// append-only file — the live uploader is never interrupted, exactly like
|
|
2085
|
+
/// a watchdog stream rebuild.
|
|
2086
|
+
pub(crate) fn resume(&self, paused_for: Duration) -> Result<(), String> {
|
|
2087
|
+
let writer = &self.handler.writer;
|
|
2088
|
+
let prev_offset = writer.pause_offset();
|
|
2089
|
+
|
|
2090
|
+
// Build the replacement stream FIRST: a build failure must not shift the
|
|
2091
|
+
// timeline. The session stays paused and can be retried — and a retry
|
|
2092
|
+
// re-measures `paused_for` from the same (uncleared) pause instant, so
|
|
2093
|
+
// advancing the offset here would compound on every failed attempt.
|
|
2094
|
+
let new_stream = build_custom_scstream(&self.params, &self.handler, &self.watch)?;
|
|
2095
|
+
|
|
2096
|
+
// Apply the pause gap just before the stream starts delivering, so the
|
|
2097
|
+
// first rebased frame already skips it. Roll back if startup fails so
|
|
2098
|
+
// the failed attempt leaves the offset exactly as it was.
|
|
2099
|
+
writer.set_pause_offset(prev_offset + paused_for.as_secs_f64());
|
|
2100
|
+
if let Err(err) = new_stream.start_capture() {
|
|
2101
|
+
writer.set_pause_offset(prev_offset);
|
|
2102
|
+
return Err(format!("resume start_capture failed: {err:?}"));
|
|
2103
|
+
}
|
|
2104
|
+
|
|
2105
|
+
// Stop any lingering paused stream, then swap the fresh one in under
|
|
2106
|
+
// the same lock the watchdog uses so the two never feed at once.
|
|
2107
|
+
if let Ok(guard) = self.stream.lock() {
|
|
2108
|
+
let _ = guard.stop_capture();
|
|
2109
|
+
}
|
|
2110
|
+
if let Ok(mut guard) = self.stream.lock() {
|
|
2111
|
+
*guard = new_stream;
|
|
2112
|
+
}
|
|
2113
|
+
// Discard the stop note our own pause/stop raised and reset the
|
|
2114
|
+
// activity clock so the watchdog doesn't immediately treat the just-
|
|
2115
|
+
// rebuilt stream as stalled, then hand supervision back.
|
|
2116
|
+
let _ = self.watch.take_stream_stopped();
|
|
2117
|
+
self.watch.note_activity();
|
|
2118
|
+
self.watch.set_paused(false);
|
|
2119
|
+
eprintln!(
|
|
2120
|
+
"[mixer] capture resumed; fresh stream spliced onto same writer (paused {}ms)",
|
|
2121
|
+
paused_for.as_millis()
|
|
2122
|
+
);
|
|
2123
|
+
Ok(())
|
|
2124
|
+
}
|
|
2125
|
+
}
|
|
2126
|
+
|
|
2127
|
+
/// Stream lifecycle delegate for the custom pipeline. ScreenCaptureKit calls
|
|
2128
|
+
/// this when it stops the stream (e.g. the captured display changed Spaces or a
|
|
2129
|
+
/// full-screen app took over). Without it those stops are invisible and the
|
|
2130
|
+
/// recording silently freezes. We only flag the watchdog here — rebuilding the
|
|
2131
|
+
/// stream from an SCK callback thread is unsafe, so recovery happens off-thread.
|
|
2132
|
+
struct CustomCaptureStreamDelegate {
|
|
2133
|
+
watch: Arc<CaptureWatch>,
|
|
2134
|
+
}
|
|
2135
|
+
|
|
2136
|
+
impl SCStreamDelegateTrait for CustomCaptureStreamDelegate {
|
|
2137
|
+
fn did_stop_with_error(&self, error: SCError) {
|
|
2138
|
+
// The user stopping capture via macOS (menu-bar "Stop Sharing") is a
|
|
2139
|
+
// request, not a failure — rebuilding the stream would fight the
|
|
2140
|
+
// user. Everything else (SystemStoppedStream on lid close / display
|
|
2141
|
+
// sleep, connection failures, ...) goes to the rebuild path.
|
|
2142
|
+
if error.stream_error_code()
|
|
2143
|
+
== Some(screencapturekit::error::SCStreamErrorCode::UserStopped)
|
|
2144
|
+
{
|
|
2145
|
+
eprintln!("[mixer] capture stopped by the user via macOS; requesting recording stop");
|
|
2146
|
+
self.watch.note_user_stopped();
|
|
2147
|
+
return;
|
|
2148
|
+
}
|
|
2149
|
+
let reason = format!("ScreenCaptureKit stream stopped with error: {error}");
|
|
2150
|
+
eprintln!("[mixer] {reason}");
|
|
2151
|
+
self.watch.note_stream_stopped(reason);
|
|
2152
|
+
}
|
|
2153
|
+
|
|
2154
|
+
fn stream_did_stop(&self, error: Option<String>) {
|
|
2155
|
+
let detail = error.unwrap_or_else(|| "no detail".to_string());
|
|
2156
|
+
let reason = format!("ScreenCaptureKit stream stopped: {detail}");
|
|
2157
|
+
eprintln!("[mixer] {reason}");
|
|
2158
|
+
self.watch.note_stream_stopped(reason);
|
|
2159
|
+
}
|
|
2160
|
+
}
|
|
2161
|
+
|
|
2162
|
+
/// Build (but do not start) a fresh SCStream for the custom pipeline from plain
|
|
2163
|
+
/// parameters. Shared by the initial start and every watchdog rebuild so the
|
|
2164
|
+
/// filter/config/handler/delegate wiring can never drift between them.
|
|
2165
|
+
fn build_custom_scstream(
|
|
2166
|
+
params: &RestartParams,
|
|
2167
|
+
handler: &CustomScreenCaptureOutputHandler,
|
|
2168
|
+
watch: &Arc<CaptureWatch>,
|
|
2169
|
+
) -> Result<SCStream, String> {
|
|
2170
|
+
let content =
|
|
2171
|
+
SCShareableContent::get().map_err(|e| format!("shareable content lookup failed: {e:?}"))?;
|
|
2172
|
+
let displays = content.displays();
|
|
2173
|
+
let display = params
|
|
2174
|
+
.target_display_id
|
|
2175
|
+
.and_then(|id| displays.iter().find(|d| d.display_id() == id))
|
|
2176
|
+
.or_else(|| displays.first())
|
|
2177
|
+
.ok_or_else(|| "No displays available for ScreenCaptureKit recording.".to_string())?;
|
|
2178
|
+
|
|
2179
|
+
let region_rect = region_source_rect(params.capture_region, display.width(), display.height())?;
|
|
2180
|
+
let filter_builder = SCContentFilter::create()
|
|
2181
|
+
.with_display(display)
|
|
2182
|
+
.with_excluding_windows(&[]);
|
|
2183
|
+
let filter = if let Some((rect, _, _)) = region_rect {
|
|
2184
|
+
filter_builder.with_content_rect(rect).build()
|
|
2185
|
+
} else {
|
|
2186
|
+
filter_builder.build()
|
|
2187
|
+
};
|
|
2188
|
+
|
|
2189
|
+
let selected_mic = if params.include_audio {
|
|
2190
|
+
resolve_microphone_capture_device(
|
|
2191
|
+
params.mic_device_id.as_deref(),
|
|
2192
|
+
params.mic_device_label.as_deref(),
|
|
2193
|
+
)?
|
|
2194
|
+
} else {
|
|
2195
|
+
None
|
|
2196
|
+
};
|
|
2197
|
+
|
|
2198
|
+
let mut config = SCStreamConfiguration::new()
|
|
2199
|
+
.with_width(params.width)
|
|
2200
|
+
.with_height(params.height)
|
|
2201
|
+
.with_fps(NATIVE_CAPTURE_FPS)
|
|
2202
|
+
.with_queue_depth(8)
|
|
2203
|
+
.with_shows_cursor(true)
|
|
2204
|
+
.with_captures_audio(params.capture_system_audio)
|
|
2205
|
+
.with_captures_microphone(params.include_audio)
|
|
2206
|
+
.with_excludes_current_process_audio(true)
|
|
2207
|
+
.with_sample_rate(48000)
|
|
2208
|
+
.with_channel_count(2);
|
|
2209
|
+
// Pin SDR NV12 (video-range 4:2:0) delivery, matching Cap. Two reasons:
|
|
2210
|
+
// - Without a pin, ScreenCaptureKit switches the delivered pixel format
|
|
2211
|
+
// to HDR/EDR variants (half-float / 10-bit) when the frontmost app
|
|
2212
|
+
// renders EDR content; the SDR H.264 writer input then rejects every
|
|
2213
|
+
// appended frame (-11800 / OSStatus -16122) and the writer dies —
|
|
2214
|
+
// including on rebuilt streams while that app stays frontmost.
|
|
2215
|
+
// - NV12 is VideoToolbox's native encoder input and half the memory
|
|
2216
|
+
// bandwidth of BGRA.
|
|
2217
|
+
// NOTE: do not add `set_color_space_name` — the crate's ObjC shim for it
|
|
2218
|
+
// raises an uncatchable Objective-C exception and aborts the process;
|
|
2219
|
+
// the pixel-format pin alone keeps the encoder input format stable.
|
|
2220
|
+
config.set_pixel_format(screencapturekit::stream::configuration::PixelFormat::YCbCr_420v);
|
|
2221
|
+
if let Some((rect, _, _)) = region_rect {
|
|
2222
|
+
config.set_source_rect(rect);
|
|
2223
|
+
}
|
|
2224
|
+
if let Some(device) = selected_mic.as_ref() {
|
|
2225
|
+
config.set_microphone_capture_device_id(&device.id);
|
|
2226
|
+
}
|
|
2227
|
+
config.set_stream_name(Some("Clips custom full-screen recording"));
|
|
2228
|
+
|
|
2229
|
+
let delegate = CustomCaptureStreamDelegate {
|
|
2230
|
+
watch: Arc::clone(watch),
|
|
2231
|
+
};
|
|
2232
|
+
let mut stream = SCStream::new_with_delegate(&filter, &config, delegate);
|
|
2233
|
+
stream.add_output_handler(handler.clone(), SCStreamOutputType::Screen);
|
|
2234
|
+
if params.capture_system_audio {
|
|
2235
|
+
stream.add_output_handler(handler.clone(), SCStreamOutputType::Audio);
|
|
2236
|
+
}
|
|
2237
|
+
if params.include_audio {
|
|
2238
|
+
stream.add_output_handler(handler.clone(), SCStreamOutputType::Microphone);
|
|
2239
|
+
}
|
|
2240
|
+
Ok(stream)
|
|
2241
|
+
}
|
|
2242
|
+
|
|
2243
|
+
/// Supervise a running custom capture and rebuild the SCStream when it stops or
|
|
2244
|
+
/// goes silent (the Spaces/full-screen interruption). Runs on its own thread;
|
|
2245
|
+
/// exits when recording is torn down (`shutdown`) or the writer is closed.
|
|
2246
|
+
fn spawn_capture_watchdog(
|
|
2247
|
+
app: AppHandle,
|
|
2248
|
+
stream: Arc<Mutex<SCStream>>,
|
|
2249
|
+
writer: CustomScreenCaptureWriter,
|
|
2250
|
+
handler: CustomScreenCaptureOutputHandler,
|
|
2251
|
+
watch: Arc<CaptureWatch>,
|
|
2252
|
+
recording_enabled: Arc<AtomicBool>,
|
|
2253
|
+
shutdown: Arc<AtomicBool>,
|
|
2254
|
+
params: RestartParams,
|
|
2255
|
+
) {
|
|
2256
|
+
std::thread::spawn(move || {
|
|
2257
|
+
let mut restart_streak: u32 = 0;
|
|
2258
|
+
// After a rebuild, hold off re-evaluating until the new stream has had a
|
|
2259
|
+
// fair chance to deliver its first frames.
|
|
2260
|
+
let mut cooldown_until: Option<Instant> = None;
|
|
2261
|
+
|
|
2262
|
+
loop {
|
|
2263
|
+
std::thread::sleep(CAPTURE_WATCHDOG_POLL);
|
|
2264
|
+
if shutdown.load(Ordering::SeqCst) {
|
|
2265
|
+
return;
|
|
2266
|
+
}
|
|
2267
|
+
if writer.appends_closed.load(Ordering::SeqCst) {
|
|
2268
|
+
// Appends closed while we're still supervising. A clean stop
|
|
2269
|
+
// sets `watchdog_shutdown` BEFORE closing appends (SeqCst), so
|
|
2270
|
+
// re-check it: if it's still unset, appends were closed by an
|
|
2271
|
+
// unrecoverable capture failure (an Objective-C exception or
|
|
2272
|
+
// panic in the sample callback, an `appendSampleBuffer` error,
|
|
2273
|
+
// or a writer-session failure) with the SCStream still running
|
|
2274
|
+
// and the UI still showing "recording". Stop capture and fire
|
|
2275
|
+
// the normal stop so the partial — still a valid fragmented
|
|
2276
|
+
// file — is finalized/uploaded instead of silently truncating.
|
|
2277
|
+
if shutdown.load(Ordering::SeqCst) {
|
|
2278
|
+
return;
|
|
2279
|
+
}
|
|
2280
|
+
eprintln!(
|
|
2281
|
+
"[mixer] appends closed without a stop request; treating as fatal capture failure and finalizing partial recording"
|
|
2282
|
+
);
|
|
2283
|
+
if let Ok(guard) = stream.lock() {
|
|
2284
|
+
let _ = guard.stop_capture();
|
|
2285
|
+
}
|
|
2286
|
+
let _ = app.emit("clips:recorder-stop", ());
|
|
2287
|
+
return;
|
|
2288
|
+
}
|
|
2289
|
+
// Nothing to supervise until output is enabled and the writer
|
|
2290
|
+
// session has actually begun; the handler keeps `last_activity`
|
|
2291
|
+
// fresh from the first delivered buffer, so there is no false stall
|
|
2292
|
+
// when we start evaluating.
|
|
2293
|
+
if !recording_enabled.load(Ordering::SeqCst) || !writer.started.load(Ordering::SeqCst) {
|
|
2294
|
+
continue;
|
|
2295
|
+
}
|
|
2296
|
+
|
|
2297
|
+
// Paused: the capture source is intentionally stopped while the
|
|
2298
|
+
// writer/file/uploader stay alive. Don't read the silence as a
|
|
2299
|
+
// stall or rebuild — resume splices a fresh stream back in.
|
|
2300
|
+
if watch.is_paused() {
|
|
2301
|
+
restart_streak = 0;
|
|
2302
|
+
cooldown_until = None;
|
|
2303
|
+
continue;
|
|
2304
|
+
}
|
|
2305
|
+
|
|
2306
|
+
// User stopped capture from the macOS UI: trigger the normal stop
|
|
2307
|
+
// flow (same event the toolbar Stop button emits, so the clip is
|
|
2308
|
+
// finalized and uploaded) and stop supervising. Never rebuild.
|
|
2309
|
+
if watch.user_stopped() {
|
|
2310
|
+
eprintln!("[mixer] user stopped capture via macOS; emitting recorder stop");
|
|
2311
|
+
let _ = app.emit("clips:recorder-stop", ());
|
|
2312
|
+
return;
|
|
2313
|
+
}
|
|
2314
|
+
|
|
2315
|
+
if let Some(until) = cooldown_until {
|
|
2316
|
+
if Instant::now() < until {
|
|
2317
|
+
continue;
|
|
2318
|
+
}
|
|
2319
|
+
cooldown_until = None;
|
|
2320
|
+
if watch.since_activity() < CAPTURE_STALL_TIMEOUT {
|
|
2321
|
+
// Rebuild recovered — frames are flowing again.
|
|
2322
|
+
restart_streak = 0;
|
|
2323
|
+
continue;
|
|
2324
|
+
}
|
|
2325
|
+
// Still no frames after the rebuild; fall through and retry.
|
|
2326
|
+
}
|
|
2327
|
+
|
|
2328
|
+
let reported_stop = watch.take_stream_stopped();
|
|
2329
|
+
let stalled = watch.since_activity() >= CAPTURE_STALL_TIMEOUT;
|
|
2330
|
+
if reported_stop.is_none() && !stalled {
|
|
2331
|
+
restart_streak = 0;
|
|
2332
|
+
continue;
|
|
2333
|
+
}
|
|
2334
|
+
|
|
2335
|
+
let reason = reported_stop.unwrap_or_else(|| {
|
|
2336
|
+
format!(
|
|
2337
|
+
"no capture frames for {:?} (display may have moved to another Space)",
|
|
2338
|
+
watch.since_activity()
|
|
2339
|
+
)
|
|
2340
|
+
});
|
|
2341
|
+
restart_streak += 1;
|
|
2342
|
+
if restart_streak > CAPTURE_MAX_RESTARTS {
|
|
2343
|
+
eprintln!(
|
|
2344
|
+
"[mixer] capture interrupted ({reason}) and did not recover after {CAPTURE_MAX_RESTARTS} restarts; finalizing partial recording"
|
|
2345
|
+
);
|
|
2346
|
+
// Close appends (but leave `failed` unset) so the stop path
|
|
2347
|
+
// still finalizes everything captured before the interruption
|
|
2348
|
+
// instead of discarding it.
|
|
2349
|
+
writer.appends_closed.store(true, Ordering::SeqCst);
|
|
2350
|
+
if let Ok(guard) = stream.lock() {
|
|
2351
|
+
let _ = guard.stop_capture();
|
|
2352
|
+
}
|
|
2353
|
+
// Fire the normal stop so the partial clip is finalized/uploaded
|
|
2354
|
+
// and the UI leaves the recording state, rather than sitting on
|
|
2355
|
+
// a frozen recording after the watchdog gives up.
|
|
2356
|
+
let _ = app.emit("clips:recorder-stop", ());
|
|
2357
|
+
return;
|
|
2358
|
+
}
|
|
2359
|
+
|
|
2360
|
+
eprintln!(
|
|
2361
|
+
"[mixer] capture interrupted ({reason}); rebuilding stream (attempt {restart_streak}/{CAPTURE_MAX_RESTARTS})"
|
|
2362
|
+
);
|
|
2363
|
+
|
|
2364
|
+
// Stop the dead/wedged stream before starting a replacement so two
|
|
2365
|
+
// streams never feed the writer at once. Slow build/start work is
|
|
2366
|
+
// done without holding the stream lock.
|
|
2367
|
+
if let Ok(guard) = stream.lock() {
|
|
2368
|
+
let _ = guard.stop_capture();
|
|
2369
|
+
}
|
|
2370
|
+
if shutdown.load(Ordering::SeqCst) || writer.appends_closed.load(Ordering::SeqCst) {
|
|
2371
|
+
return;
|
|
2372
|
+
}
|
|
2373
|
+
|
|
2374
|
+
match build_custom_scstream(¶ms, &handler, &watch).and_then(|s| {
|
|
2375
|
+
s.start_capture()
|
|
2376
|
+
.map(|()| s)
|
|
2377
|
+
.map_err(|e| format!("start_capture failed: {e:?}"))
|
|
2378
|
+
}) {
|
|
2379
|
+
Ok(new_stream) => {
|
|
2380
|
+
if shutdown.load(Ordering::SeqCst)
|
|
2381
|
+
|| writer.appends_closed.load(Ordering::SeqCst)
|
|
2382
|
+
{
|
|
2383
|
+
let _ = new_stream.stop_capture();
|
|
2384
|
+
return;
|
|
2385
|
+
}
|
|
2386
|
+
if let Ok(mut guard) = stream.lock() {
|
|
2387
|
+
*guard = new_stream;
|
|
2388
|
+
}
|
|
2389
|
+
// Discard any stop note the deliberate teardown of the old
|
|
2390
|
+
// stream raised; otherwise a later poll would consume it as
|
|
2391
|
+
// a fresh failure and rebuild the healthy stream again.
|
|
2392
|
+
let _ = watch.take_stream_stopped();
|
|
2393
|
+
cooldown_until = Some(Instant::now() + CAPTURE_STALL_TIMEOUT);
|
|
2394
|
+
eprintln!("[mixer] capture stream rebuilt; waiting for frames");
|
|
2395
|
+
}
|
|
2396
|
+
Err(err) => {
|
|
2397
|
+
eprintln!("[mixer] capture rebuild failed: {err}");
|
|
2398
|
+
// Short cooldown before the next attempt so a hard failure
|
|
2399
|
+
// (e.g. display gone) doesn't spin the CPU.
|
|
2400
|
+
cooldown_until = Some(Instant::now() + CAPTURE_WATCHDOG_POLL);
|
|
2401
|
+
}
|
|
2402
|
+
}
|
|
2403
|
+
}
|
|
2404
|
+
});
|
|
2405
|
+
}
|
|
2406
|
+
|
|
2407
|
+
/// Start the custom capture backend: create the fragmented-MP4 writer,
|
|
2408
|
+
/// build + start the SCStream from rebuildable params, and spawn the
|
|
2409
|
+
/// capture watchdog that supervises it. Returns the backend handle plus
|
|
2410
|
+
/// the output dimensions.
|
|
2411
|
+
pub(super) fn start_custom_screencapturekit_backend_at(
|
|
2412
|
+
app: &AppHandle,
|
|
2413
|
+
output_path: &Path,
|
|
2414
|
+
include_audio: bool,
|
|
2415
|
+
capture_system_audio: bool,
|
|
2416
|
+
mic_device_id: Option<&str>,
|
|
2417
|
+
mic_device_label: Option<&str>,
|
|
2418
|
+
target_display_id: Option<u32>,
|
|
2419
|
+
capture_region: Option<NativeCaptureRegion>,
|
|
2420
|
+
defer_recording_output: bool,
|
|
2421
|
+
) -> Result<(NativeFullscreenBackend, Option<u32>, Option<u32>), String> {
|
|
2422
|
+
eprintln!("[clips-tray] starting custom screen capture backend");
|
|
2423
|
+
let content =
|
|
2424
|
+
SCShareableContent::get().map_err(|e| format!("shareable content lookup failed: {e:?}"))?;
|
|
2425
|
+
let displays = content.displays();
|
|
2426
|
+
let display = target_display_id
|
|
2427
|
+
.and_then(|id| displays.iter().find(|d| d.display_id() == id))
|
|
2428
|
+
.or_else(|| displays.first())
|
|
2429
|
+
.ok_or_else(|| "No displays available for ScreenCaptureKit recording.".to_string())?;
|
|
2430
|
+
|
|
2431
|
+
let source_width = display.width();
|
|
2432
|
+
let source_height = display.height();
|
|
2433
|
+
let region_rect = region_source_rect(capture_region, source_width, source_height)?;
|
|
2434
|
+
let (capture_width, capture_height) = region_rect
|
|
2435
|
+
.as_ref()
|
|
2436
|
+
.map(|(_, width, height)| (*width, *height))
|
|
2437
|
+
.unwrap_or((source_width, source_height));
|
|
2438
|
+
let (width, height) = native_capture_dimensions(capture_width, capture_height);
|
|
2439
|
+
// The display handle here is only used to size the output; the actual
|
|
2440
|
+
// (rebuildable) stream is constructed from plain params via
|
|
2441
|
+
// `build_custom_scstream` so the watchdog can recreate it later.
|
|
2442
|
+
|
|
2443
|
+
let params = RestartParams {
|
|
2444
|
+
include_audio,
|
|
2445
|
+
capture_system_audio,
|
|
2446
|
+
mic_device_id: mic_device_id.map(str::to_string),
|
|
2447
|
+
mic_device_label: mic_device_label.map(str::to_string),
|
|
2448
|
+
target_display_id,
|
|
2449
|
+
capture_region,
|
|
2450
|
+
width,
|
|
2451
|
+
height,
|
|
2452
|
+
};
|
|
2453
|
+
|
|
2454
|
+
// Live-mix only when both audio sources are present; a single source needs
|
|
2455
|
+
// no mixing and goes straight to its own track.
|
|
2456
|
+
let mix_live = include_audio && capture_system_audio;
|
|
2457
|
+
let writer = CustomScreenCaptureWriter::new(
|
|
2458
|
+
output_path,
|
|
2459
|
+
width,
|
|
2460
|
+
height,
|
|
2461
|
+
capture_system_audio,
|
|
2462
|
+
include_audio,
|
|
2463
|
+
mix_live,
|
|
2464
|
+
)?;
|
|
2465
|
+
let recording_enabled = Arc::new(AtomicBool::new(!defer_recording_output));
|
|
2466
|
+
let mic_ready = include_audio.then(|| Arc::new(AtomicBool::new(false)));
|
|
2467
|
+
let watch = Arc::new(CaptureWatch::new());
|
|
2468
|
+
let handler = CustomScreenCaptureOutputHandler {
|
|
2469
|
+
writer: writer.clone(),
|
|
2470
|
+
recording_enabled: Arc::clone(&recording_enabled),
|
|
2471
|
+
mic_ready: mic_ready.clone(),
|
|
2472
|
+
watch: Arc::clone(&watch),
|
|
2473
|
+
};
|
|
2474
|
+
|
|
2475
|
+
let stream = build_custom_scstream(¶ms, &handler, &watch)?;
|
|
2476
|
+
if let Err(err) = stream.start_capture() {
|
|
2477
|
+
let _ = std::fs::remove_file(output_path);
|
|
2478
|
+
return Err(format!("custom capture start failed: {err:?}"));
|
|
2479
|
+
}
|
|
2480
|
+
eprintln!(
|
|
2481
|
+
"[clips-tray] custom ScreenCaptureKit recording started: {width}x{height} @ {NATIVE_CAPTURE_FPS}fps from {capture_width}x{capture_height} (display {source_width}x{source_height}), mic={include_audio} system_audio={capture_system_audio} deferred_output={defer_recording_output}"
|
|
2482
|
+
);
|
|
2483
|
+
|
|
2484
|
+
let stream = Arc::new(Mutex::new(stream));
|
|
2485
|
+
let watchdog_shutdown = Arc::new(AtomicBool::new(false));
|
|
2486
|
+
// Snapshot the handles pause/resume needs before the watchdog consumes
|
|
2487
|
+
// `handler` and `params`; the fresh stream it builds on resume feeds the
|
|
2488
|
+
// same writer as the watchdog's own rebuilds.
|
|
2489
|
+
let resume = CustomCaptureResume {
|
|
2490
|
+
stream: Arc::clone(&stream),
|
|
2491
|
+
handler: handler.clone(),
|
|
2492
|
+
watch: Arc::clone(&watch),
|
|
2493
|
+
params: params.clone(),
|
|
2494
|
+
};
|
|
2495
|
+
spawn_capture_watchdog(
|
|
2496
|
+
app.clone(),
|
|
2497
|
+
Arc::clone(&stream),
|
|
2498
|
+
writer.clone(),
|
|
2499
|
+
handler,
|
|
2500
|
+
Arc::clone(&watch),
|
|
2501
|
+
Arc::clone(&recording_enabled),
|
|
2502
|
+
Arc::clone(&watchdog_shutdown),
|
|
2503
|
+
params,
|
|
2504
|
+
);
|
|
2505
|
+
|
|
2506
|
+
Ok((
|
|
2507
|
+
NativeFullscreenBackend::CustomScreenCaptureKit {
|
|
2508
|
+
stream,
|
|
2509
|
+
writer,
|
|
2510
|
+
mic_ready,
|
|
2511
|
+
recording_enabled,
|
|
2512
|
+
watchdog_shutdown,
|
|
2513
|
+
resume,
|
|
2514
|
+
},
|
|
2515
|
+
Some(width),
|
|
2516
|
+
Some(height),
|
|
2517
|
+
))
|
|
2518
|
+
}
|