@agent-native/core 0.100.2 → 0.101.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (388) hide show
  1. package/corpus/README.md +2 -2
  2. package/corpus/core/CHANGELOG.md +83 -0
  3. package/corpus/core/docs/content/locales/ar-SA/pr-visual-recap.mdx +48 -30
  4. package/corpus/core/docs/content/locales/ar-SA/template-plan.mdx +7 -0
  5. package/corpus/core/docs/content/locales/de-DE/pr-visual-recap.mdx +48 -30
  6. package/corpus/core/docs/content/locales/de-DE/template-plan.mdx +7 -0
  7. package/corpus/core/docs/content/locales/es-ES/pr-visual-recap.mdx +48 -30
  8. package/corpus/core/docs/content/locales/es-ES/template-plan.mdx +14 -0
  9. package/corpus/core/docs/content/locales/fr-FR/pr-visual-recap.mdx +48 -30
  10. package/corpus/core/docs/content/locales/fr-FR/template-plan.mdx +7 -0
  11. package/corpus/core/docs/content/locales/hi-IN/pr-visual-recap.mdx +48 -30
  12. package/corpus/core/docs/content/locales/hi-IN/template-plan.mdx +7 -0
  13. package/corpus/core/docs/content/locales/ja-JP/pr-visual-recap.mdx +48 -30
  14. package/corpus/core/docs/content/locales/ja-JP/template-plan.mdx +7 -0
  15. package/corpus/core/docs/content/locales/ko-KR/pr-visual-recap.mdx +48 -30
  16. package/corpus/core/docs/content/locales/ko-KR/template-plan.mdx +7 -0
  17. package/corpus/core/docs/content/locales/pt-BR/pr-visual-recap.mdx +48 -30
  18. package/corpus/core/docs/content/locales/pt-BR/template-plan.mdx +14 -0
  19. package/corpus/core/docs/content/locales/zh-CN/pr-visual-recap.mdx +48 -30
  20. package/corpus/core/docs/content/locales/zh-CN/template-plan.mdx +7 -0
  21. package/corpus/core/docs/content/locales/zh-TW/pr-visual-recap.mdx +48 -30
  22. package/corpus/core/docs/content/locales/zh-TW/template-plan.mdx +7 -0
  23. package/corpus/core/docs/content/pr-visual-recap.mdx +50 -32
  24. package/corpus/core/docs/content/template-plan.mdx +14 -0
  25. package/corpus/core/docs/content/toolkit-comments-review.mdx +16 -2
  26. package/corpus/core/package.json +4 -1
  27. package/corpus/core/src/a2a/index.ts +2 -1
  28. package/corpus/core/src/a2a/server.ts +34 -9
  29. package/corpus/core/src/agent/production-agent.ts +56 -10
  30. package/corpus/core/src/cli/create.ts +47 -19
  31. package/corpus/core/src/cli/pr-visual-recap-workflow.ts +2 -3
  32. package/corpus/core/src/cli/recap.ts +3 -4932
  33. package/corpus/core/src/cli/skills-content/visual-plan-skill.ts +6 -4
  34. package/corpus/core/src/cli/skills-content/visual-recap-skill.ts +7 -1
  35. package/corpus/core/src/client/AgentPanel.tsx +19 -17
  36. package/corpus/core/src/client/analytics.ts +8 -1
  37. package/corpus/core/src/client/chat/markdown-renderer.tsx +54 -8
  38. package/corpus/core/src/client/chat/message-components.tsx +8 -2
  39. package/corpus/core/src/client/clipboard.ts +28 -1
  40. package/corpus/core/src/client/guided-questions.tsx +4 -0
  41. package/corpus/core/src/client/i18n.tsx +2 -1
  42. package/corpus/core/src/client/index.ts +4 -0
  43. package/corpus/core/src/client/review/ReviewCommentComposer.tsx +108 -0
  44. package/corpus/core/src/client/review/ReviewThreadPanel.tsx +482 -150
  45. package/corpus/core/src/client/review/index.ts +8 -0
  46. package/corpus/core/src/client/review/use-review.ts +35 -1
  47. package/corpus/core/src/client/use-agent-chat-context.ts +3 -2
  48. package/corpus/core/src/credentials/index.ts +41 -8
  49. package/corpus/core/src/extensions/fetch-tool.ts +33 -3
  50. package/corpus/core/src/notifications/channels.ts +20 -6
  51. package/corpus/core/src/review/actions/create-review-comment.ts +2 -1
  52. package/corpus/core/src/review/actions/get-review-feedback.ts +6 -11
  53. package/corpus/core/src/review/actions/list-review-comments.ts +38 -4
  54. package/corpus/core/src/review/actions/reply-review-comment.ts +33 -22
  55. package/corpus/core/src/review/actions/resolve-review-thread.ts +29 -2
  56. package/corpus/core/src/review/actions/send-review-thread-to-agent.ts +62 -0
  57. package/corpus/core/src/review/identity.ts +101 -0
  58. package/corpus/core/src/review/index.ts +14 -0
  59. package/corpus/core/src/review/store.ts +315 -8
  60. package/corpus/core/src/review/types.ts +4 -0
  61. package/corpus/core/src/secrets/crypto.ts +26 -10
  62. package/corpus/core/src/secrets/storage.ts +16 -3
  63. package/corpus/core/src/server/action-discovery.ts +4 -0
  64. package/corpus/core/src/server/action-routes.ts +125 -4
  65. package/corpus/core/src/server/agent-chat/plugin-options.ts +16 -0
  66. package/corpus/core/src/server/agent-chat-plugin.ts +61 -7
  67. package/corpus/core/src/server/derived-secret.ts +5 -1
  68. package/corpus/core/src/server/index.ts +8 -0
  69. package/corpus/core/src/vite/action-types-plugin.ts +4 -0
  70. package/corpus/templates/analytics/changelog/2026-07-14-daily-dashboard-email-captures-authenticate-with-a-sessi.md +6 -0
  71. package/corpus/templates/analytics/changelog/2026-07-14-long-running-cross-app-analyses-now-continue-reliably-in-the.md +6 -0
  72. package/corpus/templates/analytics/server/lib/dashboard-report.ts +156 -3
  73. package/corpus/templates/analytics/server/plugins/agent-chat.ts +8 -4
  74. package/corpus/templates/assets/app/components/generation/GenerationResults.tsx +402 -142
  75. package/corpus/templates/assets/app/i18n/zh-TW.ts +1 -0
  76. package/corpus/templates/assets/app/i18n-data.ts +1 -0
  77. package/corpus/templates/assets/app/routes/library.tsx +187 -48
  78. package/corpus/templates/assets/changelog/2026-07-13-merged-the-candidates-panel-into-the-library-as-a-drafts-tab.md +6 -0
  79. package/corpus/templates/chat/app/lib/agent-page.tsx +40 -0
  80. package/corpus/templates/chat/app/routes/agent.tsx +5 -3
  81. package/corpus/templates/chat/changelog/2026-07-14-fixed-chat-template-startup-with-older-core-versions.md +6 -0
  82. package/corpus/templates/clips/.agents/skills/ai-video-tools/SKILL.md +5 -0
  83. package/corpus/templates/clips/AGENTS.md +4 -0
  84. package/corpus/templates/clips/actions/get-feature-flags.ts +30 -0
  85. package/corpus/templates/clips/actions/lib/transcript-preview.ts +48 -0
  86. package/corpus/templates/clips/actions/request-transcript.ts +41 -1
  87. package/corpus/templates/clips/actions/view-screen.ts +5 -9
  88. package/corpus/templates/clips/app/components/library/library-layout.tsx +1 -1
  89. package/corpus/templates/clips/app/components/player/share-dialog.tsx +44 -81
  90. package/corpus/templates/clips/app/components/player/video-player.tsx +39 -1
  91. package/corpus/templates/clips/app/hooks/use-desktop-promo.ts +6 -12
  92. package/corpus/templates/clips/app/hooks/use-mse-video-source.ts +146 -0
  93. package/corpus/templates/clips/app/i18n/ar-SA.ts +0 -8
  94. package/corpus/templates/clips/app/i18n/de-DE.ts +0 -9
  95. package/corpus/templates/clips/app/i18n/en-US.ts +0 -8
  96. package/corpus/templates/clips/app/i18n/es-ES.ts +0 -9
  97. package/corpus/templates/clips/app/i18n/fr-FR.ts +0 -9
  98. package/corpus/templates/clips/app/i18n/hi-IN.ts +0 -8
  99. package/corpus/templates/clips/app/i18n/ja-JP.ts +0 -9
  100. package/corpus/templates/clips/app/i18n/ko-KR.ts +0 -9
  101. package/corpus/templates/clips/app/i18n/pt-BR.ts +0 -9
  102. package/corpus/templates/clips/app/i18n/zh-CN.ts +0 -7
  103. package/corpus/templates/clips/app/i18n/zh-TW.ts +0 -7
  104. package/corpus/templates/clips/app/lib/capture-install-options.ts +21 -0
  105. package/corpus/templates/clips/app/lib/fmp4.ts +271 -0
  106. package/corpus/templates/clips/app/lib/mse-video-loader.ts +585 -0
  107. package/corpus/templates/clips/app/routes/_app.dictate.tsx +0 -4
  108. package/corpus/templates/clips/app/routes/_app.meetings.$meetingId.tsx +5 -13
  109. package/corpus/templates/clips/app/routes/_app.meetings._index.tsx +2 -37
  110. package/corpus/templates/clips/app/routes/download.tsx +3 -2
  111. package/corpus/templates/clips/changelog/2026-07-10-shared-clips-now-start-playing-instantly-instead-of-showing-.md +6 -0
  112. package/corpus/templates/clips/changelog/2026-07-14-clips-now-distinguishes-transcript-previews-from-incomplete-.md +6 -0
  113. package/corpus/templates/clips/changelog/2026-07-14-desktop-app-prompts-stay-hidden-after-you-download-the-insta.md +6 -0
  114. package/corpus/templates/clips/changelog/2026-07-14-share-dialogs-show-separate-links-for-people-and-agents.md +6 -0
  115. package/corpus/templates/clips/desktop/src/lib/recorder.ts +5 -0
  116. package/corpus/templates/clips/desktop/src/overlays/recording-pill.tsx +10 -0
  117. package/corpus/templates/clips/desktop/src-tauri/src/lib.rs +4 -0
  118. package/corpus/templates/clips/desktop/src-tauri/src/native_screen/custom_capture.rs +2518 -0
  119. package/corpus/templates/clips/desktop/src-tauri/src/native_screen/live_upload.rs +353 -0
  120. package/corpus/templates/clips/desktop/src-tauri/src/native_screen.rs +558 -67
  121. package/corpus/templates/clips/desktop/src-tauri/src/recording_indicator.rs +11 -1
  122. package/corpus/templates/clips/desktop/src-tauri/src/remote_flags.rs +188 -0
  123. package/corpus/templates/clips/desktop/src-tauri/src/whisper_speech.rs +1 -1
  124. package/corpus/templates/clips/server/lib/post-finalize-dispatch.ts +1 -0
  125. package/corpus/templates/clips/server/plugins/agent-chat.ts +5 -0
  126. package/corpus/templates/clips/server/routes/api/_agent-native-background/post-finalize-worker.post.ts +5 -1
  127. package/corpus/templates/clips/shared/feature-flags.ts +41 -0
  128. package/corpus/templates/design/.agents/skills/design-generation/SKILL.md +3 -0
  129. package/corpus/templates/design/.agents/skills/design-review-feedback/SKILL.md +42 -0
  130. package/corpus/templates/design/AGENTS.md +6 -0
  131. package/corpus/templates/design/README.md +1 -0
  132. package/corpus/templates/design/actions/navigate.ts +4 -4
  133. package/corpus/templates/design/actions/view-screen.ts +116 -2
  134. package/corpus/templates/design/agent-native.app-skill.json +5 -0
  135. package/corpus/templates/design/app/components/design/DesignCanvas.tsx +52 -59
  136. package/corpus/templates/design/app/components/design/DesignExtensionsPanel.tsx +13 -9
  137. package/corpus/templates/design/app/components/design/EditPanel.tsx +79 -11
  138. package/corpus/templates/design/app/components/design/LayersPanel.tsx +78 -65
  139. package/corpus/templates/design/app/components/design/ReviewCommentsPanel.tsx +161 -0
  140. package/corpus/templates/design/app/components/design/ReviewStatusControl.tsx +96 -0
  141. package/corpus/templates/design/app/components/design/bridge/hit-test.bridge.ts +166 -0
  142. package/corpus/templates/design/app/components/layout/Layout.tsx +1 -0
  143. package/corpus/templates/design/app/components/visual-editor/ReviewCanvasPins.tsx +1077 -0
  144. package/corpus/templates/design/app/components/visual-editor/index.ts +1 -0
  145. package/corpus/templates/design/app/components/visual-editor/review-canvas-state.ts +42 -0
  146. package/corpus/templates/design/app/hooks/use-design-systems.ts +2 -2
  147. package/corpus/templates/design/app/hooks/use-navigation-state.ts +9 -6
  148. package/corpus/templates/design/app/hooks/use-question-flow.ts +7 -1
  149. package/corpus/templates/design/app/i18n/ar-SA.ts +56 -1
  150. package/corpus/templates/design/app/i18n/de-DE.ts +57 -1
  151. package/corpus/templates/design/app/i18n/en-US.ts +56 -1
  152. package/corpus/templates/design/app/i18n/es-ES.ts +56 -1
  153. package/corpus/templates/design/app/i18n/fr-FR.ts +57 -1
  154. package/corpus/templates/design/app/i18n/hi-IN.ts +56 -1
  155. package/corpus/templates/design/app/i18n/ja-JP.ts +56 -1
  156. package/corpus/templates/design/app/i18n/ko-KR.ts +56 -1
  157. package/corpus/templates/design/app/i18n/pt-BR.ts +57 -1
  158. package/corpus/templates/design/app/i18n/zh-CN.ts +56 -1
  159. package/corpus/templates/design/app/i18n/zh-TW.ts +51 -0
  160. package/corpus/templates/design/app/pages/Present.tsx +172 -14
  161. package/corpus/templates/design/app/pages/design-editor/overview-camera.ts +23 -0
  162. package/corpus/templates/design/app/pages/design-editor/tool-state.ts +10 -0
  163. package/corpus/templates/design/app/pages/present-review-state.ts +18 -0
  164. package/corpus/templates/design/app/public-routes.ts +8 -0
  165. package/corpus/templates/design/app/root.tsx +7 -6
  166. package/corpus/templates/design/changelog/2026-07-13-dragged-design-assets-now-appear-at-the-visible-drop-locatio.md +6 -0
  167. package/corpus/templates/design/changelog/2026-07-13-reviewers-can-pin-comments-on-shared-designs-and-apply-verif.md +6 -0
  168. package/corpus/templates/design/server/plugins/agent-chat.ts +10 -0
  169. package/corpus/templates/design/server/plugins/auth.ts +7 -5
  170. package/corpus/templates/design/server/plugins/review.ts +9 -0
  171. package/corpus/templates/design/shared/review-anchor.ts +83 -0
  172. package/corpus/templates/design/shared/review-summary.ts +25 -0
  173. package/corpus/templates/dispatch/changelog/2026-07-14-dispatch-now-distinguishes-connected-agents-from-locally-mou.md +6 -0
  174. package/corpus/templates/forms/.agents/skills/form-publishing/SKILL.md +2 -0
  175. package/corpus/templates/forms/AGENTS.md +4 -0
  176. package/corpus/templates/forms/actions/create-form.ts +2 -1
  177. package/corpus/templates/forms/actions/update-form.ts +5 -2
  178. package/corpus/templates/forms/app/i18n/ar-SA.ts +3 -0
  179. package/corpus/templates/forms/app/i18n/de-DE.ts +3 -0
  180. package/corpus/templates/forms/app/i18n/en-US.ts +3 -0
  181. package/corpus/templates/forms/app/i18n/es-ES.ts +3 -0
  182. package/corpus/templates/forms/app/i18n/fr-FR.ts +3 -0
  183. package/corpus/templates/forms/app/i18n/hi-IN.ts +3 -0
  184. package/corpus/templates/forms/app/i18n/ja-JP.ts +3 -0
  185. package/corpus/templates/forms/app/i18n/ko-KR.ts +3 -0
  186. package/corpus/templates/forms/app/i18n/pt-BR.ts +3 -0
  187. package/corpus/templates/forms/app/i18n/zh-CN.ts +3 -0
  188. package/corpus/templates/forms/app/i18n/zh-TW.ts +3 -0
  189. package/corpus/templates/forms/app/pages/FormBuilderPage.tsx +19 -0
  190. package/corpus/templates/forms/changelog/2026-07-14-fixed-the-ask-forms-loading-input-appearing-above-the-welcom.md +6 -0
  191. package/corpus/templates/forms/changelog/2026-07-14-form-social-previews-now-show-the-form-title-description-and.md +6 -0
  192. package/corpus/templates/forms/changelog/2026-07-14-forms-can-email-the-form-owner-when-new-responses-arrive.md +6 -0
  193. package/corpus/templates/forms/server/handlers/submissions.ts +17 -0
  194. package/corpus/templates/forms/server/lib/form-og-image.ts +64 -6
  195. package/corpus/templates/forms/server/lib/public-form-ssr.ts +6 -2
  196. package/corpus/templates/forms/server/lib/response-email.ts +65 -0
  197. package/corpus/templates/forms/server/plugins/agent-chat.ts +1 -0
  198. package/corpus/templates/forms/server/routes/api/forms/og/[...slug]/og.png.get.ts +184 -0
  199. package/corpus/templates/forms/shared/types.ts +2 -0
  200. package/corpus/templates/plan/.agents/skills/visual-plan/SKILL.md +6 -4
  201. package/corpus/templates/plan/.agents/skills/visual-recap/SKILL.md +7 -1
  202. package/dist/a2a/index.d.ts +2 -1
  203. package/dist/a2a/index.d.ts.map +1 -1
  204. package/dist/a2a/index.js +1 -1
  205. package/dist/a2a/index.js.map +1 -1
  206. package/dist/a2a/server.d.ts +25 -0
  207. package/dist/a2a/server.d.ts.map +1 -1
  208. package/dist/a2a/server.js +30 -6
  209. package/dist/a2a/server.js.map +1 -1
  210. package/dist/agent/production-agent.d.ts +2 -0
  211. package/dist/agent/production-agent.d.ts.map +1 -1
  212. package/dist/agent/production-agent.js +37 -12
  213. package/dist/agent/production-agent.js.map +1 -1
  214. package/dist/cli/create.d.ts +5 -3
  215. package/dist/cli/create.d.ts.map +1 -1
  216. package/dist/cli/create.js +44 -19
  217. package/dist/cli/create.js.map +1 -1
  218. package/dist/cli/pr-visual-recap-workflow.d.ts +2 -2
  219. package/dist/cli/pr-visual-recap-workflow.d.ts.map +1 -1
  220. package/dist/cli/pr-visual-recap-workflow.js +2 -2
  221. package/dist/cli/pr-visual-recap-workflow.js.map +1 -1
  222. package/dist/cli/recap.d.ts +3 -562
  223. package/dist/cli/recap.d.ts.map +1 -1
  224. package/dist/cli/recap.js +3 -3875
  225. package/dist/cli/recap.js.map +1 -1
  226. package/dist/cli/skills-content/visual-plan-skill.d.ts +1 -1
  227. package/dist/cli/skills-content/visual-plan-skill.d.ts.map +1 -1
  228. package/dist/cli/skills-content/visual-plan-skill.js +6 -4
  229. package/dist/cli/skills-content/visual-plan-skill.js.map +1 -1
  230. package/dist/cli/skills-content/visual-recap-skill.d.ts +1 -1
  231. package/dist/cli/skills-content/visual-recap-skill.d.ts.map +1 -1
  232. package/dist/cli/skills-content/visual-recap-skill.js +7 -1
  233. package/dist/cli/skills-content/visual-recap-skill.js.map +1 -1
  234. package/dist/client/AgentPanel.d.ts.map +1 -1
  235. package/dist/client/AgentPanel.js +3 -3
  236. package/dist/client/AgentPanel.js.map +1 -1
  237. package/dist/client/analytics.d.ts +5 -0
  238. package/dist/client/analytics.d.ts.map +1 -1
  239. package/dist/client/analytics.js +3 -1
  240. package/dist/client/analytics.js.map +1 -1
  241. package/dist/client/chat/markdown-renderer.d.ts +1 -0
  242. package/dist/client/chat/markdown-renderer.d.ts.map +1 -1
  243. package/dist/client/chat/markdown-renderer.js +36 -1
  244. package/dist/client/chat/markdown-renderer.js.map +1 -1
  245. package/dist/client/chat/message-components.d.ts.map +1 -1
  246. package/dist/client/chat/message-components.js +5 -2
  247. package/dist/client/chat/message-components.js.map +1 -1
  248. package/dist/client/clipboard.d.ts +3 -1
  249. package/dist/client/clipboard.d.ts.map +1 -1
  250. package/dist/client/clipboard.js +24 -1
  251. package/dist/client/clipboard.js.map +1 -1
  252. package/dist/client/guided-questions.d.ts +3 -1
  253. package/dist/client/guided-questions.d.ts.map +1 -1
  254. package/dist/client/guided-questions.js +2 -1
  255. package/dist/client/guided-questions.js.map +1 -1
  256. package/dist/client/i18n.d.ts.map +1 -1
  257. package/dist/client/i18n.js +3 -1
  258. package/dist/client/i18n.js.map +1 -1
  259. package/dist/client/index.d.ts +1 -1
  260. package/dist/client/index.d.ts.map +1 -1
  261. package/dist/client/index.js +1 -1
  262. package/dist/client/index.js.map +1 -1
  263. package/dist/client/review/ReviewCommentComposer.d.ts +18 -0
  264. package/dist/client/review/ReviewCommentComposer.d.ts.map +1 -0
  265. package/dist/client/review/ReviewCommentComposer.js +30 -0
  266. package/dist/client/review/ReviewCommentComposer.js.map +1 -0
  267. package/dist/client/review/ReviewThreadPanel.d.ts +28 -1
  268. package/dist/client/review/ReviewThreadPanel.d.ts.map +1 -1
  269. package/dist/client/review/ReviewThreadPanel.js +151 -39
  270. package/dist/client/review/ReviewThreadPanel.js.map +1 -1
  271. package/dist/client/review/index.d.ts +3 -2
  272. package/dist/client/review/index.d.ts.map +1 -1
  273. package/dist/client/review/index.js +2 -1
  274. package/dist/client/review/index.js.map +1 -1
  275. package/dist/client/review/use-review.d.ts +23 -0
  276. package/dist/client/review/use-review.d.ts.map +1 -1
  277. package/dist/client/review/use-review.js +3 -0
  278. package/dist/client/review/use-review.js.map +1 -1
  279. package/dist/client/use-agent-chat-context.d.ts +1 -1
  280. package/dist/client/use-agent-chat-context.d.ts.map +1 -1
  281. package/dist/client/use-agent-chat-context.js +4 -2
  282. package/dist/client/use-agent-chat-context.js.map +1 -1
  283. package/dist/collab/awareness.d.ts +2 -2
  284. package/dist/collab/awareness.d.ts.map +1 -1
  285. package/dist/collab/struct-routes.d.ts +1 -1
  286. package/dist/credentials/index.d.ts +12 -7
  287. package/dist/credentials/index.d.ts.map +1 -1
  288. package/dist/credentials/index.js +33 -8
  289. package/dist/credentials/index.js.map +1 -1
  290. package/dist/extensions/fetch-tool.d.ts +13 -1
  291. package/dist/extensions/fetch-tool.d.ts.map +1 -1
  292. package/dist/extensions/fetch-tool.js +9 -2
  293. package/dist/extensions/fetch-tool.js.map +1 -1
  294. package/dist/notifications/channels.js +20 -4
  295. package/dist/notifications/channels.js.map +1 -1
  296. package/dist/observability/routes.d.ts +5 -5
  297. package/dist/progress/routes.d.ts +1 -1
  298. package/dist/provider-api/corpus-jobs.d.ts +2 -2
  299. package/dist/resources/handlers.d.ts +3 -3
  300. package/dist/review/actions/create-review-comment.js +2 -1
  301. package/dist/review/actions/create-review-comment.js.map +1 -1
  302. package/dist/review/actions/get-review-feedback.js +6 -11
  303. package/dist/review/actions/get-review-feedback.js.map +1 -1
  304. package/dist/review/actions/list-review-comments.d.ts +1 -0
  305. package/dist/review/actions/list-review-comments.js +25 -4
  306. package/dist/review/actions/list-review-comments.js.map +1 -1
  307. package/dist/review/actions/reply-review-comment.js +12 -5
  308. package/dist/review/actions/reply-review-comment.js.map +1 -1
  309. package/dist/review/actions/resolve-review-thread.d.ts +3 -0
  310. package/dist/review/actions/resolve-review-thread.js +20 -3
  311. package/dist/review/actions/resolve-review-thread.js.map +1 -1
  312. package/dist/review/actions/send-review-thread-to-agent.d.ts +17 -0
  313. package/dist/review/actions/send-review-thread-to-agent.d.ts.map +1 -0
  314. package/dist/review/actions/send-review-thread-to-agent.js +49 -0
  315. package/dist/review/actions/send-review-thread-to-agent.js.map +1 -0
  316. package/dist/review/identity.d.ts +6 -0
  317. package/dist/review/identity.d.ts.map +1 -0
  318. package/dist/review/identity.js +68 -0
  319. package/dist/review/identity.js.map +1 -0
  320. package/dist/review/index.d.ts +3 -1
  321. package/dist/review/index.d.ts.map +1 -1
  322. package/dist/review/index.js +2 -1
  323. package/dist/review/index.js.map +1 -1
  324. package/dist/review/store.d.ts +36 -0
  325. package/dist/review/store.d.ts.map +1 -1
  326. package/dist/review/store.js +223 -8
  327. package/dist/review/store.js.map +1 -1
  328. package/dist/review/types.d.ts +4 -0
  329. package/dist/review/types.d.ts.map +1 -1
  330. package/dist/review/types.js.map +1 -1
  331. package/dist/secrets/crypto.d.ts +15 -7
  332. package/dist/secrets/crypto.d.ts.map +1 -1
  333. package/dist/secrets/crypto.js +23 -10
  334. package/dist/secrets/crypto.js.map +1 -1
  335. package/dist/secrets/routes.d.ts +9 -9
  336. package/dist/secrets/storage.d.ts.map +1 -1
  337. package/dist/secrets/storage.js +16 -3
  338. package/dist/secrets/storage.js.map +1 -1
  339. package/dist/server/action-discovery.d.ts.map +1 -1
  340. package/dist/server/action-discovery.js +4 -0
  341. package/dist/server/action-discovery.js.map +1 -1
  342. package/dist/server/action-routes.d.ts +45 -0
  343. package/dist/server/action-routes.d.ts.map +1 -1
  344. package/dist/server/action-routes.js +71 -5
  345. package/dist/server/action-routes.js.map +1 -1
  346. package/dist/server/agent-chat/plugin-options.d.ts +16 -0
  347. package/dist/server/agent-chat/plugin-options.d.ts.map +1 -1
  348. package/dist/server/agent-chat/plugin-options.js.map +1 -1
  349. package/dist/server/agent-chat-plugin.d.ts +20 -0
  350. package/dist/server/agent-chat-plugin.d.ts.map +1 -1
  351. package/dist/server/agent-chat-plugin.js +39 -4
  352. package/dist/server/agent-chat-plugin.js.map +1 -1
  353. package/dist/server/derived-secret.d.ts +1 -1
  354. package/dist/server/derived-secret.d.ts.map +1 -1
  355. package/dist/server/derived-secret.js.map +1 -1
  356. package/dist/server/index.d.ts +3 -1
  357. package/dist/server/index.d.ts.map +1 -1
  358. package/dist/server/index.js +2 -0
  359. package/dist/server/index.js.map +1 -1
  360. package/dist/vite/action-types-plugin.d.ts.map +1 -1
  361. package/dist/vite/action-types-plugin.js +4 -0
  362. package/dist/vite/action-types-plugin.js.map +1 -1
  363. package/docs/content/locales/ar-SA/pr-visual-recap.mdx +48 -30
  364. package/docs/content/locales/ar-SA/template-plan.mdx +7 -0
  365. package/docs/content/locales/de-DE/pr-visual-recap.mdx +48 -30
  366. package/docs/content/locales/de-DE/template-plan.mdx +7 -0
  367. package/docs/content/locales/es-ES/pr-visual-recap.mdx +48 -30
  368. package/docs/content/locales/es-ES/template-plan.mdx +14 -0
  369. package/docs/content/locales/fr-FR/pr-visual-recap.mdx +48 -30
  370. package/docs/content/locales/fr-FR/template-plan.mdx +7 -0
  371. package/docs/content/locales/hi-IN/pr-visual-recap.mdx +48 -30
  372. package/docs/content/locales/hi-IN/template-plan.mdx +7 -0
  373. package/docs/content/locales/ja-JP/pr-visual-recap.mdx +48 -30
  374. package/docs/content/locales/ja-JP/template-plan.mdx +7 -0
  375. package/docs/content/locales/ko-KR/pr-visual-recap.mdx +48 -30
  376. package/docs/content/locales/ko-KR/template-plan.mdx +7 -0
  377. package/docs/content/locales/pt-BR/pr-visual-recap.mdx +48 -30
  378. package/docs/content/locales/pt-BR/template-plan.mdx +14 -0
  379. package/docs/content/locales/zh-CN/pr-visual-recap.mdx +48 -30
  380. package/docs/content/locales/zh-CN/template-plan.mdx +7 -0
  381. package/docs/content/locales/zh-TW/pr-visual-recap.mdx +48 -30
  382. package/docs/content/locales/zh-TW/template-plan.mdx +7 -0
  383. package/docs/content/pr-visual-recap.mdx +50 -32
  384. package/docs/content/template-plan.mdx +14 -0
  385. package/docs/content/toolkit-comments-review.mdx +16 -2
  386. package/package.json +4 -1
  387. package/corpus/templates/clips/app/components/sharing/slack-share-hint.tsx +0 -68
  388. package/corpus/templates/forms/server/routes/api/forms/og/[slug]/og.png.get.ts +0 -45
@@ -0,0 +1,2518 @@
1
+ //! Custom ScreenCaptureKit capture engine.
2
+ //!
3
+ //! Owns everything between "SCStream hands us a sample buffer" and "a
4
+ //! fragmented MP4 grows on disk": the AVAssetWriter wrapper
5
+ //! (`CustomScreenCaptureWriter`), the realtime `LiveAudioMixer` that combines
6
+ //! mic + system audio into one track, the PCM decode/resample helpers, the
7
+ //! AVFoundation/CoreMedia FFI glue, and the capture watchdog that rebuilds a
8
+ //! stopped/stalled SCStream in place. Session management and live upload stay
9
+ //! in the parent `native_screen` module and the `live_upload` sibling.
10
+
11
+ use super::*;
12
+ use screencapturekit::error::SCError;
13
+ use screencapturekit::stream::delegate_trait::SCStreamDelegateTrait;
14
+
15
+ /// `AVAssetWriter.status` raw value for `.completed`.
16
+ const AV_WRITER_STATUS_COMPLETED: i64 = 2;
17
+ /// `kAudioFormatMPEG4AAC` FourCC ('aac ') for the writer's audio output.
18
+ const AUDIO_FORMAT_AAC: i64 = 0x6161_6320;
19
+ /// Cadence of delegate-produced output segments in segmented mode (seconds of
20
+ /// media time). Smaller = data becomes uploadable sooner, at a small
21
+ /// container-overhead cost.
22
+ const OUTPUT_SEGMENT_INTERVAL_SECONDS: i64 = 1;
23
+ /// Capture-time H.264 budget in bits per pixel per frame. 0.15 bpp matches
24
+ /// Cap's "instant" quality tier (~3.3 Mbps at 1280x720@24) and keeps most
25
+ /// recordings small enough to upload without a post-capture transcode.
26
+ const CAPTURE_VIDEO_BPP: f64 = 0.15;
27
+
28
+ // Self-healing capture watchdog. ScreenCaptureKit can silently stop feeding a
29
+ // display stream when the captured display changes Spaces (virtual desktops),
30
+ // a full-screen app takes over, or the display config changes — sometimes with
31
+ // a `did_stop_with_error` callback, sometimes by just going quiet. Either way
32
+ // the recording file stops growing with no app-level signal. The watchdog
33
+ // notices the gap and rebuilds the SCStream in place so recording continues.
34
+ //
35
+ // How long without any delivered sample buffer before we treat the stream as
36
+ // dead and rebuild it. Generous enough not to trip on a momentarily idle
37
+ // screen (SCK still delivers idle frames, which count as activity here).
38
+ const CAPTURE_STALL_TIMEOUT: Duration = Duration::from_secs(4);
39
+ // Watchdog poll cadence.
40
+ const CAPTURE_WATCHDOG_POLL: Duration = Duration::from_millis(1000);
41
+ // Consecutive failed restarts (rebuild or start error, or an immediate
42
+ // re-stall) before giving up and finalizing whatever was captured. A single
43
+ // successful stretch of frames resets the counter.
44
+ const CAPTURE_MAX_RESTARTS: u32 = 5;
45
+
46
+ /// Liveness state shared between the SCK output handler (which records that a
47
+ /// sample arrived), the stream delegate (which records an OS-reported stop),
48
+ /// and the watchdog thread (which reads both to decide when to rebuild the
49
+ /// stream).
50
+ pub(crate) struct CaptureWatch {
51
+ /// Wall-clock time of the most recent delivered sample buffer.
52
+ last_activity: Mutex<Instant>,
53
+ /// Set by the stream delegate when ScreenCaptureKit reports the stream
54
+ /// stopped. Consumed by the watchdog to force an immediate rebuild.
55
+ stream_stopped: Mutex<Option<String>>,
56
+ /// The user stopped capture through macOS itself (menu-bar "Stop
57
+ /// Sharing", SCStreamError code -3817). The watchdog must treat this as
58
+ /// a clean stop request, never as a failure to rebuild from.
59
+ user_stopped: AtomicBool,
60
+ /// True between pause and resume. The capture source (SCStream) is
61
+ /// intentionally stopped while the writer/file/uploader stay alive, so
62
+ /// the watchdog must not read the silence as a stall and rebuild — resume
63
+ /// brings a fresh stream back and clears this.
64
+ paused: AtomicBool,
65
+ }
66
+
67
+ impl CaptureWatch {
68
+ fn new() -> Self {
69
+ Self {
70
+ last_activity: Mutex::new(Instant::now()),
71
+ stream_stopped: Mutex::new(None),
72
+ user_stopped: AtomicBool::new(false),
73
+ paused: AtomicBool::new(false),
74
+ }
75
+ }
76
+
77
+ fn note_activity(&self) {
78
+ if let Ok(mut guard) = self.last_activity.lock() {
79
+ *guard = Instant::now();
80
+ }
81
+ }
82
+
83
+ fn since_activity(&self) -> Duration {
84
+ self.last_activity
85
+ .lock()
86
+ .map(|t| t.elapsed())
87
+ .unwrap_or_default()
88
+ }
89
+
90
+ fn note_stream_stopped(&self, reason: String) {
91
+ if let Ok(mut guard) = self.stream_stopped.lock() {
92
+ if guard.is_none() {
93
+ *guard = Some(reason);
94
+ }
95
+ }
96
+ }
97
+
98
+ fn take_stream_stopped(&self) -> Option<String> {
99
+ self.stream_stopped.lock().ok().and_then(|mut g| g.take())
100
+ }
101
+
102
+ fn note_user_stopped(&self) {
103
+ self.user_stopped.store(true, Ordering::SeqCst);
104
+ }
105
+
106
+ fn user_stopped(&self) -> bool {
107
+ self.user_stopped.load(Ordering::SeqCst)
108
+ }
109
+
110
+ fn set_paused(&self, paused: bool) {
111
+ self.paused.store(paused, Ordering::SeqCst);
112
+ }
113
+
114
+ fn is_paused(&self) -> bool {
115
+ self.paused.load(Ordering::SeqCst)
116
+ }
117
+ }
118
+
119
+ // ---------------------------------------------------------------------------
120
+ // Segmented output (live-upload mode)
121
+ //
122
+ // With `movieFragmentInterval` + a writer-owned file, `finishWriting`
123
+ // DEFRAGMENTS the file in place (fragmented layout -> classic mdat+moov), so
124
+ // byte ranges streamed to the server during recording no longer match the
125
+ // final file and the uploaded clip comes back corrupt. The fix is Apple's
126
+ // segment API: the writer is created WITHOUT an output URL, produces discrete
127
+ // fMP4 segments through `AVAssetWriterDelegate`, and WE append them to the
128
+ // local file ourselves. AVFoundation never owns the file, so nothing is ever
129
+ // rewritten — the file is append-only by construction and the live uploader
130
+ // can safely tail it forever.
131
+ // ---------------------------------------------------------------------------
132
+
133
+ /// Owns the local recording file in segmented mode. The ObjC delegate appends
134
+ /// every segment it receives; append order matches delivery order (delegate
135
+ /// callbacks are serial, and the file mutex serializes any stragglers).
136
+ pub(super) struct SegmentSink {
137
+ file: Mutex<std::fs::File>,
138
+ /// First write error, if any; surfaced by `finish()` so a failing disk
139
+ /// turns into a reported error instead of a silently truncated file.
140
+ failed: Mutex<Option<String>>,
141
+ segments: AtomicU64,
142
+ }
143
+
144
+ impl SegmentSink {
145
+ fn create(path: &Path) -> Result<Arc<Self>, String> {
146
+ let file = std::fs::File::create(path)
147
+ .map_err(|e| format!("could not create recording file {}: {e}", path.display()))?;
148
+ Ok(Arc::new(Self {
149
+ file: Mutex::new(file),
150
+ failed: Mutex::new(None),
151
+ segments: AtomicU64::new(0),
152
+ }))
153
+ }
154
+
155
+ fn append(&self, bytes: &[u8]) {
156
+ use std::io::Write;
157
+ let Ok(mut file) = self.file.lock() else {
158
+ return;
159
+ };
160
+ if let Err(err) = file.write_all(bytes) {
161
+ if let Ok(mut failed) = self.failed.lock() {
162
+ if failed.is_none() {
163
+ eprintln!("[mixer] segment write failed: {err}");
164
+ *failed = Some(format!("segment write failed: {err}"));
165
+ }
166
+ }
167
+ return;
168
+ }
169
+ let n = self.segments.fetch_add(1, Ordering::Relaxed) + 1;
170
+ if n == 1 {
171
+ eprintln!(
172
+ "[mixer] first output segment written ({} bytes)",
173
+ bytes.len()
174
+ );
175
+ }
176
+ }
177
+
178
+ fn failure(&self) -> Option<String> {
179
+ self.failed.lock().ok().and_then(|guard| guard.clone())
180
+ }
181
+ }
182
+
183
+ /// Instance variables for the segment delegate: just the shared sink.
184
+ struct SegmentDelegateIvars {
185
+ sink: Arc<SegmentSink>,
186
+ }
187
+
188
+ objc2::define_class!(
189
+ // SAFETY: NSObject has no subclassing requirements and the type has no
190
+ // Drop impl. Methods are called by AVFoundation on its own serial queue.
191
+ #[unsafe(super(objc2::runtime::NSObject))]
192
+ #[name = "ClipsSegmentWriterDelegate"]
193
+ #[ivars = SegmentDelegateIvars]
194
+ struct SegmentWriterDelegate;
195
+
196
+ impl SegmentWriterDelegate {
197
+ /// `AVAssetWriterDelegate` — receives each fMP4 segment (type 1 =
198
+ /// initialization, 2 = separable media) as it is produced.
199
+ #[unsafe(method(assetWriter:didOutputSegmentData:segmentType:))]
200
+ fn did_output_segment(
201
+ &self,
202
+ _writer: *mut objc2::runtime::AnyObject,
203
+ data: *mut objc2::runtime::AnyObject,
204
+ _segment_type: isize,
205
+ ) {
206
+ // Crossing the ObjC boundary: a Rust panic here would abort the
207
+ // whole process, so contain it.
208
+ let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
209
+ if data.is_null() {
210
+ return;
211
+ }
212
+ let (ptr, len): (*const std::ffi::c_void, usize) = unsafe {
213
+ (
214
+ objc2::msg_send![&*data, bytes],
215
+ objc2::msg_send![&*data, length],
216
+ )
217
+ };
218
+ if ptr.is_null() || len == 0 {
219
+ return;
220
+ }
221
+ let bytes = unsafe { std::slice::from_raw_parts(ptr as *const u8, len) };
222
+ use objc2::DefinedClass;
223
+ self.ivars().sink.append(bytes);
224
+ }));
225
+ }
226
+ }
227
+ );
228
+
229
+ impl SegmentWriterDelegate {
230
+ fn new(sink: Arc<SegmentSink>) -> objc2::rc::Retained<Self> {
231
+ use objc2::AllocAnyThread;
232
+ let this = Self::alloc().set_ivars(SegmentDelegateIvars { sink });
233
+ unsafe { objc2::msg_send![super(this), init] }
234
+ }
235
+ }
236
+
237
+ /// Cheap-to-clone handle around one `AVAssetWriter` producing a fragmented
238
+ /// MP4. SCK callbacks append through it from multiple dispatch queues; the
239
+ /// stop path calls [`Self::finish`]. All clones share the same underlying
240
+ /// state (everything inside is an `Arc`).
241
+ pub(crate) struct CustomScreenCaptureWriter {
242
+ inner: Arc<Mutex<CustomScreenCaptureWriterState>>,
243
+ /// Present only in live-mixing mode; combines the two incoming audio
244
+ /// streams into the one track written to `mixed_audio_input`. Lives on its
245
+ /// own lock so PCM mixing on the audio callbacks never blocks video frame
246
+ /// appends (which only need `inner`). Lock order is always `mixer` →
247
+ /// `inner`; never take `mixer` while holding `inner`.
248
+ mixer: Option<Arc<Mutex<LiveAudioMixer>>>,
249
+ /// The writer session has started (first video frame ran
250
+ /// `startSessionAtSourceTime:`). Written under the `inner` lock, read
251
+ /// lock-free by the audio callbacks.
252
+ started: Arc<AtomicBool>,
253
+ /// Source-time (seconds, as f64 bits) the writer session started at.
254
+ /// `f64::NAN` until known; mixed audio earlier than this must be dropped
255
+ /// or the writer rejects it. Written before `started` flips true.
256
+ session_start_bits: Arc<AtomicU64>,
257
+ /// Once true the writer accepts no more samples — set on stop, cancel, and
258
+ /// any append failure.
259
+ appends_closed: Arc<AtomicBool>,
260
+ /// Samples skipped because an AVAssetWriterInput wasn't ready; logged
261
+ /// periodically so realtime backpressure is visible.
262
+ dropped_samples: Arc<AtomicU64>,
263
+ /// Accumulated pause time (seconds, as f64 bits) to subtract from the
264
+ /// zero-based session timeline so a pause/resume leaves no gap in the
265
+ /// append-only file. Zero until the first resume. Read by the video
266
+ /// retime path and the live audio mixer.
267
+ pause_offset_bits: Arc<AtomicU64>,
268
+ }
269
+
270
+ /// The lock-guarded half of the writer: the retained AVFoundation objects
271
+ /// plus lifecycle flags. Every Objective-C call on these handles happens
272
+ /// while holding this state's mutex (see the SAFETY note below).
273
+ struct CustomScreenCaptureWriterState {
274
+ writer: objc2::rc::Retained<objc2::runtime::AnyObject>,
275
+ video_input: objc2::rc::Retained<objc2::runtime::AnyObject>,
276
+ system_audio_input: Option<objc2::rc::Retained<objc2::runtime::AnyObject>>,
277
+ mic_audio_input: Option<objc2::rc::Retained<objc2::runtime::AnyObject>>,
278
+ /// Single output track used when mic + system audio are mixed live into one
279
+ /// stream. Mutually exclusive with `system_audio_input` / `mic_audio_input`.
280
+ mixed_audio_input: Option<objc2::rc::Retained<objc2::runtime::AnyObject>>,
281
+ /// Segmented (delegate-fed) output mode; see the "Segmented output"
282
+ /// section. When true, `initialSegmentStartTime` must be set before
283
+ /// `startWriting`.
284
+ segmented: bool,
285
+ /// Local-file writer for segmented mode (we own the file, not
286
+ /// AVFoundation). `None` in plain file mode.
287
+ segment_sink: Option<Arc<SegmentSink>>,
288
+ /// Keeps the ObjC delegate alive — `AVAssetWriter.delegate` is weak.
289
+ #[allow(dead_code)]
290
+ segment_delegate: Option<objc2::rc::Retained<SegmentWriterDelegate>>,
291
+ /// First video frame's PTS as (value, timescale). In segmented mode the
292
+ /// writer session starts at ZERO and every sample is rebased against this
293
+ /// — the segment API preserves source timestamps verbatim, so appending
294
+ /// raw host-clock PTS would give the clip a media timeline starting at
295
+ /// "seconds since boot" (browsers then show wall-clock-like times).
296
+ session_start_time: Option<(i64, i32)>,
297
+ finished: bool,
298
+ failed: Option<String>,
299
+ }
300
+
301
+ // SAFETY: `Retained<AnyObject>` is `!Send`/`!Sync` by default because objc2
302
+ // cannot know an arbitrary object's threading contract. Here every
303
+ // Objective-C call on the retained writer/input handles (`startWriting`,
304
+ // `startSessionAtSourceTime:`, `appendSampleBuffer:`, `markAsFinished`,
305
+ // `finishWriting…`, `status`/`error` reads) happens while holding the `inner`
306
+ // mutex, so access is serialized even though callbacks arrive on multiple SCK
307
+ // dispatch queues. The remaining shared fields are lock-free atomics
308
+ // (`started`, `session_start_bits`, `appends_closed`, `dropped_samples`), and
309
+ // the mixer holds no Objective-C state and lives behind its own mutex.
310
+ // Objective-C retain/release itself is atomic, so moving the retained
311
+ // pointers across threads is sound.
312
+ unsafe impl Send for CustomScreenCaptureWriter {}
313
+ unsafe impl Sync for CustomScreenCaptureWriter {}
314
+ unsafe impl Send for CustomScreenCaptureWriterState {}
315
+
316
+ #[derive(Clone)]
317
+ /// The `SCStreamOutputTrait` sink registered for all three output types
318
+ /// (screen / system audio / microphone). One instance is shared across the
319
+ /// registrations and across watchdog stream rebuilds, so the same writer
320
+ /// keeps receiving samples over the whole recording.
321
+ struct CustomScreenCaptureOutputHandler {
322
+ writer: CustomScreenCaptureWriter,
323
+ recording_enabled: Arc<AtomicBool>,
324
+ mic_ready: Option<Arc<AtomicBool>>,
325
+ watch: Arc<CaptureWatch>,
326
+ }
327
+
328
+ impl Clone for CustomScreenCaptureWriter {
329
+ fn clone(&self) -> Self {
330
+ Self {
331
+ inner: Arc::clone(&self.inner),
332
+ mixer: self.mixer.clone(),
333
+ started: Arc::clone(&self.started),
334
+ session_start_bits: Arc::clone(&self.session_start_bits),
335
+ appends_closed: Arc::clone(&self.appends_closed),
336
+ dropped_samples: Arc::clone(&self.dropped_samples),
337
+ pause_offset_bits: Arc::clone(&self.pause_offset_bits),
338
+ }
339
+ }
340
+ }
341
+
342
+ impl SCStreamOutputTrait for CustomScreenCaptureOutputHandler {
343
+ fn did_output_sample_buffer(
344
+ &self,
345
+ sample_buffer: screencapturekit::cm::CMSampleBuffer,
346
+ of_type: SCStreamOutputType,
347
+ ) {
348
+ if matches!(of_type, SCStreamOutputType::Microphone) {
349
+ if let Some(mic_ready) = &self.mic_ready {
350
+ mic_ready.store(true, Ordering::Relaxed);
351
+ }
352
+ }
353
+ if !self.recording_enabled.load(Ordering::SeqCst) {
354
+ return;
355
+ }
356
+ // A delivered buffer (even a content-less idle frame) proves the stream
357
+ // is still alive; record it so the watchdog can tell a genuinely stalled
358
+ // stream apart from a quiet one.
359
+ self.watch.note_activity();
360
+ if self.writer.appends_closed.load(Ordering::SeqCst) {
361
+ return;
362
+ }
363
+ // Append every screen frame that carries an image buffer, INCLUDING
364
+ // idle/blank ones. On a static screen (e.g. after switching to another
365
+ // Space) ScreenCaptureKit delivers Idle frames; dropping them starves
366
+ // the video track, and fragmented-MP4 interleaving then can't complete
367
+ // a fragment — the file stops growing even though audio keeps flowing.
368
+ // Idle frames still reference the current surface, and the encoder
369
+ // turns repeats into tiny P-frames, so appending them is cheap. Only
370
+ // frames with no image buffer are skipped (nothing to encode).
371
+ if matches!(of_type, SCStreamOutputType::Screen) {
372
+ use std::sync::atomic::AtomicU64;
373
+ static SCREEN_SEEN: AtomicU64 = AtomicU64::new(0);
374
+ static SCREEN_BUFFERLESS: AtomicU64 = AtomicU64::new(0);
375
+ let seen = SCREEN_SEEN.fetch_add(1, Ordering::Relaxed) + 1;
376
+ if sample_buffer.image_buffer().is_none() {
377
+ let skipped = SCREEN_BUFFERLESS.fetch_add(1, Ordering::Relaxed) + 1;
378
+ if skipped == 1 || skipped % 100 == 0 {
379
+ eprintln!(
380
+ "[mixer] screen frame without image buffer skipped ({skipped} so far, status={:?})",
381
+ sample_buffer.frame_status()
382
+ );
383
+ }
384
+ return;
385
+ }
386
+ if seen == 1 || seen % 512 == 0 {
387
+ eprintln!(
388
+ "[mixer] screen frames delivered: seen={seen} bufferless={} status_now={:?}",
389
+ SCREEN_BUFFERLESS.load(Ordering::Relaxed),
390
+ sample_buffer.frame_status()
391
+ );
392
+ }
393
+ }
394
+ // ScreenCaptureKit invokes this from its own dispatch queues through an
395
+ // Objective-C boundary. Two failure modes can abort the whole process:
396
+ // - an Objective-C exception (e.g. AVFoundation), which `catch_unwind`
397
+ // CANNOT catch ("Rust cannot catch foreign exceptions"), so we wrap
398
+ // the body in `objc2::exception::catch` first; and
399
+ // - a Rust panic, contained by the outer `catch_unwind`.
400
+ // Either way we log the cause and cancel the capture instead of dying.
401
+ let panic_result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
402
+ let exc_result = objc2::exception::catch(std::panic::AssertUnwindSafe(|| {
403
+ self.writer.append_sample(&sample_buffer, of_type);
404
+ }));
405
+ if let Err(exc) = exc_result {
406
+ let detail = describe_objc_exception(exc);
407
+ eprintln!(
408
+ "[mixer] append_sample raised Objective-C exception for {of_type:?}: {detail}; cancelling capture"
409
+ );
410
+ self.writer.appends_closed.store(true, Ordering::SeqCst);
411
+ }
412
+ }));
413
+ if panic_result.is_err() {
414
+ eprintln!("[mixer] panic while appending {of_type:?} sample; cancelling capture");
415
+ self.writer.appends_closed.store(true, Ordering::SeqCst);
416
+ }
417
+ }
418
+ }
419
+
420
+ impl CustomScreenCaptureWriter {
421
+ /// Build the `AVAssetWriter` + inputs for one recording file. With live
422
+ /// mixing (mic + system) a single mixed audio track is created;
423
+ /// otherwise each captured source gets its own track. In live-upload
424
+ /// mode the writer produces movie fragments (append-only file); without
425
+ /// it, a regular MP4 with faststart.
426
+ fn new(
427
+ output_path: &Path,
428
+ width: u32,
429
+ height: u32,
430
+ capture_system_audio: bool,
431
+ include_audio: bool,
432
+ mix_live: bool,
433
+ ) -> Result<Self, String> {
434
+ use objc2::msg_send;
435
+ use objc2::runtime::AnyObject;
436
+
437
+ #[link(name = "AVFoundation", kind = "framework")]
438
+ extern "C" {
439
+ static AVFileTypeMPEG4: *const AnyObject;
440
+ static AVMediaTypeVideo: *const AnyObject;
441
+ static AVFileTypeProfileMPEG4AppleHLS: *const AnyObject;
442
+ }
443
+ // Force-load UniformTypeIdentifiers so the runtime UTType lookup in
444
+ // segmented mode resolves.
445
+ #[link(name = "UniformTypeIdentifiers", kind = "framework")]
446
+ extern "C" {}
447
+
448
+ let segmented = crate::remote_flags::current().custom_sck_pipeline_live_upload_enabled;
449
+ unsafe {
450
+ let writer_cls = av_class_named("AVAssetWriter")
451
+ .ok_or_else(|| "AVAssetWriter missing".to_string())?;
452
+
453
+ let (writer, segment_sink, segment_delegate) = if segmented {
454
+ // Segmented mode: the writer has NO output URL. It produces
455
+ // discrete fMP4 segments through the delegate, and WE append
456
+ // them to the local file — so the file is append-only by
457
+ // construction and `finishWriting` cannot defragment/rewrite
458
+ // it (which would invalidate live-uploaded byte ranges).
459
+ let ident = av_ns_string_from("public.mpeg-4")
460
+ .ok_or_else(|| "NSString for UTType failed".to_string())?;
461
+ let ut_cls =
462
+ av_class_named("UTType").ok_or_else(|| "UTType missing".to_string())?;
463
+ let ut: *mut AnyObject = msg_send![ut_cls, typeWithIdentifier: &*ident];
464
+ if ut.is_null() {
465
+ return Err("UTType public.mpeg-4 unavailable".into());
466
+ }
467
+ let allocated: *mut AnyObject = msg_send![writer_cls, alloc];
468
+ let writer_raw: *mut AnyObject = msg_send![allocated, initWithContentType: ut];
469
+ if writer_raw.is_null() {
470
+ if !allocated.is_null() {
471
+ let _ = objc2::rc::Retained::from_raw(allocated);
472
+ }
473
+ return Err("AVAssetWriter initWithContentType failed".into());
474
+ }
475
+ let writer = objc2::rc::Retained::from_raw(writer_raw)
476
+ .ok_or_else(|| "AVAssetWriter retain failed".to_string())?;
477
+
478
+ let interval = ObjcCMTime {
479
+ value: OUTPUT_SEGMENT_INTERVAL_SECONDS,
480
+ timescale: 1,
481
+ flags: 1,
482
+ epoch: 0,
483
+ };
484
+ let _: () = msg_send![&*writer, setPreferredOutputSegmentInterval: interval];
485
+ let _: () =
486
+ msg_send![&*writer, setOutputFileTypeProfile: AVFileTypeProfileMPEG4AppleHLS];
487
+
488
+ let sink = SegmentSink::create(output_path)?;
489
+ let delegate = SegmentWriterDelegate::new(Arc::clone(&sink));
490
+ // `AVAssetWriter.delegate` is weak — the Retained delegate is
491
+ // stored in the writer state to keep it alive.
492
+ let _: () = msg_send![&*writer, setDelegate: &*delegate];
493
+ (writer, Some(sink), Some(delegate))
494
+ } else {
495
+ // Plain file mode: AVFoundation owns the file; faststart is
496
+ // safe because nothing tails the file during recording.
497
+ let url = av_file_url(output_path).ok_or_else(|| {
498
+ format!("could not build output URL for {}", output_path.display())
499
+ })?;
500
+ let allocated: *mut AnyObject = msg_send![writer_cls, alloc];
501
+ let mut err_ptr: *mut AnyObject = std::ptr::null_mut();
502
+ let writer_raw: *mut AnyObject = msg_send![
503
+ allocated,
504
+ initWithURL: &*url,
505
+ fileType: AVFileTypeMPEG4,
506
+ error: &mut err_ptr
507
+ ];
508
+ if writer_raw.is_null() {
509
+ let detail = av_error_suffix(err_ptr);
510
+ if !allocated.is_null() {
511
+ let _ = objc2::rc::Retained::from_raw(allocated);
512
+ }
513
+ return Err(format!("AVAssetWriter init failed{detail}"));
514
+ }
515
+ let writer = objc2::rc::Retained::from_raw(writer_raw)
516
+ .ok_or_else(|| "AVAssetWriter retain failed".to_string())?;
517
+ let _: () = msg_send![&*writer, setShouldOptimizeForNetworkUse: true];
518
+ (writer, None, None)
519
+ };
520
+ let input_cls = av_class_named("AVAssetWriterInput")
521
+ .ok_or_else(|| "AVAssetWriterInput missing".to_string())?;
522
+ let video_settings = av_video_output_settings(width, height)?;
523
+ let video_raw: *mut AnyObject = msg_send![
524
+ input_cls,
525
+ assetWriterInputWithMediaType: AVMediaTypeVideo,
526
+ outputSettings: &*video_settings
527
+ ];
528
+ if video_raw.is_null() {
529
+ return Err("AVAssetWriterInput video allocation failed".into());
530
+ }
531
+ let video_input = objc2::rc::Retained::retain(video_raw)
532
+ .ok_or_else(|| "AVAssetWriterInput video retain failed".to_string())?;
533
+ let _: () = msg_send![&*video_input, setExpectsMediaDataInRealTime: true];
534
+ let can_add_video: bool = msg_send![&*writer, canAddInput: &*video_input];
535
+ if !can_add_video {
536
+ return Err("AVAssetWriter cannot add video input".into());
537
+ }
538
+ let _: () = msg_send![&*writer, addInput: &*video_input];
539
+
540
+ // Live-mixing mode writes one combined audio track; otherwise each
541
+ // captured source gets its own track (mixed later by ffmpeg, or
542
+ // left as the single captured source).
543
+ let (system_audio_input, mic_audio_input, mixed_audio_input, mixer) = if mix_live {
544
+ let mixed = av_make_audio_writer_input(input_cls, &writer)?;
545
+ (
546
+ None,
547
+ None,
548
+ Some(mixed),
549
+ Some(LiveAudioMixer::new(segmented)?),
550
+ )
551
+ } else {
552
+ let system_audio_input = if capture_system_audio {
553
+ Some(av_make_audio_writer_input(input_cls, &writer)?)
554
+ } else {
555
+ None
556
+ };
557
+ let mic_audio_input = if include_audio {
558
+ Some(av_make_audio_writer_input(input_cls, &writer)?)
559
+ } else {
560
+ None
561
+ };
562
+ (system_audio_input, mic_audio_input, None, None)
563
+ };
564
+
565
+ Ok(Self {
566
+ inner: Arc::new(Mutex::new(CustomScreenCaptureWriterState {
567
+ writer,
568
+ video_input,
569
+ system_audio_input,
570
+ mic_audio_input,
571
+ mixed_audio_input,
572
+ segmented,
573
+ segment_sink,
574
+ segment_delegate,
575
+ session_start_time: None,
576
+ finished: false,
577
+ failed: None,
578
+ })),
579
+ mixer: mixer.map(|m| Arc::new(Mutex::new(m))),
580
+ started: Arc::new(AtomicBool::new(false)),
581
+ session_start_bits: Arc::new(AtomicU64::new(f64::NAN.to_bits())),
582
+ appends_closed: Arc::new(AtomicBool::new(false)),
583
+ dropped_samples: Arc::new(AtomicU64::new(0)),
584
+ pause_offset_bits: Arc::new(AtomicU64::new(0.0_f64.to_bits())),
585
+ })
586
+ }
587
+ }
588
+
589
+ /// Whether the writer runs the segmented (zero-based, append-only) output
590
+ /// used by live upload. Only that mode rebases sample timestamps, so it's
591
+ /// the only mode where a fresh SCStream can be spliced onto the same file.
592
+ pub(super) fn segmented(&self) -> bool {
593
+ self.inner.lock().map(|g| g.segmented).unwrap_or(false)
594
+ }
595
+
596
+ /// Whether the writer session has begun (first video frame appended).
597
+ pub(super) fn is_started(&self) -> bool {
598
+ self.started.load(Ordering::SeqCst)
599
+ }
600
+
601
+ /// Accumulated pause offset in seconds. Every appended sample skips this
602
+ /// much wall-clock time so a pause/resume leaves no gap in the file.
603
+ pub(super) fn pause_offset(&self) -> f64 {
604
+ f64::from_bits(self.pause_offset_bits.load(Ordering::SeqCst))
605
+ }
606
+
607
+ /// Set the accumulated pause offset (clamped to non-negative). Resume reads
608
+ /// the prior value, applies `prior + paused_for` once the replacement stream
609
+ /// is about to start, and restores the prior value if startup fails — so a
610
+ /// failed-then-retried resume never double-counts the same pause.
611
+ pub(super) fn set_pause_offset(&self, seconds: f64) {
612
+ self.pause_offset_bits
613
+ .store(seconds.max(0.0).to_bits(), Ordering::SeqCst);
614
+ }
615
+
616
+ /// Route one SCK sample buffer to the right writer input. Mixed-mode
617
+ /// audio goes through [`Self::append_mixed_audio`] (no writer lock while
618
+ /// decoding); everything else appends directly under the `inner` lock.
619
+ /// The first video frame starts the writer session.
620
+ fn append_sample(
621
+ &self,
622
+ sample: &screencapturekit::cm::CMSampleBuffer,
623
+ of_type: SCStreamOutputType,
624
+ ) {
625
+ // Live-mixing mode: mic + system audio are combined into one track
626
+ // before being written, instead of going to separate inputs. Handled
627
+ // before taking the writer lock — the decode/resample/mix work happens
628
+ // on the audio callbacks and must not delay video frame appends.
629
+ if self.mixer.is_some()
630
+ && matches!(
631
+ of_type,
632
+ SCStreamOutputType::Audio | SCStreamOutputType::Microphone
633
+ )
634
+ {
635
+ self.append_mixed_audio(sample, of_type);
636
+ return;
637
+ }
638
+
639
+ let Ok(mut guard) = self.inner.lock() else {
640
+ return;
641
+ };
642
+ if guard.finished || guard.failed.is_some() {
643
+ return;
644
+ }
645
+ let input = match of_type {
646
+ SCStreamOutputType::Screen => Some(guard.video_input.clone()),
647
+ SCStreamOutputType::Audio => guard.system_audio_input.clone(),
648
+ SCStreamOutputType::Microphone => guard.mic_audio_input.clone(),
649
+ };
650
+ let Some(input) = input else {
651
+ return;
652
+ };
653
+ let timing = match sample.sample_timing_info(0) {
654
+ Ok(timing) if timing.presentation_time_stamp.is_valid() => timing,
655
+ _ => return,
656
+ };
657
+
658
+ unsafe {
659
+ if !self.ensure_session_started(&mut guard, timing.presentation_time_stamp) {
660
+ return;
661
+ }
662
+ if guard.segmented {
663
+ // Segmented mode runs a ZERO-based session timeline: append a
664
+ // copy with PTS/DTS rebased against the session start.
665
+ // Appending the raw buffer here would put host-clock times in
666
+ // the track — the writer then buffers/starves (frozen video)
667
+ // and browsers show wall-clock timestamps.
668
+ let Some(base) = guard.session_start_time else {
669
+ return;
670
+ };
671
+ let pause_offset = self.pause_offset();
672
+ match retimed_sample_copy(sample, &timing, base, pause_offset) {
673
+ Ok(copy) => {
674
+ self.append_sample_ptr(&mut guard, &input, copy.as_ptr());
675
+ }
676
+ Err(err) => {
677
+ drop(guard);
678
+ self.fail(format!("sample retime failed: {err}"));
679
+ }
680
+ }
681
+ } else {
682
+ self.append_sample_ptr(&mut guard, &input, sample.as_ptr());
683
+ }
684
+ }
685
+ }
686
+
687
+ /// Pull PCM out of an audio sample, push it into the mixer, then append any
688
+ /// mixed buffers the mixer is ready to emit. The session is started by the
689
+ /// video track, so emitted audio is held until that happens.
690
+ ///
691
+ /// Locking: PCM decode + resample run with no lock held; timeline placement
692
+ /// and draining run under the mixer lock, which stays held through the
693
+ /// appends below so two concurrently-draining audio callbacks can't
694
+ /// interleave out-of-PTS-order appends; the writer (`inner`) lock is taken
695
+ /// last, only around the actual appends.
696
+ fn append_mixed_audio(
697
+ &self,
698
+ sample: &screencapturekit::cm::CMSampleBuffer,
699
+ of_type: SCStreamOutputType,
700
+ ) {
701
+ let (source, label) = match of_type {
702
+ SCStreamOutputType::Audio => (MixSource::System, "system"),
703
+ SCStreamOutputType::Microphone => (MixSource::Mic, "mic"),
704
+ _ => return,
705
+ };
706
+ // Heavy part (decode + resample) — pure function of the sample.
707
+ let Some((interleaved, pts_seconds)) = extract_interleaved_stereo(sample, label) else {
708
+ return;
709
+ };
710
+
711
+ let Some(mixer) = self.mixer.as_ref() else {
712
+ return;
713
+ };
714
+ let Ok(mut mixer_guard) = mixer.lock() else {
715
+ return;
716
+ };
717
+ mixer_guard.set_pause_offset(self.pause_offset());
718
+ mixer_guard.push(source, &interleaved, pts_seconds);
719
+ if !self.started.load(Ordering::SeqCst) {
720
+ return;
721
+ }
722
+ let start_secs = f64::from_bits(self.session_start_bits.load(Ordering::SeqCst));
723
+ if !start_secs.is_nan() {
724
+ mixer_guard.set_min_start(start_secs);
725
+ }
726
+ let emitted = match mixer_guard.drain_ready(false) {
727
+ Ok(buffers) => buffers,
728
+ Err(err) => {
729
+ drop(mixer_guard);
730
+ self.fail(err);
731
+ return;
732
+ }
733
+ };
734
+ if emitted.is_empty() {
735
+ return;
736
+ }
737
+
738
+ let Ok(mut guard) = self.inner.lock() else {
739
+ return;
740
+ };
741
+ if guard.finished || guard.failed.is_some() {
742
+ return;
743
+ }
744
+ let Some(input) = guard.mixed_audio_input.clone() else {
745
+ return;
746
+ };
747
+ for buffer in &emitted {
748
+ unsafe {
749
+ self.append_sample_ptr(&mut guard, &input, buffer.as_ptr());
750
+ }
751
+ if guard.failed.is_some() {
752
+ break;
753
+ }
754
+ }
755
+ }
756
+
757
+ /// Close the writer to further samples and record the failure reason.
758
+ fn fail(&self, err: String) {
759
+ self.appends_closed.store(true, Ordering::SeqCst);
760
+ if let Ok(mut guard) = self.inner.lock() {
761
+ if guard.failed.is_none() {
762
+ guard.failed = Some(err);
763
+ }
764
+ }
765
+ }
766
+
767
+ /// Start the writer session at the first video frame's PTS (once). Also
768
+ /// publishes the session start time to the lock-free atomics the audio
769
+ /// path reads. Returns false (and records the failure) if AVFoundation
770
+ /// refuses to start.
771
+ unsafe fn ensure_session_started(
772
+ &self,
773
+ guard: &mut CustomScreenCaptureWriterState,
774
+ pts: screencapturekit::cm::CMTime,
775
+ ) -> bool {
776
+ use objc2::msg_send;
777
+
778
+ if self.started.load(Ordering::SeqCst) {
779
+ return true;
780
+ }
781
+ // `startWriting` / `startSessionAtSourceTime:` can raise Objective-C
782
+ // exceptions (e.g. invalid state). Catch them so they don't abort the
783
+ // process from inside the realtime callback.
784
+ let writer_ptr = &*guard.writer as *const objc2::runtime::AnyObject;
785
+ let segmented = guard.segmented;
786
+ // Segmented mode: the session runs on a ZERO-based timeline and every
787
+ // appended sample is rebased (see `session_start_time`). Plain file
788
+ // mode keeps the source clock — `startSessionAtSourceTime:` writes an
789
+ // implicit edit so playback still starts at zero there.
790
+ let start = if segmented {
791
+ ObjcCMTime {
792
+ value: 0,
793
+ timescale: pts.timescale.max(1),
794
+ flags: 1,
795
+ epoch: 0,
796
+ }
797
+ } else {
798
+ ObjcCMTime::from(pts)
799
+ };
800
+ let outcome = objc2::exception::catch(std::panic::AssertUnwindSafe(|| unsafe {
801
+ // Segmented output requires the initial segment start time to be
802
+ // set before writing starts; it must equal the session start.
803
+ if segmented {
804
+ let _: () = msg_send![&*writer_ptr, setInitialSegmentStartTime: start];
805
+ }
806
+ let ok: bool = msg_send![&*writer_ptr, startWriting];
807
+ if !ok {
808
+ return false;
809
+ }
810
+ let _: () = msg_send![&*writer_ptr, startSessionAtSourceTime: start];
811
+ true
812
+ }));
813
+ match outcome {
814
+ Ok(true) => {
815
+ guard.session_start_time = Some((pts.value, pts.timescale.max(1)));
816
+ // Publish the session start time before flipping `started` so
817
+ // lock-free readers that observe `started == true` always see
818
+ // a valid start time.
819
+ if let Some(secs) = pts.as_seconds() {
820
+ self.session_start_bits
821
+ .store(secs.to_bits(), Ordering::SeqCst);
822
+ }
823
+ self.started.store(true, Ordering::SeqCst);
824
+ true
825
+ }
826
+ Ok(false) => {
827
+ self.appends_closed.store(true, Ordering::SeqCst);
828
+ guard.failed = Some(format!(
829
+ "AVAssetWriter startWriting failed{}",
830
+ av_writer_error_suffix(&guard.writer)
831
+ ));
832
+ false
833
+ }
834
+ Err(exc) => {
835
+ self.appends_closed.store(true, Ordering::SeqCst);
836
+ let detail = describe_objc_exception(exc);
837
+ eprintln!("[mixer] startWriting raised Objective-C exception: {detail}");
838
+ guard.failed = Some(format!("AVAssetWriter startWriting raised: {detail}"));
839
+ false
840
+ }
841
+ }
842
+ }
843
+
844
+ /// Append one CMSampleBuffer to a writer input, containing Objective-C
845
+ /// exceptions and recording any failure into `guard.failed`. Skips (and
846
+ /// counts) samples when the input reports not-ready — realtime capture
847
+ /// must never block the SCK callback.
848
+ unsafe fn append_sample_ptr(
849
+ &self,
850
+ guard: &mut CustomScreenCaptureWriterState,
851
+ input: &objc2::rc::Retained<objc2::runtime::AnyObject>,
852
+ sample_ptr: *mut std::ffi::c_void,
853
+ ) {
854
+ use objc2::msg_send;
855
+
856
+ let ready: bool = msg_send![&**input, isReadyForMoreMediaData];
857
+ if !ready {
858
+ // Realtime mode: skip rather than block the capture callback, but
859
+ // count and periodically log so sustained backpressure is visible.
860
+ let dropped = self.dropped_samples.fetch_add(1, Ordering::Relaxed) + 1;
861
+ if dropped == 1 || dropped % 100 == 0 {
862
+ eprintln!("[mixer] writer input not ready; dropped {dropped} sample(s) so far");
863
+ }
864
+ return;
865
+ }
866
+ // `appendSampleBuffer:` throws Objective-C exceptions on bad input
867
+ // (format/timestamp/state). Those can't be caught by `catch_unwind`
868
+ // and would abort the app, so contain them here.
869
+ let input_ptr = &**input as *const objc2::runtime::AnyObject;
870
+ let outcome = objc2::exception::catch(std::panic::AssertUnwindSafe(|| unsafe {
871
+ let appended: bool = msg_send![&*input_ptr, appendSampleBuffer: sample_ptr];
872
+ appended
873
+ }));
874
+ match outcome {
875
+ Ok(true) => {}
876
+ Ok(false) => {
877
+ self.appends_closed.store(true, Ordering::SeqCst);
878
+ guard.failed = Some(format!(
879
+ "AVAssetWriter appendSampleBuffer failed{}",
880
+ av_writer_error_suffix(&guard.writer)
881
+ ));
882
+ }
883
+ Err(exc) => {
884
+ self.appends_closed.store(true, Ordering::SeqCst);
885
+ let detail = describe_objc_exception(exc);
886
+ eprintln!("[mixer] appendSampleBuffer raised Objective-C exception: {detail}");
887
+ guard.failed = Some(format!("AVAssetWriter appendSampleBuffer raised: {detail}"));
888
+ }
889
+ }
890
+ }
891
+
892
+ /// Stop accepting samples, flush audio still held by the mixer, mark all
893
+ /// inputs finished, and finalize the file. With `wait_for_finalize` the
894
+ /// call blocks (bounded) until AVFoundation completes and verifies the
895
+ /// writer status; without it, finalization completes in the background
896
+ /// (the fragmented file is already playable up to the last fragment).
897
+ pub(super) fn finish(&self, wait_for_finalize: bool) -> Result<(), String> {
898
+ self.appends_closed.store(true, Ordering::SeqCst);
899
+ let dropped = self.dropped_samples.load(Ordering::Relaxed);
900
+ eprintln!(
901
+ "[mixer] writer finish requested (wait={wait_for_finalize}, dropped_samples={dropped})"
902
+ );
903
+
904
+ // Flush any audio still held in the mixer (treating a missing source as
905
+ // silence) before we tear the writer down. Lock order: mixer → inner.
906
+ if let Some(mixer) = self.mixer.as_ref() {
907
+ if self.started.load(Ordering::SeqCst) {
908
+ let mut mixer_guard = mixer.lock().map_err(|e| e.to_string())?;
909
+ let start_secs = f64::from_bits(self.session_start_bits.load(Ordering::SeqCst));
910
+ if !start_secs.is_nan() {
911
+ mixer_guard.set_min_start(start_secs);
912
+ }
913
+ let emitted = mixer_guard.drain_ready(true);
914
+ let mut guard = self.inner.lock().map_err(|e| e.to_string())?;
915
+ if !guard.finished && guard.failed.is_none() {
916
+ match emitted {
917
+ Ok(buffers) => {
918
+ if let Some(input) = guard.mixed_audio_input.clone() {
919
+ for buffer in &buffers {
920
+ unsafe {
921
+ self.append_sample_ptr(&mut guard, &input, buffer.as_ptr());
922
+ }
923
+ if guard.failed.is_some() {
924
+ break;
925
+ }
926
+ }
927
+ }
928
+ }
929
+ Err(err) => {
930
+ guard.failed = Some(err);
931
+ }
932
+ }
933
+ }
934
+ }
935
+ }
936
+
937
+ let (
938
+ writer,
939
+ video_input,
940
+ system_audio_input,
941
+ mic_audio_input,
942
+ mixed_audio_input,
943
+ segment_sink,
944
+ started,
945
+ failed,
946
+ ) = {
947
+ let mut guard = self.inner.lock().map_err(|e| e.to_string())?;
948
+ if guard.finished {
949
+ return guard.failed.clone().map_or(Ok(()), Err);
950
+ }
951
+ guard.finished = true;
952
+ (
953
+ guard.writer.clone(),
954
+ guard.video_input.clone(),
955
+ guard.system_audio_input.clone(),
956
+ guard.mic_audio_input.clone(),
957
+ guard.mixed_audio_input.clone(),
958
+ guard.segment_sink.clone(),
959
+ self.started.load(Ordering::SeqCst),
960
+ guard.failed.clone(),
961
+ )
962
+ };
963
+ if let Some(err) = failed {
964
+ return Err(err);
965
+ }
966
+ unsafe {
967
+ use block2::RcBlock;
968
+ use objc2::msg_send;
969
+ use std::sync::mpsc;
970
+ use std::time::Duration as StdDuration;
971
+
972
+ if !started {
973
+ let _: () = msg_send![&*writer, cancelWriting];
974
+ return Err("AVAssetWriter received no samples".into());
975
+ }
976
+ let _: () = msg_send![&*video_input, markAsFinished];
977
+ if let Some(input) = system_audio_input.as_ref() {
978
+ let _: () = msg_send![&**input, markAsFinished];
979
+ }
980
+ if let Some(input) = mic_audio_input.as_ref() {
981
+ let _: () = msg_send![&**input, markAsFinished];
982
+ }
983
+ if let Some(input) = mixed_audio_input.as_ref() {
984
+ let _: () = msg_send![&**input, markAsFinished];
985
+ }
986
+
987
+ let (tx, rx) = mpsc::sync_channel::<()>(1);
988
+ let block = RcBlock::new(move || {
989
+ let _ = tx.send(());
990
+ });
991
+ let _: () = msg_send![&*writer, finishWritingWithCompletionHandler: &*block];
992
+ if wait_for_finalize && rx.recv_timeout(StdDuration::from_secs(15)).is_err() {
993
+ let _: () = msg_send![&*writer, cancelWriting];
994
+ return Err("AVAssetWriter finalize timed out".into());
995
+ }
996
+ if wait_for_finalize {
997
+ let status: i64 = msg_send![&*writer, status];
998
+ if status != AV_WRITER_STATUS_COMPLETED {
999
+ return Err(format!(
1000
+ "AVAssetWriter finalize failed (status={status}{})",
1001
+ av_writer_error_suffix(&writer)
1002
+ ));
1003
+ }
1004
+ }
1005
+ }
1006
+ // Segmented mode: the delegate delivers the final segment before the
1007
+ // completion handler fires, so by now the local file is complete —
1008
+ // unless a disk write failed along the way.
1009
+ if let Some(sink) = segment_sink.as_ref() {
1010
+ if let Some(err) = sink.failure() {
1011
+ return Err(err);
1012
+ }
1013
+ }
1014
+ eprintln!("[mixer] writer finish completed (wait={wait_for_finalize})");
1015
+ Ok(())
1016
+ }
1017
+ }
1018
+
1019
+ // ---------------------------------------------------------------------------
1020
+ // Live audio mixer
1021
+ //
1022
+ // Combines the two ScreenCaptureKit audio streams (system + mic) into a single
1023
+ // interleaved-stereo track in real time, so the recorded file already has one
1024
+ // mixed audio track and no post-recording ffmpeg mixing pass is needed.
1025
+ //
1026
+ // The hard part is that the two streams arrive on independent callbacks with
1027
+ // their own timestamps and can start late (mic warmup) or stall. We place each
1028
+ // stream on a shared sample timeline (anchored to the first audio sample),
1029
+ // zero-filling gaps, and only emit output up to the point where we have data we
1030
+ // trust from every still-active source.
1031
+ // ---------------------------------------------------------------------------
1032
+
1033
+ /// Frames per emitted mixed buffer (~85ms at 48kHz).
1034
+ const MIX_CHUNK_FRAMES: i64 = 4096;
1035
+ /// A source with no data for this long stops bounding output (treated as
1036
+ /// silent) so one stalled source can't freeze the mixed track.
1037
+ const MIX_STALL_TIMEOUT: Duration = Duration::from_millis(250);
1038
+ /// How long after mixer creation to keep waiting for a source that hasn't
1039
+ /// produced its first buffer (mic warmup); afterwards it counts as absent.
1040
+ const MIX_SOURCE_GRACE: Duration = Duration::from_millis(2000);
1041
+ const AUDIO_FORMAT_LPCM: u32 = 0x6C70_636D; // 'lpcm'
1042
+ const AUDIO_FORMAT_FLAGS_FLOAT_PACKED: u32 = 1 | 8; // float + packed, interleaved, little-endian
1043
+
1044
+ #[derive(Clone, Copy, PartialEq, Eq)]
1045
+ /// Which capture stream a pushed PCM chunk came from.
1046
+ enum MixSource {
1047
+ System,
1048
+ Mic,
1049
+ }
1050
+
1051
+ /// One source's contiguous PCM ring on the shared output timeline. Gaps
1052
+ /// between pushes are zero-filled; consumed frames are dropped from the
1053
+ /// front as the mixer emits.
1054
+ struct MixerTimeline {
1055
+ /// Absolute frame index (anchor = 0) of `samples[0]`.
1056
+ base_frame: i64,
1057
+ /// Interleaved stereo f32 samples from `base_frame` onward.
1058
+ samples: Vec<f32>,
1059
+ started: bool,
1060
+ last_push: Option<Instant>,
1061
+ }
1062
+
1063
+ impl MixerTimeline {
1064
+ fn new() -> Self {
1065
+ Self {
1066
+ base_frame: 0,
1067
+ samples: Vec::new(),
1068
+ started: false,
1069
+ last_push: None,
1070
+ }
1071
+ }
1072
+
1073
+ /// One past the last frame this source has data for (absolute index).
1074
+ fn end_frame(&self) -> i64 {
1075
+ self.base_frame + (self.samples.len() / 2) as i64
1076
+ }
1077
+
1078
+ /// Stereo sample at an absolute frame index; silence outside the
1079
+ /// buffered range, so callers never need bounds checks.
1080
+ fn sample_at(&self, frame: i64) -> (f32, f32) {
1081
+ if frame < self.base_frame {
1082
+ return (0.0, 0.0);
1083
+ }
1084
+ let offset = ((frame - self.base_frame) as usize) * 2;
1085
+ if offset + 1 < self.samples.len() {
1086
+ (self.samples[offset], self.samples[offset + 1])
1087
+ } else {
1088
+ (0.0, 0.0)
1089
+ }
1090
+ }
1091
+ }
1092
+
1093
+ enum SourceBound {
1094
+ /// Started and recently fed; bounds output to `end_frame`.
1095
+ Active(i64),
1096
+ /// Started but no recent data; treat beyond its data as silence.
1097
+ Stalled,
1098
+ /// Not started yet but still within the warmup grace window; hold output.
1099
+ Pending,
1100
+ /// Not started and past grace; treat as silent.
1101
+ Absent,
1102
+ }
1103
+
1104
+ /// Realtime mic + system mixer. Places each source on a shared 48kHz
1105
+ /// stereo timeline anchored at the first audio PTS, then emits summed
1106
+ /// chunks up to the point every still-active source has data for. See the
1107
+ /// section comment above for the full design rationale.
1108
+ struct LiveAudioMixer {
1109
+ format_desc: screencapturekit::cm::CMFormatDescription,
1110
+ sample_rate: i32,
1111
+ /// Subtract the writer session start from emitted PTS (segmented mode,
1112
+ /// where the session timeline is zero-based). Plain file mode keeps
1113
+ /// absolute source time to match `startSessionAtSourceTime:`.
1114
+ rebase_output: bool,
1115
+ /// Session start in source seconds, recorded by `set_min_start`.
1116
+ session_start_seconds: Option<f64>,
1117
+ /// Accumulated pause time (seconds) subtracted from every incoming source
1118
+ /// PTS so a pause/resume leaves no gap on the audio timeline — mirrors the
1119
+ /// video path's `pause_offset`.
1120
+ pause_offset_seconds: f64,
1121
+ anchor_seconds: Option<f64>,
1122
+ out_pos: i64,
1123
+ system: MixerTimeline,
1124
+ mic: MixerTimeline,
1125
+ created_at: Instant,
1126
+ }
1127
+
1128
+ impl LiveAudioMixer {
1129
+ fn new(rebase_output: bool) -> Result<Self, String> {
1130
+ let sample_rate = AUDIO_OUTPUT_SAMPLE_RATE as i32;
1131
+ let asbd = AudioStreamBasicDescription {
1132
+ sample_rate: AUDIO_OUTPUT_SAMPLE_RATE as f64,
1133
+ format_id: AUDIO_FORMAT_LPCM,
1134
+ format_flags: AUDIO_FORMAT_FLAGS_FLOAT_PACKED,
1135
+ bytes_per_packet: 8,
1136
+ frames_per_packet: 1,
1137
+ bytes_per_frame: 8,
1138
+ channels_per_frame: 2,
1139
+ bits_per_channel: 32,
1140
+ reserved: 0,
1141
+ };
1142
+ let mut desc: *mut std::ffi::c_void = std::ptr::null_mut();
1143
+ let status = unsafe {
1144
+ CMAudioFormatDescriptionCreate(
1145
+ std::ptr::null(),
1146
+ &asbd,
1147
+ 0,
1148
+ std::ptr::null(),
1149
+ 0,
1150
+ std::ptr::null(),
1151
+ std::ptr::null(),
1152
+ &mut desc,
1153
+ )
1154
+ };
1155
+ if status != 0 || desc.is_null() {
1156
+ return Err(format!(
1157
+ "CMAudioFormatDescriptionCreate failed (status={status})"
1158
+ ));
1159
+ }
1160
+ let format_desc = screencapturekit::cm::CMFormatDescription::from_raw(desc)
1161
+ .ok_or_else(|| "CMAudioFormatDescription wrap failed".to_string())?;
1162
+ Ok(Self {
1163
+ format_desc,
1164
+ sample_rate,
1165
+ rebase_output,
1166
+ session_start_seconds: None,
1167
+ pause_offset_seconds: 0.0,
1168
+ anchor_seconds: None,
1169
+ out_pos: 0,
1170
+ system: MixerTimeline::new(),
1171
+ mic: MixerTimeline::new(),
1172
+ created_at: Instant::now(),
1173
+ })
1174
+ }
1175
+
1176
+ /// Place decoded PCM on the source's timeline at its PTS-derived frame
1177
+ /// position, zero-filling any gap since the previous push (capped so a
1178
+ /// glitched timestamp can't allocate gigabytes).
1179
+ fn push(&mut self, source: MixSource, interleaved: &[f32], pts_seconds: f64) {
1180
+ let frames = interleaved.len() / 2;
1181
+ if frames == 0 {
1182
+ return;
1183
+ }
1184
+ // Collapse the paused gap: pull post-resume audio back onto the
1185
+ // continuous timeline so its frame positions abut the pre-pause data
1186
+ // instead of leaving a silence hole (which `push` would otherwise cap
1187
+ // at `max_gap` and desync from the video track).
1188
+ let pts_seconds = pts_seconds - self.pause_offset_seconds;
1189
+ let anchor = *self.anchor_seconds.get_or_insert(pts_seconds);
1190
+ let frame_index =
1191
+ (((pts_seconds - anchor) * self.sample_rate as f64).round() as i64).max(0);
1192
+ let out_pos = self.out_pos;
1193
+ // Cap silence inserted for a timestamp gap. A glitched/discontinuous
1194
+ // PTS could otherwise compute a multi-billion-frame gap and try to
1195
+ // allocate gigabytes of zeros, aborting the process.
1196
+ let max_gap = self.sample_rate as i64 * 2;
1197
+ let timeline = match source {
1198
+ MixSource::System => &mut self.system,
1199
+ MixSource::Mic => &mut self.mic,
1200
+ };
1201
+ if !timeline.started {
1202
+ timeline.started = true;
1203
+ // Never start behind already-emitted output.
1204
+ timeline.base_frame = frame_index.max(out_pos);
1205
+ }
1206
+ let cur_end = timeline.end_frame();
1207
+ if frame_index > cur_end {
1208
+ let gap = (frame_index - cur_end).min(max_gap) as usize;
1209
+ timeline
1210
+ .samples
1211
+ .extend(std::iter::repeat(0.0_f32).take(gap * 2));
1212
+ }
1213
+ timeline.samples.extend_from_slice(interleaved);
1214
+ timeline.last_push = Some(Instant::now());
1215
+ }
1216
+
1217
+ /// Update the accumulated pause offset (seconds) applied to incoming
1218
+ /// source PTS in `push`. Set from the writer's shared offset on resume.
1219
+ fn set_pause_offset(&mut self, seconds: f64) {
1220
+ self.pause_offset_seconds = seconds;
1221
+ }
1222
+
1223
+ /// Advance the output cursor so we never emit audio earlier than the
1224
+ /// writer session start (the writer rejects samples before it).
1225
+ fn set_min_start(&mut self, start_seconds: f64) {
1226
+ self.session_start_seconds = Some(start_seconds);
1227
+ if let Some(anchor) = self.anchor_seconds {
1228
+ // ceil, not round: rounding down would place the first emitted
1229
+ // sample a fraction of a frame BEFORE the session start, which
1230
+ // AVAssetWriter can reject when it writes the fragment.
1231
+ let floor = (((start_seconds - anchor) * self.sample_rate as f64).ceil() as i64).max(0);
1232
+ if floor > self.out_pos {
1233
+ self.out_pos = floor;
1234
+ self.drain_consumed();
1235
+ }
1236
+ }
1237
+ }
1238
+
1239
+ /// How a source currently bounds output: actively feeding (bound to its
1240
+ /// data end), stalled/absent (ignored), or still warming up (holds all
1241
+ /// output back).
1242
+ fn classify(&self, timeline: &MixerTimeline, now: Instant) -> SourceBound {
1243
+ if !timeline.started {
1244
+ if now.duration_since(self.created_at) < MIX_SOURCE_GRACE {
1245
+ SourceBound::Pending
1246
+ } else {
1247
+ SourceBound::Absent
1248
+ }
1249
+ } else if timeline
1250
+ .last_push
1251
+ .map_or(true, |t| now.duration_since(t) >= MIX_STALL_TIMEOUT)
1252
+ {
1253
+ SourceBound::Stalled
1254
+ } else {
1255
+ SourceBound::Active(timeline.end_frame())
1256
+ }
1257
+ }
1258
+
1259
+ /// The frame up to which mixing is safe: the minimum data end across
1260
+ /// active sources (or everything buffered when flushing at stop).
1261
+ fn compute_safe_end(&self, flush: bool) -> i64 {
1262
+ if flush {
1263
+ return self.system.end_frame().max(self.mic.end_frame());
1264
+ }
1265
+ let now = Instant::now();
1266
+ let sys = self.classify(&self.system, now);
1267
+ let mic = self.classify(&self.mic, now);
1268
+ if matches!(sys, SourceBound::Pending) || matches!(mic, SourceBound::Pending) {
1269
+ // Still expecting a source to start; don't run ahead of it.
1270
+ return self.out_pos;
1271
+ }
1272
+ let mut bound = i64::MAX;
1273
+ let mut any_active = false;
1274
+ for b in [&sys, &mic] {
1275
+ if let SourceBound::Active(end) = b {
1276
+ bound = bound.min(*end);
1277
+ any_active = true;
1278
+ }
1279
+ }
1280
+ if any_active {
1281
+ bound
1282
+ } else {
1283
+ // Everything stalled/absent: drain whatever frozen data we have.
1284
+ self.system.end_frame().max(self.mic.end_frame())
1285
+ }
1286
+ }
1287
+
1288
+ /// Emit mixed sample buffers covering `out_pos..safe_end` in
1289
+ /// `MIX_CHUNK_FRAMES` chunks: sum system + mic per frame, clamp, wrap as
1290
+ /// LPCM `CMSampleBuffer`s with contiguous PTS.
1291
+ fn drain_ready(
1292
+ &mut self,
1293
+ flush: bool,
1294
+ ) -> Result<Vec<screencapturekit::cm::CMSampleBuffer>, String> {
1295
+ if self.anchor_seconds.is_none() {
1296
+ return Ok(Vec::new());
1297
+ }
1298
+ let safe_end = self.compute_safe_end(flush);
1299
+ if safe_end <= self.out_pos {
1300
+ return Ok(Vec::new());
1301
+ }
1302
+ let mut emitted = Vec::new();
1303
+ let mut a = self.out_pos;
1304
+ while a < safe_end {
1305
+ let b = (a + MIX_CHUNK_FRAMES).min(safe_end);
1306
+ let n = (b - a) as usize;
1307
+ let mut interleaved = vec![0.0_f32; n * 2];
1308
+ for f in 0..n {
1309
+ let frame = a + f as i64;
1310
+ let (sl, sr) = self.system.sample_at(frame);
1311
+ let (ml, mr) = self.mic.sample_at(frame);
1312
+ interleaved[f * 2] = (sl + ml).clamp(-1.0, 1.0);
1313
+ interleaved[f * 2 + 1] = (sr + mr).clamp(-1.0, 1.0);
1314
+ }
1315
+ emitted.push(self.build_sample_buffer(&interleaved, a)?);
1316
+ a = b;
1317
+ }
1318
+ self.out_pos = safe_end;
1319
+ self.drain_consumed();
1320
+ Ok(emitted)
1321
+ }
1322
+
1323
+ /// Free PCM below the output cursor from both timelines (already mixed
1324
+ /// and emitted; `sample_at` treats it as silence if ever re-read).
1325
+ fn drain_consumed(&mut self) {
1326
+ let out_pos = self.out_pos;
1327
+ for timeline in [&mut self.system, &mut self.mic] {
1328
+ if out_pos > timeline.base_frame {
1329
+ let drop_frames = (out_pos - timeline.base_frame) as usize;
1330
+ let drop_samples = (drop_frames * 2).min(timeline.samples.len());
1331
+ timeline.samples.drain(0..drop_samples);
1332
+ timeline.base_frame = out_pos;
1333
+ }
1334
+ }
1335
+ }
1336
+
1337
+ /// Wrap raw interleaved f32 PCM as a ready-to-append LPCM
1338
+ /// `CMSampleBuffer` whose PTS continues the mixer's output timeline.
1339
+ fn build_sample_buffer(
1340
+ &self,
1341
+ interleaved: &[f32],
1342
+ start_frame: i64,
1343
+ ) -> Result<screencapturekit::cm::CMSampleBuffer, String> {
1344
+ let frames = interleaved.len() / 2;
1345
+ let bytes = unsafe {
1346
+ std::slice::from_raw_parts(
1347
+ interleaved.as_ptr() as *const u8,
1348
+ std::mem::size_of_val(interleaved),
1349
+ )
1350
+ };
1351
+ let block = screencapturekit::cm::CMBlockBuffer::create(bytes)
1352
+ .ok_or_else(|| "CMBlockBuffer create failed".to_string())?;
1353
+ let anchor = self.anchor_seconds.unwrap_or(0.0);
1354
+ // Zero-based session timeline: emit PTS relative to the session start
1355
+ // (set_min_start guarantees emitted frames are never earlier than it).
1356
+ let base = if self.rebase_output {
1357
+ self.session_start_seconds.unwrap_or(anchor)
1358
+ } else {
1359
+ 0.0
1360
+ };
1361
+ let pts_value = ((anchor - base) * self.sample_rate as f64).round() as i64 + start_frame;
1362
+ let pts = ObjcCMTime {
1363
+ value: pts_value,
1364
+ timescale: self.sample_rate,
1365
+ flags: 1,
1366
+ epoch: 0,
1367
+ };
1368
+ let mut out: *mut std::ffi::c_void = std::ptr::null_mut();
1369
+ let status = unsafe {
1370
+ CMAudioSampleBufferCreateReadyWithPacketDescriptions(
1371
+ std::ptr::null(),
1372
+ block.as_ptr(),
1373
+ self.format_desc.as_ptr(),
1374
+ frames as isize,
1375
+ pts,
1376
+ std::ptr::null(),
1377
+ &mut out,
1378
+ )
1379
+ };
1380
+ if status != 0 || out.is_null() {
1381
+ return Err(format!(
1382
+ "CMAudioSampleBufferCreate failed (status={status})"
1383
+ ));
1384
+ }
1385
+ screencapturekit::cm::CMSampleBuffer::from_raw(out)
1386
+ .ok_or_else(|| "CMSampleBuffer wrap failed".to_string())
1387
+ }
1388
+ }
1389
+
1390
+ /// Decode little-endian f32 samples from a CoreMedia byte buffer. Copies
1391
+ /// instead of reinterpreting the pointer: CoreAudio makes no alignment
1392
+ /// guarantee, and casting an unaligned `*const u8` to `&[f32]` is undefined
1393
+ /// behavior.
1394
+ fn bytes_to_f32_vec(bytes: &[u8]) -> Vec<f32> {
1395
+ bytes
1396
+ .chunks_exact(4)
1397
+ .map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]]))
1398
+ .collect()
1399
+ }
1400
+
1401
+ // CoreAudio format flags (AudioFormatFlags).
1402
+ const K_AUDIO_FLAG_IS_FLOAT: u32 = 1 << 0;
1403
+ const K_AUDIO_FLAG_IS_SIGNED_INT: u32 = 1 << 2;
1404
+
1405
+ /// Render a caught Objective-C exception (name + reason) for logging.
1406
+ fn describe_objc_exception(
1407
+ exc: Option<objc2::rc::Retained<objc2::exception::Exception>>,
1408
+ ) -> String {
1409
+ match exc {
1410
+ Some(e) => format!("{e:?}"),
1411
+ None => "unknown Objective-C exception".to_string(),
1412
+ }
1413
+ }
1414
+
1415
+ /// Read the source stream's `AudioStreamBasicDescription` (rate, channels,
1416
+ /// sample format) off a sample buffer; `None` when unavailable.
1417
+ fn source_asbd(
1418
+ sample: &screencapturekit::cm::CMSampleBuffer,
1419
+ ) -> Option<AudioStreamBasicDescription> {
1420
+ let format = sample.format_description()?;
1421
+ let asbd = unsafe { CMAudioFormatDescriptionGetStreamBasicDescription(format.as_ptr()) };
1422
+ if asbd.is_null() {
1423
+ return None;
1424
+ }
1425
+ Some(unsafe { std::ptr::read(asbd) })
1426
+ }
1427
+
1428
+ /// Decode one audio buffer's raw bytes into f32 samples according to the
1429
+ /// stream's sample format (float32/float64 or signed int16/int32).
1430
+ fn decode_samples_to_f32(bytes: &[u8], is_float: bool, bits: u32) -> Vec<f32> {
1431
+ match (is_float, bits) {
1432
+ (true, 32) => bytes_to_f32_vec(bytes),
1433
+ (true, 64) => bytes
1434
+ .chunks_exact(8)
1435
+ .map(|c| f64::from_le_bytes(c.try_into().unwrap()) as f32)
1436
+ .collect(),
1437
+ (false, 16) => bytes
1438
+ .chunks_exact(2)
1439
+ .map(|c| i16::from_le_bytes([c[0], c[1]]) as f32 / 32768.0)
1440
+ .collect(),
1441
+ (false, 32) => bytes
1442
+ .chunks_exact(4)
1443
+ .map(|c| i32::from_le_bytes(c.try_into().unwrap()) as f32 / 2_147_483_648.0)
1444
+ .collect(),
1445
+ // Unknown: best-effort treat as float32.
1446
+ _ => bytes_to_f32_vec(bytes),
1447
+ }
1448
+ }
1449
+
1450
+ /// Resample interleaved-stereo f32 from `src_rate` to `dst_rate` with linear
1451
+ /// interpolation. ScreenCaptureKit delivers the microphone at its native rate
1452
+ /// (often 44.1 kHz / mono-upmixed), which is not the 48 kHz the mixer assumes;
1453
+ /// without this the mic plays back pitch-shifted and unintelligible.
1454
+ fn resample_interleaved_stereo(input: &[f32], src_rate: f64, dst_rate: f64) -> Vec<f32> {
1455
+ let in_frames = input.len() / 2;
1456
+ if in_frames == 0 || src_rate <= 0.0 || dst_rate <= 0.0 || (src_rate - dst_rate).abs() < 1.0 {
1457
+ return input.to_vec();
1458
+ }
1459
+ let ratio = dst_rate / src_rate;
1460
+ let out_frames = ((in_frames as f64) * ratio).round() as usize;
1461
+ if out_frames == 0 {
1462
+ return Vec::new();
1463
+ }
1464
+ let mut out = vec![0.0_f32; out_frames * 2];
1465
+ let step = src_rate / dst_rate;
1466
+ for f in 0..out_frames {
1467
+ let pos = f as f64 * step;
1468
+ let i = pos.floor() as usize;
1469
+ let frac = (pos - i as f64) as f32;
1470
+ let i1 = (i + 1).min(in_frames - 1);
1471
+ for ch in 0..2 {
1472
+ let a = input[i * 2 + ch];
1473
+ let b = input[i1 * 2 + ch];
1474
+ out[f * 2 + ch] = a + (b - a) * frac;
1475
+ }
1476
+ }
1477
+ out
1478
+ }
1479
+
1480
+ fn extract_interleaved_stereo(
1481
+ sample: &screencapturekit::cm::CMSampleBuffer,
1482
+ label: &str,
1483
+ ) -> Option<(Vec<f32>, f64)> {
1484
+ let frames = sample.num_samples();
1485
+ if frames == 0 {
1486
+ return None;
1487
+ }
1488
+ let timing = sample.sample_timing_info(0).ok()?;
1489
+ let pts_seconds = timing.presentation_time_stamp.as_seconds()?;
1490
+ let abl = sample.audio_buffer_list()?;
1491
+ let num_buffers = abl.num_buffers();
1492
+ if num_buffers == 0 {
1493
+ return None;
1494
+ }
1495
+
1496
+ let asbd = source_asbd(sample);
1497
+ let (is_float, bits, src_rate) = match asbd {
1498
+ Some(a) => {
1499
+ // If neither float nor signed-int is flagged, assume float (SCK default).
1500
+ let is_float = a.format_flags & K_AUDIO_FLAG_IS_FLOAT != 0
1501
+ || a.format_flags & K_AUDIO_FLAG_IS_SIGNED_INT == 0;
1502
+ (is_float, a.bits_per_channel, a.sample_rate)
1503
+ }
1504
+ None => (true, 32, AUDIO_OUTPUT_SAMPLE_RATE as f64),
1505
+ };
1506
+
1507
+ log_audio_format_once(label, &asbd, num_buffers, &abl, frames);
1508
+
1509
+ let mut out = vec![0.0_f32; frames * 2];
1510
+ if num_buffers >= 2 {
1511
+ // Non-interleaved (planar): one channel per buffer.
1512
+ let left = decode_samples_to_f32(abl.get(0)?.data(), is_float, bits);
1513
+ let right = decode_samples_to_f32(abl.get(1)?.data(), is_float, bits);
1514
+ let n = frames.min(left.len()).min(right.len());
1515
+ for i in 0..n {
1516
+ out[i * 2] = left[i];
1517
+ out[i * 2 + 1] = right[i];
1518
+ }
1519
+ } else {
1520
+ let buf = abl.get(0)?;
1521
+ let channels = buf.number_channels.max(1) as usize;
1522
+ let decoded = decode_samples_to_f32(buf.data(), is_float, bits);
1523
+ if channels >= 2 {
1524
+ // Interleaved multi-channel: take the first two channels.
1525
+ let n = frames.min(decoded.len() / channels);
1526
+ for i in 0..n {
1527
+ out[i * 2] = decoded[i * channels];
1528
+ out[i * 2 + 1] = decoded[i * channels + 1];
1529
+ }
1530
+ } else {
1531
+ // Mono: duplicate into both channels.
1532
+ let n = frames.min(decoded.len());
1533
+ for i in 0..n {
1534
+ out[i * 2] = decoded[i];
1535
+ out[i * 2 + 1] = decoded[i];
1536
+ }
1537
+ }
1538
+ }
1539
+
1540
+ // Normalize every source to the mixer's output rate so contiguous placement
1541
+ // on the timeline matches real time (otherwise the mic is pitch-shifted).
1542
+ out = resample_interleaved_stereo(&out, src_rate, AUDIO_OUTPUT_SAMPLE_RATE as f64);
1543
+ Some((out, pts_seconds))
1544
+ }
1545
+
1546
+ /// Logs the decoded audio format the first time each source is seen, so format
1547
+ /// mismatches (rate / channels / int-vs-float / interleaving) are diagnosable.
1548
+ fn log_audio_format_once(
1549
+ label: &str,
1550
+ asbd: &Option<AudioStreamBasicDescription>,
1551
+ num_buffers: usize,
1552
+ abl: &screencapturekit::cm::AudioBufferList,
1553
+ frames: usize,
1554
+ ) {
1555
+ use std::sync::atomic::AtomicBool;
1556
+ static SYS_LOGGED: AtomicBool = AtomicBool::new(false);
1557
+ static MIC_LOGGED: AtomicBool = AtomicBool::new(false);
1558
+ let flag = if label == "mic" {
1559
+ &MIC_LOGGED
1560
+ } else {
1561
+ &SYS_LOGGED
1562
+ };
1563
+ if flag.swap(true, Ordering::SeqCst) {
1564
+ return;
1565
+ }
1566
+ let ch0 = abl.get(0).map(|b| (b.number_channels, b.data_byte_size()));
1567
+ let ch1 = abl.get(1).map(|b| (b.number_channels, b.data_byte_size()));
1568
+ eprintln!(
1569
+ "[mixer] {label} format: asbd={asbd:?} num_buffers={num_buffers} frames={frames} buf0={ch0:?} buf1={ch1:?}"
1570
+ );
1571
+ }
1572
+
1573
+ /// Mirror of CoreAudio's `AudioStreamBasicDescription` (repr(C) so it can
1574
+ /// cross the FFI boundary by value).
1575
+ #[repr(C)]
1576
+ #[derive(Copy, Clone, Debug)]
1577
+ struct AudioStreamBasicDescription {
1578
+ sample_rate: f64,
1579
+ format_id: u32,
1580
+ format_flags: u32,
1581
+ bytes_per_packet: u32,
1582
+ frames_per_packet: u32,
1583
+ bytes_per_frame: u32,
1584
+ channels_per_frame: u32,
1585
+ bits_per_channel: u32,
1586
+ reserved: u32,
1587
+ }
1588
+
1589
+ /// `CMSampleTimingInfo` mirror for the retiming FFI call.
1590
+ #[repr(C)]
1591
+ #[derive(Copy, Clone)]
1592
+ struct ObjcCMSampleTimingInfo {
1593
+ duration: ObjcCMTime,
1594
+ presentation_time_stamp: ObjcCMTime,
1595
+ decode_time_stamp: ObjcCMTime,
1596
+ }
1597
+
1598
+ /// Rebase a timestamp onto the zero-based session timeline (subtract the
1599
+ /// first video frame's PTS). `pause_offset_seconds` is additionally subtracted
1600
+ /// so time spent paused collapses to nothing — the file stays gapless across a
1601
+ /// pause/resume. Invalid times pass through untouched.
1602
+ fn rebased_time(
1603
+ t: screencapturekit::cm::CMTime,
1604
+ base: (i64, i32),
1605
+ pause_offset_seconds: f64,
1606
+ ) -> ObjcCMTime {
1607
+ const K_CMTIME_FLAG_VALID: u32 = 1;
1608
+ if t.flags & K_CMTIME_FLAG_VALID == 0 {
1609
+ return ObjcCMTime::from(t);
1610
+ }
1611
+ let (base_value, base_timescale) = base;
1612
+ let base_in_t = if t.timescale == base_timescale {
1613
+ base_value
1614
+ } else {
1615
+ // Convert the base into this timestamp's timescale before subtracting.
1616
+ (i128::from(base_value) * i128::from(t.timescale) / i128::from(base_timescale.max(1)))
1617
+ as i64
1618
+ };
1619
+ // Convert the accumulated pause time into this timestamp's timescale.
1620
+ let pause_in_t = (pause_offset_seconds * f64::from(t.timescale)).round() as i64;
1621
+ ObjcCMTime {
1622
+ value: t.value - base_in_t - pause_in_t,
1623
+ timescale: t.timescale,
1624
+ flags: t.flags,
1625
+ epoch: t.epoch,
1626
+ }
1627
+ }
1628
+
1629
+ /// Copy a sample buffer with its PTS/DTS rebased onto the session timeline.
1630
+ /// One timing entry applies to every sample in the buffer (SCK video buffers
1631
+ /// hold one frame; audio buffers have uniform per-sample timing).
1632
+ fn retimed_sample_copy(
1633
+ sample: &screencapturekit::cm::CMSampleBuffer,
1634
+ timing: &screencapturekit::cm::CMSampleTimingInfo,
1635
+ base: (i64, i32),
1636
+ pause_offset_seconds: f64,
1637
+ ) -> Result<screencapturekit::cm::CMSampleBuffer, String> {
1638
+ let new_timing = ObjcCMSampleTimingInfo {
1639
+ duration: ObjcCMTime::from(timing.duration),
1640
+ presentation_time_stamp: rebased_time(
1641
+ timing.presentation_time_stamp,
1642
+ base,
1643
+ pause_offset_seconds,
1644
+ ),
1645
+ decode_time_stamp: rebased_time(timing.decode_time_stamp, base, pause_offset_seconds),
1646
+ };
1647
+ let mut out: *mut std::ffi::c_void = std::ptr::null_mut();
1648
+ let status = unsafe {
1649
+ CMSampleBufferCreateCopyWithNewTiming(
1650
+ std::ptr::null(),
1651
+ sample.as_ptr(),
1652
+ 1,
1653
+ &new_timing,
1654
+ &mut out,
1655
+ )
1656
+ };
1657
+ if status != 0 || out.is_null() {
1658
+ return Err(format!(
1659
+ "CMSampleBufferCreateCopyWithNewTiming failed (status={status})"
1660
+ ));
1661
+ }
1662
+ screencapturekit::cm::CMSampleBuffer::from_raw(out)
1663
+ .ok_or_else(|| "retimed CMSampleBuffer wrap failed".to_string())
1664
+ }
1665
+
1666
+ // CoreMedia C API used to hand-build the mixer's LPCM output buffers —
1667
+ // the screencapturekit crate has no constructors for these.
1668
+ #[link(name = "CoreMedia", kind = "framework")]
1669
+ extern "C" {
1670
+ fn CMAudioFormatDescriptionCreate(
1671
+ allocator: *const std::ffi::c_void,
1672
+ asbd: *const AudioStreamBasicDescription,
1673
+ layout_size: usize,
1674
+ layout: *const std::ffi::c_void,
1675
+ magic_cookie_size: usize,
1676
+ magic_cookie: *const std::ffi::c_void,
1677
+ extensions: *const std::ffi::c_void,
1678
+ format_description_out: *mut *mut std::ffi::c_void,
1679
+ ) -> i32;
1680
+
1681
+ fn CMAudioSampleBufferCreateReadyWithPacketDescriptions(
1682
+ allocator: *const std::ffi::c_void,
1683
+ data_buffer: *mut std::ffi::c_void,
1684
+ format_description: *mut std::ffi::c_void,
1685
+ num_samples: isize,
1686
+ presentation_time_stamp: ObjcCMTime,
1687
+ packet_descriptions: *const std::ffi::c_void,
1688
+ sample_buffer_out: *mut *mut std::ffi::c_void,
1689
+ ) -> i32;
1690
+
1691
+ fn CMAudioFormatDescriptionGetStreamBasicDescription(
1692
+ format_description: *mut std::ffi::c_void,
1693
+ ) -> *const AudioStreamBasicDescription;
1694
+
1695
+ fn CMSampleBufferCreateCopyWithNewTiming(
1696
+ allocator: *const std::ffi::c_void,
1697
+ original: *mut std::ffi::c_void,
1698
+ num_timing_entries: isize,
1699
+ timing_array: *const ObjcCMSampleTimingInfo,
1700
+ sample_buffer_out: *mut *mut std::ffi::c_void,
1701
+ ) -> i32;
1702
+ }
1703
+
1704
+ /// Mirror of CoreMedia's `CMTime`, with objc2 `Encode` impls so it can be
1705
+ /// passed by value through `msg_send!` (e.g. `startSessionAtSourceTime:`,
1706
+ /// `setPreferredOutputSegmentInterval:`).
1707
+ #[repr(C)]
1708
+ #[derive(Copy, Clone)]
1709
+ struct ObjcCMTime {
1710
+ value: i64,
1711
+ timescale: i32,
1712
+ flags: u32,
1713
+ epoch: i64,
1714
+ }
1715
+
1716
+ unsafe impl objc2::encode::RefEncode for ObjcCMTime {
1717
+ const ENCODING_REF: objc2::encode::Encoding =
1718
+ objc2::encode::Encoding::Pointer(&<Self as objc2::encode::Encode>::ENCODING);
1719
+ }
1720
+
1721
+ unsafe impl objc2::encode::Encode for ObjcCMTime {
1722
+ const ENCODING: objc2::encode::Encoding = objc2::encode::Encoding::Struct(
1723
+ "CMTime",
1724
+ &[
1725
+ <i64 as objc2::encode::Encode>::ENCODING,
1726
+ <i32 as objc2::encode::Encode>::ENCODING,
1727
+ <u32 as objc2::encode::Encode>::ENCODING,
1728
+ <i64 as objc2::encode::Encode>::ENCODING,
1729
+ ],
1730
+ );
1731
+ }
1732
+
1733
+ impl From<screencapturekit::cm::CMTime> for ObjcCMTime {
1734
+ fn from(value: screencapturekit::cm::CMTime) -> Self {
1735
+ Self {
1736
+ value: value.value,
1737
+ timescale: value.timescale,
1738
+ flags: value.flags,
1739
+ epoch: value.epoch,
1740
+ }
1741
+ }
1742
+ }
1743
+
1744
+ /// Look up an Objective-C class by name at runtime.
1745
+ unsafe fn av_class_named(name: &str) -> Option<&'static objc2::runtime::AnyClass> {
1746
+ let bytes = std::ffi::CString::new(name).ok()?;
1747
+ objc2::runtime::AnyClass::get(&bytes)
1748
+ }
1749
+
1750
+ /// Build a retained `NSString` from a Rust string.
1751
+ unsafe fn av_ns_string_from(s: &str) -> Option<objc2::rc::Retained<objc2::runtime::AnyObject>> {
1752
+ use objc2::runtime::AnyObject;
1753
+ use objc2::{class, msg_send};
1754
+
1755
+ let cstr = std::ffi::CString::new(s).ok()?;
1756
+ let allocated: *mut AnyObject = msg_send![class!(NSString), alloc];
1757
+ if allocated.is_null() {
1758
+ return None;
1759
+ }
1760
+ let inited: *mut AnyObject = msg_send![allocated, initWithUTF8String: cstr.as_ptr()];
1761
+ if inited.is_null() {
1762
+ let _ = objc2::rc::Retained::from_raw(allocated);
1763
+ None
1764
+ } else {
1765
+ objc2::rc::Retained::from_raw(inited)
1766
+ }
1767
+ }
1768
+
1769
+ /// Build a retained `NSURL` file URL for the writer's output path.
1770
+ unsafe fn av_file_url(path: &Path) -> Option<objc2::rc::Retained<objc2::runtime::AnyObject>> {
1771
+ use objc2::runtime::AnyObject;
1772
+ use objc2::{class, msg_send};
1773
+
1774
+ let path_str = path.to_str()?;
1775
+ let nsstr = av_ns_string_from(path_str)?;
1776
+ let url: *mut AnyObject = msg_send![class!(NSURL), fileURLWithPath: &*nsstr];
1777
+ if url.is_null() {
1778
+ None
1779
+ } else {
1780
+ objc2::rc::Retained::retain(url)
1781
+ }
1782
+ }
1783
+
1784
+ /// Boxed `NSNumber` for integer settings-dictionary values.
1785
+ unsafe fn av_number_i64(value: i64) -> Option<objc2::rc::Retained<objc2::runtime::AnyObject>> {
1786
+ use objc2::runtime::AnyObject;
1787
+ use objc2::{class, msg_send};
1788
+
1789
+ let raw: *mut AnyObject = msg_send![class!(NSNumber), numberWithLongLong: value];
1790
+ if raw.is_null() {
1791
+ None
1792
+ } else {
1793
+ objc2::rc::Retained::retain(raw)
1794
+ }
1795
+ }
1796
+
1797
+ /// Boxed boolean `NSNumber` for settings-dictionary flags.
1798
+ unsafe fn av_number_bool(value: bool) -> Option<objc2::rc::Retained<objc2::runtime::AnyObject>> {
1799
+ use objc2::runtime::AnyObject;
1800
+ use objc2::{class, msg_send};
1801
+
1802
+ let raw: *mut AnyObject = msg_send![class!(NSNumber), numberWithBool: value];
1803
+ if raw.is_null() {
1804
+ None
1805
+ } else {
1806
+ objc2::rc::Retained::retain(raw)
1807
+ }
1808
+ }
1809
+
1810
+ /// Fresh `NSMutableDictionary` for AVFoundation output settings.
1811
+ unsafe fn av_dict() -> Result<objc2::rc::Retained<objc2::runtime::AnyObject>, String> {
1812
+ use objc2::runtime::AnyObject;
1813
+ use objc2::{class, msg_send};
1814
+
1815
+ let raw: *mut AnyObject = msg_send![class!(NSMutableDictionary), dictionary];
1816
+ if raw.is_null() {
1817
+ Err("NSMutableDictionary allocation failed".into())
1818
+ } else {
1819
+ objc2::rc::Retained::retain(raw)
1820
+ .ok_or_else(|| "NSMutableDictionary retain failed".to_string())
1821
+ }
1822
+ }
1823
+
1824
+ /// `dict[key] = value` on an `NSMutableDictionary`.
1825
+ unsafe fn av_dict_set(
1826
+ dict: &objc2::runtime::AnyObject,
1827
+ key: *const objc2::runtime::AnyObject,
1828
+ value: &objc2::runtime::AnyObject,
1829
+ ) {
1830
+ use objc2::msg_send;
1831
+
1832
+ let _: () = msg_send![dict, setObject: value, forKey: key];
1833
+ }
1834
+
1835
+ /// H.264 output settings for the video writer input, sized to the capture
1836
+ /// dimensions and with frame reordering (B-frames) disabled — required for
1837
+ /// stable fragmented-MP4 writing (see the comment inside).
1838
+ unsafe fn av_video_output_settings(
1839
+ width: u32,
1840
+ height: u32,
1841
+ ) -> Result<objc2::rc::Retained<objc2::runtime::AnyObject>, String> {
1842
+ use objc2::runtime::AnyObject;
1843
+
1844
+ #[link(name = "AVFoundation", kind = "framework")]
1845
+ extern "C" {
1846
+ static AVVideoCodecKey: *const AnyObject;
1847
+ static AVVideoCodecTypeH264: *const AnyObject;
1848
+ static AVVideoHeightKey: *const AnyObject;
1849
+ static AVVideoWidthKey: *const AnyObject;
1850
+ static AVVideoCompressionPropertiesKey: *const AnyObject;
1851
+ static AVVideoAllowFrameReorderingKey: *const AnyObject;
1852
+ static AVVideoAverageBitRateKey: *const AnyObject;
1853
+ static AVVideoExpectedSourceFrameRateKey: *const AnyObject;
1854
+ static AVVideoMaxKeyFrameIntervalKey: *const AnyObject;
1855
+ }
1856
+
1857
+ let settings = av_dict()?;
1858
+ let width_value =
1859
+ av_number_i64(width as i64).ok_or_else(|| "NSNumber width failed".to_string())?;
1860
+ let height_value =
1861
+ av_number_i64(height as i64).ok_or_else(|| "NSNumber height failed".to_string())?;
1862
+ av_dict_set(&settings, AVVideoCodecKey, &*AVVideoCodecTypeH264);
1863
+ av_dict_set(&settings, AVVideoWidthKey, &width_value);
1864
+ av_dict_set(&settings, AVVideoHeightKey, &height_value);
1865
+
1866
+ // Segmented output (preferredOutputSegmentInterval) needs monotonic video timing
1867
+ // inside each fragment. The encoder's default B-frames (frame reordering)
1868
+ // intermittently kill the writer with -11800 / OSStatus -16341 when a
1869
+ // reordered frame group straddles a fragment boundary, truncating the
1870
+ // recording at an exact fragment-boundary timestamp. Screen capture gains
1871
+ // almost nothing from B-frames; disable reordering.
1872
+ let compression = av_dict()?;
1873
+ let no_reordering =
1874
+ av_number_bool(false).ok_or_else(|| "NSNumber reordering flag failed".to_string())?;
1875
+ av_dict_set(&compression, AVVideoAllowFrameReorderingKey, &no_reordering);
1876
+
1877
+ // Capture-time rate control (mirrors Cap's AVAssetWriter setup). Without
1878
+ // an explicit budget the encoder default runs several times larger and
1879
+ // every upload needs an ffmpeg transcode pass to shrink it.
1880
+ let average_bit_rate =
1881
+ (CAPTURE_VIDEO_BPP * f64::from(width) * f64::from(height) * f64::from(NATIVE_CAPTURE_FPS))
1882
+ as i64;
1883
+ let bit_rate =
1884
+ av_number_i64(average_bit_rate).ok_or_else(|| "NSNumber bit rate failed".to_string())?;
1885
+ av_dict_set(&compression, AVVideoAverageBitRateKey, &bit_rate);
1886
+ let expected_fps = av_number_i64(i64::from(NATIVE_CAPTURE_FPS))
1887
+ .ok_or_else(|| "NSNumber expected fps failed".to_string())?;
1888
+ av_dict_set(
1889
+ &compression,
1890
+ AVVideoExpectedSourceFrameRateKey,
1891
+ &expected_fps,
1892
+ );
1893
+ // ~0.75s keyframe cadence: guarantees at least one sync sample per movie
1894
+ // fragment (1s interval) so every fragment stays independently seekable.
1895
+ let keyframe_interval = av_number_i64((i64::from(NATIVE_CAPTURE_FPS) * 3 / 4).max(1))
1896
+ .ok_or_else(|| "NSNumber keyframe interval failed".to_string())?;
1897
+ av_dict_set(
1898
+ &compression,
1899
+ AVVideoMaxKeyFrameIntervalKey,
1900
+ &keyframe_interval,
1901
+ );
1902
+
1903
+ av_dict_set(&settings, AVVideoCompressionPropertiesKey, &compression);
1904
+ Ok(settings)
1905
+ }
1906
+
1907
+ /// AAC 48kHz stereo 128kbps output settings for the audio writer inputs
1908
+ /// (both the mixed track and per-source tracks).
1909
+ unsafe fn av_audio_output_settings(
1910
+ ) -> Result<objc2::rc::Retained<objc2::runtime::AnyObject>, String> {
1911
+ use objc2::runtime::AnyObject;
1912
+
1913
+ #[link(name = "AVFoundation", kind = "framework")]
1914
+ extern "C" {
1915
+ static AVEncoderBitRateKey: *const AnyObject;
1916
+ static AVFormatIDKey: *const AnyObject;
1917
+ static AVNumberOfChannelsKey: *const AnyObject;
1918
+ static AVSampleRateKey: *const AnyObject;
1919
+ }
1920
+
1921
+ let settings = av_dict()?;
1922
+ let format = av_number_i64(AUDIO_FORMAT_AAC)
1923
+ .ok_or_else(|| "NSNumber audio format failed".to_string())?;
1924
+ let sample_rate =
1925
+ av_number_i64(48_000).ok_or_else(|| "NSNumber sample rate failed".to_string())?;
1926
+ let channels = av_number_i64(2).ok_or_else(|| "NSNumber channels failed".to_string())?;
1927
+ let bitrate = av_number_i64(128_000).ok_or_else(|| "NSNumber bitrate failed".to_string())?;
1928
+ av_dict_set(&settings, AVFormatIDKey, &format);
1929
+ av_dict_set(&settings, AVSampleRateKey, &sample_rate);
1930
+ av_dict_set(&settings, AVNumberOfChannelsKey, &channels);
1931
+ av_dict_set(&settings, AVEncoderBitRateKey, &bitrate);
1932
+ Ok(settings)
1933
+ }
1934
+
1935
+ /// Create an AAC `AVAssetWriterInput` (realtime mode), attach it to the
1936
+ /// writer, and return it retained.
1937
+ unsafe fn av_make_audio_writer_input(
1938
+ input_cls: &objc2::runtime::AnyClass,
1939
+ writer: &objc2::runtime::AnyObject,
1940
+ ) -> Result<objc2::rc::Retained<objc2::runtime::AnyObject>, String> {
1941
+ use objc2::msg_send;
1942
+ use objc2::runtime::AnyObject;
1943
+
1944
+ #[link(name = "AVFoundation", kind = "framework")]
1945
+ extern "C" {
1946
+ static AVMediaTypeAudio: *const AnyObject;
1947
+ }
1948
+
1949
+ let settings = av_audio_output_settings()?;
1950
+ let raw: *mut AnyObject = msg_send![
1951
+ input_cls,
1952
+ assetWriterInputWithMediaType: AVMediaTypeAudio,
1953
+ outputSettings: &*settings
1954
+ ];
1955
+ if raw.is_null() {
1956
+ return Err("AVAssetWriterInput audio allocation failed".into());
1957
+ }
1958
+ let input = objc2::rc::Retained::retain(raw)
1959
+ .ok_or_else(|| "AVAssetWriterInput audio retain failed".to_string())?;
1960
+ let _: () = msg_send![&*input, setExpectsMediaDataInRealTime: true];
1961
+ let can_add: bool = msg_send![writer, canAddInput: &*input];
1962
+ if !can_add {
1963
+ return Err("AVAssetWriter cannot add audio input".into());
1964
+ }
1965
+ let _: () = msg_send![writer, addInput: &*input];
1966
+ Ok(input)
1967
+ }
1968
+
1969
+ /// Render an `NSError` for logs: description, domain+code, failure reason,
1970
+ /// and the underlying error's domain+code (where the raw OSStatus that
1971
+ /// names the real cause usually hides).
1972
+ unsafe fn av_error_suffix(err_obj: *mut objc2::runtime::AnyObject) -> String {
1973
+ if err_obj.is_null() {
1974
+ return String::new();
1975
+ }
1976
+ let desc_obj: *mut objc2::runtime::AnyObject = objc2::msg_send![err_obj, localizedDescription];
1977
+ let mut out = av_string_suffix(desc_obj);
1978
+
1979
+ // Domain + code identify the error class; the localizedDescription alone
1980
+ // is usually a generic "The operation could not be completed".
1981
+ let domain_obj: *mut objc2::runtime::AnyObject = objc2::msg_send![err_obj, domain];
1982
+ let code: i64 = objc2::msg_send![err_obj, code];
1983
+ out.push_str(&format!(
1984
+ " [domain{} code={code}]",
1985
+ av_string_suffix(domain_obj)
1986
+ ));
1987
+
1988
+ let reason_obj: *mut objc2::runtime::AnyObject =
1989
+ objc2::msg_send![err_obj, localizedFailureReason];
1990
+ if !reason_obj.is_null() {
1991
+ out.push_str(&format!(" reason{}", av_string_suffix(reason_obj)));
1992
+ }
1993
+
1994
+ // The underlying error carries the raw OSStatus naming the real cause
1995
+ // (e.g. CoreMedia -12780); surface its domain + code too.
1996
+ if let Some(cls) = av_class_named("NSString") {
1997
+ let key_cstr = b"NSUnderlyingError\0".as_ptr() as *const i8;
1998
+ let key: *mut objc2::runtime::AnyObject =
1999
+ objc2::msg_send![cls, stringWithUTF8String: key_cstr];
2000
+ if !key.is_null() {
2001
+ let user_info: *mut objc2::runtime::AnyObject = objc2::msg_send![err_obj, userInfo];
2002
+ if !user_info.is_null() {
2003
+ let underlying: *mut objc2::runtime::AnyObject =
2004
+ objc2::msg_send![user_info, objectForKey: key];
2005
+ if !underlying.is_null() {
2006
+ let u_domain: *mut objc2::runtime::AnyObject =
2007
+ objc2::msg_send![underlying, domain];
2008
+ let u_code: i64 = objc2::msg_send![underlying, code];
2009
+ out.push_str(&format!(
2010
+ " underlying=[domain{} code={u_code}]",
2011
+ av_string_suffix(u_domain)
2012
+ ));
2013
+ }
2014
+ }
2015
+ }
2016
+ }
2017
+ out
2018
+ }
2019
+
2020
+ /// Render the writer's current status + `error` property for logs.
2021
+ unsafe fn av_writer_error_suffix(writer: &objc2::runtime::AnyObject) -> String {
2022
+ let status: i64 = objc2::msg_send![writer, status];
2023
+ let err_obj: *mut objc2::runtime::AnyObject = objc2::msg_send![writer, error];
2024
+ format!(" (writer status={status}){}", av_error_suffix(err_obj))
2025
+ }
2026
+
2027
+ /// `": <string>"` from an `NSString` pointer, or empty when nil.
2028
+ unsafe fn av_string_suffix(obj: *mut objc2::runtime::AnyObject) -> String {
2029
+ if obj.is_null() {
2030
+ return String::new();
2031
+ }
2032
+ let utf8: *const i8 = objc2::msg_send![obj, UTF8String];
2033
+ if utf8.is_null() {
2034
+ return String::new();
2035
+ }
2036
+ let cstr = std::ffi::CStr::from_ptr(utf8);
2037
+ format!(": {}", cstr.to_string_lossy())
2038
+ }
2039
+
2040
+ /// Plain, `Send` capture parameters kept so the watchdog can rebuild the
2041
+ /// SCStream from scratch after an interruption without holding on to any
2042
+ /// non-`Send` ScreenCaptureKit handles. `width`/`height` are fixed to the
2043
+ /// dimensions the writer was created with so a rebuilt stream keeps producing
2044
+ /// frames the existing video input accepts.
2045
+ #[derive(Clone)]
2046
+ struct RestartParams {
2047
+ include_audio: bool,
2048
+ capture_system_audio: bool,
2049
+ mic_device_id: Option<String>,
2050
+ mic_device_label: Option<String>,
2051
+ target_display_id: Option<u32>,
2052
+ capture_region: Option<NativeCaptureRegion>,
2053
+ width: u32,
2054
+ height: u32,
2055
+ }
2056
+
2057
+ /// Everything the pause/resume path needs to stop the capture source without
2058
+ /// tearing down the writer, and to splice a fresh SCStream onto the same
2059
+ /// (append-only) file on resume. Cheap to clone — all handles are `Arc`s or
2060
+ /// plain data — so it lives alongside the backend and shares the watchdog's
2061
+ /// stream/handler/watch.
2062
+ pub(crate) struct CustomCaptureResume {
2063
+ stream: Arc<Mutex<SCStream>>,
2064
+ handler: CustomScreenCaptureOutputHandler,
2065
+ watch: Arc<CaptureWatch>,
2066
+ params: RestartParams,
2067
+ }
2068
+
2069
+ impl CustomCaptureResume {
2070
+ /// Pause: stop only the capture source (SCStream). The writer, file, and
2071
+ /// live uploader stay alive; mic/screen go cold. The watchdog is told to
2072
+ /// hold so it never reads the silence as a stall and rebuilds.
2073
+ pub(crate) fn pause(&self) {
2074
+ self.watch.set_paused(true);
2075
+ if let Ok(guard) = self.stream.lock() {
2076
+ let _ = guard.stop_capture();
2077
+ }
2078
+ eprintln!("[mixer] capture paused; source stream stopped, writer/file kept open");
2079
+ }
2080
+
2081
+ /// Resume: build a fresh SCStream wired to the SAME writer and start it,
2082
+ /// after advancing the writer's pause offset by `paused_for` so the new
2083
+ /// samples rebase past the pause gap. The result is one continuous
2084
+ /// append-only file — the live uploader is never interrupted, exactly like
2085
+ /// a watchdog stream rebuild.
2086
+ pub(crate) fn resume(&self, paused_for: Duration) -> Result<(), String> {
2087
+ let writer = &self.handler.writer;
2088
+ let prev_offset = writer.pause_offset();
2089
+
2090
+ // Build the replacement stream FIRST: a build failure must not shift the
2091
+ // timeline. The session stays paused and can be retried — and a retry
2092
+ // re-measures `paused_for` from the same (uncleared) pause instant, so
2093
+ // advancing the offset here would compound on every failed attempt.
2094
+ let new_stream = build_custom_scstream(&self.params, &self.handler, &self.watch)?;
2095
+
2096
+ // Apply the pause gap just before the stream starts delivering, so the
2097
+ // first rebased frame already skips it. Roll back if startup fails so
2098
+ // the failed attempt leaves the offset exactly as it was.
2099
+ writer.set_pause_offset(prev_offset + paused_for.as_secs_f64());
2100
+ if let Err(err) = new_stream.start_capture() {
2101
+ writer.set_pause_offset(prev_offset);
2102
+ return Err(format!("resume start_capture failed: {err:?}"));
2103
+ }
2104
+
2105
+ // Stop any lingering paused stream, then swap the fresh one in under
2106
+ // the same lock the watchdog uses so the two never feed at once.
2107
+ if let Ok(guard) = self.stream.lock() {
2108
+ let _ = guard.stop_capture();
2109
+ }
2110
+ if let Ok(mut guard) = self.stream.lock() {
2111
+ *guard = new_stream;
2112
+ }
2113
+ // Discard the stop note our own pause/stop raised and reset the
2114
+ // activity clock so the watchdog doesn't immediately treat the just-
2115
+ // rebuilt stream as stalled, then hand supervision back.
2116
+ let _ = self.watch.take_stream_stopped();
2117
+ self.watch.note_activity();
2118
+ self.watch.set_paused(false);
2119
+ eprintln!(
2120
+ "[mixer] capture resumed; fresh stream spliced onto same writer (paused {}ms)",
2121
+ paused_for.as_millis()
2122
+ );
2123
+ Ok(())
2124
+ }
2125
+ }
2126
+
2127
+ /// Stream lifecycle delegate for the custom pipeline. ScreenCaptureKit calls
2128
+ /// this when it stops the stream (e.g. the captured display changed Spaces or a
2129
+ /// full-screen app took over). Without it those stops are invisible and the
2130
+ /// recording silently freezes. We only flag the watchdog here — rebuilding the
2131
+ /// stream from an SCK callback thread is unsafe, so recovery happens off-thread.
2132
+ struct CustomCaptureStreamDelegate {
2133
+ watch: Arc<CaptureWatch>,
2134
+ }
2135
+
2136
+ impl SCStreamDelegateTrait for CustomCaptureStreamDelegate {
2137
+ fn did_stop_with_error(&self, error: SCError) {
2138
+ // The user stopping capture via macOS (menu-bar "Stop Sharing") is a
2139
+ // request, not a failure — rebuilding the stream would fight the
2140
+ // user. Everything else (SystemStoppedStream on lid close / display
2141
+ // sleep, connection failures, ...) goes to the rebuild path.
2142
+ if error.stream_error_code()
2143
+ == Some(screencapturekit::error::SCStreamErrorCode::UserStopped)
2144
+ {
2145
+ eprintln!("[mixer] capture stopped by the user via macOS; requesting recording stop");
2146
+ self.watch.note_user_stopped();
2147
+ return;
2148
+ }
2149
+ let reason = format!("ScreenCaptureKit stream stopped with error: {error}");
2150
+ eprintln!("[mixer] {reason}");
2151
+ self.watch.note_stream_stopped(reason);
2152
+ }
2153
+
2154
+ fn stream_did_stop(&self, error: Option<String>) {
2155
+ let detail = error.unwrap_or_else(|| "no detail".to_string());
2156
+ let reason = format!("ScreenCaptureKit stream stopped: {detail}");
2157
+ eprintln!("[mixer] {reason}");
2158
+ self.watch.note_stream_stopped(reason);
2159
+ }
2160
+ }
2161
+
2162
+ /// Build (but do not start) a fresh SCStream for the custom pipeline from plain
2163
+ /// parameters. Shared by the initial start and every watchdog rebuild so the
2164
+ /// filter/config/handler/delegate wiring can never drift between them.
2165
+ fn build_custom_scstream(
2166
+ params: &RestartParams,
2167
+ handler: &CustomScreenCaptureOutputHandler,
2168
+ watch: &Arc<CaptureWatch>,
2169
+ ) -> Result<SCStream, String> {
2170
+ let content =
2171
+ SCShareableContent::get().map_err(|e| format!("shareable content lookup failed: {e:?}"))?;
2172
+ let displays = content.displays();
2173
+ let display = params
2174
+ .target_display_id
2175
+ .and_then(|id| displays.iter().find(|d| d.display_id() == id))
2176
+ .or_else(|| displays.first())
2177
+ .ok_or_else(|| "No displays available for ScreenCaptureKit recording.".to_string())?;
2178
+
2179
+ let region_rect = region_source_rect(params.capture_region, display.width(), display.height())?;
2180
+ let filter_builder = SCContentFilter::create()
2181
+ .with_display(display)
2182
+ .with_excluding_windows(&[]);
2183
+ let filter = if let Some((rect, _, _)) = region_rect {
2184
+ filter_builder.with_content_rect(rect).build()
2185
+ } else {
2186
+ filter_builder.build()
2187
+ };
2188
+
2189
+ let selected_mic = if params.include_audio {
2190
+ resolve_microphone_capture_device(
2191
+ params.mic_device_id.as_deref(),
2192
+ params.mic_device_label.as_deref(),
2193
+ )?
2194
+ } else {
2195
+ None
2196
+ };
2197
+
2198
+ let mut config = SCStreamConfiguration::new()
2199
+ .with_width(params.width)
2200
+ .with_height(params.height)
2201
+ .with_fps(NATIVE_CAPTURE_FPS)
2202
+ .with_queue_depth(8)
2203
+ .with_shows_cursor(true)
2204
+ .with_captures_audio(params.capture_system_audio)
2205
+ .with_captures_microphone(params.include_audio)
2206
+ .with_excludes_current_process_audio(true)
2207
+ .with_sample_rate(48000)
2208
+ .with_channel_count(2);
2209
+ // Pin SDR NV12 (video-range 4:2:0) delivery, matching Cap. Two reasons:
2210
+ // - Without a pin, ScreenCaptureKit switches the delivered pixel format
2211
+ // to HDR/EDR variants (half-float / 10-bit) when the frontmost app
2212
+ // renders EDR content; the SDR H.264 writer input then rejects every
2213
+ // appended frame (-11800 / OSStatus -16122) and the writer dies —
2214
+ // including on rebuilt streams while that app stays frontmost.
2215
+ // - NV12 is VideoToolbox's native encoder input and half the memory
2216
+ // bandwidth of BGRA.
2217
+ // NOTE: do not add `set_color_space_name` — the crate's ObjC shim for it
2218
+ // raises an uncatchable Objective-C exception and aborts the process;
2219
+ // the pixel-format pin alone keeps the encoder input format stable.
2220
+ config.set_pixel_format(screencapturekit::stream::configuration::PixelFormat::YCbCr_420v);
2221
+ if let Some((rect, _, _)) = region_rect {
2222
+ config.set_source_rect(rect);
2223
+ }
2224
+ if let Some(device) = selected_mic.as_ref() {
2225
+ config.set_microphone_capture_device_id(&device.id);
2226
+ }
2227
+ config.set_stream_name(Some("Clips custom full-screen recording"));
2228
+
2229
+ let delegate = CustomCaptureStreamDelegate {
2230
+ watch: Arc::clone(watch),
2231
+ };
2232
+ let mut stream = SCStream::new_with_delegate(&filter, &config, delegate);
2233
+ stream.add_output_handler(handler.clone(), SCStreamOutputType::Screen);
2234
+ if params.capture_system_audio {
2235
+ stream.add_output_handler(handler.clone(), SCStreamOutputType::Audio);
2236
+ }
2237
+ if params.include_audio {
2238
+ stream.add_output_handler(handler.clone(), SCStreamOutputType::Microphone);
2239
+ }
2240
+ Ok(stream)
2241
+ }
2242
+
2243
+ /// Supervise a running custom capture and rebuild the SCStream when it stops or
2244
+ /// goes silent (the Spaces/full-screen interruption). Runs on its own thread;
2245
+ /// exits when recording is torn down (`shutdown`) or the writer is closed.
2246
+ fn spawn_capture_watchdog(
2247
+ app: AppHandle,
2248
+ stream: Arc<Mutex<SCStream>>,
2249
+ writer: CustomScreenCaptureWriter,
2250
+ handler: CustomScreenCaptureOutputHandler,
2251
+ watch: Arc<CaptureWatch>,
2252
+ recording_enabled: Arc<AtomicBool>,
2253
+ shutdown: Arc<AtomicBool>,
2254
+ params: RestartParams,
2255
+ ) {
2256
+ std::thread::spawn(move || {
2257
+ let mut restart_streak: u32 = 0;
2258
+ // After a rebuild, hold off re-evaluating until the new stream has had a
2259
+ // fair chance to deliver its first frames.
2260
+ let mut cooldown_until: Option<Instant> = None;
2261
+
2262
+ loop {
2263
+ std::thread::sleep(CAPTURE_WATCHDOG_POLL);
2264
+ if shutdown.load(Ordering::SeqCst) {
2265
+ return;
2266
+ }
2267
+ if writer.appends_closed.load(Ordering::SeqCst) {
2268
+ // Appends closed while we're still supervising. A clean stop
2269
+ // sets `watchdog_shutdown` BEFORE closing appends (SeqCst), so
2270
+ // re-check it: if it's still unset, appends were closed by an
2271
+ // unrecoverable capture failure (an Objective-C exception or
2272
+ // panic in the sample callback, an `appendSampleBuffer` error,
2273
+ // or a writer-session failure) with the SCStream still running
2274
+ // and the UI still showing "recording". Stop capture and fire
2275
+ // the normal stop so the partial — still a valid fragmented
2276
+ // file — is finalized/uploaded instead of silently truncating.
2277
+ if shutdown.load(Ordering::SeqCst) {
2278
+ return;
2279
+ }
2280
+ eprintln!(
2281
+ "[mixer] appends closed without a stop request; treating as fatal capture failure and finalizing partial recording"
2282
+ );
2283
+ if let Ok(guard) = stream.lock() {
2284
+ let _ = guard.stop_capture();
2285
+ }
2286
+ let _ = app.emit("clips:recorder-stop", ());
2287
+ return;
2288
+ }
2289
+ // Nothing to supervise until output is enabled and the writer
2290
+ // session has actually begun; the handler keeps `last_activity`
2291
+ // fresh from the first delivered buffer, so there is no false stall
2292
+ // when we start evaluating.
2293
+ if !recording_enabled.load(Ordering::SeqCst) || !writer.started.load(Ordering::SeqCst) {
2294
+ continue;
2295
+ }
2296
+
2297
+ // Paused: the capture source is intentionally stopped while the
2298
+ // writer/file/uploader stay alive. Don't read the silence as a
2299
+ // stall or rebuild — resume splices a fresh stream back in.
2300
+ if watch.is_paused() {
2301
+ restart_streak = 0;
2302
+ cooldown_until = None;
2303
+ continue;
2304
+ }
2305
+
2306
+ // User stopped capture from the macOS UI: trigger the normal stop
2307
+ // flow (same event the toolbar Stop button emits, so the clip is
2308
+ // finalized and uploaded) and stop supervising. Never rebuild.
2309
+ if watch.user_stopped() {
2310
+ eprintln!("[mixer] user stopped capture via macOS; emitting recorder stop");
2311
+ let _ = app.emit("clips:recorder-stop", ());
2312
+ return;
2313
+ }
2314
+
2315
+ if let Some(until) = cooldown_until {
2316
+ if Instant::now() < until {
2317
+ continue;
2318
+ }
2319
+ cooldown_until = None;
2320
+ if watch.since_activity() < CAPTURE_STALL_TIMEOUT {
2321
+ // Rebuild recovered — frames are flowing again.
2322
+ restart_streak = 0;
2323
+ continue;
2324
+ }
2325
+ // Still no frames after the rebuild; fall through and retry.
2326
+ }
2327
+
2328
+ let reported_stop = watch.take_stream_stopped();
2329
+ let stalled = watch.since_activity() >= CAPTURE_STALL_TIMEOUT;
2330
+ if reported_stop.is_none() && !stalled {
2331
+ restart_streak = 0;
2332
+ continue;
2333
+ }
2334
+
2335
+ let reason = reported_stop.unwrap_or_else(|| {
2336
+ format!(
2337
+ "no capture frames for {:?} (display may have moved to another Space)",
2338
+ watch.since_activity()
2339
+ )
2340
+ });
2341
+ restart_streak += 1;
2342
+ if restart_streak > CAPTURE_MAX_RESTARTS {
2343
+ eprintln!(
2344
+ "[mixer] capture interrupted ({reason}) and did not recover after {CAPTURE_MAX_RESTARTS} restarts; finalizing partial recording"
2345
+ );
2346
+ // Close appends (but leave `failed` unset) so the stop path
2347
+ // still finalizes everything captured before the interruption
2348
+ // instead of discarding it.
2349
+ writer.appends_closed.store(true, Ordering::SeqCst);
2350
+ if let Ok(guard) = stream.lock() {
2351
+ let _ = guard.stop_capture();
2352
+ }
2353
+ // Fire the normal stop so the partial clip is finalized/uploaded
2354
+ // and the UI leaves the recording state, rather than sitting on
2355
+ // a frozen recording after the watchdog gives up.
2356
+ let _ = app.emit("clips:recorder-stop", ());
2357
+ return;
2358
+ }
2359
+
2360
+ eprintln!(
2361
+ "[mixer] capture interrupted ({reason}); rebuilding stream (attempt {restart_streak}/{CAPTURE_MAX_RESTARTS})"
2362
+ );
2363
+
2364
+ // Stop the dead/wedged stream before starting a replacement so two
2365
+ // streams never feed the writer at once. Slow build/start work is
2366
+ // done without holding the stream lock.
2367
+ if let Ok(guard) = stream.lock() {
2368
+ let _ = guard.stop_capture();
2369
+ }
2370
+ if shutdown.load(Ordering::SeqCst) || writer.appends_closed.load(Ordering::SeqCst) {
2371
+ return;
2372
+ }
2373
+
2374
+ match build_custom_scstream(&params, &handler, &watch).and_then(|s| {
2375
+ s.start_capture()
2376
+ .map(|()| s)
2377
+ .map_err(|e| format!("start_capture failed: {e:?}"))
2378
+ }) {
2379
+ Ok(new_stream) => {
2380
+ if shutdown.load(Ordering::SeqCst)
2381
+ || writer.appends_closed.load(Ordering::SeqCst)
2382
+ {
2383
+ let _ = new_stream.stop_capture();
2384
+ return;
2385
+ }
2386
+ if let Ok(mut guard) = stream.lock() {
2387
+ *guard = new_stream;
2388
+ }
2389
+ // Discard any stop note the deliberate teardown of the old
2390
+ // stream raised; otherwise a later poll would consume it as
2391
+ // a fresh failure and rebuild the healthy stream again.
2392
+ let _ = watch.take_stream_stopped();
2393
+ cooldown_until = Some(Instant::now() + CAPTURE_STALL_TIMEOUT);
2394
+ eprintln!("[mixer] capture stream rebuilt; waiting for frames");
2395
+ }
2396
+ Err(err) => {
2397
+ eprintln!("[mixer] capture rebuild failed: {err}");
2398
+ // Short cooldown before the next attempt so a hard failure
2399
+ // (e.g. display gone) doesn't spin the CPU.
2400
+ cooldown_until = Some(Instant::now() + CAPTURE_WATCHDOG_POLL);
2401
+ }
2402
+ }
2403
+ }
2404
+ });
2405
+ }
2406
+
2407
+ /// Start the custom capture backend: create the fragmented-MP4 writer,
2408
+ /// build + start the SCStream from rebuildable params, and spawn the
2409
+ /// capture watchdog that supervises it. Returns the backend handle plus
2410
+ /// the output dimensions.
2411
+ pub(super) fn start_custom_screencapturekit_backend_at(
2412
+ app: &AppHandle,
2413
+ output_path: &Path,
2414
+ include_audio: bool,
2415
+ capture_system_audio: bool,
2416
+ mic_device_id: Option<&str>,
2417
+ mic_device_label: Option<&str>,
2418
+ target_display_id: Option<u32>,
2419
+ capture_region: Option<NativeCaptureRegion>,
2420
+ defer_recording_output: bool,
2421
+ ) -> Result<(NativeFullscreenBackend, Option<u32>, Option<u32>), String> {
2422
+ eprintln!("[clips-tray] starting custom screen capture backend");
2423
+ let content =
2424
+ SCShareableContent::get().map_err(|e| format!("shareable content lookup failed: {e:?}"))?;
2425
+ let displays = content.displays();
2426
+ let display = target_display_id
2427
+ .and_then(|id| displays.iter().find(|d| d.display_id() == id))
2428
+ .or_else(|| displays.first())
2429
+ .ok_or_else(|| "No displays available for ScreenCaptureKit recording.".to_string())?;
2430
+
2431
+ let source_width = display.width();
2432
+ let source_height = display.height();
2433
+ let region_rect = region_source_rect(capture_region, source_width, source_height)?;
2434
+ let (capture_width, capture_height) = region_rect
2435
+ .as_ref()
2436
+ .map(|(_, width, height)| (*width, *height))
2437
+ .unwrap_or((source_width, source_height));
2438
+ let (width, height) = native_capture_dimensions(capture_width, capture_height);
2439
+ // The display handle here is only used to size the output; the actual
2440
+ // (rebuildable) stream is constructed from plain params via
2441
+ // `build_custom_scstream` so the watchdog can recreate it later.
2442
+
2443
+ let params = RestartParams {
2444
+ include_audio,
2445
+ capture_system_audio,
2446
+ mic_device_id: mic_device_id.map(str::to_string),
2447
+ mic_device_label: mic_device_label.map(str::to_string),
2448
+ target_display_id,
2449
+ capture_region,
2450
+ width,
2451
+ height,
2452
+ };
2453
+
2454
+ // Live-mix only when both audio sources are present; a single source needs
2455
+ // no mixing and goes straight to its own track.
2456
+ let mix_live = include_audio && capture_system_audio;
2457
+ let writer = CustomScreenCaptureWriter::new(
2458
+ output_path,
2459
+ width,
2460
+ height,
2461
+ capture_system_audio,
2462
+ include_audio,
2463
+ mix_live,
2464
+ )?;
2465
+ let recording_enabled = Arc::new(AtomicBool::new(!defer_recording_output));
2466
+ let mic_ready = include_audio.then(|| Arc::new(AtomicBool::new(false)));
2467
+ let watch = Arc::new(CaptureWatch::new());
2468
+ let handler = CustomScreenCaptureOutputHandler {
2469
+ writer: writer.clone(),
2470
+ recording_enabled: Arc::clone(&recording_enabled),
2471
+ mic_ready: mic_ready.clone(),
2472
+ watch: Arc::clone(&watch),
2473
+ };
2474
+
2475
+ let stream = build_custom_scstream(&params, &handler, &watch)?;
2476
+ if let Err(err) = stream.start_capture() {
2477
+ let _ = std::fs::remove_file(output_path);
2478
+ return Err(format!("custom capture start failed: {err:?}"));
2479
+ }
2480
+ eprintln!(
2481
+ "[clips-tray] custom ScreenCaptureKit recording started: {width}x{height} @ {NATIVE_CAPTURE_FPS}fps from {capture_width}x{capture_height} (display {source_width}x{source_height}), mic={include_audio} system_audio={capture_system_audio} deferred_output={defer_recording_output}"
2482
+ );
2483
+
2484
+ let stream = Arc::new(Mutex::new(stream));
2485
+ let watchdog_shutdown = Arc::new(AtomicBool::new(false));
2486
+ // Snapshot the handles pause/resume needs before the watchdog consumes
2487
+ // `handler` and `params`; the fresh stream it builds on resume feeds the
2488
+ // same writer as the watchdog's own rebuilds.
2489
+ let resume = CustomCaptureResume {
2490
+ stream: Arc::clone(&stream),
2491
+ handler: handler.clone(),
2492
+ watch: Arc::clone(&watch),
2493
+ params: params.clone(),
2494
+ };
2495
+ spawn_capture_watchdog(
2496
+ app.clone(),
2497
+ Arc::clone(&stream),
2498
+ writer.clone(),
2499
+ handler,
2500
+ Arc::clone(&watch),
2501
+ Arc::clone(&recording_enabled),
2502
+ Arc::clone(&watchdog_shutdown),
2503
+ params,
2504
+ );
2505
+
2506
+ Ok((
2507
+ NativeFullscreenBackend::CustomScreenCaptureKit {
2508
+ stream,
2509
+ writer,
2510
+ mic_ready,
2511
+ recording_enabled,
2512
+ watchdog_shutdown,
2513
+ resume,
2514
+ },
2515
+ Some(width),
2516
+ Some(height),
2517
+ ))
2518
+ }