@agent-native/core 0.70.3 → 0.72.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (221) hide show
  1. package/corpus/README.md +2 -2
  2. package/corpus/core/CHANGELOG.md +51 -0
  3. package/corpus/core/docs/content/actions.md +23 -0
  4. package/corpus/core/docs/content/audit-log.md +111 -0
  5. package/corpus/core/docs/content/multi-app-workspace.md +2 -2
  6. package/corpus/core/docs/design/durable-agent-runs.md +458 -4
  7. package/corpus/core/package.json +2 -1
  8. package/corpus/core/src/action.ts +80 -1
  9. package/corpus/core/src/agent/durable-background.ts +192 -0
  10. package/corpus/core/src/agent/production-agent.ts +357 -13
  11. package/corpus/core/src/agent/run-manager.ts +66 -3
  12. package/corpus/core/src/agent/run-store.ts +143 -23
  13. package/corpus/core/src/agent/types.ts +20 -0
  14. package/corpus/core/src/application-state/store.ts +5 -0
  15. package/corpus/core/src/audit/actions/get-audit-event.ts +23 -0
  16. package/corpus/core/src/audit/actions/list-audit-events.ts +65 -0
  17. package/corpus/core/src/audit/cleanup-job.ts +100 -0
  18. package/corpus/core/src/audit/config.ts +91 -0
  19. package/corpus/core/src/audit/index.ts +43 -0
  20. package/corpus/core/src/audit/record.ts +143 -0
  21. package/corpus/core/src/audit/redact.ts +109 -0
  22. package/corpus/core/src/audit/store.ts +244 -0
  23. package/corpus/core/src/audit/types.ts +125 -0
  24. package/corpus/core/src/chat-threads/store.ts +10 -0
  25. package/corpus/core/src/client/AgentPanel.tsx +12 -9
  26. package/corpus/core/src/client/AssistantChat.tsx +6 -0
  27. package/corpus/core/src/client/blocks/library/FileTreeBlock.tsx +72 -14
  28. package/corpus/core/src/client/chat/repo-helpers.ts +38 -0
  29. package/corpus/core/src/db/client.ts +4 -0
  30. package/corpus/core/src/db/widen-columns.ts +75 -0
  31. package/corpus/core/src/deploy/build.ts +96 -0
  32. package/corpus/core/src/deploy/workspace-deploy.ts +121 -0
  33. package/corpus/core/src/mcp/build-server.ts +23 -13
  34. package/corpus/core/src/oauth-tokens/store.ts +6 -0
  35. package/corpus/core/src/provider-api/custom-registry.ts +8 -0
  36. package/corpus/core/src/resources/store.ts +10 -0
  37. package/corpus/core/src/scripts/runner.ts +6 -3
  38. package/corpus/core/src/server/action-discovery.ts +6 -0
  39. package/corpus/core/src/server/action-routes.ts +1 -0
  40. package/corpus/core/src/server/agent-chat-plugin.ts +133 -66
  41. package/corpus/core/src/server/auth.ts +5 -0
  42. package/corpus/core/src/server/core-routes-plugin.ts +14 -0
  43. package/corpus/core/src/server/security-headers.ts +9 -6
  44. package/corpus/core/src/settings/store.ts +6 -0
  45. package/corpus/core/src/templates/workspace-core/.agents/skills/audit-log/SKILL.md +93 -0
  46. package/corpus/core/src/usage/store.ts +6 -0
  47. package/corpus/templates/analytics/.agents/skills/dashboard-management/SKILL.md +19 -0
  48. package/corpus/templates/analytics/AGENTS.md +8 -0
  49. package/corpus/templates/analytics/actions/compose-dashboard.ts +317 -0
  50. package/corpus/templates/analytics/changelog/2026-06-23-build-large-first-party-analytics-dashboards-in-one-fast-cal.md +6 -0
  51. package/corpus/templates/analytics/server/lib/first-party-metric-catalog.ts +574 -0
  52. package/corpus/templates/calendar/app/components/calendar/GoogleConnectBanner.tsx +60 -6
  53. package/corpus/templates/calendar/changelog/2026-06-23-added-a-heads-up-explaining-google-s-app-not-verified-screen.md +6 -0
  54. package/corpus/templates/clips/app/components/recorder/recorder-engine.ts +20 -43
  55. package/corpus/templates/clips/app/components/recorder/recording-toolbar.tsx +5 -1
  56. package/corpus/templates/clips/app/lib/countdown-audio-cue.ts +3 -51
  57. package/corpus/templates/clips/changelog/2026-06-23-dragging-the-desktop-camera-bubble-now-glides-to-a-stop-at-t.md +6 -0
  58. package/corpus/templates/clips/changelog/2026-06-23-public-clips-now-play-inline-in-slack-connect-a-workspac.md +6 -0
  59. package/corpus/templates/clips/chrome-extension/PERMISSIONS.md +72 -0
  60. package/corpus/templates/clips/chrome-extension/public/manifest.json +1 -10
  61. package/corpus/templates/clips/chrome-extension/src/background.ts +163 -32
  62. package/corpus/templates/clips/chrome-extension/src/content-script.ts +301 -24
  63. package/corpus/templates/clips/chrome-extension/src/offscreen.ts +245 -67
  64. package/corpus/templates/clips/chrome-extension/src/overlay.css +77 -43
  65. package/corpus/templates/clips/chrome-extension/src/overlay.html +7 -1
  66. package/corpus/templates/clips/chrome-extension/src/overlay.ts +102 -38
  67. package/corpus/templates/clips/chrome-extension/src/popup.html +62 -5
  68. package/corpus/templates/clips/chrome-extension/src/popup.ts +275 -4
  69. package/corpus/templates/clips/chrome-extension/src/styles.css +34 -0
  70. package/corpus/templates/clips/chrome-extension/vite.config.ts +5 -0
  71. package/corpus/templates/clips/desktop/src/lib/audio-cue.ts +21 -16
  72. package/corpus/templates/clips/desktop/src/lib/recorder.ts +10 -17
  73. package/corpus/templates/clips/desktop/src/overlays/bubble.tsx +73 -23
  74. package/corpus/templates/clips/desktop/src/overlays/countdown.tsx +0 -6
  75. package/corpus/templates/clips/desktop/src/overlays/toolbar.tsx +56 -46
  76. package/corpus/templates/clips/desktop/src/styles.css +33 -5
  77. package/corpus/templates/clips/desktop/src-tauri/src/clips/mod.rs +113 -0
  78. package/corpus/templates/clips/desktop/src-tauri/src/lib.rs +3 -0
  79. package/corpus/templates/clips/server/lib/media-permissions.ts +5 -1
  80. package/corpus/templates/clips/shared/recording-audio.ts +62 -0
  81. package/corpus/templates/clips/shared/recording-core.ts +94 -0
  82. package/dist/action.d.ts +31 -0
  83. package/dist/action.d.ts.map +1 -1
  84. package/dist/action.js +45 -1
  85. package/dist/action.js.map +1 -1
  86. package/dist/agent/context-xray/schema.d.ts +1 -1
  87. package/dist/agent/durable-background.d.ts +60 -0
  88. package/dist/agent/durable-background.d.ts.map +1 -0
  89. package/dist/agent/durable-background.js +144 -0
  90. package/dist/agent/durable-background.js.map +1 -0
  91. package/dist/agent/observational-memory/schema.d.ts +1 -1
  92. package/dist/agent/production-agent.d.ts +20 -0
  93. package/dist/agent/production-agent.d.ts.map +1 -1
  94. package/dist/agent/production-agent.js +320 -14
  95. package/dist/agent/production-agent.js.map +1 -1
  96. package/dist/agent/run-manager.d.ts +48 -0
  97. package/dist/agent/run-manager.d.ts.map +1 -1
  98. package/dist/agent/run-manager.js +45 -3
  99. package/dist/agent/run-manager.js.map +1 -1
  100. package/dist/agent/run-store.d.ts +30 -1
  101. package/dist/agent/run-store.d.ts.map +1 -1
  102. package/dist/agent/run-store.js +138 -24
  103. package/dist/agent/run-store.js.map +1 -1
  104. package/dist/agent/types.d.ts +20 -0
  105. package/dist/agent/types.d.ts.map +1 -1
  106. package/dist/agent/types.js.map +1 -1
  107. package/dist/application-state/store.d.ts.map +1 -1
  108. package/dist/application-state/store.js +5 -0
  109. package/dist/application-state/store.js.map +1 -1
  110. package/dist/audit/actions/get-audit-event.d.ts +11 -0
  111. package/dist/audit/actions/get-audit-event.d.ts.map +1 -0
  112. package/dist/audit/actions/get-audit-event.js +22 -0
  113. package/dist/audit/actions/get-audit-event.js.map +1 -0
  114. package/dist/audit/actions/list-audit-events.d.ts +22 -0
  115. package/dist/audit/actions/list-audit-events.d.ts.map +1 -0
  116. package/dist/audit/actions/list-audit-events.js +61 -0
  117. package/dist/audit/actions/list-audit-events.js.map +1 -0
  118. package/dist/audit/cleanup-job.d.ts +12 -0
  119. package/dist/audit/cleanup-job.d.ts.map +1 -0
  120. package/dist/audit/cleanup-job.js +93 -0
  121. package/dist/audit/cleanup-job.js.map +1 -0
  122. package/dist/audit/config.d.ts +30 -0
  123. package/dist/audit/config.d.ts.map +1 -0
  124. package/dist/audit/config.js +65 -0
  125. package/dist/audit/config.js.map +1 -0
  126. package/dist/audit/index.d.ts +13 -0
  127. package/dist/audit/index.d.ts.map +1 -0
  128. package/dist/audit/index.js +6 -0
  129. package/dist/audit/index.js.map +1 -0
  130. package/dist/audit/record.d.ts +24 -0
  131. package/dist/audit/record.d.ts.map +1 -0
  132. package/dist/audit/record.js +106 -0
  133. package/dist/audit/record.js.map +1 -0
  134. package/dist/audit/redact.d.ts +28 -0
  135. package/dist/audit/redact.d.ts.map +1 -0
  136. package/dist/audit/redact.js +109 -0
  137. package/dist/audit/redact.js.map +1 -0
  138. package/dist/audit/store.d.ts +14 -0
  139. package/dist/audit/store.d.ts.map +1 -0
  140. package/dist/audit/store.js +219 -0
  141. package/dist/audit/store.js.map +1 -0
  142. package/dist/audit/types.d.ts +114 -0
  143. package/dist/audit/types.d.ts.map +1 -0
  144. package/dist/audit/types.js +15 -0
  145. package/dist/audit/types.js.map +1 -0
  146. package/dist/chat-threads/store.d.ts.map +1 -1
  147. package/dist/chat-threads/store.js +10 -0
  148. package/dist/chat-threads/store.js.map +1 -1
  149. package/dist/client/AgentPanel.d.ts.map +1 -1
  150. package/dist/client/AgentPanel.js +2 -2
  151. package/dist/client/AgentPanel.js.map +1 -1
  152. package/dist/client/AssistantChat.d.ts.map +1 -1
  153. package/dist/client/AssistantChat.js +6 -1
  154. package/dist/client/AssistantChat.js.map +1 -1
  155. package/dist/client/blocks/library/FileTreeBlock.d.ts.map +1 -1
  156. package/dist/client/blocks/library/FileTreeBlock.js +37 -4
  157. package/dist/client/blocks/library/FileTreeBlock.js.map +1 -1
  158. package/dist/client/chat/repo-helpers.d.ts +16 -0
  159. package/dist/client/chat/repo-helpers.d.ts.map +1 -1
  160. package/dist/client/chat/repo-helpers.js +40 -0
  161. package/dist/client/chat/repo-helpers.js.map +1 -1
  162. package/dist/db/client.d.ts.map +1 -1
  163. package/dist/db/client.js +3 -0
  164. package/dist/db/client.js.map +1 -1
  165. package/dist/db/widen-columns.d.ts +39 -0
  166. package/dist/db/widen-columns.d.ts.map +1 -0
  167. package/dist/db/widen-columns.js +73 -0
  168. package/dist/db/widen-columns.js.map +1 -0
  169. package/dist/deploy/build.d.ts +29 -0
  170. package/dist/deploy/build.d.ts.map +1 -1
  171. package/dist/deploy/build.js +85 -0
  172. package/dist/deploy/build.js.map +1 -1
  173. package/dist/deploy/workspace-deploy.d.ts.map +1 -1
  174. package/dist/deploy/workspace-deploy.js +108 -0
  175. package/dist/deploy/workspace-deploy.js.map +1 -1
  176. package/dist/mcp/build-server.d.ts.map +1 -1
  177. package/dist/mcp/build-server.js +24 -9
  178. package/dist/mcp/build-server.js.map +1 -1
  179. package/dist/oauth-tokens/store.d.ts.map +1 -1
  180. package/dist/oauth-tokens/store.js +6 -0
  181. package/dist/oauth-tokens/store.js.map +1 -1
  182. package/dist/provider-api/custom-registry.d.ts.map +1 -1
  183. package/dist/provider-api/custom-registry.js +8 -0
  184. package/dist/provider-api/custom-registry.js.map +1 -1
  185. package/dist/resources/store.d.ts.map +1 -1
  186. package/dist/resources/store.js +9 -0
  187. package/dist/resources/store.js.map +1 -1
  188. package/dist/scripts/runner.js +4 -3
  189. package/dist/scripts/runner.js.map +1 -1
  190. package/dist/server/action-discovery.d.ts.map +1 -1
  191. package/dist/server/action-discovery.js +6 -0
  192. package/dist/server/action-discovery.js.map +1 -1
  193. package/dist/server/action-routes.d.ts.map +1 -1
  194. package/dist/server/action-routes.js +1 -0
  195. package/dist/server/action-routes.js.map +1 -1
  196. package/dist/server/agent-chat-plugin.d.ts.map +1 -1
  197. package/dist/server/agent-chat-plugin.js +68 -13
  198. package/dist/server/agent-chat-plugin.js.map +1 -1
  199. package/dist/server/auth.d.ts.map +1 -1
  200. package/dist/server/auth.js +5 -0
  201. package/dist/server/auth.js.map +1 -1
  202. package/dist/server/core-routes-plugin.d.ts.map +1 -1
  203. package/dist/server/core-routes-plugin.js +13 -0
  204. package/dist/server/core-routes-plugin.js.map +1 -1
  205. package/dist/server/security-headers.d.ts +8 -5
  206. package/dist/server/security-headers.d.ts.map +1 -1
  207. package/dist/server/security-headers.js +9 -6
  208. package/dist/server/security-headers.js.map +1 -1
  209. package/dist/settings/store.d.ts.map +1 -1
  210. package/dist/settings/store.js +6 -0
  211. package/dist/settings/store.js.map +1 -1
  212. package/dist/templates/workspace-core/.agents/skills/audit-log/SKILL.md +93 -0
  213. package/dist/usage/store.d.ts.map +1 -1
  214. package/dist/usage/store.js +5 -0
  215. package/dist/usage/store.js.map +1 -1
  216. package/docs/content/actions.md +23 -0
  217. package/docs/content/audit-log.md +111 -0
  218. package/docs/content/multi-app-workspace.md +2 -2
  219. package/docs/design/durable-agent-runs.md +458 -4
  220. package/package.json +2 -1
  221. package/src/templates/workspace-core/.agents/skills/audit-log/SKILL.md +93 -0
@@ -3,15 +3,21 @@
3
3
  // This document holds the getDisplayMedia()/getUserMedia() stream and the
4
4
  // MediaRecorder. Living in an offscreen document (reason DISPLAY_MEDIA) is what
5
5
  // lets a recording survive page navigations — the capture is decoupled from any
6
- // tab. The camera bubble and controls are rendered as on-page overlays by the
7
- // content script, so we no longer composite the camera into a canvas here; for
8
- // screen modes we record the display stream directly (+ mixed mic audio), and
9
- // the bubble shows up in the recording naturally when a full screen/window/tab
10
- // surface that contains it is captured.
6
+ // tab. Controls render as on-page overlays. For SCREEN+CAMERA we capture the
7
+ // camera HERE and composite it (canvas) into the recording, because the on-page
8
+ // bubble can't get the camera on pages that send `Permissions-Policy: camera=()`
9
+ // (an iframe can't escape its parent's policy). Screen-only records the display
10
+ // directly; camera-only records the webcam directly.
11
11
  //
12
12
  // Lifecycle: ACQUIRE (show picker, hold stream) → BEGIN (start recorder after
13
13
  // the countdown) → PAUSE/RESUME → STOP/CANCEL, plus RESTART (discard and start
14
14
  // over on the same stream).
15
+ //
16
+ // MIME selection and the chunk-upload URL/param protocol are shared with the web
17
+ // app recorder via @shared/recording-core so the server contract can't drift.
18
+
19
+ import { chunkUploadUrl, pickMimeType } from "@shared/recording-core";
20
+ import { scheduleReadyChime } from "@shared/recording-audio";
15
21
 
16
22
  type CaptureMode = "screen" | "camera";
17
23
 
@@ -21,6 +27,9 @@ type AcquireMessage = {
21
27
  mode: CaptureMode;
22
28
  surface: "browser" | "window" | "monitor";
23
29
  includeMicrophone: boolean;
30
+ // Screen+camera: capture the camera here and composite it into the recording
31
+ // (the on-page bubble can be blocked by the page's Permissions-Policy).
32
+ includeCamera: boolean;
24
33
  };
25
34
 
26
35
  type BeginMessage = {
@@ -94,6 +103,13 @@ type ActiveRecording = {
94
103
  restarting: boolean;
95
104
  // Pending pre-roll timer; non-null means the recorder hasn't started yet.
96
105
  startTimer: ReturnType<typeof setTimeout> | null;
106
+ // Canvas compositor draw loop (screen+camera only); stop it on teardown.
107
+ stopCompositor: (() => void) | null;
108
+ // Original capture streams, kept distinct so restart can re-home exactly the
109
+ // right ones (sourceStreams also holds the derived canvas stream).
110
+ displaySource: MediaStream | null;
111
+ cameraSource: MediaStream | null;
112
+ micSource: MediaStream | null;
97
113
  dimensions: { width: number; height: number };
98
114
  hasAudio: boolean;
99
115
  hasCamera: boolean;
@@ -118,20 +134,6 @@ function reportStatus(
118
134
  });
119
135
  }
120
136
 
121
- function chooseMimeType(): string {
122
- const preferred = [
123
- "video/webm;codecs=vp9,opus",
124
- "video/webm;codecs=vp8,opus",
125
- "video/webm;codecs=opus",
126
- "video/webm",
127
- ];
128
- if (typeof MediaRecorder === "undefined") return "video/webm";
129
- return (
130
- preferred.find((type) => MediaRecorder.isTypeSupported(type)) ??
131
- "video/webm"
132
- );
133
- }
134
-
135
137
  function waitForMetadata(video: HTMLVideoElement): Promise<void> {
136
138
  if (video.readyState >= HTMLMediaElement.HAVE_METADATA) {
137
139
  return Promise.resolve();
@@ -194,22 +196,140 @@ function displayConstraints(
194
196
  } as MediaStreamConstraints;
195
197
  }
196
198
 
197
- async function getMicStream(): Promise<MediaStream | null> {
199
+ // The user's chosen camera/mic devices (set in the popup, saved to storage).
200
+ async function readDeviceIds(): Promise<{ video: string; audio: string }> {
198
201
  try {
199
- return await navigator.mediaDevices.getUserMedia({
200
- audio: {
201
- echoCancellation: true,
202
- noiseSuppression: true,
203
- autoGainControl: true,
204
- channelCount: 1,
205
- },
206
- video: false,
207
- });
202
+ const v = await chrome.storage.sync.get(["videoDeviceId", "audioDeviceId"]);
203
+ return {
204
+ video: typeof v.videoDeviceId === "string" ? v.videoDeviceId : "",
205
+ audio: typeof v.audioDeviceId === "string" ? v.audioDeviceId : "",
206
+ };
207
+ } catch {
208
+ return { video: "", audio: "" };
209
+ }
210
+ }
211
+
212
+ function cameraConstraints(deviceId: string): MediaTrackConstraints {
213
+ const base: MediaTrackConstraints = {
214
+ width: { ideal: 1280 },
215
+ height: { ideal: 720 },
216
+ };
217
+ if (deviceId) base.deviceId = { exact: deviceId };
218
+ else base.facingMode = "user";
219
+ return base;
220
+ }
221
+
222
+ async function getMicStream(deviceId: string): Promise<MediaStream | null> {
223
+ const audio: MediaTrackConstraints = {
224
+ echoCancellation: true,
225
+ noiseSuppression: true,
226
+ autoGainControl: true,
227
+ channelCount: 1,
228
+ };
229
+ if (deviceId) audio.deviceId = { exact: deviceId };
230
+ try {
231
+ return await navigator.mediaDevices.getUserMedia({ audio, video: false });
208
232
  } catch {
209
233
  return null;
210
234
  }
211
235
  }
212
236
 
237
+ /* ----------------------------------------------------- camera compositing --- */
238
+
239
+ // On many pages the on-page camera bubble can't run (the page sets
240
+ // `Permissions-Policy: camera=()`, which an iframe cannot escape). So for
241
+ // screen+camera we capture the camera HERE in the offscreen document (extension
242
+ // origin — always allowed) and draw it into a canvas on top of the screen, then
243
+ // record the canvas. The face ends up in the video on every page.
244
+
245
+ async function readyVideo(stream: MediaStream): Promise<HTMLVideoElement> {
246
+ const video = document.createElement("video");
247
+ video.muted = true;
248
+ video.playsInline = true;
249
+ video.srcObject = stream;
250
+ await waitForMetadata(video).catch(() => undefined);
251
+ await video.play().catch(() => undefined);
252
+ return video;
253
+ }
254
+
255
+ function drawCameraBubble(
256
+ ctx: CanvasRenderingContext2D,
257
+ camera: HTMLVideoElement,
258
+ canvas: HTMLCanvasElement,
259
+ ): void {
260
+ const size = Math.round(Math.min(canvas.width, canvas.height) * 0.22);
261
+ const margin = Math.round(Math.min(canvas.width, canvas.height) * 0.035);
262
+ const cx = margin + size / 2;
263
+ const cy = canvas.height - margin - size / 2;
264
+ const r = size / 2;
265
+ const vw = camera.videoWidth || size;
266
+ const vh = camera.videoHeight || size;
267
+ const scale = Math.max(size / vw, size / vh);
268
+ const dw = vw * scale;
269
+ const dh = vh * scale;
270
+
271
+ ctx.save();
272
+ ctx.beginPath();
273
+ ctx.arc(cx, cy, r, 0, Math.PI * 2);
274
+ ctx.closePath();
275
+ ctx.clip();
276
+ // Mirror the camera horizontally to match how people expect to see themselves.
277
+ ctx.translate(cx, cy);
278
+ ctx.scale(-1, 1);
279
+ ctx.drawImage(camera, -dw / 2, -dh / 2, dw, dh);
280
+ ctx.restore();
281
+
282
+ ctx.save();
283
+ ctx.beginPath();
284
+ ctx.arc(cx, cy, r - 1, 0, Math.PI * 2);
285
+ ctx.lineWidth = Math.max(3, Math.round(size * 0.03));
286
+ ctx.strokeStyle = "rgba(255,255,255,0.92)";
287
+ ctx.stroke();
288
+ ctx.restore();
289
+ }
290
+
291
+ type Compositor = {
292
+ videoTrack: MediaStreamTrack;
293
+ canvasStream: MediaStream;
294
+ stop: () => void;
295
+ };
296
+
297
+ async function buildCompositor(
298
+ displayStream: MediaStream,
299
+ cameraStream: MediaStream,
300
+ dims: { width: number; height: number },
301
+ ): Promise<Compositor> {
302
+ const displayVideo = await readyVideo(displayStream);
303
+ const cameraVideo = await readyVideo(cameraStream);
304
+ const canvas = document.createElement("canvas");
305
+ canvas.width = dims.width;
306
+ canvas.height = dims.height;
307
+ const ctx = canvas.getContext("2d");
308
+ if (!ctx) throw new Error("Canvas 2D context unavailable for compositing.");
309
+
310
+ let running = true;
311
+ let raf = 0;
312
+ const draw = () => {
313
+ if (!running) return;
314
+ ctx.drawImage(displayVideo, 0, 0, canvas.width, canvas.height);
315
+ if (cameraVideo.readyState >= 2) drawCameraBubble(ctx, cameraVideo, canvas);
316
+ raf = requestAnimationFrame(draw);
317
+ };
318
+ raf = requestAnimationFrame(draw);
319
+
320
+ const canvasStream = canvas.captureStream(30);
321
+ return {
322
+ videoTrack: canvasStream.getVideoTracks()[0],
323
+ canvasStream,
324
+ stop: () => {
325
+ running = false;
326
+ cancelAnimationFrame(raf);
327
+ displayVideo.srcObject = null;
328
+ cameraVideo.srcObject = null;
329
+ },
330
+ };
331
+ }
332
+
213
333
  async function createMixedAudio(
214
334
  streams: MediaStream[],
215
335
  ): Promise<{ audioContext: AudioContext | null; tracks: MediaStreamTrack[] }> {
@@ -230,21 +350,6 @@ async function createMixedAudio(
230
350
  return { audioContext, tracks: destination.stream.getAudioTracks() };
231
351
  }
232
352
 
233
- function appendUploadParams(
234
- uploadUrl: string,
235
- params: Record<string, string | number | boolean | undefined>,
236
- ): string {
237
- const url = new URL(uploadUrl);
238
- for (const [key, value] of Object.entries(params)) {
239
- if (value === undefined) continue;
240
- url.searchParams.set(
241
- key,
242
- typeof value === "boolean" ? (value ? "1" : "0") : String(value),
243
- );
244
- }
245
- return url.toString();
246
- }
247
-
248
353
  async function uploadChunk(
249
354
  recording: ActiveRecording,
250
355
  blob: Blob,
@@ -259,10 +364,10 @@ async function uploadChunk(
259
364
  hasCamera?: boolean;
260
365
  } = {},
261
366
  ): Promise<UploadResult> {
262
- const url = appendUploadParams(recording.uploadUrl, {
367
+ const url = chunkUploadUrl(recording.uploadUrl, {
263
368
  index,
264
369
  total: extra.total,
265
- isFinal: extra.isFinal ? 1 : 0,
370
+ isFinal: extra.isFinal,
266
371
  mimeType: recording.mimeType,
267
372
  durationMs: extra.durationMs,
268
373
  width: extra.width,
@@ -287,6 +392,13 @@ async function uploadChunk(
287
392
  const text = await res.text().catch(() => "");
288
393
  const data = text ? (JSON.parse(text) as UploadResult) : {};
289
394
  if (!res.ok) {
395
+ console.error(
396
+ "[clips-offscreen] chunk upload failed",
397
+ res.status,
398
+ "hadAuth:",
399
+ Boolean(recording.authToken),
400
+ text.slice(0, 200),
401
+ );
290
402
  throw new Error(
291
403
  data?.error || `Upload failed (${res.status}): ${text || res.statusText}`,
292
404
  );
@@ -310,6 +422,7 @@ function disposePrepared(): void {
310
422
  }
311
423
 
312
424
  function cleanup(recording: ActiveRecording): void {
425
+ recording.stopCompositor?.();
313
426
  stopStreams([recording.outputStream, ...recording.sourceStreams]);
314
427
  void recording.audioContext?.close().catch(() => undefined);
315
428
  }
@@ -329,22 +442,29 @@ async function acquire(message: AcquireMessage): Promise<{
329
442
  let displayStream: MediaStream | null = null;
330
443
  let micStream: MediaStream | null = null;
331
444
  let cameraStream: MediaStream | null = null;
445
+ const devices = await readDeviceIds();
332
446
 
333
447
  if (message.mode === "camera") {
334
448
  cameraStream = await navigator.mediaDevices.getUserMedia({
335
- video: {
336
- width: { ideal: 1280 },
337
- height: { ideal: 720 },
338
- facingMode: "user",
339
- },
340
- audio: message.includeMicrophone,
449
+ video: cameraConstraints(devices.video),
450
+ audio: message.includeMicrophone
451
+ ? devices.audio
452
+ ? { deviceId: { exact: devices.audio } }
453
+ : true
454
+ : false,
341
455
  });
342
456
  } else {
343
457
  // Native "Choose what to share" picker. This is the screenshot Steve showed.
344
458
  displayStream = await navigator.mediaDevices.getDisplayMedia(
345
459
  displayConstraints(message.surface),
346
460
  );
347
- if (message.includeMicrophone) micStream = await getMicStream();
461
+ if (message.includeMicrophone)
462
+ micStream = await getMicStream(devices.audio);
463
+ // The screen+camera face comes from the on-page bubble (captured in the
464
+ // display pixels), NOT composited here: canvas/requestAnimationFrame does
465
+ // not run in a hidden offscreen document, so compositing produced an empty
466
+ // recording ("No chunks found"). We record the display stream directly.
467
+ void message.includeCamera;
348
468
  }
349
469
 
350
470
  const videoStream = displayStream ?? cameraStream;
@@ -401,8 +521,24 @@ async function begin(message: BeginMessage): Promise<{
401
521
  if (activeRecording) throw new Error("Clips is already recording.");
402
522
 
403
523
  const videoStream = ready.displayStream ?? ready.cameraStream;
404
- const videoTrack = videoStream?.getVideoTracks()[0];
405
- if (!videoTrack) throw new Error("Capture video track was lost.");
524
+ const directVideoTrack = videoStream?.getVideoTracks()[0];
525
+ if (!directVideoTrack) throw new Error("Capture video track was lost.");
526
+
527
+ // Screen + camera → composite the camera bubble into a canvas and record that,
528
+ // so the face is in the video even on pages that block the on-page bubble.
529
+ let compositor: Compositor | null = null;
530
+ let videoTrack = directVideoTrack;
531
+ if (ready.displayStream && ready.cameraStream) {
532
+ compositor = await buildCompositor(
533
+ ready.displayStream,
534
+ ready.cameraStream,
535
+ {
536
+ width: ready.width,
537
+ height: ready.height,
538
+ },
539
+ );
540
+ videoTrack = compositor.videoTrack;
541
+ }
406
542
 
407
543
  const audioInputs = [
408
544
  ...(ready.displayStream ? [ready.displayStream] : []),
@@ -414,7 +550,7 @@ async function begin(message: BeginMessage): Promise<{
414
550
  const mixedAudio = await createMixedAudio(audioInputs);
415
551
 
416
552
  const outputStream = new MediaStream([videoTrack, ...mixedAudio.tracks]);
417
- const mimeType = chooseMimeType();
553
+ const mimeType = pickMimeType() || "video/webm";
418
554
  const recorder = new MediaRecorder(outputStream, {
419
555
  mimeType,
420
556
  // Crisp 1080p capture — matches the web/desktop recorders. Files upload
@@ -444,6 +580,7 @@ async function begin(message: BeginMessage): Promise<{
444
580
  ...(ready.displayStream ? [ready.displayStream] : []),
445
581
  ...(ready.micStream ? [ready.micStream] : []),
446
582
  ...(ready.cameraStream ? [ready.cameraStream] : []),
583
+ ...(compositor ? [compositor.canvasStream] : []),
447
584
  ],
448
585
  audioContext: mixedAudio.audioContext,
449
586
  chunkIndex: 0,
@@ -452,6 +589,10 @@ async function begin(message: BeginMessage): Promise<{
452
589
  cancelled: false,
453
590
  restarting: false,
454
591
  startTimer: null,
592
+ stopCompositor: compositor?.stop ?? null,
593
+ displaySource: ready.displayStream,
594
+ cameraSource: ready.cameraStream,
595
+ micSource: ready.micStream,
455
596
  dimensions: { width: ready.width, height: ready.height },
456
597
  hasAudio: outputStream.getAudioTracks().length > 0,
457
598
  hasCamera:
@@ -476,6 +617,10 @@ async function begin(message: BeginMessage): Promise<{
476
617
  return;
477
618
  }
478
619
  const index = recording.chunkIndex++;
620
+ // Record the failure and stop, but do NOT re-throw: re-throwing leaves a
621
+ // rejected promise that surfaces as an "Uncaught (in promise)" error (bad
622
+ // look in a Chrome Web Store review). finalizeStop reads recording.upload-
623
+ // Failure and surfaces it through the normal error path instead.
479
624
  const upload = uploadChunk(recording, event.data, index).catch((err) => {
480
625
  recording.uploadFailure =
481
626
  err instanceof Error ? err : new Error(String(err));
@@ -483,7 +628,6 @@ async function begin(message: BeginMessage): Promise<{
483
628
  error: recording.uploadFailure.message,
484
629
  });
485
630
  if (recorder.state !== "inactive") recorder.stop();
486
- throw recording.uploadFailure;
487
631
  });
488
632
  recording.uploadPromises.push(upload);
489
633
  });
@@ -513,11 +657,28 @@ async function begin(message: BeginMessage): Promise<{
513
657
  };
514
658
  }
515
659
 
660
+ // The canonical Clips "ready" chime when recording starts — shared with the web
661
+ // app recorder (and matching the desktop app) so every surface sounds the same.
662
+ function playStartChime(): void {
663
+ try {
664
+ const ctx = new AudioContext();
665
+ void ctx.resume().catch(() => undefined);
666
+ void scheduleReadyChime(ctx).finally(() => {
667
+ void ctx.close().catch(() => undefined);
668
+ });
669
+ } catch {
670
+ /* audio unavailable */
671
+ }
672
+ }
673
+
516
674
  function startRecorderNow(recording: ActiveRecording): void {
517
675
  if (recording.cancelled) return;
518
676
  try {
677
+ // Chime first (on "Go") so it mostly lands before the recording begins.
678
+ playStartChime();
519
679
  recording.recorder.start(2000);
520
680
  recording.startedAtMs = Date.now();
681
+ console.log("[clips-offscreen] recorder.start ok");
521
682
  reportStatus(recording.sessionId, "recording", {
522
683
  recordingId: recording.recordingId,
523
684
  width: recording.dimensions.width,
@@ -559,6 +720,27 @@ async function finalizeStop(recording: ActiveRecording): Promise<void> {
559
720
  : new Error(String(rejected.reason));
560
721
  }
561
722
  const durationMs = Math.max(0, Date.now() - recording.startedAtMs);
723
+ // Surface WHY a finalize might fail before the server's cryptic "No chunks
724
+ // found": 0 chunks means the recorder emitted no non-empty data (empty
725
+ // capture / never started), which is a different problem than an auth 401.
726
+ if (recording.chunkIndex === 0) {
727
+ console.warn(
728
+ "[clips-offscreen] finalizing with 0 chunks — empty recording.",
729
+ "recorderStarted:",
730
+ recording.startedAtMs > 0,
731
+ "durationMs:",
732
+ durationMs,
733
+ "hadAuth:",
734
+ Boolean(recording.authToken),
735
+ );
736
+ } else {
737
+ console.log(
738
+ "[clips-offscreen] finalizing",
739
+ recording.chunkIndex,
740
+ "chunk(s), durationMs:",
741
+ durationMs,
742
+ );
743
+ }
562
744
  const result = await uploadChunk(
563
745
  recording,
564
746
  new Blob([], { type: recording.mimeType }),
@@ -685,22 +867,18 @@ async function restart(
685
867
  recording.restarting = true;
686
868
  recording.cancelled = true;
687
869
  if (recording.recorder.state !== "inactive") recording.recorder.stop();
688
- // Close only the old mixing context; keep the capture tracks alive.
870
+ // Tear down the old compositor + mixing context; keep the capture tracks alive
871
+ // so the next BEGIN can build a fresh compositor/recorder on the same streams.
872
+ recording.stopCompositor?.();
689
873
  void recording.audioContext?.close().catch(() => undefined);
690
874
  activeRecording = null;
691
875
 
692
- const videoStream =
693
- recording.sourceStreams.find((s) => s.getVideoTracks().length) ?? null;
694
- const isCamera = recording.mode === "camera";
695
876
  prepared = {
696
877
  sessionId: recording.sessionId,
697
878
  mode: recording.mode,
698
- displayStream: isCamera ? null : videoStream,
699
- cameraStream: isCamera ? videoStream : null,
700
- micStream:
701
- recording.sourceStreams.find(
702
- (s) => s.getAudioTracks().length && s !== videoStream,
703
- ) ?? null,
879
+ displayStream: recording.displaySource,
880
+ cameraStream: recording.cameraSource,
881
+ micStream: recording.micSource,
704
882
  width: recording.dimensions.width,
705
883
  height: recording.dimensions.height,
706
884
  endedListener: null,
@@ -1,5 +1,8 @@
1
1
  :root {
2
- color-scheme: light dark;
2
+ /* "dark" to match the dark overlay: a declared color-scheme always paints an
3
+ opaque iframe canvas, so we keep it dark rather than white. Shaped parts clip
4
+ the canvas away (see content-script styleFrame); the countdown uses it. */
5
+ color-scheme: dark;
3
6
  }
4
7
 
5
8
  * {
@@ -39,10 +42,8 @@ body {
39
42
  height: 100%;
40
43
  border-radius: 50%;
41
44
  overflow: hidden;
42
- background: #0f172a;
43
- box-shadow:
44
- 0 10px 30px rgba(15, 23, 42, 0.35),
45
- 0 0 0 4px rgba(255, 255, 255, 0.92);
45
+ background: #18181b;
46
+ border: 3px solid rgba(255, 255, 255, 0.92);
46
47
  display: flex;
47
48
  align-items: center;
48
49
  justify-content: center;
@@ -75,7 +76,7 @@ body {
75
76
  display: flex;
76
77
  align-items: center;
77
78
  justify-content: center;
78
- background: rgba(2, 6, 23, 0.55);
79
+ background: rgba(9, 9, 11, 0.55);
79
80
  backdrop-filter: blur(2px);
80
81
  }
81
82
 
@@ -126,9 +127,9 @@ body {
126
127
  font-size: clamp(96px, 22vmin, 240px);
127
128
  font-weight: 700;
128
129
  color: #fff;
129
- text-shadow: 0 6px 30px rgba(2, 6, 23, 0.6);
130
+ text-shadow: 0 6px 30px rgba(9, 9, 11, 0.6);
130
131
  line-height: 1;
131
- animation: countdown-pop 1s ease-out forwards;
132
+ animation: countdown-pop 0.45s ease-out forwards;
132
133
  }
133
134
 
134
135
  .countdown-go {
@@ -138,21 +139,17 @@ body {
138
139
 
139
140
  @keyframes countdown-pop {
140
141
  0% {
141
- transform: scale(0.45);
142
+ transform: scale(0.55);
142
143
  opacity: 0;
143
144
  }
144
- 25% {
145
- transform: scale(1);
145
+ 45% {
146
+ transform: scale(1.08);
146
147
  opacity: 1;
147
148
  }
148
- 70% {
149
+ 100% {
149
150
  transform: scale(1);
150
151
  opacity: 1;
151
152
  }
152
- 100% {
153
- transform: scale(1.25);
154
- opacity: 0;
155
- }
156
153
  }
157
154
 
158
155
  .countdown-hint {
@@ -163,7 +160,7 @@ body {
163
160
  color: rgba(255, 255, 255, 0.85);
164
161
  font-size: 15px;
165
162
  font-weight: 500;
166
- background: rgba(2, 6, 23, 0.45);
163
+ background: rgba(9, 9, 11, 0.45);
167
164
  padding: 8px 16px;
168
165
  border-radius: 999px;
169
166
  }
@@ -181,19 +178,22 @@ body {
181
178
  }
182
179
 
183
180
  .toolbar-v {
181
+ /* Fill the iframe so the (opaque) iframe canvas is fully covered by the pill —
182
+ the iframe is clipped to this same 20px radius in styleFrame, so no canvas
183
+ box shows. Top-anchored content (justify-content:flex-start) keeps Stop/time/
184
+ pause put as the pill grows DOWNWARD on hover (centering caused misclicks).
185
+ Shadow lives on the iframe element so the clip doesn't cut it off. */
184
186
  position: absolute;
185
- top: 50%;
186
- left: 50%;
187
- transform: translate(-50%, -50%);
187
+ inset: 0;
188
188
  display: flex;
189
189
  flex-direction: column;
190
190
  align-items: center;
191
+ justify-content: flex-start;
191
192
  gap: 8px;
192
193
  padding: 9px;
193
194
  border-radius: 20px;
194
- background: rgba(15, 23, 42, 0.94);
195
+ background: rgba(24, 24, 27, 0.94);
195
196
  border: 1px solid rgba(255, 255, 255, 0.1);
196
- box-shadow: 0 10px 28px rgba(2, 6, 23, 0.45);
197
197
  color: #fff;
198
198
  -webkit-user-select: none;
199
199
  user-select: none;
@@ -323,30 +323,11 @@ body {
323
323
  gap: 9px;
324
324
  padding: 14px 16px;
325
325
  border-radius: 14px;
326
- background: rgba(15, 23, 42, 0.95);
326
+ background: rgba(24, 24, 27, 0.95);
327
327
  border: 1px solid rgba(255, 255, 255, 0.1);
328
- box-shadow: 0 10px 28px rgba(2, 6, 23, 0.45);
329
328
  color: #fff;
330
329
  }
331
330
 
332
- .saving-card .saving-spinner {
333
- position: absolute;
334
- top: 14px;
335
- right: 14px;
336
- width: 16px;
337
- height: 16px;
338
- border-radius: 50%;
339
- border: 2px solid rgba(255, 255, 255, 0.25);
340
- border-top-color: #fff;
341
- animation: saving-spin 0.8s linear infinite;
342
- }
343
-
344
- @keyframes saving-spin {
345
- to {
346
- transform: rotate(360deg);
347
- }
348
- }
349
-
350
331
  .saving-caption {
351
332
  font-size: 14px;
352
333
  font-weight: 600;
@@ -363,7 +344,7 @@ body {
363
344
  height: 100%;
364
345
  width: 40%;
365
346
  border-radius: 999px;
366
- background: #6366f1;
347
+ background: #d4d4d8;
367
348
  animation: saving-indeterminate 1.1s ease-in-out infinite;
368
349
  }
369
350
 
@@ -375,3 +356,56 @@ body {
375
356
  margin-left: 100%;
376
357
  }
377
358
  }
359
+
360
+ /* Bubble size dots (small / large), revealed on hover — desktop-style. */
361
+ .bubble-sizes {
362
+ position: absolute;
363
+ bottom: 7%;
364
+ left: 50%;
365
+ transform: translateX(-50%);
366
+ display: flex;
367
+ align-items: center;
368
+ gap: 9px;
369
+ padding: 5px 9px;
370
+ border-radius: 999px;
371
+ background: rgba(9, 9, 11, 0.55);
372
+ -webkit-backdrop-filter: blur(4px);
373
+ backdrop-filter: blur(4px);
374
+ opacity: 0;
375
+ transition: opacity 0.15s ease;
376
+ pointer-events: none;
377
+ }
378
+
379
+ .bubble:hover .bubble-sizes {
380
+ opacity: 1;
381
+ pointer-events: auto;
382
+ }
383
+
384
+ .bubble-size-dot {
385
+ border: none;
386
+ padding: 0;
387
+ border-radius: 50%;
388
+ background: rgba(255, 255, 255, 0.55);
389
+ cursor: pointer;
390
+ transition:
391
+ background 0.12s ease,
392
+ transform 0.1s ease;
393
+ }
394
+
395
+ .bubble-size-dot:hover {
396
+ background: #fff;
397
+ }
398
+
399
+ .bubble-size-dot:active {
400
+ transform: scale(0.9);
401
+ }
402
+
403
+ .bubble-size-sm {
404
+ width: 8px;
405
+ height: 8px;
406
+ }
407
+
408
+ .bubble-size-lg {
409
+ width: 13px;
410
+ height: 13px;
411
+ }
@@ -2,7 +2,13 @@
2
2
  <html lang="en">
3
3
  <head>
4
4
  <meta charset="UTF-8" />
5
- <meta name="color-scheme" content="light dark" />
5
+ <!-- A declared color-scheme always makes Chrome paint an OPAQUE iframe canvas
6
+ (white for light, dark for dark) — it can't be made transparent. The
7
+ overlay is dark-themed, so we use "dark" to keep that canvas dark rather
8
+ than a jarring white. The shaped parts (bubble/toolbar/saving) clip their
9
+ iframe to the content shape so the canvas never shows; only the
10
+ full-screen countdown relies on this dark value. -->
11
+ <meta name="color-scheme" content="dark" />
6
12
  <title>Clips overlay</title>
7
13
  </head>
8
14
  <body>