@agent-native/core 0.76.11 → 0.76.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/corpus/README.md +2 -2
  2. package/corpus/core/CHANGELOG.md +19 -0
  3. package/corpus/core/package.json +1 -1
  4. package/corpus/core/src/agent/production-agent.ts +7 -4
  5. package/corpus/core/src/client/FeedbackButton.tsx +162 -18
  6. package/corpus/core/src/client/blocks/library/AnnotatedCodeBlock.tsx +11 -2
  7. package/corpus/core/src/client/blocks/library/ApiEndpointBlock.tsx +46 -19
  8. package/corpus/core/src/client/blocks/library/MermaidBlock.tsx +7 -5
  9. package/corpus/core/src/client/blocks/library/block-copy.ts +367 -0
  10. package/corpus/core/src/client/blocks/library/diagram.tsx +8 -6
  11. package/corpus/core/src/client/blocks/library/wireframe.tsx +7 -2
  12. package/corpus/core/src/client/i18n.tsx +4 -0
  13. package/corpus/templates/clips/app/components/recorder/camera-visualizer.tsx +88 -10
  14. package/corpus/templates/clips/app/components/recorder/pre-record-panel.tsx +64 -0
  15. package/corpus/templates/clips/app/components/recorder/recorder-engine.ts +94 -9
  16. package/corpus/templates/clips/app/lib/camera-blur.ts +341 -0
  17. package/corpus/templates/clips/app/lib/camera-composite.ts +6 -1
  18. package/corpus/templates/clips/app/lib/recorder-preferences.ts +15 -0
  19. package/corpus/templates/clips/app/routes/record.tsx +13 -0
  20. package/corpus/templates/clips/desktop/src/app.tsx +101 -69
  21. package/corpus/templates/clips/desktop/src/lib/voice-dictation.ts +109 -11
  22. package/corpus/templates/clips/desktop/src/styles.css +19 -4
  23. package/corpus/templates/clips/desktop/src-tauri/Cargo.toml +1 -1
  24. package/corpus/templates/clips/desktop/src-tauri/src/native_speech.rs +143 -86
  25. package/corpus/templates/clips/package.json +1 -0
  26. package/corpus/templates/clips/vite.config.ts +38 -1
  27. package/corpus/templates/mail/app/root.tsx +90 -6
  28. package/dist/agent/production-agent.d.ts +7 -4
  29. package/dist/agent/production-agent.d.ts.map +1 -1
  30. package/dist/agent/production-agent.js +7 -4
  31. package/dist/agent/production-agent.js.map +1 -1
  32. package/dist/client/FeedbackButton.d.ts.map +1 -1
  33. package/dist/client/FeedbackButton.js +142 -14
  34. package/dist/client/FeedbackButton.js.map +1 -1
  35. package/dist/client/blocks/library/AnnotatedCodeBlock.d.ts.map +1 -1
  36. package/dist/client/blocks/library/AnnotatedCodeBlock.js +6 -2
  37. package/dist/client/blocks/library/AnnotatedCodeBlock.js.map +1 -1
  38. package/dist/client/blocks/library/ApiEndpointBlock.d.ts.map +1 -1
  39. package/dist/client/blocks/library/ApiEndpointBlock.js +14 -6
  40. package/dist/client/blocks/library/ApiEndpointBlock.js.map +1 -1
  41. package/dist/client/blocks/library/MermaidBlock.d.ts.map +1 -1
  42. package/dist/client/blocks/library/MermaidBlock.js +6 -4
  43. package/dist/client/blocks/library/MermaidBlock.js.map +1 -1
  44. package/dist/client/blocks/library/block-copy.d.ts +33 -0
  45. package/dist/client/blocks/library/block-copy.d.ts.map +1 -0
  46. package/dist/client/blocks/library/block-copy.js +329 -0
  47. package/dist/client/blocks/library/block-copy.js.map +1 -0
  48. package/dist/client/blocks/library/diagram.d.ts.map +1 -1
  49. package/dist/client/blocks/library/diagram.js +7 -5
  50. package/dist/client/blocks/library/diagram.js.map +1 -1
  51. package/dist/client/blocks/library/wireframe.d.ts.map +1 -1
  52. package/dist/client/blocks/library/wireframe.js +4 -2
  53. package/dist/client/blocks/library/wireframe.js.map +1 -1
  54. package/dist/client/i18n.d.ts +1 -0
  55. package/dist/client/i18n.d.ts.map +1 -1
  56. package/dist/client/i18n.js +3 -0
  57. package/dist/client/i18n.js.map +1 -1
  58. package/package.json +1 -1
@@ -7,6 +7,7 @@ import {
7
7
  type FormEvent,
8
8
  } from "react";
9
9
  import {
10
+ IconBlur,
10
11
  IconBrowser,
11
12
  IconCamera,
12
13
  IconChevronDown,
@@ -40,6 +41,7 @@ import {
40
41
  SelectValue,
41
42
  } from "@/components/ui/select";
42
43
  import { Switch } from "@/components/ui/switch";
44
+ import { Slider } from "@/components/ui/slider";
43
45
  import { cn } from "@/lib/utils";
44
46
  import {
45
47
  NO_CAMERA_DEVICE_ID,
@@ -48,6 +50,7 @@ import {
48
50
  type RecordingMode,
49
51
  } from "./recorder-engine";
50
52
  import type { CameraBubbleSize } from "./camera-bubble";
53
+ import { DEFAULT_BLUR_PX, MAX_BLUR_PX, MIN_BLUR_PX } from "@/lib/camera-blur";
51
54
  import {
52
55
  loadRecorderPreferences,
53
56
  saveRecorderPreferences,
@@ -65,6 +68,8 @@ export interface PreRecordPanelProps {
65
68
  displaySurface: DisplaySurface;
66
69
  micDeviceId: string | null;
67
70
  cameraDeviceId: string | null;
71
+ cameraBlur: boolean;
72
+ cameraBlurRadius: number;
68
73
  }) => void;
69
74
  initialMode?: RecordingMode | null;
70
75
  initialDisplaySurface?: DisplaySurface | null;
@@ -210,6 +215,12 @@ export function PreRecordPanel({
210
215
  const [cameraId, setCameraId] = useState<string>(
211
216
  () => savedPrefs.cameraId ?? "default",
212
217
  );
218
+ const [cameraBlur, setCameraBlur] = useState(
219
+ () => savedPrefs.cameraBlur ?? false,
220
+ );
221
+ const [cameraBlurRadius, setCameraBlurRadius] = useState(
222
+ () => savedPrefs.cameraBlurRadius ?? DEFAULT_BLUR_PX,
223
+ );
213
224
  const [enumError, setEnumError] = useState<string | null>(null);
214
225
  const [micAccessStatus, setMicAccessStatus] =
215
226
  useState<DeviceAccessStatus>("idle");
@@ -335,6 +346,10 @@ export function PreRecordPanel({
335
346
  setCameraId(value);
336
347
  saveRecorderPreferences({ cameraId: value });
337
348
  }, []);
349
+ const chooseCameraBlur = useCallback((value: boolean) => {
350
+ setCameraBlur(value);
351
+ saveRecorderPreferences({ cameraBlur: value });
352
+ }, []);
338
353
 
339
354
  const requestMicrophoneChoices = useCallback(async () => {
340
355
  if (!navigator.mediaDevices?.getUserMedia) {
@@ -826,12 +841,59 @@ export function PreRecordPanel({
826
841
  <CameraVisualizer
827
842
  deviceId={cameraId === "default" ? null : cameraId}
828
843
  disabled={busy}
844
+ blur={cameraBlur}
845
+ blurRadius={cameraBlurRadius}
829
846
  size={cameraSize}
830
847
  onSizeChange={onCameraSizeChange}
831
848
  onStatusChange={handleCameraStatusChange}
832
849
  onPreviewChange={handleCameraPreviewChange}
833
850
  />
834
851
  ) : null}
852
+
853
+ {needsCamera ? (
854
+ <label className="flex cursor-pointer items-center gap-2 rounded-lg px-1 py-1.5 hover:bg-muted/45">
855
+ <div className="flex h-8 w-8 shrink-0 items-center justify-center rounded-lg bg-muted text-muted-foreground">
856
+ <IconBlur className="h-4 w-4" />
857
+ </div>
858
+ <div className="min-w-0 flex-1">
859
+ <p className="text-sm">Blur background</p>
860
+ <p className="text-[11px] text-muted-foreground">
861
+ Keep yourself sharp, blur what's behind you
862
+ </p>
863
+ </div>
864
+ <Switch
865
+ checked={cameraBlur}
866
+ onCheckedChange={chooseCameraBlur}
867
+ disabled={busy}
868
+ aria-label="Blur the camera background"
869
+ />
870
+ </label>
871
+ ) : null}
872
+
873
+ {needsCamera && cameraBlur ? (
874
+ <div className="flex items-center gap-3 px-1 pb-1">
875
+ <span className="w-14 shrink-0 text-[11px] text-muted-foreground">
876
+ Intensity
877
+ </span>
878
+ <Slider
879
+ value={[cameraBlurRadius]}
880
+ min={MIN_BLUR_PX}
881
+ max={MAX_BLUR_PX}
882
+ step={1}
883
+ disabled={busy}
884
+ onValueChange={(value) =>
885
+ setCameraBlurRadius(value[0] ?? DEFAULT_BLUR_PX)
886
+ }
887
+ onValueCommit={(value) =>
888
+ saveRecorderPreferences({
889
+ cameraBlurRadius: value[0] ?? DEFAULT_BLUR_PX,
890
+ })
891
+ }
892
+ aria-label="Background blur intensity"
893
+ className="flex-1"
894
+ />
895
+ </div>
896
+ ) : null}
835
897
  </div>
836
898
  ) : null}
837
899
 
@@ -870,6 +932,8 @@ export function PreRecordPanel({
870
932
  micDeviceId: micId === "default" ? null : micId,
871
933
  cameraDeviceId:
872
934
  needsCamera && cameraId !== "default" ? cameraId : null,
935
+ cameraBlur: needsCamera ? cameraBlur : false,
936
+ cameraBlurRadius,
873
937
  })
874
938
  }
875
939
  className={cn("h-12 gap-2", onCancel ? "flex-1" : "w-full")}
@@ -19,6 +19,10 @@ import {
19
19
  createCameraCompositeStream,
20
20
  type CameraCompositeHandle,
21
21
  } from "@/lib/camera-composite";
22
+ import {
23
+ createBackgroundBlurStream,
24
+ type CameraBlurHandle,
25
+ } from "@/lib/camera-blur";
22
26
  import {
23
27
  chunkUploadUrl,
24
28
  pickMimeType,
@@ -75,6 +79,15 @@ export interface RecorderEngineOptions {
75
79
  cameraDeviceId?: string | null;
76
80
  /** Camera bubble size selected in the pre-record UI. */
77
81
  cameraBubbleSize?: "sm" | "md" | "lg";
82
+ /**
83
+ * Blur the camera background (sharp person, blurred surroundings) for both the
84
+ * live preview bubble and the baked-in recording composite. Resolved at
85
+ * `acquire()` time. Silently no-ops (records un-blurred) if segmentation is
86
+ * unavailable in the browser.
87
+ */
88
+ cameraBlur?: boolean;
89
+ /** Background blur radius in px when `cameraBlur` is on. Defaults to ~12. */
90
+ cameraBlurRadius?: number;
78
91
  /** Chunk size in ms (MediaRecorder timeslice). Default 2000. */
79
92
  chunkIntervalMs?: number;
80
93
  /** Base URL for the chunk upload endpoint. Default `/api/uploads/:id/chunk`. */
@@ -99,6 +112,11 @@ export interface RecorderEngineOptions {
99
112
  * `onError`, this does NOT transition the engine into the `error` state.
100
113
  */
101
114
  onWarning?: (message: string) => void;
115
+ /**
116
+ * Fired when the camera ends (unplugged / revoked) any time after acquire,
117
+ * so the UI can drop the on-page camera bubble to match the recorded output.
118
+ */
119
+ onCameraEnded?: () => void;
102
120
  /**
103
121
  * Called when the display stream's video track ends because the user clicked
104
122
  * the browser's native "Stop sharing" button. When provided, the engine
@@ -360,6 +378,17 @@ export class RecorderEngine {
360
378
 
361
379
  private displayStream: MediaStream | null = null;
362
380
  private cameraStream: MediaStream | null = null;
381
+ /**
382
+ * Raw getUserMedia camera stream. When background blur is active,
383
+ * `cameraStream` points at the processed (blurred) derivative and this field
384
+ * keeps the original so teardown stops the real camera tracks and the
385
+ * disconnect handler can observe the hardware ending.
386
+ */
387
+ private rawCameraStream: MediaStream | null = null;
388
+ private cameraBlur: CameraBlurHandle | null = null;
389
+ // True once the camera is acquired, through preview/countdown/recording, until
390
+ // teardown — gates disconnect handling outside the recording state.
391
+ private cameraLive = false;
363
392
  private micStream: MediaStream | null = null;
364
393
  private combinedStream: MediaStream | null = null;
365
394
  private previewStream: MediaStream | null = null;
@@ -697,13 +726,10 @@ export class RecorderEngine {
697
726
  }
698
727
  }
699
728
 
700
- // Camera / mic disconnects mid-recording (USB webcam unplugged, mic
701
- // permission revoked, a Bluetooth input dropping) are NON-fatal: the
702
- // recording continues with whatever inputs remain, and we surface a
703
- // non-blocking warning. We use the same recording/paused guard as the
704
- // display handler so the `ended` events fired by `cleanupTracks()` during
705
- // a normal stop()/cancel() (state is `stopping`/`idle` by then) are
706
- // ignored rather than treated as a disconnect.
729
+ // Camera / mic disconnects (USB webcam unplugged, permission revoked,
730
+ // Bluetooth dropped) are NON-fatal: keep whatever inputs remain and warn.
731
+ // The handlers gate on `cameraLive`/state so the `ended` events fired by
732
+ // `cleanupTracks()` during a normal stop/cancel are ignored.
707
733
  if (this.cameraStream) {
708
734
  for (const track of this.cameraStream.getVideoTracks()) {
709
735
  track.addEventListener("ended", () => {
@@ -720,6 +746,37 @@ export class RecorderEngine {
720
746
  }
721
747
  }
722
748
 
749
+ // Camera is live from here through preview/countdown/recording until
750
+ // teardown — set before the async blur setup so a disconnect during that
751
+ // window is handled, not inherited as a dead stream.
752
+ if (this.cameraStream) this.cameraLive = true;
753
+
754
+ // Swap the raw camera for its blurred derivative, which both the preview
755
+ // bubble and the recording composite read ("what you see is what's
756
+ // recorded"). createBackgroundBlurStream never throws — it falls back to
757
+ // the raw stream on failure.
758
+ if (this.opts.cameraBlur && this.cameraStream) {
759
+ this.rawCameraStream = this.cameraStream;
760
+ const handle = await createBackgroundBlurStream(this.cameraStream, {
761
+ blurPx: this.opts.cameraBlurRadius,
762
+ });
763
+ if (this.cameraDisconnectNotified) {
764
+ // Webcam ended while the pipeline was loading: discard it and drop the
765
+ // dead camera rather than inheriting a frozen processed stream.
766
+ handle.cleanup();
767
+ this.cameraStream = null;
768
+ } else {
769
+ this.cameraBlur = handle;
770
+ this.cameraStream = this.cameraBlur.stream;
771
+ }
772
+ }
773
+
774
+ if (this.opts.mode === "camera" && !this.cameraStream) {
775
+ throw new Error(
776
+ "Camera disconnected before recording could start. Reconnect it and try again.",
777
+ );
778
+ }
779
+
723
780
  this.previewStream =
724
781
  this.opts.mode === "camera" ? this.cameraStream! : this.displayStream!;
725
782
 
@@ -1414,6 +1471,12 @@ export class RecorderEngine {
1414
1471
  return combined;
1415
1472
  }
1416
1473
 
1474
+ // Camera dropped before start (disconnected during setup/countdown):
1475
+ // record screen-only rather than compositing a dead stream.
1476
+ if (!this.cameraStream) {
1477
+ return this.buildDisplayRecordingStream();
1478
+ }
1479
+
1417
1480
  // Screen + camera: display capture does not reliably include our separate
1418
1481
  // DOM bubble once the user records another app/window, so the saved
1419
1482
  // recording must composite the camera feed before MediaRecorder sees it.
@@ -1705,14 +1768,23 @@ export class RecorderEngine {
1705
1768
  }
1706
1769
 
1707
1770
  private cleanupTracks(): void {
1771
+ // Clear before stopping tracks so the `ended` events our own stop() fires
1772
+ // aren't mistaken for a disconnect.
1773
+ this.cameraLive = false;
1708
1774
  this.audioMixSources = [];
1709
1775
  this.audioMixCtx?.close().catch(() => {});
1710
1776
  this.audioMixCtx = null;
1711
1777
  this.cameraComposite?.cleanup();
1712
1778
  this.cameraComposite = null;
1779
+ // Tear down the blur pipeline (segmenter, hidden video, processed capture)
1780
+ // before stopping streams. `rawCameraStream` holds the real hardware tracks
1781
+ // when blur is active; stop those so the camera indicator clears.
1782
+ this.cameraBlur?.cleanup();
1783
+ this.cameraBlur = null;
1713
1784
  for (const s of [
1714
1785
  this.displayStream,
1715
1786
  this.cameraStream,
1787
+ this.rawCameraStream,
1716
1788
  this.micStream,
1717
1789
  this.combinedStream,
1718
1790
  ]) {
@@ -1727,6 +1799,7 @@ export class RecorderEngine {
1727
1799
  }
1728
1800
  this.displayStream = null;
1729
1801
  this.cameraStream = null;
1802
+ this.rawCameraStream = null;
1730
1803
  this.micStream = null;
1731
1804
  this.combinedStream = null;
1732
1805
  this.previewStream = null;
@@ -1764,16 +1837,28 @@ export class RecorderEngine {
1764
1837
  * warn the user.
1765
1838
  */
1766
1839
  private onCameraTrackEnded() {
1767
- if (this.state !== "recording" && this.state !== "paused") return;
1840
+ if (!this.cameraLive) return;
1768
1841
  if (this.cameraDisconnectNotified) return;
1769
1842
  this.cameraDisconnectNotified = true;
1770
- for (const track of this.cameraStream?.getVideoTracks() ?? []) {
1843
+ this.cameraLive = false;
1844
+ // Tear down the blur pipeline so the composite's camera <video> sees its
1845
+ // (blurred) track end and self-hides, instead of freezing on a stale frame.
1846
+ if (this.cameraBlur) {
1847
+ this.cameraBlur.cleanup();
1848
+ this.cameraBlur = null;
1849
+ }
1850
+ for (const track of [
1851
+ ...(this.rawCameraStream?.getVideoTracks() ?? []),
1852
+ ...(this.cameraStream?.getVideoTracks() ?? []),
1853
+ ]) {
1771
1854
  try {
1772
1855
  track.stop();
1773
1856
  } catch {
1774
1857
  // ignore — the track has already ended.
1775
1858
  }
1776
1859
  }
1860
+ // Drop the on-page bubble to match the recorded output (screen-only now).
1861
+ this.opts.onCameraEnded?.();
1777
1862
  this.emitWarning(
1778
1863
  "Camera disconnected — recording continues without webcam.",
1779
1864
  );
@@ -0,0 +1,341 @@
1
+ import { appBasePath } from "@agent-native/core/client";
2
+ import type { ImageSegmenter } from "@mediapipe/tasks-vision";
3
+
4
+ /**
5
+ * A processed camera stream whose background is blurred while the person stays
6
+ * sharp (Zoom / Loom style). Produced once in the recorder engine and shared by
7
+ * both the live preview bubble and the baked-in recording composite, so "what
8
+ * you see is what's recorded".
9
+ */
10
+ export interface CameraBlurHandle {
11
+ /** The processed stream, or the original `source` stream when `active` is false. */
12
+ stream: MediaStream;
13
+ /** False when segmentation was unavailable and we transparently fell back to raw. */
14
+ readonly active: boolean;
15
+ /**
16
+ * Update the background blur radius (px) live, without rebuilding the
17
+ * segmenter. No-op on the passthrough fallback handle.
18
+ */
19
+ setBlurPx(px: number): void;
20
+ cleanup(): void;
21
+ }
22
+
23
+ export interface CameraBlurOptions {
24
+ /** CSS blur radius applied to the background, in px. Default 12. */
25
+ blurPx?: number;
26
+ /**
27
+ * How often segmentation runs, in frames per second. Kept below the 30fps
28
+ * capture rate to bound CPU/GPU cost — the small bubble does not need a fresh
29
+ * mask every captured frame. Default 20.
30
+ */
31
+ segmentationFps?: number;
32
+ }
33
+
34
+ export const DEFAULT_BLUR_PX = 12;
35
+ export const MIN_BLUR_PX = 2;
36
+ export const MAX_BLUR_PX = 30;
37
+ const DEFAULT_SEGMENTATION_FPS = 20;
38
+ const CAPTURE_FPS = 30;
39
+ /**
40
+ * Max dimension (px) of the frame we feed the segmenter. The model resizes to
41
+ * its own 256² input internally and upsamples the mask back to this size, so
42
+ * keeping the input small bounds the per-frame mask readback + alpha loop cost
43
+ * regardless of the camera's native resolution. The soft mask is scaled back up
44
+ * to the camera resolution with bilinear filtering when compositing.
45
+ */
46
+ const SEG_MAX_DIM = 256;
47
+
48
+ const MODEL_PATH = "/mediapipe/selfie_segmenter.tflite";
49
+ const WASM_PATH = "/mediapipe/wasm";
50
+
51
+ function positive(value: unknown): value is number {
52
+ return typeof value === "number" && Number.isFinite(value) && value > 0;
53
+ }
54
+
55
+ function sourceDimensions(
56
+ video: HTMLVideoElement,
57
+ stream: MediaStream,
58
+ ): { width: number; height: number } {
59
+ if (positive(video.videoWidth) && positive(video.videoHeight)) {
60
+ return { width: video.videoWidth, height: video.videoHeight };
61
+ }
62
+ const settings = stream.getVideoTracks()[0]?.getSettings();
63
+ return {
64
+ width: positive(settings?.width) ? Math.round(settings.width) : 640,
65
+ height: positive(settings?.height) ? Math.round(settings.height) : 480,
66
+ };
67
+ }
68
+
69
+ /** Hidden, off-screen `<video>` that plays the source stream for canvas reads. */
70
+ function attachHiddenVideo(stream: MediaStream): {
71
+ video: HTMLVideoElement;
72
+ cleanup(): void;
73
+ } {
74
+ const video = document.createElement("video");
75
+ video.muted = true;
76
+ video.playsInline = true;
77
+ video.autoplay = true;
78
+ video.srcObject = stream;
79
+ video.style.position = "fixed";
80
+ video.style.left = "-10000px";
81
+ video.style.top = "0";
82
+ video.style.width = "1px";
83
+ video.style.height = "1px";
84
+ video.style.opacity = "0";
85
+ video.style.pointerEvents = "none";
86
+
87
+ const tryPlay = () => {
88
+ video.play().catch(() => undefined);
89
+ };
90
+
91
+ document.body.appendChild(video);
92
+ video.addEventListener("loadedmetadata", tryPlay);
93
+ tryPlay();
94
+
95
+ return {
96
+ video,
97
+ cleanup() {
98
+ video.removeEventListener("loadedmetadata", tryPlay);
99
+ video.pause();
100
+ video.srcObject = null;
101
+ video.remove();
102
+ },
103
+ };
104
+ }
105
+
106
+ async function createSegmenter(): Promise<ImageSegmenter> {
107
+ // Lazy-loaded so the ~11MB Wasm runtime and its loader never enter the main
108
+ // bundle or run during SSR — only when a recording actually requests blur.
109
+ const vision = await import("@mediapipe/tasks-vision");
110
+ const base = appBasePath();
111
+ const fileset = await vision.FilesetResolver.forVisionTasks(
112
+ `${base}${WASM_PATH}`,
113
+ );
114
+ const build = (delegate: "GPU" | "CPU") =>
115
+ vision.ImageSegmenter.createFromOptions(fileset, {
116
+ baseOptions: {
117
+ modelAssetPath: `${base}${MODEL_PATH}`,
118
+ delegate,
119
+ },
120
+ runningMode: "VIDEO",
121
+ outputConfidenceMasks: true,
122
+ outputCategoryMask: false,
123
+ });
124
+ try {
125
+ return await build("GPU");
126
+ } catch {
127
+ // Older GPUs / blocked WebGL contexts: fall back to the CPU delegate.
128
+ return await build("CPU");
129
+ }
130
+ }
131
+
132
+ const fallback = (source: MediaStream): CameraBlurHandle => ({
133
+ stream: source,
134
+ active: false,
135
+ setBlurPx() {},
136
+ cleanup() {},
137
+ });
138
+
139
+ /**
140
+ * Build a background-blurred derivative of `source`. Never throws: if MediaPipe,
141
+ * the Wasm fileset, the model, or canvas capture are unavailable, it resolves to
142
+ * a passthrough handle wrapping the raw `source` (`active === false`) so the
143
+ * recording always proceeds. Callers own `source`'s tracks; `cleanup()` here
144
+ * tears down only the processing pipeline, never the source.
145
+ */
146
+ export async function createBackgroundBlurStream(
147
+ source: MediaStream,
148
+ opts: CameraBlurOptions = {},
149
+ ): Promise<CameraBlurHandle> {
150
+ if (typeof document === "undefined") return fallback(source);
151
+ if (!source.getVideoTracks().length) return fallback(source);
152
+
153
+ let blurPx = positive(opts.blurPx) ? opts.blurPx : DEFAULT_BLUR_PX;
154
+ const segFps = positive(opts.segmentationFps)
155
+ ? opts.segmentationFps
156
+ : DEFAULT_SEGMENTATION_FPS;
157
+
158
+ const out = document.createElement("canvas");
159
+ const outCtx = out.getContext("2d", { alpha: false });
160
+ if (!outCtx || typeof out.captureStream !== "function") {
161
+ return fallback(source);
162
+ }
163
+
164
+ let segmenter: ImageSegmenter;
165
+ try {
166
+ segmenter = await createSegmenter();
167
+ } catch (err) {
168
+ console.warn(
169
+ "[camera-blur] Segmentation unavailable — recording without background blur:",
170
+ err,
171
+ );
172
+ return fallback(source);
173
+ }
174
+
175
+ const hidden = attachHiddenVideo(source);
176
+ const { width: w0, height: h0 } = sourceDimensions(hidden.video, source);
177
+ out.width = w0;
178
+ out.height = h0;
179
+
180
+ // Foreground compositing scratch: sharp person punched out by the mask alpha.
181
+ const fg = document.createElement("canvas");
182
+ const fgCtx = fg.getContext("2d");
183
+ // Downscaled frame we actually segment, plus the soft alpha mask it produces.
184
+ const segInput = document.createElement("canvas");
185
+ const segCtx = segInput.getContext("2d");
186
+ const maskCanvas = document.createElement("canvas");
187
+ const maskCtx = maskCanvas.getContext("2d");
188
+ if (!fgCtx || !segCtx || !maskCtx) {
189
+ hidden.cleanup();
190
+ segmenter.close();
191
+ return fallback(source);
192
+ }
193
+
194
+ let lastTimestamp = -1;
195
+ let lastSegAt = -Infinity;
196
+ let haveMask = false;
197
+ const segIntervalMs = 1000 / segFps;
198
+
199
+ // Re-segment into maskCanvas. Runs at segFps; on failure the previous mask is
200
+ // kept rather than dropping to an un-blurred frame.
201
+ const updateMask = (video: HTMLVideoElement) => {
202
+ const scale = SEG_MAX_DIM / Math.max(video.videoWidth, video.videoHeight);
203
+ const segW = Math.max(1, Math.round(video.videoWidth * scale));
204
+ const segH = Math.max(1, Math.round(video.videoHeight * scale));
205
+ if (segInput.width !== segW) segInput.width = segW;
206
+ if (segInput.height !== segH) segInput.height = segH;
207
+ try {
208
+ segCtx.drawImage(video, 0, 0, segW, segH);
209
+ const timestamp = Math.max(
210
+ Math.round(performance.now()),
211
+ lastTimestamp + 1,
212
+ );
213
+ lastTimestamp = timestamp;
214
+ const result = segmenter.segmentForVideo(segInput, timestamp);
215
+ const confidence = result.confidenceMasks?.[0];
216
+ if (confidence) {
217
+ const maskW = confidence.width;
218
+ const maskH = confidence.height;
219
+ const values = confidence.getAsFloat32Array();
220
+ if (maskCanvas.width !== maskW) maskCanvas.width = maskW;
221
+ if (maskCanvas.height !== maskH) maskCanvas.height = maskH;
222
+ const image = maskCtx.createImageData(maskW, maskH);
223
+ for (let i = 0; i < values.length; i++) {
224
+ image.data[i * 4 + 3] = Math.round(values[i] * 255);
225
+ }
226
+ maskCtx.putImageData(image, 0, 0);
227
+ haveMask = true;
228
+ }
229
+ result.close();
230
+ } catch {
231
+ // keep the previous mask
232
+ }
233
+ };
234
+
235
+ // Composite blurred background + sharp foreground (or raw until a first mask).
236
+ const composite = (video: HTMLVideoElement, outW: number, outH: number) => {
237
+ if (!haveMask) {
238
+ outCtx.drawImage(video, 0, 0, outW, outH);
239
+ return;
240
+ }
241
+ if (fg.width !== outW) fg.width = outW;
242
+ if (fg.height !== outH) fg.height = outH;
243
+ fgCtx.globalCompositeOperation = "source-over";
244
+ fgCtx.clearRect(0, 0, outW, outH);
245
+ fgCtx.drawImage(video, 0, 0, outW, outH);
246
+ fgCtx.globalCompositeOperation = "destination-in";
247
+ fgCtx.imageSmoothingEnabled = true;
248
+ fgCtx.drawImage(maskCanvas, 0, 0, outW, outH);
249
+ fgCtx.globalCompositeOperation = "source-over";
250
+
251
+ outCtx.filter = `blur(${blurPx}px)`;
252
+ outCtx.drawImage(video, 0, 0, outW, outH);
253
+ outCtx.filter = "none";
254
+ outCtx.drawImage(fg, 0, 0);
255
+ };
256
+
257
+ // Composite every tick (capture rate) for smooth motion; re-segment only at
258
+ // segFps, reusing the last mask in between.
259
+ const drawFrame = () => {
260
+ const video = hidden.video;
261
+ if (!positive(video.videoWidth) || !positive(video.videoHeight)) return;
262
+ if (out.width !== video.videoWidth) out.width = video.videoWidth;
263
+ if (out.height !== video.videoHeight) out.height = video.videoHeight;
264
+
265
+ const now = performance.now();
266
+ if (now - lastSegAt >= segIntervalMs) {
267
+ updateMask(video);
268
+ lastSegAt = now;
269
+ }
270
+ composite(video, out.width, out.height);
271
+ };
272
+
273
+ const stream = out.captureStream(CAPTURE_FPS);
274
+ const minFrameMs = 1000 / CAPTURE_FPS;
275
+
276
+ // Worker-driven timer so the loop keeps running at full rate in background
277
+ // tabs (rAF is throttled to ~1fps when hidden). Falls back to rAF when blob:
278
+ // workers are blocked by CSP — mirrors camera-composite.ts.
279
+ let worker: Worker | null = null;
280
+ let raf: number | null = null;
281
+
282
+ try {
283
+ const workerBlob = new Blob(
284
+ [
285
+ `let t=null;onmessage=e=>{if(e.data==='start'){clearInterval(t);t=setInterval(()=>postMessage('tick'),${minFrameMs});}else if(e.data==='stop'){clearInterval(t);}};`,
286
+ ],
287
+ { type: "application/javascript" },
288
+ );
289
+ const workerUrl = URL.createObjectURL(workerBlob);
290
+ try {
291
+ worker = new Worker(workerUrl);
292
+ } finally {
293
+ URL.revokeObjectURL(workerUrl);
294
+ }
295
+ worker.onmessage = () => drawFrame();
296
+ worker.postMessage("start");
297
+ } catch (err) {
298
+ console.warn(
299
+ "[camera-blur] Worker timer unavailable, falling back to rAF — blur may glitch on hidden tabs:",
300
+ err,
301
+ );
302
+ let lastFrameAt = 0;
303
+ const tick = (now: number) => {
304
+ if (raf === null) return;
305
+ if (now - lastFrameAt >= minFrameMs) {
306
+ lastFrameAt = now;
307
+ drawFrame();
308
+ }
309
+ raf = window.requestAnimationFrame(tick);
310
+ };
311
+ raf = window.requestAnimationFrame(tick);
312
+ }
313
+
314
+ drawFrame();
315
+
316
+ return {
317
+ stream,
318
+ active: true,
319
+ setBlurPx(px: number) {
320
+ if (positive(px)) blurPx = px;
321
+ },
322
+ cleanup() {
323
+ if (worker) {
324
+ worker.postMessage("stop");
325
+ worker.terminate();
326
+ worker = null;
327
+ }
328
+ if (raf !== null) {
329
+ window.cancelAnimationFrame(raf);
330
+ raf = null;
331
+ }
332
+ stream.getTracks().forEach((track) => track.stop());
333
+ hidden.cleanup();
334
+ try {
335
+ segmenter.close();
336
+ } catch {
337
+ // ignore — already closed.
338
+ }
339
+ },
340
+ };
341
+ }
@@ -244,7 +244,12 @@ export function createCameraCompositeStream(
244
244
  } catch {
245
245
  // The display video can be momentarily unavailable while metadata loads.
246
246
  }
247
- drawCameraBubble(ctx, camera.video, canvas, drawOptions);
247
+ // Hide the bubble once the camera ends (unplugged, or the blur pipeline
248
+ // stopped its captureStream) — the <video> freezes on its last frame rather
249
+ // than zeroing its dimensions, so check the track instead.
250
+ if (cameraTrack.readyState !== "ended") {
251
+ drawCameraBubble(ctx, camera.video, canvas, drawOptions);
252
+ }
248
253
  };
249
254
 
250
255
  // Use a Worker-based timer so the draw loop keeps running at the target