@agent-native/core 0.76.11 → 0.76.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/corpus/README.md +2 -2
- package/corpus/core/CHANGELOG.md +19 -0
- package/corpus/core/package.json +1 -1
- package/corpus/core/src/agent/production-agent.ts +7 -4
- package/corpus/core/src/client/FeedbackButton.tsx +162 -18
- package/corpus/core/src/client/blocks/library/AnnotatedCodeBlock.tsx +11 -2
- package/corpus/core/src/client/blocks/library/ApiEndpointBlock.tsx +46 -19
- package/corpus/core/src/client/blocks/library/MermaidBlock.tsx +7 -5
- package/corpus/core/src/client/blocks/library/block-copy.ts +367 -0
- package/corpus/core/src/client/blocks/library/diagram.tsx +8 -6
- package/corpus/core/src/client/blocks/library/wireframe.tsx +7 -2
- package/corpus/core/src/client/i18n.tsx +4 -0
- package/corpus/templates/clips/app/components/recorder/camera-visualizer.tsx +88 -10
- package/corpus/templates/clips/app/components/recorder/pre-record-panel.tsx +64 -0
- package/corpus/templates/clips/app/components/recorder/recorder-engine.ts +94 -9
- package/corpus/templates/clips/app/lib/camera-blur.ts +341 -0
- package/corpus/templates/clips/app/lib/camera-composite.ts +6 -1
- package/corpus/templates/clips/app/lib/recorder-preferences.ts +15 -0
- package/corpus/templates/clips/app/routes/record.tsx +13 -0
- package/corpus/templates/clips/desktop/src/app.tsx +101 -69
- package/corpus/templates/clips/desktop/src/lib/voice-dictation.ts +109 -11
- package/corpus/templates/clips/desktop/src/styles.css +19 -4
- package/corpus/templates/clips/desktop/src-tauri/Cargo.toml +1 -1
- package/corpus/templates/clips/desktop/src-tauri/src/native_speech.rs +143 -86
- package/corpus/templates/clips/package.json +1 -0
- package/corpus/templates/clips/vite.config.ts +38 -1
- package/corpus/templates/mail/app/root.tsx +90 -6
- package/dist/agent/production-agent.d.ts +7 -4
- package/dist/agent/production-agent.d.ts.map +1 -1
- package/dist/agent/production-agent.js +7 -4
- package/dist/agent/production-agent.js.map +1 -1
- package/dist/client/FeedbackButton.d.ts.map +1 -1
- package/dist/client/FeedbackButton.js +142 -14
- package/dist/client/FeedbackButton.js.map +1 -1
- package/dist/client/blocks/library/AnnotatedCodeBlock.d.ts.map +1 -1
- package/dist/client/blocks/library/AnnotatedCodeBlock.js +6 -2
- package/dist/client/blocks/library/AnnotatedCodeBlock.js.map +1 -1
- package/dist/client/blocks/library/ApiEndpointBlock.d.ts.map +1 -1
- package/dist/client/blocks/library/ApiEndpointBlock.js +14 -6
- package/dist/client/blocks/library/ApiEndpointBlock.js.map +1 -1
- package/dist/client/blocks/library/MermaidBlock.d.ts.map +1 -1
- package/dist/client/blocks/library/MermaidBlock.js +6 -4
- package/dist/client/blocks/library/MermaidBlock.js.map +1 -1
- package/dist/client/blocks/library/block-copy.d.ts +33 -0
- package/dist/client/blocks/library/block-copy.d.ts.map +1 -0
- package/dist/client/blocks/library/block-copy.js +329 -0
- package/dist/client/blocks/library/block-copy.js.map +1 -0
- package/dist/client/blocks/library/diagram.d.ts.map +1 -1
- package/dist/client/blocks/library/diagram.js +7 -5
- package/dist/client/blocks/library/diagram.js.map +1 -1
- package/dist/client/blocks/library/wireframe.d.ts.map +1 -1
- package/dist/client/blocks/library/wireframe.js +4 -2
- package/dist/client/blocks/library/wireframe.js.map +1 -1
- package/dist/client/i18n.d.ts +1 -0
- package/dist/client/i18n.d.ts.map +1 -1
- package/dist/client/i18n.js +3 -0
- package/dist/client/i18n.js.map +1 -1
- package/package.json +1 -1
|
@@ -7,6 +7,7 @@ import {
|
|
|
7
7
|
type FormEvent,
|
|
8
8
|
} from "react";
|
|
9
9
|
import {
|
|
10
|
+
IconBlur,
|
|
10
11
|
IconBrowser,
|
|
11
12
|
IconCamera,
|
|
12
13
|
IconChevronDown,
|
|
@@ -40,6 +41,7 @@ import {
|
|
|
40
41
|
SelectValue,
|
|
41
42
|
} from "@/components/ui/select";
|
|
42
43
|
import { Switch } from "@/components/ui/switch";
|
|
44
|
+
import { Slider } from "@/components/ui/slider";
|
|
43
45
|
import { cn } from "@/lib/utils";
|
|
44
46
|
import {
|
|
45
47
|
NO_CAMERA_DEVICE_ID,
|
|
@@ -48,6 +50,7 @@ import {
|
|
|
48
50
|
type RecordingMode,
|
|
49
51
|
} from "./recorder-engine";
|
|
50
52
|
import type { CameraBubbleSize } from "./camera-bubble";
|
|
53
|
+
import { DEFAULT_BLUR_PX, MAX_BLUR_PX, MIN_BLUR_PX } from "@/lib/camera-blur";
|
|
51
54
|
import {
|
|
52
55
|
loadRecorderPreferences,
|
|
53
56
|
saveRecorderPreferences,
|
|
@@ -65,6 +68,8 @@ export interface PreRecordPanelProps {
|
|
|
65
68
|
displaySurface: DisplaySurface;
|
|
66
69
|
micDeviceId: string | null;
|
|
67
70
|
cameraDeviceId: string | null;
|
|
71
|
+
cameraBlur: boolean;
|
|
72
|
+
cameraBlurRadius: number;
|
|
68
73
|
}) => void;
|
|
69
74
|
initialMode?: RecordingMode | null;
|
|
70
75
|
initialDisplaySurface?: DisplaySurface | null;
|
|
@@ -210,6 +215,12 @@ export function PreRecordPanel({
|
|
|
210
215
|
const [cameraId, setCameraId] = useState<string>(
|
|
211
216
|
() => savedPrefs.cameraId ?? "default",
|
|
212
217
|
);
|
|
218
|
+
const [cameraBlur, setCameraBlur] = useState(
|
|
219
|
+
() => savedPrefs.cameraBlur ?? false,
|
|
220
|
+
);
|
|
221
|
+
const [cameraBlurRadius, setCameraBlurRadius] = useState(
|
|
222
|
+
() => savedPrefs.cameraBlurRadius ?? DEFAULT_BLUR_PX,
|
|
223
|
+
);
|
|
213
224
|
const [enumError, setEnumError] = useState<string | null>(null);
|
|
214
225
|
const [micAccessStatus, setMicAccessStatus] =
|
|
215
226
|
useState<DeviceAccessStatus>("idle");
|
|
@@ -335,6 +346,10 @@ export function PreRecordPanel({
|
|
|
335
346
|
setCameraId(value);
|
|
336
347
|
saveRecorderPreferences({ cameraId: value });
|
|
337
348
|
}, []);
|
|
349
|
+
const chooseCameraBlur = useCallback((value: boolean) => {
|
|
350
|
+
setCameraBlur(value);
|
|
351
|
+
saveRecorderPreferences({ cameraBlur: value });
|
|
352
|
+
}, []);
|
|
338
353
|
|
|
339
354
|
const requestMicrophoneChoices = useCallback(async () => {
|
|
340
355
|
if (!navigator.mediaDevices?.getUserMedia) {
|
|
@@ -826,12 +841,59 @@ export function PreRecordPanel({
|
|
|
826
841
|
<CameraVisualizer
|
|
827
842
|
deviceId={cameraId === "default" ? null : cameraId}
|
|
828
843
|
disabled={busy}
|
|
844
|
+
blur={cameraBlur}
|
|
845
|
+
blurRadius={cameraBlurRadius}
|
|
829
846
|
size={cameraSize}
|
|
830
847
|
onSizeChange={onCameraSizeChange}
|
|
831
848
|
onStatusChange={handleCameraStatusChange}
|
|
832
849
|
onPreviewChange={handleCameraPreviewChange}
|
|
833
850
|
/>
|
|
834
851
|
) : null}
|
|
852
|
+
|
|
853
|
+
{needsCamera ? (
|
|
854
|
+
<label className="flex cursor-pointer items-center gap-2 rounded-lg px-1 py-1.5 hover:bg-muted/45">
|
|
855
|
+
<div className="flex h-8 w-8 shrink-0 items-center justify-center rounded-lg bg-muted text-muted-foreground">
|
|
856
|
+
<IconBlur className="h-4 w-4" />
|
|
857
|
+
</div>
|
|
858
|
+
<div className="min-w-0 flex-1">
|
|
859
|
+
<p className="text-sm">Blur background</p>
|
|
860
|
+
<p className="text-[11px] text-muted-foreground">
|
|
861
|
+
Keep yourself sharp, blur what's behind you
|
|
862
|
+
</p>
|
|
863
|
+
</div>
|
|
864
|
+
<Switch
|
|
865
|
+
checked={cameraBlur}
|
|
866
|
+
onCheckedChange={chooseCameraBlur}
|
|
867
|
+
disabled={busy}
|
|
868
|
+
aria-label="Blur the camera background"
|
|
869
|
+
/>
|
|
870
|
+
</label>
|
|
871
|
+
) : null}
|
|
872
|
+
|
|
873
|
+
{needsCamera && cameraBlur ? (
|
|
874
|
+
<div className="flex items-center gap-3 px-1 pb-1">
|
|
875
|
+
<span className="w-14 shrink-0 text-[11px] text-muted-foreground">
|
|
876
|
+
Intensity
|
|
877
|
+
</span>
|
|
878
|
+
<Slider
|
|
879
|
+
value={[cameraBlurRadius]}
|
|
880
|
+
min={MIN_BLUR_PX}
|
|
881
|
+
max={MAX_BLUR_PX}
|
|
882
|
+
step={1}
|
|
883
|
+
disabled={busy}
|
|
884
|
+
onValueChange={(value) =>
|
|
885
|
+
setCameraBlurRadius(value[0] ?? DEFAULT_BLUR_PX)
|
|
886
|
+
}
|
|
887
|
+
onValueCommit={(value) =>
|
|
888
|
+
saveRecorderPreferences({
|
|
889
|
+
cameraBlurRadius: value[0] ?? DEFAULT_BLUR_PX,
|
|
890
|
+
})
|
|
891
|
+
}
|
|
892
|
+
aria-label="Background blur intensity"
|
|
893
|
+
className="flex-1"
|
|
894
|
+
/>
|
|
895
|
+
</div>
|
|
896
|
+
) : null}
|
|
835
897
|
</div>
|
|
836
898
|
) : null}
|
|
837
899
|
|
|
@@ -870,6 +932,8 @@ export function PreRecordPanel({
|
|
|
870
932
|
micDeviceId: micId === "default" ? null : micId,
|
|
871
933
|
cameraDeviceId:
|
|
872
934
|
needsCamera && cameraId !== "default" ? cameraId : null,
|
|
935
|
+
cameraBlur: needsCamera ? cameraBlur : false,
|
|
936
|
+
cameraBlurRadius,
|
|
873
937
|
})
|
|
874
938
|
}
|
|
875
939
|
className={cn("h-12 gap-2", onCancel ? "flex-1" : "w-full")}
|
|
@@ -19,6 +19,10 @@ import {
|
|
|
19
19
|
createCameraCompositeStream,
|
|
20
20
|
type CameraCompositeHandle,
|
|
21
21
|
} from "@/lib/camera-composite";
|
|
22
|
+
import {
|
|
23
|
+
createBackgroundBlurStream,
|
|
24
|
+
type CameraBlurHandle,
|
|
25
|
+
} from "@/lib/camera-blur";
|
|
22
26
|
import {
|
|
23
27
|
chunkUploadUrl,
|
|
24
28
|
pickMimeType,
|
|
@@ -75,6 +79,15 @@ export interface RecorderEngineOptions {
|
|
|
75
79
|
cameraDeviceId?: string | null;
|
|
76
80
|
/** Camera bubble size selected in the pre-record UI. */
|
|
77
81
|
cameraBubbleSize?: "sm" | "md" | "lg";
|
|
82
|
+
/**
|
|
83
|
+
* Blur the camera background (sharp person, blurred surroundings) for both the
|
|
84
|
+
* live preview bubble and the baked-in recording composite. Resolved at
|
|
85
|
+
* `acquire()` time. Silently no-ops (records un-blurred) if segmentation is
|
|
86
|
+
* unavailable in the browser.
|
|
87
|
+
*/
|
|
88
|
+
cameraBlur?: boolean;
|
|
89
|
+
/** Background blur radius in px when `cameraBlur` is on. Defaults to ~12. */
|
|
90
|
+
cameraBlurRadius?: number;
|
|
78
91
|
/** Chunk size in ms (MediaRecorder timeslice). Default 2000. */
|
|
79
92
|
chunkIntervalMs?: number;
|
|
80
93
|
/** Base URL for the chunk upload endpoint. Default `/api/uploads/:id/chunk`. */
|
|
@@ -99,6 +112,11 @@ export interface RecorderEngineOptions {
|
|
|
99
112
|
* `onError`, this does NOT transition the engine into the `error` state.
|
|
100
113
|
*/
|
|
101
114
|
onWarning?: (message: string) => void;
|
|
115
|
+
/**
|
|
116
|
+
* Fired when the camera ends (unplugged / revoked) any time after acquire,
|
|
117
|
+
* so the UI can drop the on-page camera bubble to match the recorded output.
|
|
118
|
+
*/
|
|
119
|
+
onCameraEnded?: () => void;
|
|
102
120
|
/**
|
|
103
121
|
* Called when the display stream's video track ends because the user clicked
|
|
104
122
|
* the browser's native "Stop sharing" button. When provided, the engine
|
|
@@ -360,6 +378,17 @@ export class RecorderEngine {
|
|
|
360
378
|
|
|
361
379
|
private displayStream: MediaStream | null = null;
|
|
362
380
|
private cameraStream: MediaStream | null = null;
|
|
381
|
+
/**
|
|
382
|
+
* Raw getUserMedia camera stream. When background blur is active,
|
|
383
|
+
* `cameraStream` points at the processed (blurred) derivative and this field
|
|
384
|
+
* keeps the original so teardown stops the real camera tracks and the
|
|
385
|
+
* disconnect handler can observe the hardware ending.
|
|
386
|
+
*/
|
|
387
|
+
private rawCameraStream: MediaStream | null = null;
|
|
388
|
+
private cameraBlur: CameraBlurHandle | null = null;
|
|
389
|
+
// True once the camera is acquired, through preview/countdown/recording, until
|
|
390
|
+
// teardown — gates disconnect handling outside the recording state.
|
|
391
|
+
private cameraLive = false;
|
|
363
392
|
private micStream: MediaStream | null = null;
|
|
364
393
|
private combinedStream: MediaStream | null = null;
|
|
365
394
|
private previewStream: MediaStream | null = null;
|
|
@@ -697,13 +726,10 @@ export class RecorderEngine {
|
|
|
697
726
|
}
|
|
698
727
|
}
|
|
699
728
|
|
|
700
|
-
// Camera / mic disconnects
|
|
701
|
-
//
|
|
702
|
-
//
|
|
703
|
-
//
|
|
704
|
-
// display handler so the `ended` events fired by `cleanupTracks()` during
|
|
705
|
-
// a normal stop()/cancel() (state is `stopping`/`idle` by then) are
|
|
706
|
-
// ignored rather than treated as a disconnect.
|
|
729
|
+
// Camera / mic disconnects (USB webcam unplugged, permission revoked,
|
|
730
|
+
// Bluetooth dropped) are NON-fatal: keep whatever inputs remain and warn.
|
|
731
|
+
// The handlers gate on `cameraLive`/state so the `ended` events fired by
|
|
732
|
+
// `cleanupTracks()` during a normal stop/cancel are ignored.
|
|
707
733
|
if (this.cameraStream) {
|
|
708
734
|
for (const track of this.cameraStream.getVideoTracks()) {
|
|
709
735
|
track.addEventListener("ended", () => {
|
|
@@ -720,6 +746,37 @@ export class RecorderEngine {
|
|
|
720
746
|
}
|
|
721
747
|
}
|
|
722
748
|
|
|
749
|
+
// Camera is live from here through preview/countdown/recording until
|
|
750
|
+
// teardown — set before the async blur setup so a disconnect during that
|
|
751
|
+
// window is handled, not inherited as a dead stream.
|
|
752
|
+
if (this.cameraStream) this.cameraLive = true;
|
|
753
|
+
|
|
754
|
+
// Swap the raw camera for its blurred derivative, which both the preview
|
|
755
|
+
// bubble and the recording composite read ("what you see is what's
|
|
756
|
+
// recorded"). createBackgroundBlurStream never throws — it falls back to
|
|
757
|
+
// the raw stream on failure.
|
|
758
|
+
if (this.opts.cameraBlur && this.cameraStream) {
|
|
759
|
+
this.rawCameraStream = this.cameraStream;
|
|
760
|
+
const handle = await createBackgroundBlurStream(this.cameraStream, {
|
|
761
|
+
blurPx: this.opts.cameraBlurRadius,
|
|
762
|
+
});
|
|
763
|
+
if (this.cameraDisconnectNotified) {
|
|
764
|
+
// Webcam ended while the pipeline was loading: discard it and drop the
|
|
765
|
+
// dead camera rather than inheriting a frozen processed stream.
|
|
766
|
+
handle.cleanup();
|
|
767
|
+
this.cameraStream = null;
|
|
768
|
+
} else {
|
|
769
|
+
this.cameraBlur = handle;
|
|
770
|
+
this.cameraStream = this.cameraBlur.stream;
|
|
771
|
+
}
|
|
772
|
+
}
|
|
773
|
+
|
|
774
|
+
if (this.opts.mode === "camera" && !this.cameraStream) {
|
|
775
|
+
throw new Error(
|
|
776
|
+
"Camera disconnected before recording could start. Reconnect it and try again.",
|
|
777
|
+
);
|
|
778
|
+
}
|
|
779
|
+
|
|
723
780
|
this.previewStream =
|
|
724
781
|
this.opts.mode === "camera" ? this.cameraStream! : this.displayStream!;
|
|
725
782
|
|
|
@@ -1414,6 +1471,12 @@ export class RecorderEngine {
|
|
|
1414
1471
|
return combined;
|
|
1415
1472
|
}
|
|
1416
1473
|
|
|
1474
|
+
// Camera dropped before start (disconnected during setup/countdown):
|
|
1475
|
+
// record screen-only rather than compositing a dead stream.
|
|
1476
|
+
if (!this.cameraStream) {
|
|
1477
|
+
return this.buildDisplayRecordingStream();
|
|
1478
|
+
}
|
|
1479
|
+
|
|
1417
1480
|
// Screen + camera: display capture does not reliably include our separate
|
|
1418
1481
|
// DOM bubble once the user records another app/window, so the saved
|
|
1419
1482
|
// recording must composite the camera feed before MediaRecorder sees it.
|
|
@@ -1705,14 +1768,23 @@ export class RecorderEngine {
|
|
|
1705
1768
|
}
|
|
1706
1769
|
|
|
1707
1770
|
private cleanupTracks(): void {
|
|
1771
|
+
// Clear before stopping tracks so the `ended` events our own stop() fires
|
|
1772
|
+
// aren't mistaken for a disconnect.
|
|
1773
|
+
this.cameraLive = false;
|
|
1708
1774
|
this.audioMixSources = [];
|
|
1709
1775
|
this.audioMixCtx?.close().catch(() => {});
|
|
1710
1776
|
this.audioMixCtx = null;
|
|
1711
1777
|
this.cameraComposite?.cleanup();
|
|
1712
1778
|
this.cameraComposite = null;
|
|
1779
|
+
// Tear down the blur pipeline (segmenter, hidden video, processed capture)
|
|
1780
|
+
// before stopping streams. `rawCameraStream` holds the real hardware tracks
|
|
1781
|
+
// when blur is active; stop those so the camera indicator clears.
|
|
1782
|
+
this.cameraBlur?.cleanup();
|
|
1783
|
+
this.cameraBlur = null;
|
|
1713
1784
|
for (const s of [
|
|
1714
1785
|
this.displayStream,
|
|
1715
1786
|
this.cameraStream,
|
|
1787
|
+
this.rawCameraStream,
|
|
1716
1788
|
this.micStream,
|
|
1717
1789
|
this.combinedStream,
|
|
1718
1790
|
]) {
|
|
@@ -1727,6 +1799,7 @@ export class RecorderEngine {
|
|
|
1727
1799
|
}
|
|
1728
1800
|
this.displayStream = null;
|
|
1729
1801
|
this.cameraStream = null;
|
|
1802
|
+
this.rawCameraStream = null;
|
|
1730
1803
|
this.micStream = null;
|
|
1731
1804
|
this.combinedStream = null;
|
|
1732
1805
|
this.previewStream = null;
|
|
@@ -1764,16 +1837,28 @@ export class RecorderEngine {
|
|
|
1764
1837
|
* warn the user.
|
|
1765
1838
|
*/
|
|
1766
1839
|
private onCameraTrackEnded() {
|
|
1767
|
-
if (this.
|
|
1840
|
+
if (!this.cameraLive) return;
|
|
1768
1841
|
if (this.cameraDisconnectNotified) return;
|
|
1769
1842
|
this.cameraDisconnectNotified = true;
|
|
1770
|
-
|
|
1843
|
+
this.cameraLive = false;
|
|
1844
|
+
// Tear down the blur pipeline so the composite's camera <video> sees its
|
|
1845
|
+
// (blurred) track end and self-hides, instead of freezing on a stale frame.
|
|
1846
|
+
if (this.cameraBlur) {
|
|
1847
|
+
this.cameraBlur.cleanup();
|
|
1848
|
+
this.cameraBlur = null;
|
|
1849
|
+
}
|
|
1850
|
+
for (const track of [
|
|
1851
|
+
...(this.rawCameraStream?.getVideoTracks() ?? []),
|
|
1852
|
+
...(this.cameraStream?.getVideoTracks() ?? []),
|
|
1853
|
+
]) {
|
|
1771
1854
|
try {
|
|
1772
1855
|
track.stop();
|
|
1773
1856
|
} catch {
|
|
1774
1857
|
// ignore — the track has already ended.
|
|
1775
1858
|
}
|
|
1776
1859
|
}
|
|
1860
|
+
// Drop the on-page bubble to match the recorded output (screen-only now).
|
|
1861
|
+
this.opts.onCameraEnded?.();
|
|
1777
1862
|
this.emitWarning(
|
|
1778
1863
|
"Camera disconnected — recording continues without webcam.",
|
|
1779
1864
|
);
|
|
@@ -0,0 +1,341 @@
|
|
|
1
|
+
import { appBasePath } from "@agent-native/core/client";
|
|
2
|
+
import type { ImageSegmenter } from "@mediapipe/tasks-vision";
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* A processed camera stream whose background is blurred while the person stays
|
|
6
|
+
* sharp (Zoom / Loom style). Produced once in the recorder engine and shared by
|
|
7
|
+
* both the live preview bubble and the baked-in recording composite, so "what
|
|
8
|
+
* you see is what's recorded".
|
|
9
|
+
*/
|
|
10
|
+
export interface CameraBlurHandle {
|
|
11
|
+
/** The processed stream, or the original `source` stream when `active` is false. */
|
|
12
|
+
stream: MediaStream;
|
|
13
|
+
/** False when segmentation was unavailable and we transparently fell back to raw. */
|
|
14
|
+
readonly active: boolean;
|
|
15
|
+
/**
|
|
16
|
+
* Update the background blur radius (px) live, without rebuilding the
|
|
17
|
+
* segmenter. No-op on the passthrough fallback handle.
|
|
18
|
+
*/
|
|
19
|
+
setBlurPx(px: number): void;
|
|
20
|
+
cleanup(): void;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export interface CameraBlurOptions {
|
|
24
|
+
/** CSS blur radius applied to the background, in px. Default 12. */
|
|
25
|
+
blurPx?: number;
|
|
26
|
+
/**
|
|
27
|
+
* How often segmentation runs, in frames per second. Kept below the 30fps
|
|
28
|
+
* capture rate to bound CPU/GPU cost — the small bubble does not need a fresh
|
|
29
|
+
* mask every captured frame. Default 20.
|
|
30
|
+
*/
|
|
31
|
+
segmentationFps?: number;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export const DEFAULT_BLUR_PX = 12;
|
|
35
|
+
export const MIN_BLUR_PX = 2;
|
|
36
|
+
export const MAX_BLUR_PX = 30;
|
|
37
|
+
const DEFAULT_SEGMENTATION_FPS = 20;
|
|
38
|
+
const CAPTURE_FPS = 30;
|
|
39
|
+
/**
|
|
40
|
+
* Max dimension (px) of the frame we feed the segmenter. The model resizes to
|
|
41
|
+
* its own 256² input internally and upsamples the mask back to this size, so
|
|
42
|
+
* keeping the input small bounds the per-frame mask readback + alpha loop cost
|
|
43
|
+
* regardless of the camera's native resolution. The soft mask is scaled back up
|
|
44
|
+
* to the camera resolution with bilinear filtering when compositing.
|
|
45
|
+
*/
|
|
46
|
+
const SEG_MAX_DIM = 256;
|
|
47
|
+
|
|
48
|
+
const MODEL_PATH = "/mediapipe/selfie_segmenter.tflite";
|
|
49
|
+
const WASM_PATH = "/mediapipe/wasm";
|
|
50
|
+
|
|
51
|
+
function positive(value: unknown): value is number {
|
|
52
|
+
return typeof value === "number" && Number.isFinite(value) && value > 0;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
function sourceDimensions(
|
|
56
|
+
video: HTMLVideoElement,
|
|
57
|
+
stream: MediaStream,
|
|
58
|
+
): { width: number; height: number } {
|
|
59
|
+
if (positive(video.videoWidth) && positive(video.videoHeight)) {
|
|
60
|
+
return { width: video.videoWidth, height: video.videoHeight };
|
|
61
|
+
}
|
|
62
|
+
const settings = stream.getVideoTracks()[0]?.getSettings();
|
|
63
|
+
return {
|
|
64
|
+
width: positive(settings?.width) ? Math.round(settings.width) : 640,
|
|
65
|
+
height: positive(settings?.height) ? Math.round(settings.height) : 480,
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** Hidden, off-screen `<video>` that plays the source stream for canvas reads. */
|
|
70
|
+
function attachHiddenVideo(stream: MediaStream): {
|
|
71
|
+
video: HTMLVideoElement;
|
|
72
|
+
cleanup(): void;
|
|
73
|
+
} {
|
|
74
|
+
const video = document.createElement("video");
|
|
75
|
+
video.muted = true;
|
|
76
|
+
video.playsInline = true;
|
|
77
|
+
video.autoplay = true;
|
|
78
|
+
video.srcObject = stream;
|
|
79
|
+
video.style.position = "fixed";
|
|
80
|
+
video.style.left = "-10000px";
|
|
81
|
+
video.style.top = "0";
|
|
82
|
+
video.style.width = "1px";
|
|
83
|
+
video.style.height = "1px";
|
|
84
|
+
video.style.opacity = "0";
|
|
85
|
+
video.style.pointerEvents = "none";
|
|
86
|
+
|
|
87
|
+
const tryPlay = () => {
|
|
88
|
+
video.play().catch(() => undefined);
|
|
89
|
+
};
|
|
90
|
+
|
|
91
|
+
document.body.appendChild(video);
|
|
92
|
+
video.addEventListener("loadedmetadata", tryPlay);
|
|
93
|
+
tryPlay();
|
|
94
|
+
|
|
95
|
+
return {
|
|
96
|
+
video,
|
|
97
|
+
cleanup() {
|
|
98
|
+
video.removeEventListener("loadedmetadata", tryPlay);
|
|
99
|
+
video.pause();
|
|
100
|
+
video.srcObject = null;
|
|
101
|
+
video.remove();
|
|
102
|
+
},
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
async function createSegmenter(): Promise<ImageSegmenter> {
|
|
107
|
+
// Lazy-loaded so the ~11MB Wasm runtime and its loader never enter the main
|
|
108
|
+
// bundle or run during SSR — only when a recording actually requests blur.
|
|
109
|
+
const vision = await import("@mediapipe/tasks-vision");
|
|
110
|
+
const base = appBasePath();
|
|
111
|
+
const fileset = await vision.FilesetResolver.forVisionTasks(
|
|
112
|
+
`${base}${WASM_PATH}`,
|
|
113
|
+
);
|
|
114
|
+
const build = (delegate: "GPU" | "CPU") =>
|
|
115
|
+
vision.ImageSegmenter.createFromOptions(fileset, {
|
|
116
|
+
baseOptions: {
|
|
117
|
+
modelAssetPath: `${base}${MODEL_PATH}`,
|
|
118
|
+
delegate,
|
|
119
|
+
},
|
|
120
|
+
runningMode: "VIDEO",
|
|
121
|
+
outputConfidenceMasks: true,
|
|
122
|
+
outputCategoryMask: false,
|
|
123
|
+
});
|
|
124
|
+
try {
|
|
125
|
+
return await build("GPU");
|
|
126
|
+
} catch {
|
|
127
|
+
// Older GPUs / blocked WebGL contexts: fall back to the CPU delegate.
|
|
128
|
+
return await build("CPU");
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
const fallback = (source: MediaStream): CameraBlurHandle => ({
|
|
133
|
+
stream: source,
|
|
134
|
+
active: false,
|
|
135
|
+
setBlurPx() {},
|
|
136
|
+
cleanup() {},
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
/**
|
|
140
|
+
* Build a background-blurred derivative of `source`. Never throws: if MediaPipe,
|
|
141
|
+
* the Wasm fileset, the model, or canvas capture are unavailable, it resolves to
|
|
142
|
+
* a passthrough handle wrapping the raw `source` (`active === false`) so the
|
|
143
|
+
* recording always proceeds. Callers own `source`'s tracks; `cleanup()` here
|
|
144
|
+
* tears down only the processing pipeline, never the source.
|
|
145
|
+
*/
|
|
146
|
+
export async function createBackgroundBlurStream(
|
|
147
|
+
source: MediaStream,
|
|
148
|
+
opts: CameraBlurOptions = {},
|
|
149
|
+
): Promise<CameraBlurHandle> {
|
|
150
|
+
if (typeof document === "undefined") return fallback(source);
|
|
151
|
+
if (!source.getVideoTracks().length) return fallback(source);
|
|
152
|
+
|
|
153
|
+
let blurPx = positive(opts.blurPx) ? opts.blurPx : DEFAULT_BLUR_PX;
|
|
154
|
+
const segFps = positive(opts.segmentationFps)
|
|
155
|
+
? opts.segmentationFps
|
|
156
|
+
: DEFAULT_SEGMENTATION_FPS;
|
|
157
|
+
|
|
158
|
+
const out = document.createElement("canvas");
|
|
159
|
+
const outCtx = out.getContext("2d", { alpha: false });
|
|
160
|
+
if (!outCtx || typeof out.captureStream !== "function") {
|
|
161
|
+
return fallback(source);
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
let segmenter: ImageSegmenter;
|
|
165
|
+
try {
|
|
166
|
+
segmenter = await createSegmenter();
|
|
167
|
+
} catch (err) {
|
|
168
|
+
console.warn(
|
|
169
|
+
"[camera-blur] Segmentation unavailable — recording without background blur:",
|
|
170
|
+
err,
|
|
171
|
+
);
|
|
172
|
+
return fallback(source);
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
const hidden = attachHiddenVideo(source);
|
|
176
|
+
const { width: w0, height: h0 } = sourceDimensions(hidden.video, source);
|
|
177
|
+
out.width = w0;
|
|
178
|
+
out.height = h0;
|
|
179
|
+
|
|
180
|
+
// Foreground compositing scratch: sharp person punched out by the mask alpha.
|
|
181
|
+
const fg = document.createElement("canvas");
|
|
182
|
+
const fgCtx = fg.getContext("2d");
|
|
183
|
+
// Downscaled frame we actually segment, plus the soft alpha mask it produces.
|
|
184
|
+
const segInput = document.createElement("canvas");
|
|
185
|
+
const segCtx = segInput.getContext("2d");
|
|
186
|
+
const maskCanvas = document.createElement("canvas");
|
|
187
|
+
const maskCtx = maskCanvas.getContext("2d");
|
|
188
|
+
if (!fgCtx || !segCtx || !maskCtx) {
|
|
189
|
+
hidden.cleanup();
|
|
190
|
+
segmenter.close();
|
|
191
|
+
return fallback(source);
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
let lastTimestamp = -1;
|
|
195
|
+
let lastSegAt = -Infinity;
|
|
196
|
+
let haveMask = false;
|
|
197
|
+
const segIntervalMs = 1000 / segFps;
|
|
198
|
+
|
|
199
|
+
// Re-segment into maskCanvas. Runs at segFps; on failure the previous mask is
|
|
200
|
+
// kept rather than dropping to an un-blurred frame.
|
|
201
|
+
const updateMask = (video: HTMLVideoElement) => {
|
|
202
|
+
const scale = SEG_MAX_DIM / Math.max(video.videoWidth, video.videoHeight);
|
|
203
|
+
const segW = Math.max(1, Math.round(video.videoWidth * scale));
|
|
204
|
+
const segH = Math.max(1, Math.round(video.videoHeight * scale));
|
|
205
|
+
if (segInput.width !== segW) segInput.width = segW;
|
|
206
|
+
if (segInput.height !== segH) segInput.height = segH;
|
|
207
|
+
try {
|
|
208
|
+
segCtx.drawImage(video, 0, 0, segW, segH);
|
|
209
|
+
const timestamp = Math.max(
|
|
210
|
+
Math.round(performance.now()),
|
|
211
|
+
lastTimestamp + 1,
|
|
212
|
+
);
|
|
213
|
+
lastTimestamp = timestamp;
|
|
214
|
+
const result = segmenter.segmentForVideo(segInput, timestamp);
|
|
215
|
+
const confidence = result.confidenceMasks?.[0];
|
|
216
|
+
if (confidence) {
|
|
217
|
+
const maskW = confidence.width;
|
|
218
|
+
const maskH = confidence.height;
|
|
219
|
+
const values = confidence.getAsFloat32Array();
|
|
220
|
+
if (maskCanvas.width !== maskW) maskCanvas.width = maskW;
|
|
221
|
+
if (maskCanvas.height !== maskH) maskCanvas.height = maskH;
|
|
222
|
+
const image = maskCtx.createImageData(maskW, maskH);
|
|
223
|
+
for (let i = 0; i < values.length; i++) {
|
|
224
|
+
image.data[i * 4 + 3] = Math.round(values[i] * 255);
|
|
225
|
+
}
|
|
226
|
+
maskCtx.putImageData(image, 0, 0);
|
|
227
|
+
haveMask = true;
|
|
228
|
+
}
|
|
229
|
+
result.close();
|
|
230
|
+
} catch {
|
|
231
|
+
// keep the previous mask
|
|
232
|
+
}
|
|
233
|
+
};
|
|
234
|
+
|
|
235
|
+
// Composite blurred background + sharp foreground (or raw until a first mask).
|
|
236
|
+
const composite = (video: HTMLVideoElement, outW: number, outH: number) => {
|
|
237
|
+
if (!haveMask) {
|
|
238
|
+
outCtx.drawImage(video, 0, 0, outW, outH);
|
|
239
|
+
return;
|
|
240
|
+
}
|
|
241
|
+
if (fg.width !== outW) fg.width = outW;
|
|
242
|
+
if (fg.height !== outH) fg.height = outH;
|
|
243
|
+
fgCtx.globalCompositeOperation = "source-over";
|
|
244
|
+
fgCtx.clearRect(0, 0, outW, outH);
|
|
245
|
+
fgCtx.drawImage(video, 0, 0, outW, outH);
|
|
246
|
+
fgCtx.globalCompositeOperation = "destination-in";
|
|
247
|
+
fgCtx.imageSmoothingEnabled = true;
|
|
248
|
+
fgCtx.drawImage(maskCanvas, 0, 0, outW, outH);
|
|
249
|
+
fgCtx.globalCompositeOperation = "source-over";
|
|
250
|
+
|
|
251
|
+
outCtx.filter = `blur(${blurPx}px)`;
|
|
252
|
+
outCtx.drawImage(video, 0, 0, outW, outH);
|
|
253
|
+
outCtx.filter = "none";
|
|
254
|
+
outCtx.drawImage(fg, 0, 0);
|
|
255
|
+
};
|
|
256
|
+
|
|
257
|
+
// Composite every tick (capture rate) for smooth motion; re-segment only at
|
|
258
|
+
// segFps, reusing the last mask in between.
|
|
259
|
+
const drawFrame = () => {
|
|
260
|
+
const video = hidden.video;
|
|
261
|
+
if (!positive(video.videoWidth) || !positive(video.videoHeight)) return;
|
|
262
|
+
if (out.width !== video.videoWidth) out.width = video.videoWidth;
|
|
263
|
+
if (out.height !== video.videoHeight) out.height = video.videoHeight;
|
|
264
|
+
|
|
265
|
+
const now = performance.now();
|
|
266
|
+
if (now - lastSegAt >= segIntervalMs) {
|
|
267
|
+
updateMask(video);
|
|
268
|
+
lastSegAt = now;
|
|
269
|
+
}
|
|
270
|
+
composite(video, out.width, out.height);
|
|
271
|
+
};
|
|
272
|
+
|
|
273
|
+
const stream = out.captureStream(CAPTURE_FPS);
|
|
274
|
+
const minFrameMs = 1000 / CAPTURE_FPS;
|
|
275
|
+
|
|
276
|
+
// Worker-driven timer so the loop keeps running at full rate in background
|
|
277
|
+
// tabs (rAF is throttled to ~1fps when hidden). Falls back to rAF when blob:
|
|
278
|
+
// workers are blocked by CSP — mirrors camera-composite.ts.
|
|
279
|
+
let worker: Worker | null = null;
|
|
280
|
+
let raf: number | null = null;
|
|
281
|
+
|
|
282
|
+
try {
|
|
283
|
+
const workerBlob = new Blob(
|
|
284
|
+
[
|
|
285
|
+
`let t=null;onmessage=e=>{if(e.data==='start'){clearInterval(t);t=setInterval(()=>postMessage('tick'),${minFrameMs});}else if(e.data==='stop'){clearInterval(t);}};`,
|
|
286
|
+
],
|
|
287
|
+
{ type: "application/javascript" },
|
|
288
|
+
);
|
|
289
|
+
const workerUrl = URL.createObjectURL(workerBlob);
|
|
290
|
+
try {
|
|
291
|
+
worker = new Worker(workerUrl);
|
|
292
|
+
} finally {
|
|
293
|
+
URL.revokeObjectURL(workerUrl);
|
|
294
|
+
}
|
|
295
|
+
worker.onmessage = () => drawFrame();
|
|
296
|
+
worker.postMessage("start");
|
|
297
|
+
} catch (err) {
|
|
298
|
+
console.warn(
|
|
299
|
+
"[camera-blur] Worker timer unavailable, falling back to rAF — blur may glitch on hidden tabs:",
|
|
300
|
+
err,
|
|
301
|
+
);
|
|
302
|
+
let lastFrameAt = 0;
|
|
303
|
+
const tick = (now: number) => {
|
|
304
|
+
if (raf === null) return;
|
|
305
|
+
if (now - lastFrameAt >= minFrameMs) {
|
|
306
|
+
lastFrameAt = now;
|
|
307
|
+
drawFrame();
|
|
308
|
+
}
|
|
309
|
+
raf = window.requestAnimationFrame(tick);
|
|
310
|
+
};
|
|
311
|
+
raf = window.requestAnimationFrame(tick);
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
drawFrame();
|
|
315
|
+
|
|
316
|
+
return {
|
|
317
|
+
stream,
|
|
318
|
+
active: true,
|
|
319
|
+
setBlurPx(px: number) {
|
|
320
|
+
if (positive(px)) blurPx = px;
|
|
321
|
+
},
|
|
322
|
+
cleanup() {
|
|
323
|
+
if (worker) {
|
|
324
|
+
worker.postMessage("stop");
|
|
325
|
+
worker.terminate();
|
|
326
|
+
worker = null;
|
|
327
|
+
}
|
|
328
|
+
if (raf !== null) {
|
|
329
|
+
window.cancelAnimationFrame(raf);
|
|
330
|
+
raf = null;
|
|
331
|
+
}
|
|
332
|
+
stream.getTracks().forEach((track) => track.stop());
|
|
333
|
+
hidden.cleanup();
|
|
334
|
+
try {
|
|
335
|
+
segmenter.close();
|
|
336
|
+
} catch {
|
|
337
|
+
// ignore — already closed.
|
|
338
|
+
}
|
|
339
|
+
},
|
|
340
|
+
};
|
|
341
|
+
}
|
|
@@ -244,7 +244,12 @@ export function createCameraCompositeStream(
|
|
|
244
244
|
} catch {
|
|
245
245
|
// The display video can be momentarily unavailable while metadata loads.
|
|
246
246
|
}
|
|
247
|
-
|
|
247
|
+
// Hide the bubble once the camera ends (unplugged, or the blur pipeline
|
|
248
|
+
// stopped its captureStream) — the <video> freezes on its last frame rather
|
|
249
|
+
// than zeroing its dimensions, so check the track instead.
|
|
250
|
+
if (cameraTrack.readyState !== "ended") {
|
|
251
|
+
drawCameraBubble(ctx, camera.video, canvas, drawOptions);
|
|
252
|
+
}
|
|
248
253
|
};
|
|
249
254
|
|
|
250
255
|
// Use a Worker-based timer so the draw loop keeps running at the target
|