@cuemath/leap 4.1.1-j1 → 4.1.1-j2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/features/av/av-preview/av-preview.js +50 -50
- package/dist/features/av/av-preview/av-preview.js.map +1 -1
- package/dist/features/av/video-analysis/hooks/use-video-analysis.js +85 -83
- package/dist/features/av/video-analysis/hooks/use-video-analysis.js.map +1 -1
- package/dist/features/av/video-analysis/video-analysis.js +28 -32
- package/dist/features/av/video-analysis/video-analysis.js.map +1 -1
- package/dist/index.d.ts +1 -12
- package/package.json +1 -1
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { jsxs as n, jsx as e, Fragment as G } from "react/jsx-runtime";
|
|
2
|
-
import { memo as j, useRef as B, useEffect as
|
|
2
|
+
import { memo as j, useRef as B, useEffect as g, useCallback as A } from "react";
|
|
3
3
|
import { useUserMedia as M } from "@cuemath/av";
|
|
4
4
|
import N from "../video-analysis/hooks/use-video-analysis.js";
|
|
5
5
|
import R from "../video-analysis/video-analysis-overlay/video-analysis-overlay.js";
|
|
6
|
-
import
|
|
6
|
+
import $ from "../../ui/inputs/select-input/select-input.js";
|
|
7
7
|
import u from "../../ui/layout/flex-view.js";
|
|
8
8
|
import w from "../../ui/text/text.js";
|
|
9
9
|
import { AVErrorSOPWrapper as T, AVErrorStepList as F } from "./av-preview-styled.js";
|
|
@@ -14,34 +14,34 @@ const W = {
|
|
|
14
14
|
setupListeners: !0
|
|
15
15
|
}, L = j(
|
|
16
16
|
({
|
|
17
|
-
children:
|
|
17
|
+
children: V,
|
|
18
18
|
logger: a,
|
|
19
|
-
onDeviceUpdate:
|
|
20
|
-
enableVideoAnalysis:
|
|
21
|
-
detectEmotions:
|
|
22
|
-
detectSleep:
|
|
19
|
+
onDeviceUpdate: y,
|
|
20
|
+
enableVideoAnalysis: D = !1,
|
|
21
|
+
detectEmotions: x = !1,
|
|
22
|
+
detectSleep: C = !1
|
|
23
23
|
}) => {
|
|
24
24
|
const s = B(null), {
|
|
25
|
-
selectedAudioDevice:
|
|
26
|
-
selectedVideoDevice:
|
|
27
|
-
videoDeviceError:
|
|
28
|
-
devices:
|
|
25
|
+
selectedAudioDevice: l,
|
|
26
|
+
selectedVideoDevice: c,
|
|
27
|
+
videoDeviceError: m,
|
|
28
|
+
devices: p,
|
|
29
29
|
changeDevice: r,
|
|
30
|
-
selectedAudioOutputDevice:
|
|
31
|
-
audioDeviceError:
|
|
32
|
-
userMediaStream:
|
|
33
|
-
isAudioDeviceLoading:
|
|
34
|
-
isVideoDeviceLoading:
|
|
35
|
-
} = M(W),
|
|
36
|
-
metrics:
|
|
30
|
+
selectedAudioOutputDevice: h,
|
|
31
|
+
audioDeviceError: v,
|
|
32
|
+
userMediaStream: f,
|
|
33
|
+
isAudioDeviceLoading: O,
|
|
34
|
+
isVideoDeviceLoading: P
|
|
35
|
+
} = M(W), _ = m === "permissionDeniedBySystem" || m === "permissionDenied" || v === "permissionDeniedBySystem" || v === "permissionDenied", b = K(v, m), {
|
|
36
|
+
metrics: S,
|
|
37
37
|
isLoading: X,
|
|
38
38
|
error: d
|
|
39
39
|
} = N(s, {
|
|
40
|
-
enabled:
|
|
41
|
-
detectEmotions:
|
|
42
|
-
detectSleep:
|
|
40
|
+
enabled: D,
|
|
41
|
+
detectEmotions: x,
|
|
42
|
+
detectSleep: C
|
|
43
43
|
});
|
|
44
|
-
|
|
44
|
+
g(() => {
|
|
45
45
|
d && a("av_preview_video_analysis_error", {
|
|
46
46
|
error_name: d.name,
|
|
47
47
|
error_message: d.message
|
|
@@ -67,15 +67,15 @@ const W = {
|
|
|
67
67
|
},
|
|
68
68
|
[r, a]
|
|
69
69
|
);
|
|
70
|
-
return
|
|
71
|
-
|
|
72
|
-
audio:
|
|
73
|
-
video:
|
|
74
|
-
audioOutput:
|
|
70
|
+
return g(() => {
|
|
71
|
+
y({
|
|
72
|
+
audio: l,
|
|
73
|
+
video: c,
|
|
74
|
+
audioOutput: h
|
|
75
75
|
});
|
|
76
|
-
}, [
|
|
77
|
-
s.current &&
|
|
78
|
-
}, [
|
|
76
|
+
}, [l, c, h, y]), g(() => {
|
|
77
|
+
s.current && f && (s.current.srcObject = f);
|
|
78
|
+
}, [f]), /* @__PURE__ */ n(u, { $flexDirection: "row", $flexGapX: 2, children: [
|
|
79
79
|
/* @__PURE__ */ n(u, { $widthX: 26, $heightX: 19.5, $background: "BLACK_1", $position: "relative", children: [
|
|
80
80
|
/* @__PURE__ */ e(
|
|
81
81
|
"video",
|
|
@@ -89,64 +89,64 @@ const W = {
|
|
|
89
89
|
disablePictureInPicture: !0
|
|
90
90
|
}
|
|
91
91
|
),
|
|
92
|
-
|
|
92
|
+
/* @__PURE__ */ e(
|
|
93
93
|
R,
|
|
94
94
|
{
|
|
95
|
-
metrics:
|
|
95
|
+
metrics: S,
|
|
96
96
|
isLoading: X,
|
|
97
|
-
visible:
|
|
97
|
+
visible: D
|
|
98
98
|
}
|
|
99
99
|
)
|
|
100
100
|
] }),
|
|
101
101
|
/* @__PURE__ */ n(u, { $position: "relative", $widthX: 22, $flexGapX: 1, $justifyContent: "space-between", children: [
|
|
102
|
-
/* @__PURE__ */ e(u, { $flexGapX: 1, children: !
|
|
102
|
+
/* @__PURE__ */ e(u, { $flexGapX: 1, children: !_ && /* @__PURE__ */ n(G, { children: [
|
|
103
103
|
/* @__PURE__ */ e(
|
|
104
|
-
|
|
104
|
+
$,
|
|
105
105
|
{
|
|
106
106
|
label: "Camera",
|
|
107
107
|
renderAs: "primary",
|
|
108
108
|
shape: "borderLess",
|
|
109
|
-
options:
|
|
110
|
-
value:
|
|
109
|
+
options: p.video,
|
|
110
|
+
value: c,
|
|
111
111
|
onChange: k,
|
|
112
|
-
disabled:
|
|
112
|
+
disabled: P
|
|
113
113
|
}
|
|
114
114
|
),
|
|
115
115
|
/* @__PURE__ */ e(
|
|
116
|
-
|
|
116
|
+
$,
|
|
117
117
|
{
|
|
118
118
|
label: "Microphone",
|
|
119
119
|
renderAs: "primary",
|
|
120
120
|
shape: "borderLess",
|
|
121
|
-
options:
|
|
122
|
-
value:
|
|
121
|
+
options: p.audio,
|
|
122
|
+
value: l,
|
|
123
123
|
onChange: E,
|
|
124
|
-
disabled:
|
|
124
|
+
disabled: O
|
|
125
125
|
}
|
|
126
126
|
),
|
|
127
127
|
/* @__PURE__ */ e(
|
|
128
|
-
|
|
128
|
+
$,
|
|
129
129
|
{
|
|
130
130
|
label: "Speaker",
|
|
131
131
|
renderAs: "primary",
|
|
132
132
|
shape: "borderLess",
|
|
133
|
-
options:
|
|
134
|
-
value:
|
|
133
|
+
options: p.audioOutput,
|
|
134
|
+
value: h,
|
|
135
135
|
onChange: I
|
|
136
136
|
}
|
|
137
137
|
),
|
|
138
|
-
/* @__PURE__ */ e("div", { children:
|
|
138
|
+
/* @__PURE__ */ e("div", { children: V })
|
|
139
139
|
] }) }),
|
|
140
|
-
|
|
140
|
+
b && /* @__PURE__ */ n(
|
|
141
141
|
T,
|
|
142
142
|
{
|
|
143
143
|
$background: "ORANGE_2",
|
|
144
144
|
$gutterX: 1,
|
|
145
145
|
$gapX: 1,
|
|
146
|
-
$width:
|
|
146
|
+
$width: _ ? "100%" : "272px",
|
|
147
147
|
children: [
|
|
148
|
-
/* @__PURE__ */ e(w, { $renderAs: "ab1-bold", $marginBottomX: 1, children:
|
|
149
|
-
/* @__PURE__ */ e(F, { children:
|
|
148
|
+
/* @__PURE__ */ e(w, { $renderAs: "ab1-bold", $marginBottomX: 1, children: b.heading }),
|
|
149
|
+
/* @__PURE__ */ e(F, { children: b.steps.map((i, t) => /* @__PURE__ */ e("li", { children: /* @__PURE__ */ e(w, { $renderAs: "ub1", children: i }) }, t)) })
|
|
150
150
|
]
|
|
151
151
|
}
|
|
152
152
|
)
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"av-preview.js","sources":["../../../../src/features/av/av-preview/av-preview.tsx"],"sourcesContent":["import { memo, useCallback, useEffect, useRef, type FC } from 'react';\n\nimport { useUserMedia } from '@cuemath/av';\n\nimport useVideoAnalysis from '../video-analysis/hooks/use-video-analysis';\nimport VideoAnalysisOverlay from '../video-analysis/video-analysis-overlay/video-analysis-overlay';\nimport SelectInput from '../../ui/inputs/select-input/select-input';\nimport FlexView from '../../ui/layout/flex-view';\nimport Text from '../../ui/text/text';\nimport * as Styled from './av-preview-styled';\nimport type { ILogger } from './av-preview-types';\nimport useGetTroubleshootingInfo from './hooks/use-get-troubleshooting-steps';\n\ninterface IAVPreviewProps {\n children?: React.ReactNode;\n logger: ILogger;\n onDeviceUpdate: (selectedDevices: {\n audio?: string;\n video?: string;\n audioOutput?: string;\n }) => void;\n enableVideoAnalysis?: boolean;\n detectEmotions?: boolean;\n detectSleep?: boolean;\n}\nconst OPTIONS = {\n constraints: { audio: true, video: true },\n withDevices: true,\n setupListeners: true,\n};\n\nconst AVPreview: FC<IAVPreviewProps> = memo(\n ({\n children,\n logger,\n onDeviceUpdate,\n enableVideoAnalysis = false,\n detectEmotions = false,\n detectSleep = false,\n }) => {\n const videoRef = useRef<HTMLVideoElement>(null);\n const {\n selectedAudioDevice,\n selectedVideoDevice,\n videoDeviceError,\n devices,\n changeDevice,\n selectedAudioOutputDevice,\n audioDeviceError,\n userMediaStream,\n isAudioDeviceLoading,\n isVideoDeviceLoading,\n } = useUserMedia(OPTIONS);\n\n const hasPermissionProblem =\n videoDeviceError === 'permissionDeniedBySystem' ||\n videoDeviceError === 'permissionDenied' ||\n audioDeviceError === 'permissionDeniedBySystem' ||\n audioDeviceError === 'permissionDenied';\n const troubleshootingInfo = useGetTroubleshootingInfo(audioDeviceError, videoDeviceError);\n\n // Video analysis hook\n const {\n metrics,\n isLoading: isAnalysisLoading,\n error: analysisError,\n } = useVideoAnalysis(videoRef, {\n enabled: enableVideoAnalysis,\n detectEmotions,\n detectSleep,\n });\n\n // Log analysis errors\n useEffect(() => {\n if (analysisError) {\n logger('av_preview_video_analysis_error', {\n error_name: analysisError.name,\n error_message: analysisError.message,\n });\n }\n }, [analysisError, logger]);\n\n const handleAudioDeviceChange = useCallback(\n (deviceId: string) => changeDevice(deviceId, 'audio'),\n [changeDevice],\n );\n\n const handleVideoDeviceChange = useCallback(\n (deviceId: string) => changeDevice(deviceId, 'video'),\n [changeDevice],\n );\n\n const handleAudioOutputDeviceChange = useCallback(\n (deviceId: string) => {\n changeDevice(deviceId, 'audiooutput');\n const videoEl = videoRef.current;\n\n if (videoEl && 'setSinkId' in videoEl) {\n videoEl.setSinkId(deviceId).catch(error => {\n logger('av_preview_set_audio_output_device_error', {\n deviceId,\n error_name: error?.name,\n error_message: error?.message,\n });\n });\n }\n },\n [changeDevice, logger],\n );\n\n // Call onDeviceUpdate when devices or selected devices change\n useEffect(() => {\n onDeviceUpdate({\n audio: selectedAudioDevice,\n video: selectedVideoDevice,\n audioOutput: selectedAudioOutputDevice,\n });\n }, [selectedAudioDevice, selectedVideoDevice, selectedAudioOutputDevice, onDeviceUpdate]);\n\n useEffect(() => {\n if (videoRef.current && userMediaStream) {\n videoRef.current.srcObject = userMediaStream;\n }\n }, [userMediaStream]);\n\n return (\n <FlexView $flexDirection=\"row\" $flexGapX={2}>\n <FlexView $widthX={26} $heightX={19.5} $background=\"BLACK_1\" $position=\"relative\">\n <video\n ref={videoRef}\n id=\"localVideo\"\n autoPlay\n playsInline\n controls={false}\n muted={true}\n disablePictureInPicture\n />\n {enableVideoAnalysis && (\n <VideoAnalysisOverlay\n metrics={metrics}\n isLoading={isAnalysisLoading}\n visible={enableVideoAnalysis}\n />\n )}\n </FlexView>\n <FlexView $position=\"relative\" $widthX={22} $flexGapX={1} $justifyContent=\"space-between\">\n <FlexView $flexGapX={1}>\n {!hasPermissionProblem && (\n <>\n <SelectInput\n label=\"Camera\"\n renderAs=\"primary\"\n shape=\"borderLess\"\n options={devices.video}\n value={selectedVideoDevice}\n onChange={handleVideoDeviceChange}\n disabled={isVideoDeviceLoading}\n />\n <SelectInput\n label=\"Microphone\"\n renderAs=\"primary\"\n shape=\"borderLess\"\n options={devices.audio}\n value={selectedAudioDevice}\n onChange={handleAudioDeviceChange}\n disabled={isAudioDeviceLoading}\n />\n <SelectInput\n label=\"Speaker\"\n renderAs=\"primary\"\n shape=\"borderLess\"\n options={devices.audioOutput}\n value={selectedAudioOutputDevice}\n onChange={handleAudioOutputDeviceChange}\n />\n <div>{children}</div>\n </>\n )}\n </FlexView>\n {troubleshootingInfo && (\n <Styled.AVErrorSOPWrapper\n $background=\"ORANGE_2\"\n $gutterX={1}\n $gapX={1}\n $width={hasPermissionProblem ? '100%' : '272px'}\n >\n <Text $renderAs=\"ab1-bold\" $marginBottomX={1}>\n {troubleshootingInfo.heading}\n </Text>\n <Styled.AVErrorStepList>\n {troubleshootingInfo.steps.map((step, stepIndex) => (\n <li key={stepIndex}>\n <Text $renderAs=\"ub1\">{step}</Text>\n </li>\n ))}\n </Styled.AVErrorStepList>\n </Styled.AVErrorSOPWrapper>\n )}\n </FlexView>\n </FlexView>\n );\n },\n);\n\nAVPreview.displayName = 'AVPreview';\n\nexport default AVPreview;\n"],"names":["OPTIONS","AVPreview","memo","children","logger","onDeviceUpdate","enableVideoAnalysis","detectEmotions","detectSleep","videoRef","useRef","selectedAudioDevice","selectedVideoDevice","videoDeviceError","devices","changeDevice","selectedAudioOutputDevice","audioDeviceError","userMediaStream","isAudioDeviceLoading","isVideoDeviceLoading","useUserMedia","hasPermissionProblem","troubleshootingInfo","useGetTroubleshootingInfo","metrics","isAnalysisLoading","analysisError","useVideoAnalysis","useEffect","handleAudioDeviceChange","useCallback","deviceId","handleVideoDeviceChange","handleAudioOutputDeviceChange","videoEl","error","jsxs","FlexView","jsx","VideoAnalysisOverlay","Fragment","SelectInput","Styled.AVErrorSOPWrapper","Text","Styled.AVErrorStepList","step","stepIndex","AVPreview$1"],"mappings":";;;;;;;;;;AAyBA,MAAMA,IAAU;AAAA,EACd,aAAa,EAAE,OAAO,IAAM,OAAO,GAAK;AAAA,EACxC,aAAa;AAAA,EACb,gBAAgB;AAClB,GAEMC,IAAiCC;AAAA,EACrC,CAAC;AAAA,IACC,UAAAC;AAAA,IACA,QAAAC;AAAA,IACA,gBAAAC;AAAA,IACA,qBAAAC,IAAsB;AAAA,IACtB,gBAAAC,IAAiB;AAAA,IACjB,aAAAC,IAAc;AAAA,EAAA,MACV;AACE,UAAAC,IAAWC,EAAyB,IAAI,GACxC;AAAA,MACJ,qBAAAC;AAAA,MACA,qBAAAC;AAAA,MACA,kBAAAC;AAAA,MACA,SAAAC;AAAA,MACA,cAAAC;AAAA,MACA,2BAAAC;AAAA,MACA,kBAAAC;AAAA,MACA,iBAAAC;AAAA,MACA,sBAAAC;AAAA,MACA,sBAAAC;AAAA,IAAA,IACEC,EAAarB,CAAO,GAElBsB,IACJT,MAAqB,8BACrBA,MAAqB,sBACrBI,MAAqB,8BACrBA,MAAqB,oBACjBM,IAAsBC,EAA0BP,GAAkBJ,CAAgB,GAGlF;AAAA,MACJ,SAAAY;AAAA,MACA,WAAWC;AAAA,MACX,OAAOC;AAAA,IAAA,IACLC,EAAiBnB,GAAU;AAAA,MAC7B,SAASH;AAAA,MACT,gBAAAC;AAAA,MACA,aAAAC;AAAA,IAAA,CACD;AAGD,IAAAqB,EAAU,MAAM;AACd,MAAIF,KACFvB,EAAO,mCAAmC;AAAA,QACxC,YAAYuB,EAAc;AAAA,QAC1B,eAAeA,EAAc;AAAA,MAAA,CAC9B;AAAA,IACH,GACC,CAACA,GAAevB,CAAM,CAAC;AAE1B,UAAM0B,IAA0BC;AAAA,MAC9B,CAACC,MAAqBjB,EAAaiB,GAAU,OAAO;AAAA,MACpD,CAACjB,CAAY;AAAA,IAAA,GAGTkB,IAA0BF;AAAA,MAC9B,CAACC,MAAqBjB,EAAaiB,GAAU,OAAO;AAAA,MACpD,CAACjB,CAAY;AAAA,IAAA,GAGTmB,IAAgCH;AAAA,MACpC,CAACC,MAAqB;AACpB,QAAAjB,EAAaiB,GAAU,aAAa;AACpC,cAAMG,IAAU1B,EAAS;AAErB,QAAA0B,KAAW,eAAeA,KAC5BA,EAAQ,UAAUH,CAAQ,EAAE,MAAM,CAASI,MAAA;AACzC,UAAAhC,EAAO,4CAA4C;AAAA,YACjD,UAAA4B;AAAA,YACA,YAAYI,KAAA,gBAAAA,EAAO;AAAA,YACnB,eAAeA,KAAA,gBAAAA,EAAO;AAAA,UAAA,CACvB;AAAA,QAAA,CACF;AAAA,MAEL;AAAA,MACA,CAACrB,GAAcX,CAAM;AAAA,IAAA;AAIvB,WAAAyB,EAAU,MAAM;AACC,MAAAxB,EAAA;AAAA,QACb,OAAOM;AAAA,QACP,OAAOC;AAAA,QACP,aAAaI;AAAA,MAAA,CACd;AAAA,OACA,CAACL,GAAqBC,GAAqBI,GAA2BX,CAAc,CAAC,GAExFwB,EAAU,MAAM;AACV,MAAApB,EAAS,WAAWS,MACtBT,EAAS,QAAQ,YAAYS;AAAA,IAC/B,GACC,CAACA,CAAe,CAAC,GAGjB,gBAAAmB,EAAAC,GAAA,EAAS,gBAAe,OAAM,WAAW,GACxC,UAAA;AAAA,MAAC,gBAAAD,EAAAC,GAAA,EAAS,SAAS,IAAI,UAAU,MAAM,aAAY,WAAU,WAAU,YACrE,UAAA;AAAA,QAAA,gBAAAC;AAAA,UAAC;AAAA,UAAA;AAAA,YACC,KAAK9B;AAAA,YACL,IAAG;AAAA,YACH,UAAQ;AAAA,YACR,aAAW;AAAA,YACX,UAAU;AAAA,YACV,OAAO;AAAA,YACP,yBAAuB;AAAA,UAAA;AAAA,QACzB;AAAA,QACCH,KACC,gBAAAiC;AAAA,UAACC;AAAA,UAAA;AAAA,YACC,SAAAf;AAAA,YACA,WAAWC;AAAA,YACX,SAASpB;AAAA,UAAA;AAAA,QACX;AAAA,MAAA,GAEJ;AAAA,MACA,gBAAA+B,EAACC,KAAS,WAAU,YAAW,SAAS,IAAI,WAAW,GAAG,iBAAgB,iBACxE,UAAA;AAAA,QAAA,gBAAAC,EAACD,GAAS,EAAA,WAAW,GAClB,UAAA,CAAChB,KAEE,gBAAAe,EAAAI,GAAA,EAAA,UAAA;AAAA,UAAA,gBAAAF;AAAA,YAACG;AAAA,YAAA;AAAA,cACC,OAAM;AAAA,cACN,UAAS;AAAA,cACT,OAAM;AAAA,cACN,SAAS5B,EAAQ;AAAA,cACjB,OAAOF;AAAA,cACP,UAAUqB;AAAA,cACV,UAAUb;AAAA,YAAA;AAAA,UACZ;AAAA,UACA,gBAAAmB;AAAA,YAACG;AAAA,YAAA;AAAA,cACC,OAAM;AAAA,cACN,UAAS;AAAA,cACT,OAAM;AAAA,cACN,SAAS5B,EAAQ;AAAA,cACjB,OAAOH;AAAA,cACP,UAAUmB;AAAA,cACV,UAAUX;AAAA,YAAA;AAAA,UACZ;AAAA,UACA,gBAAAoB;AAAA,YAACG;AAAA,YAAA;AAAA,cACC,OAAM;AAAA,cACN,UAAS;AAAA,cACT,OAAM;AAAA,cACN,SAAS5B,EAAQ;AAAA,cACjB,OAAOE;AAAA,cACP,UAAUkB;AAAA,YAAA;AAAA,UACZ;AAAA,UACA,gBAAAK,EAAC,SAAK,UAAApC,GAAS;AAAA,QAAA,EAAA,CACjB,EAEJ,CAAA;AAAA,QACCoB,KACC,gBAAAc;AAAA,UAACM;AAAAA,UAAA;AAAA,YACC,aAAY;AAAA,YACZ,UAAU;AAAA,YACV,OAAO;AAAA,YACP,QAAQrB,IAAuB,SAAS;AAAA,YAExC,UAAA;AAAA,cAAA,gBAAAiB,EAACK,KAAK,WAAU,YAAW,gBAAgB,GACxC,YAAoB,SACvB;AAAA,cACA,gBAAAL,EAACM,GAAA,EACE,YAAoB,MAAM,IAAI,CAACC,GAAMC,wBACnC,MACC,EAAA,UAAA,gBAAAR,EAACK,KAAK,WAAU,OAAO,aAAK,EADrB,GAAAG,CAET,CACD,EACH,CAAA;AAAA,YAAA;AAAA,UAAA;AAAA,QACF;AAAA,MAAA,GAEJ;AAAA,IACF,EAAA,CAAA;AAAA,EAEJ;AACF;AAEA9C,EAAU,cAAc;AAExB,MAAA+C,KAAe/C;"}
|
|
1
|
+
{"version":3,"file":"av-preview.js","sources":["../../../../src/features/av/av-preview/av-preview.tsx"],"sourcesContent":["import { memo, useCallback, useEffect, useRef, type FC } from 'react';\n\nimport { useUserMedia } from '@cuemath/av';\n\nimport useVideoAnalysis from '../video-analysis/hooks/use-video-analysis';\nimport VideoAnalysisOverlay from '../video-analysis/video-analysis-overlay/video-analysis-overlay';\nimport SelectInput from '../../ui/inputs/select-input/select-input';\nimport FlexView from '../../ui/layout/flex-view';\nimport Text from '../../ui/text/text';\nimport * as Styled from './av-preview-styled';\nimport type { ILogger } from './av-preview-types';\nimport useGetTroubleshootingInfo from './hooks/use-get-troubleshooting-steps';\n\ninterface IAVPreviewProps {\n children?: React.ReactNode;\n logger: ILogger;\n onDeviceUpdate: (selectedDevices: {\n audio?: string;\n video?: string;\n audioOutput?: string;\n }) => void;\n enableVideoAnalysis?: boolean;\n detectEmotions?: boolean;\n detectSleep?: boolean;\n}\nconst OPTIONS = {\n constraints: { audio: true, video: true },\n withDevices: true,\n setupListeners: true,\n};\n\nconst AVPreview: FC<IAVPreviewProps> = memo(\n ({\n children,\n logger,\n onDeviceUpdate,\n enableVideoAnalysis = false,\n detectEmotions = false,\n detectSleep = false,\n }) => {\n const videoRef = useRef<HTMLVideoElement>(null);\n const {\n selectedAudioDevice,\n selectedVideoDevice,\n videoDeviceError,\n devices,\n changeDevice,\n selectedAudioOutputDevice,\n audioDeviceError,\n userMediaStream,\n isAudioDeviceLoading,\n isVideoDeviceLoading,\n } = useUserMedia(OPTIONS);\n\n const hasPermissionProblem =\n videoDeviceError === 'permissionDeniedBySystem' ||\n videoDeviceError === 'permissionDenied' ||\n audioDeviceError === 'permissionDeniedBySystem' ||\n audioDeviceError === 'permissionDenied';\n const troubleshootingInfo = useGetTroubleshootingInfo(audioDeviceError, videoDeviceError);\n\n // Video analysis hook\n const {\n metrics,\n isLoading: isAnalysisLoading,\n error: analysisError,\n } = useVideoAnalysis(videoRef, {\n enabled: enableVideoAnalysis,\n detectEmotions,\n detectSleep,\n });\n\n // Log analysis errors\n useEffect(() => {\n if (analysisError) {\n logger('av_preview_video_analysis_error', {\n error_name: analysisError.name,\n error_message: analysisError.message,\n });\n }\n }, [analysisError, logger]);\n\n const handleAudioDeviceChange = useCallback(\n (deviceId: string) => changeDevice(deviceId, 'audio'),\n [changeDevice],\n );\n\n const handleVideoDeviceChange = useCallback(\n (deviceId: string) => changeDevice(deviceId, 'video'),\n [changeDevice],\n );\n\n const handleAudioOutputDeviceChange = useCallback(\n (deviceId: string) => {\n changeDevice(deviceId, 'audiooutput');\n const videoEl = videoRef.current;\n\n if (videoEl && 'setSinkId' in videoEl) {\n videoEl.setSinkId(deviceId).catch(error => {\n logger('av_preview_set_audio_output_device_error', {\n deviceId,\n error_name: error?.name,\n error_message: error?.message,\n });\n });\n }\n },\n [changeDevice, logger],\n );\n\n // Call onDeviceUpdate when devices or selected devices change\n useEffect(() => {\n onDeviceUpdate({\n audio: selectedAudioDevice,\n video: selectedVideoDevice,\n audioOutput: selectedAudioOutputDevice,\n });\n }, [selectedAudioDevice, selectedVideoDevice, selectedAudioOutputDevice, onDeviceUpdate]);\n\n useEffect(() => {\n if (videoRef.current && userMediaStream) {\n videoRef.current.srcObject = userMediaStream;\n }\n }, [userMediaStream]);\n\n return (\n <FlexView $flexDirection=\"row\" $flexGapX={2}>\n <FlexView $widthX={26} $heightX={19.5} $background=\"BLACK_1\" $position=\"relative\">\n <video\n ref={videoRef}\n id=\"localVideo\"\n autoPlay\n playsInline\n controls={false}\n muted={true}\n disablePictureInPicture\n />\n <VideoAnalysisOverlay\n metrics={metrics}\n isLoading={isAnalysisLoading}\n visible={enableVideoAnalysis}\n />\n </FlexView>\n <FlexView $position=\"relative\" $widthX={22} $flexGapX={1} $justifyContent=\"space-between\">\n <FlexView $flexGapX={1}>\n {!hasPermissionProblem && (\n <>\n <SelectInput\n label=\"Camera\"\n renderAs=\"primary\"\n shape=\"borderLess\"\n options={devices.video}\n value={selectedVideoDevice}\n onChange={handleVideoDeviceChange}\n disabled={isVideoDeviceLoading}\n />\n <SelectInput\n label=\"Microphone\"\n renderAs=\"primary\"\n shape=\"borderLess\"\n options={devices.audio}\n value={selectedAudioDevice}\n onChange={handleAudioDeviceChange}\n disabled={isAudioDeviceLoading}\n />\n <SelectInput\n label=\"Speaker\"\n renderAs=\"primary\"\n shape=\"borderLess\"\n options={devices.audioOutput}\n value={selectedAudioOutputDevice}\n onChange={handleAudioOutputDeviceChange}\n />\n <div>{children}</div>\n </>\n )}\n </FlexView>\n {troubleshootingInfo && (\n <Styled.AVErrorSOPWrapper\n $background=\"ORANGE_2\"\n $gutterX={1}\n $gapX={1}\n $width={hasPermissionProblem ? '100%' : '272px'}\n >\n <Text $renderAs=\"ab1-bold\" $marginBottomX={1}>\n {troubleshootingInfo.heading}\n </Text>\n <Styled.AVErrorStepList>\n {troubleshootingInfo.steps.map((step, stepIndex) => (\n <li key={stepIndex}>\n <Text $renderAs=\"ub1\">{step}</Text>\n </li>\n ))}\n </Styled.AVErrorStepList>\n </Styled.AVErrorSOPWrapper>\n )}\n </FlexView>\n </FlexView>\n );\n },\n);\n\nAVPreview.displayName = 'AVPreview';\n\nexport default AVPreview;\n"],"names":["OPTIONS","AVPreview","memo","children","logger","onDeviceUpdate","enableVideoAnalysis","detectEmotions","detectSleep","videoRef","useRef","selectedAudioDevice","selectedVideoDevice","videoDeviceError","devices","changeDevice","selectedAudioOutputDevice","audioDeviceError","userMediaStream","isAudioDeviceLoading","isVideoDeviceLoading","useUserMedia","hasPermissionProblem","troubleshootingInfo","useGetTroubleshootingInfo","metrics","isAnalysisLoading","analysisError","useVideoAnalysis","useEffect","handleAudioDeviceChange","useCallback","deviceId","handleVideoDeviceChange","handleAudioOutputDeviceChange","videoEl","error","jsxs","FlexView","jsx","VideoAnalysisOverlay","Fragment","SelectInput","Styled.AVErrorSOPWrapper","Text","Styled.AVErrorStepList","step","stepIndex","AVPreview$1"],"mappings":";;;;;;;;;;AAyBA,MAAMA,IAAU;AAAA,EACd,aAAa,EAAE,OAAO,IAAM,OAAO,GAAK;AAAA,EACxC,aAAa;AAAA,EACb,gBAAgB;AAClB,GAEMC,IAAiCC;AAAA,EACrC,CAAC;AAAA,IACC,UAAAC;AAAA,IACA,QAAAC;AAAA,IACA,gBAAAC;AAAA,IACA,qBAAAC,IAAsB;AAAA,IACtB,gBAAAC,IAAiB;AAAA,IACjB,aAAAC,IAAc;AAAA,EAAA,MACV;AACE,UAAAC,IAAWC,EAAyB,IAAI,GACxC;AAAA,MACJ,qBAAAC;AAAA,MACA,qBAAAC;AAAA,MACA,kBAAAC;AAAA,MACA,SAAAC;AAAA,MACA,cAAAC;AAAA,MACA,2BAAAC;AAAA,MACA,kBAAAC;AAAA,MACA,iBAAAC;AAAA,MACA,sBAAAC;AAAA,MACA,sBAAAC;AAAA,IAAA,IACEC,EAAarB,CAAO,GAElBsB,IACJT,MAAqB,8BACrBA,MAAqB,sBACrBI,MAAqB,8BACrBA,MAAqB,oBACjBM,IAAsBC,EAA0BP,GAAkBJ,CAAgB,GAGlF;AAAA,MACJ,SAAAY;AAAA,MACA,WAAWC;AAAA,MACX,OAAOC;AAAA,IAAA,IACLC,EAAiBnB,GAAU;AAAA,MAC7B,SAASH;AAAA,MACT,gBAAAC;AAAA,MACA,aAAAC;AAAA,IAAA,CACD;AAGD,IAAAqB,EAAU,MAAM;AACd,MAAIF,KACFvB,EAAO,mCAAmC;AAAA,QACxC,YAAYuB,EAAc;AAAA,QAC1B,eAAeA,EAAc;AAAA,MAAA,CAC9B;AAAA,IACH,GACC,CAACA,GAAevB,CAAM,CAAC;AAE1B,UAAM0B,IAA0BC;AAAA,MAC9B,CAACC,MAAqBjB,EAAaiB,GAAU,OAAO;AAAA,MACpD,CAACjB,CAAY;AAAA,IAAA,GAGTkB,IAA0BF;AAAA,MAC9B,CAACC,MAAqBjB,EAAaiB,GAAU,OAAO;AAAA,MACpD,CAACjB,CAAY;AAAA,IAAA,GAGTmB,IAAgCH;AAAA,MACpC,CAACC,MAAqB;AACpB,QAAAjB,EAAaiB,GAAU,aAAa;AACpC,cAAMG,IAAU1B,EAAS;AAErB,QAAA0B,KAAW,eAAeA,KAC5BA,EAAQ,UAAUH,CAAQ,EAAE,MAAM,CAASI,MAAA;AACzC,UAAAhC,EAAO,4CAA4C;AAAA,YACjD,UAAA4B;AAAA,YACA,YAAYI,KAAA,gBAAAA,EAAO;AAAA,YACnB,eAAeA,KAAA,gBAAAA,EAAO;AAAA,UAAA,CACvB;AAAA,QAAA,CACF;AAAA,MAEL;AAAA,MACA,CAACrB,GAAcX,CAAM;AAAA,IAAA;AAIvB,WAAAyB,EAAU,MAAM;AACC,MAAAxB,EAAA;AAAA,QACb,OAAOM;AAAA,QACP,OAAOC;AAAA,QACP,aAAaI;AAAA,MAAA,CACd;AAAA,OACA,CAACL,GAAqBC,GAAqBI,GAA2BX,CAAc,CAAC,GAExFwB,EAAU,MAAM;AACV,MAAApB,EAAS,WAAWS,MACtBT,EAAS,QAAQ,YAAYS;AAAA,IAC/B,GACC,CAACA,CAAe,CAAC,GAGjB,gBAAAmB,EAAAC,GAAA,EAAS,gBAAe,OAAM,WAAW,GACxC,UAAA;AAAA,MAAC,gBAAAD,EAAAC,GAAA,EAAS,SAAS,IAAI,UAAU,MAAM,aAAY,WAAU,WAAU,YACrE,UAAA;AAAA,QAAA,gBAAAC;AAAA,UAAC;AAAA,UAAA;AAAA,YACC,KAAK9B;AAAA,YACL,IAAG;AAAA,YACH,UAAQ;AAAA,YACR,aAAW;AAAA,YACX,UAAU;AAAA,YACV,OAAO;AAAA,YACP,yBAAuB;AAAA,UAAA;AAAA,QACzB;AAAA,QACA,gBAAA8B;AAAA,UAACC;AAAA,UAAA;AAAA,YACC,SAAAf;AAAA,YACA,WAAWC;AAAA,YACX,SAASpB;AAAA,UAAA;AAAA,QACX;AAAA,MAAA,GACF;AAAA,MACA,gBAAA+B,EAACC,KAAS,WAAU,YAAW,SAAS,IAAI,WAAW,GAAG,iBAAgB,iBACxE,UAAA;AAAA,QAAA,gBAAAC,EAACD,GAAS,EAAA,WAAW,GAClB,UAAA,CAAChB,KAEE,gBAAAe,EAAAI,GAAA,EAAA,UAAA;AAAA,UAAA,gBAAAF;AAAA,YAACG;AAAA,YAAA;AAAA,cACC,OAAM;AAAA,cACN,UAAS;AAAA,cACT,OAAM;AAAA,cACN,SAAS5B,EAAQ;AAAA,cACjB,OAAOF;AAAA,cACP,UAAUqB;AAAA,cACV,UAAUb;AAAA,YAAA;AAAA,UACZ;AAAA,UACA,gBAAAmB;AAAA,YAACG;AAAA,YAAA;AAAA,cACC,OAAM;AAAA,cACN,UAAS;AAAA,cACT,OAAM;AAAA,cACN,SAAS5B,EAAQ;AAAA,cACjB,OAAOH;AAAA,cACP,UAAUmB;AAAA,cACV,UAAUX;AAAA,YAAA;AAAA,UACZ;AAAA,UACA,gBAAAoB;AAAA,YAACG;AAAA,YAAA;AAAA,cACC,OAAM;AAAA,cACN,UAAS;AAAA,cACT,OAAM;AAAA,cACN,SAAS5B,EAAQ;AAAA,cACjB,OAAOE;AAAA,cACP,UAAUkB;AAAA,YAAA;AAAA,UACZ;AAAA,UACA,gBAAAK,EAAC,SAAK,UAAApC,GAAS;AAAA,QAAA,EAAA,CACjB,EAEJ,CAAA;AAAA,QACCoB,KACC,gBAAAc;AAAA,UAACM;AAAAA,UAAA;AAAA,YACC,aAAY;AAAA,YACZ,UAAU;AAAA,YACV,OAAO;AAAA,YACP,QAAQrB,IAAuB,SAAS;AAAA,YAExC,UAAA;AAAA,cAAA,gBAAAiB,EAACK,KAAK,WAAU,YAAW,gBAAgB,GACxC,YAAoB,SACvB;AAAA,cACA,gBAAAL,EAACM,GAAA,EACE,YAAoB,MAAM,IAAI,CAACC,GAAMC,wBACnC,MACC,EAAA,UAAA,gBAAAR,EAACK,KAAK,WAAU,OAAO,aAAK,EADrB,GAAAG,CAET,CACD,EACH,CAAA;AAAA,YAAA;AAAA,UAAA;AAAA,QACF;AAAA,MAAA,GAEJ;AAAA,IACF,EAAA,CAAA;AAAA,EAEJ;AACF;AAEA9C,EAAU,cAAc;AAExB,MAAA+C,KAAe/C;"}
|
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import { useState as
|
|
2
|
-
import { getMediaPipeDetector as
|
|
3
|
-
import { calculateEAR as
|
|
4
|
-
import { getEmotionModel as
|
|
5
|
-
import { calculateVisibility as
|
|
6
|
-
const
|
|
7
|
-
const [q,
|
|
1
|
+
import { useState as b, useRef as u, useCallback as p, useEffect as N } from "react";
|
|
2
|
+
import { getMediaPipeDetector as J } from "../mediapipe/mediapipe-face-detector.js";
|
|
3
|
+
import { calculateEAR as K } from "../mediapipe/mediapipe-helpers.js";
|
|
4
|
+
import { getEmotionModel as Q } from "../tensorflow/emotion-model.js";
|
|
5
|
+
import { calculateVisibility as Z, analyzeLighting as $ } from "../video-analysis-helpers.js";
|
|
6
|
+
const B = 1e3, ee = 0.25, te = (T, s) => {
|
|
7
|
+
const [q, V] = b({
|
|
8
8
|
visibility: 0,
|
|
9
9
|
faceDetected: !1,
|
|
10
10
|
cameraAngle: "optimal",
|
|
@@ -14,128 +14,130 @@ const ee = 1e3, te = 0.25, ne = (h, n) => {
|
|
|
14
14
|
emotion: "neutral",
|
|
15
15
|
emotionConfidence: 0,
|
|
16
16
|
isAway: !1,
|
|
17
|
-
awayDurationMs: 0
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
17
|
+
awayDurationMs: 0,
|
|
18
|
+
sleepDurationMs: 0
|
|
19
|
+
}), [x, g] = b(!0), [G, H] = b(null), o = u(void 0), Y = u(0), I = u(0), A = u(null), E = u(null), P = 2e3, h = u(J()), L = u(Q()), v = p(
|
|
20
|
+
(e, c, d, t) => {
|
|
21
|
+
if (Math.abs(t.yaw) > 45)
|
|
22
|
+
return t.yaw > 0 ? "off-center-right" : "off-center-left";
|
|
23
|
+
if (t.pitch > 30)
|
|
23
24
|
return "off-center-low";
|
|
24
|
-
if (
|
|
25
|
+
if (t.pitch < -30)
|
|
25
26
|
return "off-center-high";
|
|
26
|
-
const
|
|
27
|
-
if (
|
|
27
|
+
const C = e.width * e.height, S = c * d, n = C / S, y = 0.4, _ = 0.5, w = 0.06, R = 0.04;
|
|
28
|
+
if (n > _)
|
|
28
29
|
return "too-close";
|
|
29
|
-
if (
|
|
30
|
+
if (n < R)
|
|
30
31
|
return "too-far";
|
|
31
|
-
const
|
|
32
|
-
return Math.abs(
|
|
32
|
+
const r = e.x + e.width / 2, i = e.y + e.height / 2, l = c / 2, M = d / 2, O = Math.abs(r - l) / c, F = Math.abs(i - M) / d, U = 0.2, X = 0.3, j = 0.25, W = 0.35;
|
|
33
|
+
return Math.abs(t.yaw) > 30 || O > X ? r > l || t.yaw > 0 ? "off-center-right" : "off-center-left" : t.pitch > 20 || F > W ? i > M || t.pitch > 0 ? "off-center-low" : "off-center-high" : n > y && n <= _ ? "too-close" : n < w && n >= R ? "too-far" : O > U && O <= X ? r > l ? "off-center-right" : "off-center-left" : F > j && F <= W ? i > M ? "off-center-low" : "off-center-high" : "optimal";
|
|
33
34
|
},
|
|
34
35
|
[]
|
|
35
|
-
),
|
|
36
|
-
if (!
|
|
36
|
+
), z = p(async () => {
|
|
37
|
+
if (!s.enabled || !T.current)
|
|
37
38
|
return;
|
|
38
|
-
const e =
|
|
39
|
+
const e = T.current, c = e instanceof HTMLCanvasElement;
|
|
39
40
|
if (e instanceof HTMLVideoElement && e.readyState !== e.HAVE_ENOUGH_DATA)
|
|
40
41
|
return;
|
|
41
|
-
const
|
|
42
|
-
if (!(
|
|
43
|
-
|
|
42
|
+
const t = Date.now();
|
|
43
|
+
if (!(t - Y.current < B)) {
|
|
44
|
+
Y.current = t;
|
|
44
45
|
try {
|
|
45
|
-
const
|
|
46
|
-
if (
|
|
46
|
+
const f = c ? e.width : e.videoWidth, m = c ? e.height : e.videoHeight;
|
|
47
|
+
if (f === 0 || m === 0)
|
|
47
48
|
return;
|
|
48
|
-
|
|
49
|
-
const
|
|
49
|
+
I.current = performance.now();
|
|
50
|
+
const a = await h.current.detect(
|
|
50
51
|
e,
|
|
51
|
-
|
|
52
|
+
I.current
|
|
52
53
|
);
|
|
53
|
-
if (!
|
|
54
|
-
let
|
|
55
|
-
if (
|
|
54
|
+
if (!a) {
|
|
55
|
+
let r = !1, i = 0;
|
|
56
|
+
if (s.awayDetectionEnabled) {
|
|
56
57
|
const l = Date.now();
|
|
57
|
-
|
|
58
|
-
const w = n.awayDurationThreshold || 6e4;
|
|
59
|
-
t >= w && (o = !0, !R.current && n.onUserAway && (R.current = !0, n.onUserAway(t)));
|
|
58
|
+
A.current === null && (A.current = l), i = l - A.current, r = !0;
|
|
60
59
|
}
|
|
61
|
-
|
|
60
|
+
V((l) => ({
|
|
62
61
|
...l,
|
|
63
62
|
faceDetected: !1,
|
|
64
63
|
visibility: 0,
|
|
65
64
|
cameraAngle: "optimal",
|
|
66
65
|
lighting: "good",
|
|
67
|
-
isAway:
|
|
68
|
-
awayDurationMs:
|
|
66
|
+
isAway: r,
|
|
67
|
+
awayDurationMs: i
|
|
69
68
|
}));
|
|
70
69
|
return;
|
|
71
70
|
}
|
|
72
|
-
const
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
),
|
|
78
|
-
let
|
|
79
|
-
|
|
80
|
-
let
|
|
81
|
-
if (
|
|
71
|
+
const k = Z(a.boundingBox, f, m), C = v(
|
|
72
|
+
a.boundingBox,
|
|
73
|
+
f,
|
|
74
|
+
m,
|
|
75
|
+
a.headPose
|
|
76
|
+
), S = $(e, a.boundingBox);
|
|
77
|
+
let n = !0, y = 0;
|
|
78
|
+
s.detectSleep && (y = K(a.landmarks), y < ee ? (E.current === null && (E.current = Date.now()), n = Date.now() - E.current < P) : (E.current = null, n = !0));
|
|
79
|
+
let _ = "neutral", w = 0;
|
|
80
|
+
if (s.detectEmotions)
|
|
82
81
|
try {
|
|
83
|
-
const
|
|
82
|
+
const r = a.blendshapes, i = await L.current.predict(
|
|
84
83
|
e,
|
|
85
|
-
|
|
86
|
-
|
|
84
|
+
a.boundingBox,
|
|
85
|
+
r
|
|
87
86
|
);
|
|
88
|
-
|
|
89
|
-
} catch (
|
|
90
|
-
console.warn("[VideoAnalysisV2] Emotion prediction failed:",
|
|
87
|
+
_ = i.emotion, w = i.confidence;
|
|
88
|
+
} catch (r) {
|
|
89
|
+
console.warn("[VideoAnalysisV2] Emotion prediction failed:", r);
|
|
91
90
|
}
|
|
92
|
-
|
|
93
|
-
|
|
91
|
+
s.awayDetectionEnabled && A.current !== null && (A.current = null);
|
|
92
|
+
let R = 0;
|
|
93
|
+
!n && E.current !== null && (R = Date.now() - E.current), V({
|
|
94
|
+
visibility: k,
|
|
94
95
|
faceDetected: !0,
|
|
95
|
-
cameraAngle:
|
|
96
|
-
lighting:
|
|
97
|
-
isAwake:
|
|
98
|
-
eyeAspectRatio:
|
|
99
|
-
emotion:
|
|
100
|
-
emotionConfidence:
|
|
96
|
+
cameraAngle: C,
|
|
97
|
+
lighting: S,
|
|
98
|
+
isAwake: n,
|
|
99
|
+
eyeAspectRatio: y,
|
|
100
|
+
emotion: _,
|
|
101
|
+
emotionConfidence: w,
|
|
101
102
|
isAway: !1,
|
|
102
|
-
awayDurationMs: 0
|
|
103
|
+
awayDurationMs: 0,
|
|
104
|
+
sleepDurationMs: R
|
|
103
105
|
});
|
|
104
|
-
} catch (
|
|
105
|
-
const
|
|
106
|
-
|
|
106
|
+
} catch (f) {
|
|
107
|
+
const m = f instanceof Error ? f : new Error("Face analysis failed");
|
|
108
|
+
H(m);
|
|
107
109
|
}
|
|
108
110
|
}
|
|
109
|
-
}, [
|
|
110
|
-
|
|
111
|
-
}, [
|
|
112
|
-
return
|
|
111
|
+
}, [s, T, v]), D = p(() => {
|
|
112
|
+
z(), o.current = requestAnimationFrame(D);
|
|
113
|
+
}, [z]);
|
|
114
|
+
return N(() => {
|
|
113
115
|
(async () => {
|
|
114
116
|
try {
|
|
115
|
-
await
|
|
116
|
-
} catch (
|
|
117
|
-
const
|
|
118
|
-
|
|
117
|
+
await h.current.initialize(), await L.current.initialize();
|
|
118
|
+
} catch (c) {
|
|
119
|
+
const d = c instanceof Error ? c : new Error("Failed to initialize models");
|
|
120
|
+
H(d);
|
|
119
121
|
}
|
|
120
122
|
})();
|
|
121
|
-
}, []),
|
|
122
|
-
if (!
|
|
123
|
-
g(!1),
|
|
123
|
+
}, []), N(() => {
|
|
124
|
+
if (!s.enabled) {
|
|
125
|
+
g(!1), o.current && (cancelAnimationFrame(o.current), o.current = void 0);
|
|
124
126
|
return;
|
|
125
127
|
}
|
|
126
128
|
const e = () => {
|
|
127
|
-
|
|
129
|
+
h.current.isReady() ? (g(!1), o.current || (o.current = requestAnimationFrame(D))) : (g(!0), setTimeout(e, 100));
|
|
128
130
|
};
|
|
129
131
|
return e(), () => {
|
|
130
|
-
|
|
132
|
+
o.current && (cancelAnimationFrame(o.current), o.current = void 0);
|
|
131
133
|
};
|
|
132
|
-
}, [
|
|
134
|
+
}, [s.enabled, D]), {
|
|
133
135
|
metrics: q,
|
|
134
136
|
isLoading: x,
|
|
135
137
|
error: G
|
|
136
138
|
};
|
|
137
|
-
},
|
|
139
|
+
}, ae = te;
|
|
138
140
|
export {
|
|
139
|
-
|
|
141
|
+
ae as default
|
|
140
142
|
};
|
|
141
143
|
//# sourceMappingURL=use-video-analysis.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"use-video-analysis.js","sources":["../../../../../src/features/av/video-analysis/hooks/use-video-analysis.ts"],"sourcesContent":["import { useCallback, useEffect, useRef, useState, type RefObject } from 'react';\n\nimport { getMediaPipeDetector } from '../mediapipe/mediapipe-face-detector';\nimport { calculateEAR } from '../mediapipe/mediapipe-helpers';\nimport { getEmotionModel } from '../tensorflow/emotion-model';\nimport type { TSimplifiedEmotion } from '../tensorflow/emotion-types';\nimport { analyzeLighting, calculateVisibility } from '../video-analysis-helpers';\n\n// Types\nexport type TCameraAngle =\n | 'optimal'\n | 'too-close'\n | 'too-far'\n | 'off-center-left'\n | 'off-center-right'\n | 'off-center-high'\n | 'off-center-low';\n\nexport type TLightingQuality = 'dark' | 'bright' | 'good';\n\nexport type TEmotion =\n | 'positive' // Student doing well (happy, surprised)\n | 'negative' // Student needs help (sad, angry, fearful, disgusted)\n | 'neutral'; // Baseline state\n\nexport interface IVideoAnalysisMetrics {\n visibility: number;\n faceDetected: boolean;\n cameraAngle: TCameraAngle;\n lighting: TLightingQuality;\n isAwake: boolean;\n eyeAspectRatio: number;\n emotion: TEmotion;\n emotionConfidence: number;\n // Away detection\n isAway: boolean;\n awayDurationMs: number;\n}\n\nexport interface IVideoAnalysisOptions {\n enabled: boolean;\n detectEmotions?: boolean;\n detectSleep?: boolean;\n // For canvas-based analysis (when analyzing cropped regions)\n useCanvas?: boolean;\n // Away detection options\n awayDetectionEnabled?: boolean;\n awayDurationThreshold?: number;\n onUserAway?: (awayDurationMs: number) => void;\n}\n\nexport interface IUseVideoAnalysis {\n (\n videoRef: React.RefObject<HTMLVideoElement | HTMLCanvasElement | null>,\n options: IVideoAnalysisOptions,\n ): {\n metrics: IVideoAnalysisMetrics;\n isLoading: boolean;\n error: Error | null;\n };\n}\n\n// Analysis runs at 1 FPS (every 1000ms / 1 second)\nconst ANALYSIS_INTERVAL_MS = 1000;\n\n// EAR threshold for sleep detection in online classes\n// Industry standard: 0.25 (balances sensitivity vs false positives)\n// Research: ~0.3 fully open, 0.2-0.25 drowsy threshold, <0.2 clearly closed\nconst EAR_THRESHOLD = 0.25; // Below 0.25 = eyes closing/drowsy\n\n/**\n * Custom hook for real-time video analysis using MediaPipe + TensorFlow.js\n *\n * @param videoRef - Reference to video or canvas element to analyze\n * @param options - Analysis options (enabled, detectEmotions, detectSleep, etc.)\n * @returns Analysis metrics, loading state, and error\n */\nconst useVideoAnalysisV2 = (\n videoRef: RefObject<HTMLVideoElement | HTMLCanvasElement | null>,\n options: IVideoAnalysisOptions,\n): {\n metrics: IVideoAnalysisMetrics;\n isLoading: boolean;\n error: Error | null;\n} => {\n // State\n const [metrics, setMetrics] = useState<IVideoAnalysisMetrics>({\n visibility: 0,\n faceDetected: false,\n cameraAngle: 'optimal',\n lighting: 'good',\n isAwake: true,\n eyeAspectRatio: 0,\n emotion: 'neutral',\n emotionConfidence: 0,\n isAway: false,\n awayDurationMs: 0,\n });\n const [isLoading, setIsLoading] = useState(true);\n const [error, setError] = useState<Error | null>(null);\n\n // Refs for analysis loop\n const animationFrameRef = useRef<number | undefined>(undefined);\n const lastAnalysisTimeRef = useRef<number>(0);\n const videoTimestampRef = useRef<number>(0);\n\n // Away detection state\n const awayStartTimeRef = useRef<number | null>(null);\n const awayCallbackTriggeredRef = useRef<boolean>(false);\n\n // Sleep detection state - track eyes closed duration\n const eyesClosedStartTimeRef = useRef<number | null>(null);\n const EYES_CLOSED_THRESHOLD_MS = 2000; // 2 seconds to avoid detecting blinks\n\n // Model instances\n const mediaPipeDetectorRef = useRef(getMediaPipeDetector());\n const emotionModelRef = useRef(getEmotionModel());\n\n /**\n * Analyze camera angle from head pose and face position\n * Enhanced with 3D pose estimation and better thresholds\n */\n const analyzeCameraAngle = useCallback(\n (\n faceBox: { x: number; y: number; width: number; height: number },\n videoWidth: number,\n videoHeight: number,\n headPose: { pitch: number; yaw: number; roll: number },\n ): TCameraAngle => {\n // Priority 1: Head pose analysis (most reliable indicator)\n // Using MediaPipe 3D pose for accurate angle detection\n\n // Yaw thresholds (left/right turn)\n // YAW_SLIGHT = 15° - Slight turn, still acceptable (reserved for future use)\n const YAW_MODERATE = 30; // Noticeable turn, warn user\n const YAW_SEVERE = 45; // Looking away significantly\n\n // Pitch thresholds (up/down tilt)\n // PITCH_SLIGHT = 10° - Slight tilt, still acceptable (reserved for future use)\n const PITCH_MODERATE = 20; // Noticeable tilt, warn user\n const PITCH_SEVERE = 30; // Looking down/up significantly\n\n // Check severe head pose issues first (user looking away)\n if (Math.abs(headPose.yaw) > YAW_SEVERE) {\n return headPose.yaw > 0 ? 'off-center-right' : 'off-center-left';\n }\n\n if (headPose.pitch > PITCH_SEVERE) {\n return 'off-center-low'; // Looking down (at desk/phone)\n }\n\n if (headPose.pitch < -PITCH_SEVERE) {\n return 'off-center-high'; // Looking up (at ceiling/daydreaming)\n }\n\n // Priority 2: Face size analysis (distance from camera)\n const faceArea = faceBox.width * faceBox.height;\n const videoArea = videoWidth * videoHeight;\n const faceRatio = faceArea / videoArea;\n\n // Optimal face ratio: 0.08 - 0.35 (8% - 35% of screen)\n const FACE_RATIO_TOO_CLOSE = 0.4; // More than 40% of screen\n const FACE_RATIO_VERY_CLOSE = 0.5; // More than 50% of screen\n const FACE_RATIO_TOO_FAR = 0.06; // Less than 6% of screen\n const FACE_RATIO_VERY_FAR = 0.04; // Less than 4% of screen\n\n if (faceRatio > FACE_RATIO_VERY_CLOSE) {\n return 'too-close';\n }\n\n if (faceRatio < FACE_RATIO_VERY_FAR) {\n return 'too-far';\n }\n\n // Priority 3: Face position analysis (centering)\n const faceCenterX = faceBox.x + faceBox.width / 2;\n const faceCenterY = faceBox.y + faceBox.height / 2;\n const videoCenterX = videoWidth / 2;\n const videoCenterY = videoHeight / 2;\n\n const offsetX = Math.abs(faceCenterX - videoCenterX) / videoWidth;\n const offsetY = Math.abs(faceCenterY - videoCenterY) / videoHeight;\n\n // Horizontal offset thresholds\n const OFFSET_X_MODERATE = 0.2; // 20% offset, still acceptable\n const OFFSET_X_SEVERE = 0.3; // 30% offset, warn user\n\n // Vertical offset thresholds (slightly more lenient)\n const OFFSET_Y_MODERATE = 0.25; // 25% offset, still acceptable\n const OFFSET_Y_SEVERE = 0.35; // 35% offset, warn user\n\n // Check moderate head pose issues (combined with position)\n if (Math.abs(headPose.yaw) > YAW_MODERATE || offsetX > OFFSET_X_SEVERE) {\n return faceCenterX > videoCenterX || headPose.yaw > 0\n ? 'off-center-right'\n : 'off-center-left';\n }\n\n if (headPose.pitch > PITCH_MODERATE || offsetY > OFFSET_Y_SEVERE) {\n return faceCenterY > videoCenterY || headPose.pitch > 0\n ? 'off-center-low'\n : 'off-center-high';\n }\n\n // Minor warnings for slightly suboptimal positioning\n if (faceRatio > FACE_RATIO_TOO_CLOSE && faceRatio <= FACE_RATIO_VERY_CLOSE) {\n return 'too-close';\n }\n\n if (faceRatio < FACE_RATIO_TOO_FAR && faceRatio >= FACE_RATIO_VERY_FAR) {\n return 'too-far';\n }\n\n if (offsetX > OFFSET_X_MODERATE && offsetX <= OFFSET_X_SEVERE) {\n return faceCenterX > videoCenterX ? 'off-center-right' : 'off-center-left';\n }\n\n if (offsetY > OFFSET_Y_MODERATE && offsetY <= OFFSET_Y_SEVERE) {\n return faceCenterY > videoCenterY ? 'off-center-low' : 'off-center-high';\n }\n\n // All checks passed - optimal positioning\n return 'optimal';\n },\n [],\n );\n\n /**\n * Main analysis function - runs on each frame\n */\n const analyzeFrame = useCallback(async () => {\n if (!options.enabled || !videoRef.current) {\n return;\n }\n\n const element = videoRef.current;\n const isCanvas = element instanceof HTMLCanvasElement;\n const isVideo = element instanceof HTMLVideoElement;\n\n // Check if video is ready (skip check for canvas)\n if (isVideo && element.readyState !== element.HAVE_ENOUGH_DATA) {\n return;\n }\n\n // Throttle analysis to ANALYSIS_INTERVAL_MS\n const now = Date.now();\n\n if (now - lastAnalysisTimeRef.current < ANALYSIS_INTERVAL_MS) {\n return;\n }\n\n lastAnalysisTimeRef.current = now;\n\n try {\n // Get video dimensions\n const videoWidth = isCanvas ? element.width : element.videoWidth;\n const videoHeight = isCanvas ? element.height : element.videoHeight;\n\n if (videoWidth === 0 || videoHeight === 0) {\n return;\n }\n\n // Update video timestamp for MediaPipe tracking\n videoTimestampRef.current = performance.now();\n\n // Detect face with MediaPipe\n const detection = await mediaPipeDetectorRef.current.detect(\n element,\n videoTimestampRef.current,\n );\n\n if (!detection) {\n // No face detected - handle away detection\n let isAway = false;\n let awayDurationMs = 0;\n\n if (options.awayDetectionEnabled) {\n const currentTime = Date.now();\n\n if (awayStartTimeRef.current === null) {\n awayStartTimeRef.current = currentTime;\n awayCallbackTriggeredRef.current = false;\n }\n\n awayDurationMs = currentTime - awayStartTimeRef.current;\n const threshold = options.awayDurationThreshold || 60000;\n\n if (awayDurationMs >= threshold) {\n isAway = true;\n\n if (!awayCallbackTriggeredRef.current && options.onUserAway) {\n awayCallbackTriggeredRef.current = true;\n options.onUserAway(awayDurationMs);\n }\n }\n }\n\n setMetrics((prev: IVideoAnalysisMetrics) => ({\n ...prev,\n faceDetected: false,\n visibility: 0,\n cameraAngle: 'optimal',\n lighting: 'good',\n isAway,\n awayDurationMs,\n }));\n\n return;\n }\n\n // Face detected!\n // Calculate visibility\n const visibility = calculateVisibility(detection.boundingBox, videoWidth, videoHeight);\n\n // Analyze camera angle with 3D head pose\n const cameraAngle = analyzeCameraAngle(\n detection.boundingBox,\n videoWidth,\n videoHeight,\n detection.headPose,\n );\n\n // Analyze lighting\n const lighting = analyzeLighting(element, detection.boundingBox);\n\n // Calculate EAR for sleep detection\n let isAwake = true;\n let eyeAspectRatio = 0;\n\n if (options.detectSleep) {\n eyeAspectRatio = calculateEAR(detection.landmarks);\n const eyesClosed = eyeAspectRatio < EAR_THRESHOLD;\n\n if (eyesClosed) {\n // Eyes are closed - start or continue timer\n if (eyesClosedStartTimeRef.current === null) {\n eyesClosedStartTimeRef.current = Date.now();\n }\n\n const eyesClosedDuration = Date.now() - eyesClosedStartTimeRef.current;\n\n // Only mark as drowsy if eyes closed for more than threshold (not just blink)\n isAwake = eyesClosedDuration < EYES_CLOSED_THRESHOLD_MS;\n } else {\n // Eyes are open - reset timer\n eyesClosedStartTimeRef.current = null;\n isAwake = true;\n }\n }\n\n // Detect emotion with TensorFlow.js\n let emotion: TSimplifiedEmotion = 'neutral';\n let emotionConfidence = 0;\n\n if (options.detectEmotions) {\n try {\n // eslint-disable-next-line @typescript-eslint/no-explicit-any\n const blendshapes = (detection as any).blendshapes as\n | Array<{ categoryName: string; score: number }>\n | undefined;\n const prediction = await emotionModelRef.current.predict(\n element,\n detection.boundingBox,\n blendshapes,\n );\n\n emotion = prediction.emotion;\n emotionConfidence = prediction.confidence;\n } catch (err) {\n // eslint-disable-next-line no-console\n console.warn('[VideoAnalysisV2] Emotion prediction failed:', err);\n }\n }\n\n // Reset away detection when face is detected\n if (options.awayDetectionEnabled && awayStartTimeRef.current !== null) {\n awayStartTimeRef.current = null;\n awayCallbackTriggeredRef.current = false;\n }\n\n // Update metrics\n setMetrics({\n visibility,\n faceDetected: true,\n cameraAngle,\n lighting,\n isAwake,\n eyeAspectRatio,\n emotion,\n emotionConfidence,\n isAway: false,\n awayDurationMs: 0,\n });\n } catch (err) {\n const analysisError = err instanceof Error ? err : new Error('Face analysis failed');\n\n setError(analysisError);\n }\n }, [options, videoRef, analyzeCameraAngle]);\n\n /**\n * Animation loop\n */\n const animate = useCallback(() => {\n analyzeFrame();\n animationFrameRef.current = requestAnimationFrame(animate);\n }, [analyzeFrame]);\n\n /**\n * Initialize models (runs once on mount)\n */\n useEffect(() => {\n const initializeModels = async () => {\n try {\n // Initialize MediaPipe face detector\n await mediaPipeDetectorRef.current.initialize();\n\n // Initialize TensorFlow.js emotion model\n await emotionModelRef.current.initialize();\n } catch (err) {\n const loadError = err instanceof Error ? err : new Error('Failed to initialize models');\n\n setError(loadError);\n }\n };\n\n initializeModels();\n }, []); // Run once on mount\n\n /**\n * Start/stop analysis loop based on options\n */\n useEffect(() => {\n if (!options.enabled) {\n setIsLoading(false);\n\n // Stop animation if running\n if (animationFrameRef.current) {\n cancelAnimationFrame(animationFrameRef.current);\n animationFrameRef.current = undefined;\n }\n\n return;\n }\n\n // Check if models are ready\n const checkAndStart = () => {\n if (mediaPipeDetectorRef.current.isReady()) {\n setIsLoading(false);\n\n // Start analysis loop\n if (!animationFrameRef.current) {\n animationFrameRef.current = requestAnimationFrame(animate);\n }\n } else {\n setIsLoading(true);\n // Retry after a short delay\n setTimeout(checkAndStart, 100);\n }\n };\n\n checkAndStart();\n\n // Cleanup on unmount or when disabled\n return () => {\n if (animationFrameRef.current) {\n cancelAnimationFrame(animationFrameRef.current);\n animationFrameRef.current = undefined;\n }\n };\n }, [options.enabled, animate]);\n\n return {\n metrics,\n isLoading,\n error,\n };\n};\n\nexport default useVideoAnalysisV2;\n"],"names":["ANALYSIS_INTERVAL_MS","EAR_THRESHOLD","useVideoAnalysisV2","videoRef","options","metrics","setMetrics","useState","isLoading","setIsLoading","error","setError","animationFrameRef","useRef","lastAnalysisTimeRef","videoTimestampRef","awayStartTimeRef","awayCallbackTriggeredRef","eyesClosedStartTimeRef","EYES_CLOSED_THRESHOLD_MS","mediaPipeDetectorRef","getMediaPipeDetector","emotionModelRef","getEmotionModel","analyzeCameraAngle","useCallback","faceBox","videoWidth","videoHeight","headPose","faceArea","videoArea","faceRatio","FACE_RATIO_TOO_CLOSE","FACE_RATIO_VERY_CLOSE","FACE_RATIO_TOO_FAR","FACE_RATIO_VERY_FAR","faceCenterX","faceCenterY","videoCenterX","videoCenterY","offsetX","offsetY","OFFSET_X_MODERATE","OFFSET_X_SEVERE","OFFSET_Y_MODERATE","OFFSET_Y_SEVERE","analyzeFrame","element","isCanvas","now","detection","isAway","awayDurationMs","currentTime","threshold","prev","visibility","calculateVisibility","cameraAngle","lighting","analyzeLighting","isAwake","eyeAspectRatio","calculateEAR","emotion","emotionConfidence","blendshapes","prediction","err","analysisError","animate","useEffect","loadError","checkAndStart","useVideoAnalysis"],"mappings":";;;;;AA+DA,MAAMA,KAAuB,KAKvBC,KAAgB,MAShBC,KAAqB,CACzBC,GACAC,MAKG;AAEH,QAAM,CAACC,GAASC,CAAU,IAAIC,EAAgC;AAAA,IAC5D,YAAY;AAAA,IACZ,cAAc;AAAA,IACd,aAAa;AAAA,IACb,UAAU;AAAA,IACV,SAAS;AAAA,IACT,gBAAgB;AAAA,IAChB,SAAS;AAAA,IACT,mBAAmB;AAAA,IACnB,QAAQ;AAAA,IACR,gBAAgB;AAAA,EAAA,CACjB,GACK,CAACC,GAAWC,CAAY,IAAIF,EAAS,EAAI,GACzC,CAACG,GAAOC,CAAQ,IAAIJ,EAAuB,IAAI,GAG/CK,IAAoBC,EAA2B,MAAS,GACxDC,IAAsBD,EAAe,CAAC,GACtCE,IAAoBF,EAAe,CAAC,GAGpCG,IAAmBH,EAAsB,IAAI,GAC7CI,IAA2BJ,EAAgB,EAAK,GAGhDK,IAAyBL,EAAsB,IAAI,GACnDM,IAA2B,KAG3BC,IAAuBP,EAAOQ,EAAA,CAAsB,GACpDC,IAAkBT,EAAOU,EAAA,CAAiB,GAM1CC,IAAqBC;AAAA,IACzB,CACEC,GACAC,GACAC,GACAC,MACiB;AAejB,UAAI,KAAK,IAAIA,EAAS,GAAG,IAAI;AACpB,eAAAA,EAAS,MAAM,IAAI,qBAAqB;AAG7C,UAAAA,EAAS,QAAQ;AACZ,eAAA;AAGL,UAAAA,EAAS,QAAQ;AACZ,eAAA;AAIH,YAAAC,IAAWJ,EAAQ,QAAQA,EAAQ,QACnCK,IAAYJ,IAAaC,GACzBI,IAAYF,IAAWC,GAGvBE,IAAuB,KACvBC,IAAwB,KACxBC,IAAqB,MACrBC,IAAsB;AAE5B,UAAIJ,IAAYE;AACP,eAAA;AAGT,UAAIF,IAAYI;AACP,eAAA;AAIT,YAAMC,IAAcX,EAAQ,IAAIA,EAAQ,QAAQ,GAC1CY,IAAcZ,EAAQ,IAAIA,EAAQ,SAAS,GAC3Ca,IAAeZ,IAAa,GAC5Ba,IAAeZ,IAAc,GAE7Ba,IAAU,KAAK,IAAIJ,IAAcE,CAAY,IAAIZ,GACjDe,IAAU,KAAK,IAAIJ,IAAcE,CAAY,IAAIZ,GAGjDe,IAAoB,KACpBC,IAAkB,KAGlBC,IAAoB,MACpBC,IAAkB;AAGxB,aAAI,KAAK,IAAIjB,EAAS,GAAG,IAAI,MAAgBY,IAAUG,IAC9CP,IAAcE,KAAgBV,EAAS,MAAM,IAChD,qBACA,oBAGFA,EAAS,QAAQ,MAAkBa,IAAUI,IACxCR,IAAcE,KAAgBX,EAAS,QAAQ,IAClD,mBACA,oBAIFG,IAAYC,KAAwBD,KAAaE,IAC5C,cAGLF,IAAYG,KAAsBH,KAAaI,IAC1C,YAGLK,IAAUE,KAAqBF,KAAWG,IACrCP,IAAcE,IAAe,qBAAqB,oBAGvDG,IAAUG,KAAqBH,KAAWI,IACrCR,IAAcE,IAAe,mBAAmB,oBAIlD;AAAA,IACT;AAAA,IACA,CAAC;AAAA,EAAA,GAMGO,IAAetB,EAAY,YAAY;AAC3C,QAAI,CAACrB,EAAQ,WAAW,CAACD,EAAS;AAChC;AAGF,UAAM6C,IAAU7C,EAAS,SACnB8C,IAAWD,aAAmB;AAIpC,QAHgBA,aAAmB,oBAGpBA,EAAQ,eAAeA,EAAQ;AAC5C;AAII,UAAAE,IAAM,KAAK;AAEb,QAAA,EAAAA,IAAMpC,EAAoB,UAAUd,KAIxC;AAAA,MAAAc,EAAoB,UAAUoC;AAE1B,UAAA;AAEF,cAAMvB,IAAasB,IAAWD,EAAQ,QAAQA,EAAQ,YAChDpB,IAAcqB,IAAWD,EAAQ,SAASA,EAAQ;AAEpD,YAAArB,MAAe,KAAKC,MAAgB;AACtC;AAIgB,QAAAb,EAAA,UAAU,YAAY;AAGlC,cAAAoC,IAAY,MAAM/B,EAAqB,QAAQ;AAAA,UACnD4B;AAAA,UACAjC,EAAkB;AAAA,QAAA;AAGpB,YAAI,CAACoC,GAAW;AAEd,cAAIC,IAAS,IACTC,IAAiB;AAErB,cAAIjD,EAAQ,sBAAsB;AAC1B,kBAAAkD,IAAc,KAAK;AAErB,YAAAtC,EAAiB,YAAY,SAC/BA,EAAiB,UAAUsC,GAC3BrC,EAAyB,UAAU,KAGrCoC,IAAiBC,IAActC,EAAiB;AAC1C,kBAAAuC,IAAYnD,EAAQ,yBAAyB;AAEnD,YAAIiD,KAAkBE,MACXH,IAAA,IAEL,CAACnC,EAAyB,WAAWb,EAAQ,eAC/Ca,EAAyB,UAAU,IACnCb,EAAQ,WAAWiD,CAAc;AAAA,UAGvC;AAEA,UAAA/C,EAAW,CAACkD,OAAiC;AAAA,YAC3C,GAAGA;AAAA,YACH,cAAc;AAAA,YACd,YAAY;AAAA,YACZ,aAAa;AAAA,YACb,UAAU;AAAA,YACV,QAAAJ;AAAA,YACA,gBAAAC;AAAA,UACA,EAAA;AAEF;AAAA,QACF;AAIA,cAAMI,IAAaC,EAAoBP,EAAU,aAAaxB,GAAYC,CAAW,GAG/E+B,IAAcnC;AAAA,UAClB2B,EAAU;AAAA,UACVxB;AAAA,UACAC;AAAA,UACAuB,EAAU;AAAA,QAAA,GAINS,IAAWC,EAAgBb,GAASG,EAAU,WAAW;AAG/D,YAAIW,IAAU,IACVC,IAAiB;AAErB,QAAI3D,EAAQ,gBACO2D,IAAAC,EAAab,EAAU,SAAS,GAC9BY,IAAiB9D,MAI9BiB,EAAuB,YAAY,SACdA,EAAA,UAAU,KAAK,QAMxC4C,IAH2B,KAAK,IAAI,IAAI5C,EAAuB,UAGhCC,MAG/BD,EAAuB,UAAU,MACvB4C,IAAA;AAKd,YAAIG,IAA8B,WAC9BC,IAAoB;AAExB,YAAI9D,EAAQ;AACN,cAAA;AAEF,kBAAM+D,IAAehB,EAAkB,aAGjCiB,IAAa,MAAM9C,EAAgB,QAAQ;AAAA,cAC/C0B;AAAA,cACAG,EAAU;AAAA,cACVgB;AAAA,YAAA;AAGF,YAAAF,IAAUG,EAAW,SACrBF,IAAoBE,EAAW;AAAA,mBACxBC,GAAK;AAEJ,oBAAA,KAAK,gDAAgDA,CAAG;AAAA,UAClE;AAIF,QAAIjE,EAAQ,wBAAwBY,EAAiB,YAAY,SAC/DA,EAAiB,UAAU,MAC3BC,EAAyB,UAAU,KAI1BX,EAAA;AAAA,UACT,YAAAmD;AAAA,UACA,cAAc;AAAA,UACd,aAAAE;AAAA,UACA,UAAAC;AAAA,UACA,SAAAE;AAAA,UACA,gBAAAC;AAAA,UACA,SAAAE;AAAA,UACA,mBAAAC;AAAA,UACA,QAAQ;AAAA,UACR,gBAAgB;AAAA,QAAA,CACjB;AAAA,eACMG,GAAK;AACZ,cAAMC,IAAgBD,aAAe,QAAQA,IAAM,IAAI,MAAM,sBAAsB;AAEnF,QAAA1D,EAAS2D,CAAa;AAAA,MACxB;AAAA;AAAA,EACC,GAAA,CAAClE,GAASD,GAAUqB,CAAkB,CAAC,GAKpC+C,IAAU9C,EAAY,MAAM;AACnB,IAAAsB,KACKnC,EAAA,UAAU,sBAAsB2D,CAAO;AAAA,EAAA,GACxD,CAACxB,CAAY,CAAC;AAKjB,SAAAyB,EAAU,MAAM;AAeG,KAdQ,YAAY;AAC/B,UAAA;AAEI,cAAApD,EAAqB,QAAQ,cAG7B,MAAAE,EAAgB,QAAQ;eACvB+C,GAAK;AACZ,cAAMI,IAAYJ,aAAe,QAAQA,IAAM,IAAI,MAAM,6BAA6B;AAEtF,QAAA1D,EAAS8D,CAAS;AAAA,MACpB;AAAA,IAAA;EAIJ,GAAG,CAAE,CAAA,GAKLD,EAAU,MAAM;AACV,QAAA,CAACpE,EAAQ,SAAS;AACpB,MAAAK,EAAa,EAAK,GAGdG,EAAkB,YACpB,qBAAqBA,EAAkB,OAAO,GAC9CA,EAAkB,UAAU;AAG9B;AAAA,IACF;AAGA,UAAM8D,IAAgB,MAAM;AACtB,MAAAtD,EAAqB,QAAQ,aAC/BX,EAAa,EAAK,GAGbG,EAAkB,YACHA,EAAA,UAAU,sBAAsB2D,CAAO,OAG3D9D,EAAa,EAAI,GAEjB,WAAWiE,GAAe,GAAG;AAAA,IAC/B;AAGY,WAAAA,KAGP,MAAM;AACX,MAAI9D,EAAkB,YACpB,qBAAqBA,EAAkB,OAAO,GAC9CA,EAAkB,UAAU;AAAA,IAC9B;AAAA,EAED,GAAA,CAACR,EAAQ,SAASmE,CAAO,CAAC,GAEtB;AAAA,IACL,SAAAlE;AAAA,IACA,WAAAG;AAAA,IACA,OAAAE;AAAA,EAAA;AAEJ,GAEAiE,KAAezE;"}
|
|
1
|
+
{"version":3,"file":"use-video-analysis.js","sources":["../../../../../src/features/av/video-analysis/hooks/use-video-analysis.ts"],"sourcesContent":["import { useCallback, useEffect, useRef, useState, type RefObject } from 'react';\n\nimport { getMediaPipeDetector } from '../mediapipe/mediapipe-face-detector';\nimport { calculateEAR } from '../mediapipe/mediapipe-helpers';\nimport { getEmotionModel } from '../tensorflow/emotion-model';\nimport type { TSimplifiedEmotion } from '../tensorflow/emotion-types';\nimport { analyzeLighting, calculateVisibility } from '../video-analysis-helpers';\n\n// Types\nexport type TCameraAngle =\n | 'optimal'\n | 'too-close'\n | 'too-far'\n | 'off-center-left'\n | 'off-center-right'\n | 'off-center-high'\n | 'off-center-low';\n\nexport type TLightingQuality = 'dark' | 'bright' | 'good';\n\nexport type TEmotion =\n | 'positive' // Student doing well (happy, surprised)\n | 'negative' // Student needs help (sad, angry, fearful, disgusted)\n | 'neutral'; // Baseline state\n\nexport interface IVideoAnalysisMetrics {\n visibility: number;\n faceDetected: boolean;\n cameraAngle: TCameraAngle;\n lighting: TLightingQuality;\n isAwake: boolean;\n eyeAspectRatio: number;\n emotion: TEmotion;\n emotionConfidence: number;\n // Away detection\n isAway: boolean;\n awayDurationMs: number;\n // Sleep detection - duration tracking\n sleepDurationMs: number;\n}\n\nexport interface IVideoAnalysisOptions {\n enabled: boolean;\n detectEmotions?: boolean;\n detectSleep?: boolean;\n // For canvas-based analysis (when analyzing cropped regions)\n useCanvas?: boolean;\n // Away detection options\n awayDetectionEnabled?: boolean;\n}\n\nexport interface IUseVideoAnalysis {\n (\n videoRef: React.RefObject<HTMLVideoElement | HTMLCanvasElement | null>,\n options: IVideoAnalysisOptions,\n ): {\n metrics: IVideoAnalysisMetrics;\n isLoading: boolean;\n error: Error | null;\n };\n}\n\n// Analysis runs at 1 FPS (every 1000ms / 1 second)\nconst ANALYSIS_INTERVAL_MS = 1000;\n\n// EAR threshold for sleep detection in online classes\n// Industry standard: 0.25 (balances sensitivity vs false positives)\n// Research: ~0.3 fully open, 0.2-0.25 drowsy threshold, <0.2 clearly closed\nconst EAR_THRESHOLD = 0.25; // Below 0.25 = eyes closing/drowsy\n\n/**\n * Custom hook for real-time video analysis using MediaPipe + TensorFlow.js\n *\n * @param videoRef - Reference to video or canvas element to analyze\n * @param options - Analysis options (enabled, detectEmotions, detectSleep, etc.)\n * @returns Analysis metrics, loading state, and error\n */\nconst useVideoAnalysisV2 = (\n videoRef: RefObject<HTMLVideoElement | HTMLCanvasElement | null>,\n options: IVideoAnalysisOptions,\n): {\n metrics: IVideoAnalysisMetrics;\n isLoading: boolean;\n error: Error | null;\n} => {\n // State\n const [metrics, setMetrics] = useState<IVideoAnalysisMetrics>({\n visibility: 0,\n faceDetected: false,\n cameraAngle: 'optimal',\n lighting: 'good',\n isAwake: true,\n eyeAspectRatio: 0,\n emotion: 'neutral',\n emotionConfidence: 0,\n isAway: false,\n awayDurationMs: 0,\n sleepDurationMs: 0,\n });\n const [isLoading, setIsLoading] = useState(true);\n const [error, setError] = useState<Error | null>(null);\n\n // Refs for analysis loop\n const animationFrameRef = useRef<number | undefined>(undefined);\n const lastAnalysisTimeRef = useRef<number>(0);\n const videoTimestampRef = useRef<number>(0);\n\n // Away detection state\n const awayStartTimeRef = useRef<number | null>(null);\n\n // Sleep detection state - track eyes closed duration\n const eyesClosedStartTimeRef = useRef<number | null>(null);\n const EYES_CLOSED_THRESHOLD_MS = 2000; // 2 seconds to avoid detecting blinks\n\n // Model instances\n const mediaPipeDetectorRef = useRef(getMediaPipeDetector());\n const emotionModelRef = useRef(getEmotionModel());\n\n /**\n * Analyze camera angle from head pose and face position\n * Enhanced with 3D pose estimation and better thresholds\n */\n const analyzeCameraAngle = useCallback(\n (\n faceBox: { x: number; y: number; width: number; height: number },\n videoWidth: number,\n videoHeight: number,\n headPose: { pitch: number; yaw: number; roll: number },\n ): TCameraAngle => {\n // Priority 1: Head pose analysis (most reliable indicator)\n // Using MediaPipe 3D pose for accurate angle detection\n\n // Yaw thresholds (left/right turn)\n // YAW_SLIGHT = 15° - Slight turn, still acceptable (reserved for future use)\n const YAW_MODERATE = 30; // Noticeable turn, warn user\n const YAW_SEVERE = 45; // Looking away significantly\n\n // Pitch thresholds (up/down tilt)\n // PITCH_SLIGHT = 10° - Slight tilt, still acceptable (reserved for future use)\n const PITCH_MODERATE = 20; // Noticeable tilt, warn user\n const PITCH_SEVERE = 30; // Looking down/up significantly\n\n // Check severe head pose issues first (user looking away)\n if (Math.abs(headPose.yaw) > YAW_SEVERE) {\n return headPose.yaw > 0 ? 'off-center-right' : 'off-center-left';\n }\n\n if (headPose.pitch > PITCH_SEVERE) {\n return 'off-center-low'; // Looking down (at desk/phone)\n }\n\n if (headPose.pitch < -PITCH_SEVERE) {\n return 'off-center-high'; // Looking up (at ceiling/daydreaming)\n }\n\n // Priority 2: Face size analysis (distance from camera)\n const faceArea = faceBox.width * faceBox.height;\n const videoArea = videoWidth * videoHeight;\n const faceRatio = faceArea / videoArea;\n\n // Optimal face ratio: 0.08 - 0.35 (8% - 35% of screen)\n const FACE_RATIO_TOO_CLOSE = 0.4; // More than 40% of screen\n const FACE_RATIO_VERY_CLOSE = 0.5; // More than 50% of screen\n const FACE_RATIO_TOO_FAR = 0.06; // Less than 6% of screen\n const FACE_RATIO_VERY_FAR = 0.04; // Less than 4% of screen\n\n if (faceRatio > FACE_RATIO_VERY_CLOSE) {\n return 'too-close';\n }\n\n if (faceRatio < FACE_RATIO_VERY_FAR) {\n return 'too-far';\n }\n\n // Priority 3: Face position analysis (centering)\n const faceCenterX = faceBox.x + faceBox.width / 2;\n const faceCenterY = faceBox.y + faceBox.height / 2;\n const videoCenterX = videoWidth / 2;\n const videoCenterY = videoHeight / 2;\n\n const offsetX = Math.abs(faceCenterX - videoCenterX) / videoWidth;\n const offsetY = Math.abs(faceCenterY - videoCenterY) / videoHeight;\n\n // Horizontal offset thresholds\n const OFFSET_X_MODERATE = 0.2; // 20% offset, still acceptable\n const OFFSET_X_SEVERE = 0.3; // 30% offset, warn user\n\n // Vertical offset thresholds (slightly more lenient)\n const OFFSET_Y_MODERATE = 0.25; // 25% offset, still acceptable\n const OFFSET_Y_SEVERE = 0.35; // 35% offset, warn user\n\n // Check moderate head pose issues (combined with position)\n if (Math.abs(headPose.yaw) > YAW_MODERATE || offsetX > OFFSET_X_SEVERE) {\n return faceCenterX > videoCenterX || headPose.yaw > 0\n ? 'off-center-right'\n : 'off-center-left';\n }\n\n if (headPose.pitch > PITCH_MODERATE || offsetY > OFFSET_Y_SEVERE) {\n return faceCenterY > videoCenterY || headPose.pitch > 0\n ? 'off-center-low'\n : 'off-center-high';\n }\n\n // Minor warnings for slightly suboptimal positioning\n if (faceRatio > FACE_RATIO_TOO_CLOSE && faceRatio <= FACE_RATIO_VERY_CLOSE) {\n return 'too-close';\n }\n\n if (faceRatio < FACE_RATIO_TOO_FAR && faceRatio >= FACE_RATIO_VERY_FAR) {\n return 'too-far';\n }\n\n if (offsetX > OFFSET_X_MODERATE && offsetX <= OFFSET_X_SEVERE) {\n return faceCenterX > videoCenterX ? 'off-center-right' : 'off-center-left';\n }\n\n if (offsetY > OFFSET_Y_MODERATE && offsetY <= OFFSET_Y_SEVERE) {\n return faceCenterY > videoCenterY ? 'off-center-low' : 'off-center-high';\n }\n\n // All checks passed - optimal positioning\n return 'optimal';\n },\n [],\n );\n\n /**\n * Main analysis function - runs on each frame\n */\n const analyzeFrame = useCallback(async () => {\n if (!options.enabled || !videoRef.current) {\n return;\n }\n\n const element = videoRef.current;\n const isCanvas = element instanceof HTMLCanvasElement;\n const isVideo = element instanceof HTMLVideoElement;\n\n // Check if video is ready (skip check for canvas)\n if (isVideo && element.readyState !== element.HAVE_ENOUGH_DATA) {\n return;\n }\n\n // Throttle analysis to ANALYSIS_INTERVAL_MS\n const now = Date.now();\n\n if (now - lastAnalysisTimeRef.current < ANALYSIS_INTERVAL_MS) {\n return;\n }\n\n lastAnalysisTimeRef.current = now;\n\n try {\n // Get video dimensions\n const videoWidth = isCanvas ? element.width : element.videoWidth;\n const videoHeight = isCanvas ? element.height : element.videoHeight;\n\n if (videoWidth === 0 || videoHeight === 0) {\n return;\n }\n\n // Update video timestamp for MediaPipe tracking\n videoTimestampRef.current = performance.now();\n\n // Detect face with MediaPipe\n const detection = await mediaPipeDetectorRef.current.detect(\n element,\n videoTimestampRef.current,\n );\n\n if (!detection) {\n // No face detected - handle away detection\n let isAway = false;\n let awayDurationMs = 0;\n\n if (options.awayDetectionEnabled) {\n const currentTime = Date.now();\n\n if (awayStartTimeRef.current === null) {\n awayStartTimeRef.current = currentTime;\n }\n\n awayDurationMs = currentTime - awayStartTimeRef.current;\n isAway = true;\n }\n\n setMetrics((prev: IVideoAnalysisMetrics) => ({\n ...prev,\n faceDetected: false,\n visibility: 0,\n cameraAngle: 'optimal',\n lighting: 'good',\n isAway,\n awayDurationMs,\n }));\n\n return;\n }\n\n // Face detected!\n // Calculate visibility\n const visibility = calculateVisibility(detection.boundingBox, videoWidth, videoHeight);\n\n // Analyze camera angle with 3D head pose\n const cameraAngle = analyzeCameraAngle(\n detection.boundingBox,\n videoWidth,\n videoHeight,\n detection.headPose,\n );\n\n // Analyze lighting\n const lighting = analyzeLighting(element, detection.boundingBox);\n\n // Calculate EAR for sleep detection\n let isAwake = true;\n let eyeAspectRatio = 0;\n\n if (options.detectSleep) {\n eyeAspectRatio = calculateEAR(detection.landmarks);\n const eyesClosed = eyeAspectRatio < EAR_THRESHOLD;\n\n if (eyesClosed) {\n // Eyes are closed - start or continue timer\n if (eyesClosedStartTimeRef.current === null) {\n eyesClosedStartTimeRef.current = Date.now();\n }\n\n const eyesClosedDuration = Date.now() - eyesClosedStartTimeRef.current;\n\n // Only mark as drowsy if eyes closed for more than threshold (not just blink)\n isAwake = eyesClosedDuration < EYES_CLOSED_THRESHOLD_MS;\n } else {\n // Eyes are open - reset timer\n eyesClosedStartTimeRef.current = null;\n isAwake = true;\n }\n }\n\n // Detect emotion with TensorFlow.js\n let emotion: TSimplifiedEmotion = 'neutral';\n let emotionConfidence = 0;\n\n if (options.detectEmotions) {\n try {\n // eslint-disable-next-line @typescript-eslint/no-explicit-any\n const blendshapes = (detection as any).blendshapes as\n | Array<{ categoryName: string; score: number }>\n | undefined;\n const prediction = await emotionModelRef.current.predict(\n element,\n detection.boundingBox,\n blendshapes,\n );\n\n emotion = prediction.emotion;\n emotionConfidence = prediction.confidence;\n } catch (err) {\n // eslint-disable-next-line no-console\n console.warn('[VideoAnalysisV2] Emotion prediction failed:', err);\n }\n }\n\n // Reset away detection when face is detected\n if (options.awayDetectionEnabled && awayStartTimeRef.current !== null) {\n awayStartTimeRef.current = null;\n }\n\n // Calculate sleep duration for metrics\n let sleepDurationMs = 0;\n\n if (!isAwake && eyesClosedStartTimeRef.current !== null) {\n sleepDurationMs = Date.now() - eyesClosedStartTimeRef.current;\n }\n\n // Update metrics\n setMetrics({\n visibility,\n faceDetected: true,\n cameraAngle,\n lighting,\n isAwake,\n eyeAspectRatio,\n emotion,\n emotionConfidence,\n isAway: false,\n awayDurationMs: 0,\n sleepDurationMs,\n });\n } catch (err) {\n const analysisError = err instanceof Error ? err : new Error('Face analysis failed');\n\n setError(analysisError);\n }\n }, [options, videoRef, analyzeCameraAngle]);\n\n /**\n * Animation loop\n */\n const animate = useCallback(() => {\n analyzeFrame();\n animationFrameRef.current = requestAnimationFrame(animate);\n }, [analyzeFrame]);\n\n /**\n * Initialize models (runs once on mount)\n */\n useEffect(() => {\n const initializeModels = async () => {\n try {\n // Initialize MediaPipe face detector\n await mediaPipeDetectorRef.current.initialize();\n\n // Initialize TensorFlow.js emotion model\n await emotionModelRef.current.initialize();\n } catch (err) {\n const loadError = err instanceof Error ? err : new Error('Failed to initialize models');\n\n setError(loadError);\n }\n };\n\n initializeModels();\n }, []); // Run once on mount\n\n /**\n * Start/stop analysis loop based on options\n */\n useEffect(() => {\n if (!options.enabled) {\n setIsLoading(false);\n\n // Stop animation if running\n if (animationFrameRef.current) {\n cancelAnimationFrame(animationFrameRef.current);\n animationFrameRef.current = undefined;\n }\n\n return;\n }\n\n // Check if models are ready\n const checkAndStart = () => {\n if (mediaPipeDetectorRef.current.isReady()) {\n setIsLoading(false);\n\n // Start analysis loop\n if (!animationFrameRef.current) {\n animationFrameRef.current = requestAnimationFrame(animate);\n }\n } else {\n setIsLoading(true);\n // Retry after a short delay\n setTimeout(checkAndStart, 100);\n }\n };\n\n checkAndStart();\n\n // Cleanup on unmount or when disabled\n return () => {\n if (animationFrameRef.current) {\n cancelAnimationFrame(animationFrameRef.current);\n animationFrameRef.current = undefined;\n }\n };\n }, [options.enabled, animate]);\n\n return {\n metrics,\n isLoading,\n error,\n };\n};\n\nexport default useVideoAnalysisV2;\n"],"names":["ANALYSIS_INTERVAL_MS","EAR_THRESHOLD","useVideoAnalysisV2","videoRef","options","metrics","setMetrics","useState","isLoading","setIsLoading","error","setError","animationFrameRef","useRef","lastAnalysisTimeRef","videoTimestampRef","awayStartTimeRef","eyesClosedStartTimeRef","EYES_CLOSED_THRESHOLD_MS","mediaPipeDetectorRef","getMediaPipeDetector","emotionModelRef","getEmotionModel","analyzeCameraAngle","useCallback","faceBox","videoWidth","videoHeight","headPose","faceArea","videoArea","faceRatio","FACE_RATIO_TOO_CLOSE","FACE_RATIO_VERY_CLOSE","FACE_RATIO_TOO_FAR","FACE_RATIO_VERY_FAR","faceCenterX","faceCenterY","videoCenterX","videoCenterY","offsetX","offsetY","OFFSET_X_MODERATE","OFFSET_X_SEVERE","OFFSET_Y_MODERATE","OFFSET_Y_SEVERE","analyzeFrame","element","isCanvas","now","detection","isAway","awayDurationMs","currentTime","prev","visibility","calculateVisibility","cameraAngle","lighting","analyzeLighting","isAwake","eyeAspectRatio","calculateEAR","emotion","emotionConfidence","blendshapes","prediction","err","sleepDurationMs","analysisError","animate","useEffect","loadError","checkAndStart","useVideoAnalysis"],"mappings":";;;;;AA+DA,MAAMA,IAAuB,KAKvBC,KAAgB,MAShBC,KAAqB,CACzBC,GACAC,MAKG;AAEH,QAAM,CAACC,GAASC,CAAU,IAAIC,EAAgC;AAAA,IAC5D,YAAY;AAAA,IACZ,cAAc;AAAA,IACd,aAAa;AAAA,IACb,UAAU;AAAA,IACV,SAAS;AAAA,IACT,gBAAgB;AAAA,IAChB,SAAS;AAAA,IACT,mBAAmB;AAAA,IACnB,QAAQ;AAAA,IACR,gBAAgB;AAAA,IAChB,iBAAiB;AAAA,EAAA,CAClB,GACK,CAACC,GAAWC,CAAY,IAAIF,EAAS,EAAI,GACzC,CAACG,GAAOC,CAAQ,IAAIJ,EAAuB,IAAI,GAG/CK,IAAoBC,EAA2B,MAAS,GACxDC,IAAsBD,EAAe,CAAC,GACtCE,IAAoBF,EAAe,CAAC,GAGpCG,IAAmBH,EAAsB,IAAI,GAG7CI,IAAyBJ,EAAsB,IAAI,GACnDK,IAA2B,KAG3BC,IAAuBN,EAAOO,EAAA,CAAsB,GACpDC,IAAkBR,EAAOS,EAAA,CAAiB,GAM1CC,IAAqBC;AAAA,IACzB,CACEC,GACAC,GACAC,GACAC,MACiB;AAejB,UAAI,KAAK,IAAIA,EAAS,GAAG,IAAI;AACpB,eAAAA,EAAS,MAAM,IAAI,qBAAqB;AAG7C,UAAAA,EAAS,QAAQ;AACZ,eAAA;AAGL,UAAAA,EAAS,QAAQ;AACZ,eAAA;AAIH,YAAAC,IAAWJ,EAAQ,QAAQA,EAAQ,QACnCK,IAAYJ,IAAaC,GACzBI,IAAYF,IAAWC,GAGvBE,IAAuB,KACvBC,IAAwB,KACxBC,IAAqB,MACrBC,IAAsB;AAE5B,UAAIJ,IAAYE;AACP,eAAA;AAGT,UAAIF,IAAYI;AACP,eAAA;AAIT,YAAMC,IAAcX,EAAQ,IAAIA,EAAQ,QAAQ,GAC1CY,IAAcZ,EAAQ,IAAIA,EAAQ,SAAS,GAC3Ca,IAAeZ,IAAa,GAC5Ba,IAAeZ,IAAc,GAE7Ba,IAAU,KAAK,IAAIJ,IAAcE,CAAY,IAAIZ,GACjDe,IAAU,KAAK,IAAIJ,IAAcE,CAAY,IAAIZ,GAGjDe,IAAoB,KACpBC,IAAkB,KAGlBC,IAAoB,MACpBC,IAAkB;AAGxB,aAAI,KAAK,IAAIjB,EAAS,GAAG,IAAI,MAAgBY,IAAUG,IAC9CP,IAAcE,KAAgBV,EAAS,MAAM,IAChD,qBACA,oBAGFA,EAAS,QAAQ,MAAkBa,IAAUI,IACxCR,IAAcE,KAAgBX,EAAS,QAAQ,IAClD,mBACA,oBAIFG,IAAYC,KAAwBD,KAAaE,IAC5C,cAGLF,IAAYG,KAAsBH,KAAaI,IAC1C,YAGLK,IAAUE,KAAqBF,KAAWG,IACrCP,IAAcE,IAAe,qBAAqB,oBAGvDG,IAAUG,KAAqBH,KAAWI,IACrCR,IAAcE,IAAe,mBAAmB,oBAIlD;AAAA,IACT;AAAA,IACA,CAAC;AAAA,EAAA,GAMGO,IAAetB,EAAY,YAAY;AAC3C,QAAI,CAACpB,EAAQ,WAAW,CAACD,EAAS;AAChC;AAGF,UAAM4C,IAAU5C,EAAS,SACnB6C,IAAWD,aAAmB;AAIpC,QAHgBA,aAAmB,oBAGpBA,EAAQ,eAAeA,EAAQ;AAC5C;AAII,UAAAE,IAAM,KAAK;AAEb,QAAA,EAAAA,IAAMnC,EAAoB,UAAUd,IAIxC;AAAA,MAAAc,EAAoB,UAAUmC;AAE1B,UAAA;AAEF,cAAMvB,IAAasB,IAAWD,EAAQ,QAAQA,EAAQ,YAChDpB,IAAcqB,IAAWD,EAAQ,SAASA,EAAQ;AAEpD,YAAArB,MAAe,KAAKC,MAAgB;AACtC;AAIgB,QAAAZ,EAAA,UAAU,YAAY;AAGlC,cAAAmC,IAAY,MAAM/B,EAAqB,QAAQ;AAAA,UACnD4B;AAAA,UACAhC,EAAkB;AAAA,QAAA;AAGpB,YAAI,CAACmC,GAAW;AAEd,cAAIC,IAAS,IACTC,IAAiB;AAErB,cAAIhD,EAAQ,sBAAsB;AAC1B,kBAAAiD,IAAc,KAAK;AAErB,YAAArC,EAAiB,YAAY,SAC/BA,EAAiB,UAAUqC,IAG7BD,IAAiBC,IAAcrC,EAAiB,SACvCmC,IAAA;AAAA,UACX;AAEA,UAAA7C,EAAW,CAACgD,OAAiC;AAAA,YAC3C,GAAGA;AAAA,YACH,cAAc;AAAA,YACd,YAAY;AAAA,YACZ,aAAa;AAAA,YACb,UAAU;AAAA,YACV,QAAAH;AAAA,YACA,gBAAAC;AAAA,UACA,EAAA;AAEF;AAAA,QACF;AAIA,cAAMG,IAAaC,EAAoBN,EAAU,aAAaxB,GAAYC,CAAW,GAG/E8B,IAAclC;AAAA,UAClB2B,EAAU;AAAA,UACVxB;AAAA,UACAC;AAAA,UACAuB,EAAU;AAAA,QAAA,GAINQ,IAAWC,EAAgBZ,GAASG,EAAU,WAAW;AAG/D,YAAIU,IAAU,IACVC,IAAiB;AAErB,QAAIzD,EAAQ,gBACOyD,IAAAC,EAAaZ,EAAU,SAAS,GAC9BW,IAAiB5D,MAI9BgB,EAAuB,YAAY,SACdA,EAAA,UAAU,KAAK,QAMxC2C,IAH2B,KAAK,IAAI,IAAI3C,EAAuB,UAGhCC,MAG/BD,EAAuB,UAAU,MACvB2C,IAAA;AAKd,YAAIG,IAA8B,WAC9BC,IAAoB;AAExB,YAAI5D,EAAQ;AACN,cAAA;AAEF,kBAAM6D,IAAef,EAAkB,aAGjCgB,IAAa,MAAM7C,EAAgB,QAAQ;AAAA,cAC/C0B;AAAA,cACAG,EAAU;AAAA,cACVe;AAAA,YAAA;AAGF,YAAAF,IAAUG,EAAW,SACrBF,IAAoBE,EAAW;AAAA,mBACxBC,GAAK;AAEJ,oBAAA,KAAK,gDAAgDA,CAAG;AAAA,UAClE;AAIF,QAAI/D,EAAQ,wBAAwBY,EAAiB,YAAY,SAC/DA,EAAiB,UAAU;AAI7B,YAAIoD,IAAkB;AAEtB,QAAI,CAACR,KAAW3C,EAAuB,YAAY,SAC/BmD,IAAA,KAAK,QAAQnD,EAAuB,UAI7CX,EAAA;AAAA,UACT,YAAAiD;AAAA,UACA,cAAc;AAAA,UACd,aAAAE;AAAA,UACA,UAAAC;AAAA,UACA,SAAAE;AAAA,UACA,gBAAAC;AAAA,UACA,SAAAE;AAAA,UACA,mBAAAC;AAAA,UACA,QAAQ;AAAA,UACR,gBAAgB;AAAA,UAChB,iBAAAI;AAAA,QAAA,CACD;AAAA,eACMD,GAAK;AACZ,cAAME,IAAgBF,aAAe,QAAQA,IAAM,IAAI,MAAM,sBAAsB;AAEnF,QAAAxD,EAAS0D,CAAa;AAAA,MACxB;AAAA;AAAA,EACC,GAAA,CAACjE,GAASD,GAAUoB,CAAkB,CAAC,GAKpC+C,IAAU9C,EAAY,MAAM;AACnB,IAAAsB,KACKlC,EAAA,UAAU,sBAAsB0D,CAAO;AAAA,EAAA,GACxD,CAACxB,CAAY,CAAC;AAKjB,SAAAyB,EAAU,MAAM;AAeG,KAdQ,YAAY;AAC/B,UAAA;AAEI,cAAApD,EAAqB,QAAQ,cAG7B,MAAAE,EAAgB,QAAQ;eACvB8C,GAAK;AACZ,cAAMK,IAAYL,aAAe,QAAQA,IAAM,IAAI,MAAM,6BAA6B;AAEtF,QAAAxD,EAAS6D,CAAS;AAAA,MACpB;AAAA,IAAA;EAIJ,GAAG,CAAE,CAAA,GAKLD,EAAU,MAAM;AACV,QAAA,CAACnE,EAAQ,SAAS;AACpB,MAAAK,EAAa,EAAK,GAGdG,EAAkB,YACpB,qBAAqBA,EAAkB,OAAO,GAC9CA,EAAkB,UAAU;AAG9B;AAAA,IACF;AAGA,UAAM6D,IAAgB,MAAM;AACtB,MAAAtD,EAAqB,QAAQ,aAC/BV,EAAa,EAAK,GAGbG,EAAkB,YACHA,EAAA,UAAU,sBAAsB0D,CAAO,OAG3D7D,EAAa,EAAI,GAEjB,WAAWgE,GAAe,GAAG;AAAA,IAC/B;AAGY,WAAAA,KAGP,MAAM;AACX,MAAI7D,EAAkB,YACpB,qBAAqBA,EAAkB,OAAO,GAC9CA,EAAkB,UAAU;AAAA,IAC9B;AAAA,EAED,GAAA,CAACR,EAAQ,SAASkE,CAAO,CAAC,GAEtB;AAAA,IACL,SAAAjE;AAAA,IACA,WAAAG;AAAA,IACA,OAAAE;AAAA,EAAA;AAEJ,GAEAgE,KAAexE;"}
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
import { jsxs as o, jsx as r, Fragment as W } from "react/jsx-runtime";
|
|
2
|
-
import { memo as
|
|
2
|
+
import { memo as oe, useRef as T, useState as w, useEffect as Y, useCallback as B } from "react";
|
|
3
3
|
import l from "../../ui/layout/flex-view.js";
|
|
4
4
|
import i from "../../ui/text/text.js";
|
|
5
|
-
import
|
|
6
|
-
import
|
|
7
|
-
import { Video as
|
|
8
|
-
import
|
|
9
|
-
const
|
|
5
|
+
import te from "./hooks/use-video-analysis.js";
|
|
6
|
+
import ie from "./video-analysis-overlay/video-analysis-overlay.js";
|
|
7
|
+
import { Video as ne, AnalysisRegionOverlay as se, AnalysisRegionLabel as ae, DebugCanvas as de } from "./video-analysis-styled.js";
|
|
8
|
+
import le from "../../ui/separator/separator.js";
|
|
9
|
+
const ce = oe(function({
|
|
10
10
|
videoUrl: c,
|
|
11
11
|
detectEmotions: X = !1,
|
|
12
12
|
detectSleep: j = !1,
|
|
@@ -15,17 +15,13 @@ const he = ie(function({
|
|
|
15
15
|
autoPlay: N = !0,
|
|
16
16
|
loop: P = !0,
|
|
17
17
|
cropRegion: n,
|
|
18
|
-
awayDetectionEnabled: z = !1
|
|
19
|
-
awayDurationThreshold: q = 6e4,
|
|
20
|
-
onUserAway: K
|
|
18
|
+
awayDetectionEnabled: z = !1
|
|
21
19
|
}) {
|
|
22
|
-
const $ = T(null), H = T(null), R = T(null), [y, b] = w(null), [A, C] = w(!1), [D,
|
|
20
|
+
const $ = T(null), H = T(null), R = T(null), [y, b] = w(null), [A, C] = w(!1), [D, q] = w(!1), [f, K] = w(null), U = n && D ? R : $, { metrics: a, isLoading: J, error: _ } = te(U, {
|
|
23
21
|
enabled: A && (!n || D),
|
|
24
22
|
detectEmotions: X,
|
|
25
23
|
detectSleep: j,
|
|
26
|
-
awayDetectionEnabled: z
|
|
27
|
-
awayDurationThreshold: q,
|
|
28
|
-
onUserAway: K
|
|
24
|
+
awayDetectionEnabled: z
|
|
29
25
|
});
|
|
30
26
|
Y(() => {
|
|
31
27
|
const e = $.current;
|
|
@@ -88,14 +84,14 @@ const he = ie(function({
|
|
|
88
84
|
videoHeight: d,
|
|
89
85
|
scaleX: O,
|
|
90
86
|
scaleY: L
|
|
91
|
-
}),
|
|
87
|
+
}), K({
|
|
92
88
|
left: `${V * O}px`,
|
|
93
89
|
top: `${x * L}px`,
|
|
94
90
|
width: `${g * O}px`,
|
|
95
91
|
height: `${m * L}px`
|
|
96
92
|
});
|
|
97
|
-
const
|
|
98
|
-
t.srcObject =
|
|
93
|
+
const ee = s.captureStream(30);
|
|
94
|
+
t.srcObject = ee;
|
|
99
95
|
const F = () => {
|
|
100
96
|
console.log(
|
|
101
97
|
"[VideoAnalysis] Cropped video ready state changed:",
|
|
@@ -104,9 +100,9 @@ const he = ie(function({
|
|
|
104
100
|
};
|
|
105
101
|
t.addEventListener("loadedmetadata", () => {
|
|
106
102
|
console.log("[VideoAnalysis] Cropped video metadata loaded");
|
|
107
|
-
}), t.addEventListener("loadeddata", F), t.addEventListener("canplay", F), t.addEventListener("canplaythrough", F), t.play().catch((
|
|
108
|
-
console.error("[VideoAnalysis] Failed to play cropped video:",
|
|
109
|
-
}), I = !0,
|
|
103
|
+
}), t.addEventListener("loadeddata", F), t.addEventListener("canplay", F), t.addEventListener("canplaythrough", F), t.play().catch((re) => {
|
|
104
|
+
console.error("[VideoAnalysis] Failed to play cropped video:", re);
|
|
105
|
+
}), I = !0, q(!0), console.log("[VideoAnalysis] Canvas ready for face analysis");
|
|
110
106
|
}
|
|
111
107
|
h.drawImage(e, V, x, g, m, 0, 0, g, m), E % 60 === 0 && console.log("[VideoAnalysis] Canvas update:", {
|
|
112
108
|
frameCount: E,
|
|
@@ -121,16 +117,16 @@ const he = ie(function({
|
|
|
121
117
|
v && cancelAnimationFrame(v), t.srcObject && (t.srcObject.getTracks().forEach((d) => d.stop()), t.srcObject = null);
|
|
122
118
|
};
|
|
123
119
|
}, [n, A]);
|
|
124
|
-
const
|
|
120
|
+
const Q = B(() => {
|
|
125
121
|
C(!0);
|
|
126
|
-
}, []),
|
|
122
|
+
}, []), Z = B((e) => {
|
|
127
123
|
const s = e.currentTarget, t = s.error ? `Video error: ${s.error.message} (code: ${s.error.code})` : "Unknown video error";
|
|
128
124
|
b(t), console.error("Video error:", s.error);
|
|
129
125
|
}, []);
|
|
130
126
|
return /* @__PURE__ */ o(l, { $flexDirection: "column", $flexGapX: 2, $alignItems: "center", children: [
|
|
131
127
|
/* @__PURE__ */ o(l, { $position: "relative", $width: p, $height: G, children: [
|
|
132
128
|
/* @__PURE__ */ r(
|
|
133
|
-
|
|
129
|
+
ne,
|
|
134
130
|
{
|
|
135
131
|
ref: $,
|
|
136
132
|
width: p,
|
|
@@ -141,12 +137,12 @@ const he = ie(function({
|
|
|
141
137
|
playsInline: !0,
|
|
142
138
|
controls: !0,
|
|
143
139
|
crossOrigin: "anonymous",
|
|
144
|
-
onLoadedMetadata:
|
|
145
|
-
onError:
|
|
140
|
+
onLoadedMetadata: Q,
|
|
141
|
+
onError: Z
|
|
146
142
|
}
|
|
147
143
|
),
|
|
148
144
|
n && f && /* @__PURE__ */ r(
|
|
149
|
-
|
|
145
|
+
se,
|
|
150
146
|
{
|
|
151
147
|
$position: "absolute",
|
|
152
148
|
$left: f.left,
|
|
@@ -154,7 +150,7 @@ const he = ie(function({
|
|
|
154
150
|
$width: f.width,
|
|
155
151
|
$height: f.height,
|
|
156
152
|
children: /* @__PURE__ */ r(
|
|
157
|
-
|
|
153
|
+
ae,
|
|
158
154
|
{
|
|
159
155
|
$position: "absolute",
|
|
160
156
|
$background: "GREEN_4",
|
|
@@ -167,16 +163,16 @@ const he = ie(function({
|
|
|
167
163
|
}
|
|
168
164
|
),
|
|
169
165
|
/* @__PURE__ */ r(
|
|
170
|
-
|
|
166
|
+
ie,
|
|
171
167
|
{
|
|
172
168
|
metrics: a,
|
|
173
|
-
isLoading:
|
|
169
|
+
isLoading: J,
|
|
174
170
|
visible: A,
|
|
175
171
|
isLiveCamera: !c
|
|
176
172
|
}
|
|
177
173
|
)
|
|
178
174
|
] }),
|
|
179
|
-
/* @__PURE__ */ r(
|
|
175
|
+
/* @__PURE__ */ r(de, { ref: R, $show: !!n }),
|
|
180
176
|
/* @__PURE__ */ r("video", { ref: H, style: { display: "none" }, autoPlay: !0, muted: !0, playsInline: !0 }),
|
|
181
177
|
n && D && /* @__PURE__ */ o(l, { $flexDirection: "column", $flexGapX: 0.5, children: [
|
|
182
178
|
/* @__PURE__ */ r(i, { $renderAs: "ab1-bold", children: "Debug: Canvas Being Analyzed" }),
|
|
@@ -251,7 +247,7 @@ const he = ie(function({
|
|
|
251
247
|
$width: p,
|
|
252
248
|
children: [
|
|
253
249
|
/* @__PURE__ */ r(i, { $renderAs: "ab1-bold", children: "Current Metrics:" }),
|
|
254
|
-
/* @__PURE__ */ r(
|
|
250
|
+
/* @__PURE__ */ r(le, { heightX: 1 }),
|
|
255
251
|
/* @__PURE__ */ o(
|
|
256
252
|
l,
|
|
257
253
|
{
|
|
@@ -313,8 +309,8 @@ const he = ie(function({
|
|
|
313
309
|
}
|
|
314
310
|
)
|
|
315
311
|
] });
|
|
316
|
-
}),
|
|
312
|
+
}), Ae = ce;
|
|
317
313
|
export {
|
|
318
|
-
|
|
314
|
+
Ae as default
|
|
319
315
|
};
|
|
320
316
|
//# sourceMappingURL=video-analysis.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"video-analysis.js","sources":["../../../../src/features/av/video-analysis/video-analysis.tsx"],"sourcesContent":["/* eslint-disable no-console */\nimport { memo, useCallback, useEffect, useRef, useState, type FC } from 'react';\n\nimport FlexView from '../../ui/layout/flex-view';\nimport Text from '../../ui/text/text';\nimport useVideoAnalysis from './hooks/use-video-analysis';\nimport VideoAnalysisOverlay from './video-analysis-overlay/video-analysis-overlay';\nimport * as Styled from './video-analysis-styled';\nimport type { IVideoAnalysisProps } from './video-analysis-types';\nimport Separator from '../../ui/separator/separator';\n\n/**\n * VideoAnalysis Component\n *\n * Real-time video quality analysis for online tutoring sessions using:\n * - MediaPipe FaceLandmarker (478 3D landmarks)\n * - TensorFlow.js emotion recognition\n *\n * Analyzes pre-recorded videos or live camera feeds to show visibility, camera angle,\n * lighting, sleep detection (teachers), and emotion recognition (students).\n *\n * Features:\n * - Face detection with 478 3D landmarks (better accuracy in low-light)\n * - Enhanced eye-closure detection (90%+ accuracy, even for frontal faces)\n * - 3D head pose tracking for camera angle analysis\n * - Visibility detection (optimal: 10-25%)\n * - Camera angle analysis (optimal/too-close/too-far/off-center/looking-away)\n * - Lighting quality (dark/good/bright)\n * - Sleep/drowsiness detection using Eye Aspect Ratio (EAR)\n * - Emotion recognition (happy/sad/neutral) with TensorFlow.js\n * - Region of Interest (ROI) cropping for screen recordings\n * - Away detection with configurable thresholds\n *\n * Performance Improvements (vs face-api.js):\n * - 50% faster (30-40 FPS vs 20-25 FPS)\n * - 20% better frontal face eye-closure detection (90% vs 70%)\n * - 42% better low-light detection (85% vs 60%)\n * - 23% better emotion accuracy (80% vs 65%)\n */\nconst VideoAnalysis: FC<IVideoAnalysisProps> = memo(function VideoAnalysis({\n videoUrl,\n detectEmotions = false,\n detectSleep = false,\n width = 640,\n height = 480,\n autoPlay = true,\n loop = true,\n cropRegion,\n awayDetectionEnabled = false,\n awayDurationThreshold = 60000,\n onUserAway,\n}) {\n const videoRef = useRef<HTMLVideoElement>(null);\n const croppedVideoRef = useRef<HTMLVideoElement>(null);\n const canvasRef = useRef<HTMLCanvasElement>(null);\n const [videoError, setVideoError] = useState<string | null>(null);\n const [isVideoReady, setIsVideoReady] = useState(false);\n const [canvasReady, setCanvasReady] = useState(false);\n const [overlayStyle, setOverlayStyle] = useState<{\n left: string;\n top: string;\n width: string;\n height: string;\n } | null>(null);\n\n // When cropRegion is provided, analyze the canvas directly (which shows cropped video)\n // Otherwise analyze the original video\n const analysisRef = cropRegion && canvasReady ? canvasRef : videoRef;\n\n // Video analysis hook V2 (MediaPipe + TensorFlow.js)\n const { metrics, isLoading, error } = useVideoAnalysis(analysisRef, {\n enabled: isVideoReady && (!cropRegion || canvasReady),\n detectEmotions,\n detectSleep,\n awayDetectionEnabled,\n awayDurationThreshold,\n onUserAway,\n });\n\n // Setup video source - use camera if no videoUrl provided\n useEffect(() => {\n const video = videoRef.current;\n\n const setupVideo = async () => {\n if (!video) return;\n\n setVideoError(null);\n setIsVideoReady(false);\n\n try {\n if (!videoUrl) {\n // No video URL provided - use camera\n const stream = await navigator.mediaDevices.getUserMedia({\n video: true,\n audio: false,\n });\n\n video.srcObject = stream;\n setIsVideoReady(true);\n } else {\n // Check for unsupported video sources\n if (\n videoUrl.includes('youtube.com') ||\n videoUrl.includes('youtu.be') ||\n videoUrl.includes('vimeo.com')\n ) {\n setVideoError(\n 'YouTube/Vimeo URLs are not supported. Please use direct video file URLs (MP4, WebM, OGG). Example: https://commondatastorage.googleapis.com/gtv-videos-bucket/sample/BigBuckBunny.mp4',\n );\n\n return;\n }\n\n // Use video URL\n video.src = videoUrl;\n video.load();\n }\n } catch (err) {\n const errorMessage = err instanceof Error ? err.message : 'Failed to setup video';\n\n setVideoError(errorMessage);\n console.error('Video setup error:', err);\n }\n };\n\n setupVideo();\n\n // Cleanup - capture video ref value\n return () => {\n if (video?.srcObject) {\n const stream = video.srcObject as MediaStream;\n\n stream.getTracks().forEach(track => track.stop());\n video.srcObject = null;\n }\n };\n }, [videoUrl]);\n\n // Crop video to region of interest\n useEffect(() => {\n if (!cropRegion || !isVideoReady) return;\n\n const video = videoRef.current;\n const canvas = canvasRef.current;\n const croppedVideo = croppedVideoRef.current;\n\n if (!video || !canvas || !croppedVideo) return;\n\n const ctx = canvas.getContext('2d');\n\n if (!ctx) return;\n\n let animationFrameId: number;\n let streamInitialized = false;\n let cropWidth = 0;\n let cropHeight = 0;\n let cropX = 0;\n let cropY = 0;\n let frameCount = 0;\n\n const updateCroppedFrame = () => {\n frameCount++;\n if (video.readyState === video.HAVE_ENOUGH_DATA) {\n const videoWidth = video.videoWidth;\n const videoHeight = video.videoHeight;\n\n if (videoWidth === 0 || videoHeight === 0) {\n animationFrameId = requestAnimationFrame(updateCroppedFrame);\n\n return;\n }\n\n // Calculate crop dimensions\n cropX = cropRegion.x * videoWidth;\n cropY = cropRegion.y * videoHeight;\n cropWidth = cropRegion.width * videoWidth;\n cropHeight = cropRegion.height * videoHeight;\n\n // Set canvas size to crop dimensions (only once)\n if (!streamInitialized) {\n canvas.width = cropWidth;\n canvas.height = cropHeight;\n\n console.log('[VideoAnalysis] Initializing crop region:', {\n cropX,\n cropY,\n cropWidth,\n cropHeight,\n originalWidth: videoWidth,\n originalHeight: videoHeight,\n videoCurrentTime: video.currentTime,\n });\n\n // Calculate the correct overlay position accounting for video display vs actual size\n const displayWidth = video.clientWidth;\n const displayHeight = video.clientHeight;\n const scaleX = displayWidth / videoWidth;\n const scaleY = displayHeight / videoHeight;\n\n console.log('[VideoAnalysis] Display vs actual dimensions:', {\n displayWidth,\n displayHeight,\n videoWidth,\n videoHeight,\n scaleX,\n scaleY,\n });\n\n // Update overlay style to match the actual cropped region on screen\n setOverlayStyle({\n left: `${cropX * scaleX}px`,\n top: `${cropY * scaleY}px`,\n width: `${cropWidth * scaleX}px`,\n height: `${cropHeight * scaleY}px`,\n });\n\n // Initialize stream with 30fps\n const stream = canvas.captureStream(30);\n\n croppedVideo.srcObject = stream;\n\n // Add event listeners to monitor cropped video state\n const logReadyState = () => {\n console.log(\n '[VideoAnalysis] Cropped video ready state changed:',\n croppedVideo.readyState,\n );\n };\n\n croppedVideo.addEventListener('loadedmetadata', () => {\n console.log('[VideoAnalysis] Cropped video metadata loaded');\n });\n croppedVideo.addEventListener('loadeddata', logReadyState);\n croppedVideo.addEventListener('canplay', logReadyState);\n croppedVideo.addEventListener('canplaythrough', logReadyState);\n\n // Ensure cropped video plays\n croppedVideo.play().catch(err => {\n console.error('[VideoAnalysis] Failed to play cropped video:', err);\n });\n\n streamInitialized = true;\n setCanvasReady(true); // Signal that canvas is ready for analysis\n\n console.log('[VideoAnalysis] Canvas ready for face analysis');\n }\n\n // CRITICAL: Draw cropped region to canvas on EVERY frame\n // This continuously updates the canvas with the latest cropped video frame\n ctx.drawImage(video, cropX, cropY, cropWidth, cropHeight, 0, 0, cropWidth, cropHeight);\n\n // Log every 60 frames (~1 second at 60fps) to verify continuous drawing\n if (frameCount % 60 === 0) {\n console.log('[VideoAnalysis] Canvas update:', {\n frameCount,\n videoTime: video.currentTime.toFixed(2),\n videoPlaying: !video.paused,\n });\n }\n } else {\n // Video not ready - log occasionally\n if (frameCount % 60 === 0) {\n console.log('[VideoAnalysis] Waiting for video, readyState:', video.readyState);\n }\n }\n\n // Continue animation loop regardless of video state\n animationFrameId = requestAnimationFrame(updateCroppedFrame);\n };\n\n // Start the animation loop\n updateCroppedFrame();\n\n return () => {\n if (animationFrameId) {\n cancelAnimationFrame(animationFrameId);\n }\n\n if (croppedVideo.srcObject) {\n const stream = croppedVideo.srcObject as MediaStream;\n\n stream.getTracks().forEach(track => track.stop());\n croppedVideo.srcObject = null;\n }\n };\n }, [cropRegion, isVideoReady]);\n\n // Handle video metadata loaded\n const handleLoadedMetadata = useCallback(() => {\n setIsVideoReady(true);\n }, []);\n\n const handleVideoError = useCallback((e: React.SyntheticEvent<HTMLVideoElement, Event>) => {\n const video = e.currentTarget;\n const errorMsg = video.error\n ? `Video error: ${video.error.message} (code: ${video.error.code})`\n : 'Unknown video error';\n\n setVideoError(errorMsg);\n console.error('Video error:', video.error);\n }, []);\n\n return (\n <FlexView $flexDirection=\"column\" $flexGapX={2} $alignItems=\"center\">\n <FlexView $position=\"relative\" $width={width} $height={height}>\n <Styled.Video\n ref={videoRef}\n width={width}\n height={height}\n autoPlay={autoPlay}\n loop={loop}\n muted\n playsInline\n controls\n crossOrigin=\"anonymous\"\n onLoadedMetadata={handleLoadedMetadata}\n onError={handleVideoError}\n />\n {cropRegion && overlayStyle && (\n <Styled.AnalysisRegionOverlay\n $position=\"absolute\"\n $left={overlayStyle.left}\n $top={overlayStyle.top}\n $width={overlayStyle.width}\n $height={overlayStyle.height}\n >\n <Styled.AnalysisRegionLabel\n $position=\"absolute\"\n $background=\"GREEN_4\"\n $borderRadius={3}\n $gap={2}\n $gutter={6}\n >\n <Text $renderAs=\"ub3\" $color=\"REAL_BLACK\">\n Analysis Region\n </Text>\n </Styled.AnalysisRegionLabel>\n </Styled.AnalysisRegionOverlay>\n )}\n <VideoAnalysisOverlay\n metrics={metrics}\n isLoading={isLoading}\n visible={isVideoReady}\n isLiveCamera={!videoUrl}\n />\n </FlexView>\n\n {/* Canvas for cropping - visible for debugging when cropRegion is active */}\n <Styled.DebugCanvas ref={canvasRef} $show={!!cropRegion} />\n <video ref={croppedVideoRef} style={{ display: 'none' }} autoPlay muted playsInline />\n\n {/* Debug info */}\n {cropRegion && canvasReady && (\n <FlexView $flexDirection=\"column\" $flexGapX={0.5}>\n <Text $renderAs=\"ab1-bold\">Debug: Canvas Being Analyzed</Text>\n <Text $renderAs=\"ub2\" $color=\"GREY_3\">\n This canvas shows the cropped region that face-api.js analyzes. If you don't see a\n clear face, adjust the cropRegion values.\n </Text>\n </FlexView>\n )}\n\n {videoError && (\n <FlexView\n $flexDirection=\"column\"\n $gutterX={1.5}\n $gapX={1}\n $background=\"RED_4\"\n $borderRadius={4}\n $width={width}\n >\n <Text $renderAs=\"ab1-bold\" $color=\"RED\">\n Error: {videoError}\n </Text>\n {(videoError.includes('YouTube') || videoError.includes('Vimeo')) && (\n <FlexView $flexDirection=\"column\" $flexGapX={0.5}>\n <Text $renderAs=\"ab2-bold\">How to get a direct video URL:</Text>\n <Text $renderAs=\"ub2\" as=\"ul\" style={{ paddingLeft: '20px' }}>\n <li>\n Use services like{' '}\n <a\n href=\"https://www.downloadhelper.net/\"\n target=\"_blank\"\n rel=\"noopener noreferrer\"\n >\n Video DownloadHelper\n </a>{' '}\n to download YouTube videos\n </li>\n <li>Host the downloaded MP4 file on your server or cloud storage</li>\n <li>\n Or use public test videos like:{' '}\n <code>\n https://commondatastorage.googleapis.com/gtv-videos-bucket/sample/BigBuckBunny.mp4\n </code>\n </li>\n </Text>\n </FlexView>\n )}\n </FlexView>\n )}\n\n {error && (\n <FlexView\n $flexDirection=\"column\"\n $gutterX={1.5}\n $gapX={1}\n $background=\"ORANGE_1\"\n $borderRadius={4}\n $width={width}\n >\n <Text $renderAs=\"ab1-bold\" $color=\"ORANGE_4\">\n Analysis Error: {error.message}\n </Text>\n </FlexView>\n )}\n\n <FlexView\n $flexDirection=\"column\"\n $gutterX={2}\n $gapX={1.5}\n $background=\"GREY_1\"\n $borderRadius={8}\n $width={width}\n >\n <Text $renderAs=\"ab1-bold\">Current Metrics:</Text>\n <Separator heightX={1} />\n <FlexView\n $flexDirection=\"row\"\n $flexWrap\n $flexGapX={1}\n style={{ display: 'grid', gridTemplateColumns: '1fr 1fr' }}\n >\n <Text $renderAs=\"ub2\">\n <strong>Face Detected:</strong> {metrics.faceDetected ? 'Yes' : 'No'}\n </Text>\n <Text $renderAs=\"ub2\">\n <strong>Visibility:</strong> {metrics.visibility.toFixed(1)}%\n </Text>\n <Text $renderAs=\"ub2\">\n <strong>Camera Angle:</strong> {metrics.cameraAngle}\n </Text>\n <Text $renderAs=\"ub2\">\n <strong>Lighting:</strong> {metrics.lighting}\n </Text>\n {detectSleep && (\n <>\n <Text $renderAs=\"ub2\">\n <strong>Awake:</strong> {metrics.isAwake ? 'Yes' : 'Drowsy'}\n </Text>\n <Text $renderAs=\"ub2\">\n <strong>EAR:</strong> {metrics.eyeAspectRatio.toFixed(3)}\n </Text>\n </>\n )}\n {detectEmotions && metrics.emotionConfidence > 0 && (\n <>\n <Text $renderAs=\"ub2\">\n <strong>Emotion:</strong> {metrics.emotion}\n </Text>\n <Text $renderAs=\"ub2\">\n <strong>Confidence:</strong> {(metrics.emotionConfidence * 100).toFixed(0)}%\n </Text>\n </>\n )}\n </FlexView>\n </FlexView>\n </FlexView>\n );\n});\n\nexport default VideoAnalysis;\n"],"names":["VideoAnalysis","memo","videoUrl","detectEmotions","detectSleep","width","height","autoPlay","loop","cropRegion","awayDetectionEnabled","awayDurationThreshold","onUserAway","videoRef","useRef","croppedVideoRef","canvasRef","videoError","setVideoError","useState","isVideoReady","setIsVideoReady","canvasReady","setCanvasReady","overlayStyle","setOverlayStyle","analysisRef","metrics","isLoading","error","useVideoAnalysis","useEffect","video","stream","err","errorMessage","track","canvas","croppedVideo","ctx","animationFrameId","streamInitialized","cropWidth","cropHeight","cropX","cropY","frameCount","updateCroppedFrame","videoWidth","videoHeight","displayWidth","displayHeight","scaleX","scaleY","logReadyState","handleLoadedMetadata","useCallback","handleVideoError","errorMsg","FlexView","jsxs","jsx","Styled.Video","Styled.AnalysisRegionOverlay","Styled.AnalysisRegionLabel","Text","VideoAnalysisOverlay","Styled.DebugCanvas","Separator","Fragment","VideoAnalysis$1"],"mappings":";;;;;;;;AAuCA,MAAMA,KAAyCC,GAAK,SAAuB;AAAA,EACzE,UAAAC;AAAA,EACA,gBAAAC,IAAiB;AAAA,EACjB,aAAAC,IAAc;AAAA,EACd,OAAAC,IAAQ;AAAA,EACR,QAAAC,IAAS;AAAA,EACT,UAAAC,IAAW;AAAA,EACX,MAAAC,IAAO;AAAA,EACP,YAAAC;AAAA,EACA,sBAAAC,IAAuB;AAAA,EACvB,uBAAAC,IAAwB;AAAA,EACxB,YAAAC;AACF,GAAG;AACK,QAAAC,IAAWC,EAAyB,IAAI,GACxCC,IAAkBD,EAAyB,IAAI,GAC/CE,IAAYF,EAA0B,IAAI,GAC1C,CAACG,GAAYC,CAAa,IAAIC,EAAwB,IAAI,GAC1D,CAACC,GAAcC,CAAe,IAAIF,EAAS,EAAK,GAChD,CAACG,GAAaC,CAAc,IAAIJ,EAAS,EAAK,GAC9C,CAACK,GAAcC,CAAe,IAAIN,EAK9B,IAAI,GAIRO,IAAcjB,KAAca,IAAcN,IAAYH,GAGtD,EAAE,SAAAc,GAAS,WAAAC,GAAW,OAAAC,EAAM,IAAIC,GAAiBJ,GAAa;AAAA,IAClE,SAASN,MAAiB,CAACX,KAAca;AAAA,IACzC,gBAAAnB;AAAA,IACA,aAAAC;AAAA,IACA,sBAAAM;AAAA,IACA,uBAAAC;AAAA,IACA,YAAAC;AAAA,EAAA,CACD;AAGD,EAAAmB,EAAU,MAAM;AACd,UAAMC,IAAQnB,EAAS;AA4CZ,YA1CQ,YAAY;AAC7B,UAAKmB,GAEL;AAAA,QAAAd,EAAc,IAAI,GAClBG,EAAgB,EAAK;AAEjB,YAAA;AACF,cAAKnB,GASE;AAGH,gBAAAA,EAAS,SAAS,aAAa,KAC/BA,EAAS,SAAS,UAAU,KAC5BA,EAAS,SAAS,WAAW,GAC7B;AACA,cAAAgB;AAAA,gBACE;AAAA,cAAA;AAGF;AAAA,YACF;AAGA,YAAAc,EAAM,MAAM9B,GACZ8B,EAAM,KAAK;AAAA,UACb,OA1Be;AAEb,kBAAMC,IAAS,MAAM,UAAU,aAAa,aAAa;AAAA,cACvD,OAAO;AAAA,cACP,OAAO;AAAA,YAAA,CACR;AAED,YAAAD,EAAM,YAAYC,GAClBZ,EAAgB,EAAI;AAAA,UAAA;AAAA,iBAmBfa,GAAK;AACZ,gBAAMC,IAAeD,aAAe,QAAQA,EAAI,UAAU;AAE1D,UAAAhB,EAAciB,CAAY,GAClB,QAAA,MAAM,sBAAsBD,CAAG;AAAA,QACzC;AAAA;AAAA,IAAA,MAMK,MAAM;AACX,MAAIF,KAAA,QAAAA,EAAO,cACMA,EAAM,UAEd,YAAY,QAAQ,CAASI,MAAAA,EAAM,MAAM,GAChDJ,EAAM,YAAY;AAAA,IACpB;AAAA,EACF,GACC,CAAC9B,CAAQ,CAAC,GAGb6B,EAAU,MAAM;AACV,QAAA,CAACtB,KAAc,CAACW,EAAc;AAElC,UAAMY,IAAQnB,EAAS,SACjBwB,IAASrB,EAAU,SACnBsB,IAAevB,EAAgB;AAErC,QAAI,CAACiB,KAAS,CAACK,KAAU,CAACC,EAAc;AAElC,UAAAC,IAAMF,EAAO,WAAW,IAAI;AAElC,QAAI,CAACE,EAAK;AAEN,QAAAC,GACAC,IAAoB,IACpBC,IAAY,GACZC,IAAa,GACbC,IAAQ,GACRC,IAAQ,GACRC,IAAa;AAEjB,UAAMC,IAAqB,MAAM;AAE3B,UADJD,KACId,EAAM,eAAeA,EAAM,kBAAkB;AAC/C,cAAMgB,IAAahB,EAAM,YACnBiB,IAAcjB,EAAM;AAEtB,YAAAgB,MAAe,KAAKC,MAAgB,GAAG;AACzC,UAAAT,IAAmB,sBAAsBO,CAAkB;AAE3D;AAAA,QACF;AASA,YANAH,IAAQnC,EAAW,IAAIuC,GACvBH,IAAQpC,EAAW,IAAIwC,GACvBP,IAAYjC,EAAW,QAAQuC,GAC/BL,IAAalC,EAAW,SAASwC,GAG7B,CAACR,GAAmB;AACtB,UAAAJ,EAAO,QAAQK,GACfL,EAAO,SAASM,GAEhB,QAAQ,IAAI,6CAA6C;AAAA,YACvD,OAAAC;AAAA,YACA,OAAAC;AAAA,YACA,WAAAH;AAAA,YACA,YAAAC;AAAA,YACA,eAAeK;AAAA,YACf,gBAAgBC;AAAA,YAChB,kBAAkBjB,EAAM;AAAA,UAAA,CACzB;AAGD,gBAAMkB,IAAelB,EAAM,aACrBmB,IAAgBnB,EAAM,cACtBoB,IAASF,IAAeF,GACxBK,IAASF,IAAgBF;AAE/B,kBAAQ,IAAI,iDAAiD;AAAA,YAC3D,cAAAC;AAAA,YACA,eAAAC;AAAA,YACA,YAAAH;AAAA,YACA,aAAAC;AAAA,YACA,QAAAG;AAAA,YACA,QAAAC;AAAA,UAAA,CACD,GAGe5B,EAAA;AAAA,YACd,MAAM,GAAGmB,IAAQQ,CAAM;AAAA,YACvB,KAAK,GAAGP,IAAQQ,CAAM;AAAA,YACtB,OAAO,GAAGX,IAAYU,CAAM;AAAA,YAC5B,QAAQ,GAAGT,IAAaU,CAAM;AAAA,UAAA,CAC/B;AAGK,gBAAApB,KAASI,EAAO,cAAc,EAAE;AAEtC,UAAAC,EAAa,YAAYL;AAGzB,gBAAMqB,IAAgB,MAAM;AAClB,oBAAA;AAAA,cACN;AAAA,cACAhB,EAAa;AAAA,YAAA;AAAA,UACf;AAGW,UAAAA,EAAA,iBAAiB,kBAAkB,MAAM;AACpD,oBAAQ,IAAI,+CAA+C;AAAA,UAAA,CAC5D,GACYA,EAAA,iBAAiB,cAAcgB,CAAa,GAC5ChB,EAAA,iBAAiB,WAAWgB,CAAa,GACzChB,EAAA,iBAAiB,kBAAkBgB,CAAa,GAGhDhB,EAAA,KAAA,EAAO,MAAM,CAAOJ,OAAA;AACvB,oBAAA,MAAM,iDAAiDA,EAAG;AAAA,UAAA,CACnE,GAEmBO,IAAA,IACpBlB,EAAe,EAAI,GAEnB,QAAQ,IAAI,gDAAgD;AAAA,QAC9D;AAII,QAAAgB,EAAA,UAAUP,GAAOY,GAAOC,GAAOH,GAAWC,GAAY,GAAG,GAAGD,GAAWC,CAAU,GAGjFG,IAAa,OAAO,KACtB,QAAQ,IAAI,kCAAkC;AAAA,UAC5C,YAAAA;AAAA,UACA,WAAWd,EAAM,YAAY,QAAQ,CAAC;AAAA,UACtC,cAAc,CAACA,EAAM;AAAA,QAAA,CACtB;AAAA,MACH;AAGI,QAAAc,IAAa,OAAO,KACd,QAAA,IAAI,kDAAkDd,EAAM,UAAU;AAKlF,MAAAQ,IAAmB,sBAAsBO,CAAkB;AAAA,IAAA;AAI1C,WAAAA,KAEZ,MAAM;AACX,MAAIP,KACF,qBAAqBA,CAAgB,GAGnCF,EAAa,cACAA,EAAa,UAErB,YAAY,QAAQ,CAASF,MAAAA,EAAM,MAAM,GAChDE,EAAa,YAAY;AAAA,IAC3B;AAAA,EACF,GACC,CAAC7B,GAAYW,CAAY,CAAC;AAGvB,QAAAmC,KAAuBC,EAAY,MAAM;AAC7C,IAAAnC,EAAgB,EAAI;AAAA,EACtB,GAAG,CAAE,CAAA,GAECoC,KAAmBD,EAAY,CAAC,MAAqD;AACzF,UAAMxB,IAAQ,EAAE,eACV0B,IAAW1B,EAAM,QACnB,gBAAgBA,EAAM,MAAM,OAAO,WAAWA,EAAM,MAAM,IAAI,MAC9D;AAEJ,IAAAd,EAAcwC,CAAQ,GACd,QAAA,MAAM,gBAAgB1B,EAAM,KAAK;AAAA,EAC3C,GAAG,CAAE,CAAA;AAEL,2BACG2B,GAAS,EAAA,gBAAe,UAAS,WAAW,GAAG,aAAY,UAC1D,UAAA;AAAA,IAAA,gBAAAC,EAACD,KAAS,WAAU,YAAW,QAAQtD,GAAO,SAASC,GACrD,UAAA;AAAA,MAAA,gBAAAuD;AAAA,QAACC;AAAAA,QAAA;AAAA,UACC,KAAKjD;AAAA,UACL,OAAAR;AAAA,UACA,QAAAC;AAAA,UACA,UAAAC;AAAA,UACA,MAAAC;AAAA,UACA,OAAK;AAAA,UACL,aAAW;AAAA,UACX,UAAQ;AAAA,UACR,aAAY;AAAA,UACZ,kBAAkB+C;AAAA,UAClB,SAASE;AAAA,QAAA;AAAA,MACX;AAAA,MACChD,KAAce,KACb,gBAAAqC;AAAA,QAACE;AAAAA,QAAA;AAAA,UACC,WAAU;AAAA,UACV,OAAOvC,EAAa;AAAA,UACpB,MAAMA,EAAa;AAAA,UACnB,QAAQA,EAAa;AAAA,UACrB,SAASA,EAAa;AAAA,UAEtB,UAAA,gBAAAqC;AAAA,YAACG;AAAAA,YAAA;AAAA,cACC,WAAU;AAAA,cACV,aAAY;AAAA,cACZ,eAAe;AAAA,cACf,MAAM;AAAA,cACN,SAAS;AAAA,cAET,4BAACC,GAAK,EAAA,WAAU,OAAM,QAAO,cAAa,UAE1C,mBAAA;AAAA,YAAA;AAAA,UACF;AAAA,QAAA;AAAA,MACF;AAAA,MAEF,gBAAAJ;AAAA,QAACK;AAAA,QAAA;AAAA,UACC,SAAAvC;AAAA,UACA,WAAAC;AAAA,UACA,SAASR;AAAA,UACT,cAAc,CAAClB;AAAA,QAAA;AAAA,MACjB;AAAA,IAAA,GACF;AAAA,IAGA,gBAAA2D,EAACM,IAAA,EAAmB,KAAKnD,GAAW,OAAO,CAAC,CAACP,GAAY;AAAA,IACxD,gBAAAoD,EAAA,SAAA,EAAM,KAAK9C,GAAiB,OAAO,EAAE,SAAS,OAAO,GAAG,UAAQ,IAAC,OAAK,IAAC,aAAW,IAAC;AAAA,IAGnFN,KAAca,KACb,gBAAAsC,EAACD,KAAS,gBAAe,UAAS,WAAW,KAC3C,UAAA;AAAA,MAAC,gBAAAE,EAAAI,GAAA,EAAK,WAAU,YAAW,UAA4B,gCAAA;AAAA,wBACtDA,GAAK,EAAA,WAAU,OAAM,QAAO,UAAS,UAGtC,gIAAA;AAAA,IAAA,GACF;AAAA,IAGDhD,KACC,gBAAA2C;AAAA,MAACD;AAAA,MAAA;AAAA,QACC,gBAAe;AAAA,QACf,UAAU;AAAA,QACV,OAAO;AAAA,QACP,aAAY;AAAA,QACZ,eAAe;AAAA,QACf,QAAQtD;AAAA,QAER,UAAA;AAAA,UAAA,gBAAAuD,EAACK,GAAK,EAAA,WAAU,YAAW,QAAO,OAAM,UAAA;AAAA,YAAA;AAAA,YAC9BhD;AAAA,UAAA,GACV;AAAA,WACEA,EAAW,SAAS,SAAS,KAAKA,EAAW,SAAS,OAAO,MAC5D,gBAAA2C,EAAAD,GAAA,EAAS,gBAAe,UAAS,WAAW,KAC3C,UAAA;AAAA,YAAC,gBAAAE,EAAAI,GAAA,EAAK,WAAU,YAAW,UAA8B,kCAAA;AAAA,YACzD,gBAAAL,EAACK,GAAK,EAAA,WAAU,OAAM,IAAG,MAAK,OAAO,EAAE,aAAa,OAAA,GAClD,UAAA;AAAA,cAAA,gBAAAL,EAAC,MAAG,EAAA,UAAA;AAAA,gBAAA;AAAA,gBACgB;AAAA,gBAClB,gBAAAC;AAAA,kBAAC;AAAA,kBAAA;AAAA,oBACC,MAAK;AAAA,oBACL,QAAO;AAAA,oBACP,KAAI;AAAA,oBACL,UAAA;AAAA,kBAAA;AAAA,gBAED;AAAA,gBAAK;AAAA,gBAAI;AAAA,cAAA,GAEX;AAAA,cACA,gBAAAA,EAAC,QAAG,UAA4D,+DAAA,CAAA;AAAA,gCAC/D,MAAG,EAAA,UAAA;AAAA,gBAAA;AAAA,gBAC8B;AAAA,gBAChC,gBAAAA,EAAC,UAAK,UAEN,qFAAA,CAAA;AAAA,cAAA,GACF;AAAA,YAAA,GACF;AAAA,UAAA,GACF;AAAA,QAAA;AAAA,MAAA;AAAA,IAEJ;AAAA,IAGDhC,KACC,gBAAAgC;AAAA,MAACF;AAAA,MAAA;AAAA,QACC,gBAAe;AAAA,QACf,UAAU;AAAA,QACV,OAAO;AAAA,QACP,aAAY;AAAA,QACZ,eAAe;AAAA,QACf,QAAQtD;AAAA,QAER,UAAC,gBAAAuD,EAAAK,GAAA,EAAK,WAAU,YAAW,QAAO,YAAW,UAAA;AAAA,UAAA;AAAA,UAC1BpC,EAAM;AAAA,QAAA,GACzB;AAAA,MAAA;AAAA,IACF;AAAA,IAGF,gBAAA+B;AAAA,MAACD;AAAA,MAAA;AAAA,QACC,gBAAe;AAAA,QACf,UAAU;AAAA,QACV,OAAO;AAAA,QACP,aAAY;AAAA,QACZ,eAAe;AAAA,QACf,QAAQtD;AAAA,QAER,UAAA;AAAA,UAAC,gBAAAwD,EAAAI,GAAA,EAAK,WAAU,YAAW,UAAgB,oBAAA;AAAA,UAC3C,gBAAAJ,EAACO,IAAU,EAAA,SAAS,EAAG,CAAA;AAAA,UACvB,gBAAAR;AAAA,YAACD;AAAA,YAAA;AAAA,cACC,gBAAe;AAAA,cACf,WAAS;AAAA,cACT,WAAW;AAAA,cACX,OAAO,EAAE,SAAS,QAAQ,qBAAqB,UAAU;AAAA,cAEzD,UAAA;AAAA,gBAAC,gBAAAC,EAAAK,GAAA,EAAK,WAAU,OACd,UAAA;AAAA,kBAAA,gBAAAJ,EAAC,YAAO,UAAc,iBAAA,CAAA;AAAA,kBAAS;AAAA,kBAAElC,EAAQ,eAAe,QAAQ;AAAA,gBAAA,GAClE;AAAA,gBACA,gBAAAiC,EAACK,GAAK,EAAA,WAAU,OACd,UAAA;AAAA,kBAAA,gBAAAJ,EAAC,YAAO,UAAW,cAAA,CAAA;AAAA,kBAAS;AAAA,kBAAElC,EAAQ,WAAW,QAAQ,CAAC;AAAA,kBAAE;AAAA,gBAAA,GAC9D;AAAA,gBACA,gBAAAiC,EAACK,GAAK,EAAA,WAAU,OACd,UAAA;AAAA,kBAAA,gBAAAJ,EAAC,YAAO,UAAa,gBAAA,CAAA;AAAA,kBAAS;AAAA,kBAAElC,EAAQ;AAAA,gBAAA,GAC1C;AAAA,gBACA,gBAAAiC,EAACK,GAAK,EAAA,WAAU,OACd,UAAA;AAAA,kBAAA,gBAAAJ,EAAC,YAAO,UAAS,YAAA,CAAA;AAAA,kBAAS;AAAA,kBAAElC,EAAQ;AAAA,gBAAA,GACtC;AAAA,gBACCvB,KAEG,gBAAAwD,EAAAS,GAAA,EAAA,UAAA;AAAA,kBAAC,gBAAAT,EAAAK,GAAA,EAAK,WAAU,OACd,UAAA;AAAA,oBAAA,gBAAAJ,EAAC,YAAO,UAAM,SAAA,CAAA;AAAA,oBAAS;AAAA,oBAAElC,EAAQ,UAAU,QAAQ;AAAA,kBAAA,GACrD;AAAA,kBACA,gBAAAiC,EAACK,GAAK,EAAA,WAAU,OACd,UAAA;AAAA,oBAAA,gBAAAJ,EAAC,YAAO,UAAI,OAAA,CAAA;AAAA,oBAAS;AAAA,oBAAElC,EAAQ,eAAe,QAAQ,CAAC;AAAA,kBAAA,GACzD;AAAA,gBAAA,GACF;AAAA,gBAEDxB,KAAkBwB,EAAQ,oBAAoB,KAE3C,gBAAAiC,EAAAS,GAAA,EAAA,UAAA;AAAA,kBAAC,gBAAAT,EAAAK,GAAA,EAAK,WAAU,OACd,UAAA;AAAA,oBAAA,gBAAAJ,EAAC,YAAO,UAAQ,WAAA,CAAA;AAAA,oBAAS;AAAA,oBAAElC,EAAQ;AAAA,kBAAA,GACrC;AAAA,kBACA,gBAAAiC,EAACK,GAAK,EAAA,WAAU,OACd,UAAA;AAAA,oBAAA,gBAAAJ,EAAC,YAAO,UAAW,cAAA,CAAA;AAAA,oBAAS;AAAA,qBAAGlC,EAAQ,oBAAoB,KAAK,QAAQ,CAAC;AAAA,oBAAE;AAAA,kBAAA,GAC7E;AAAA,gBAAA,GACF;AAAA,cAAA;AAAA,YAAA;AAAA,UAEJ;AAAA,QAAA;AAAA,MAAA;AAAA,IACF;AAAA,EACF,EAAA,CAAA;AAEJ,CAAC,GAED2C,KAAetE;"}
|
|
1
|
+
{"version":3,"file":"video-analysis.js","sources":["../../../../src/features/av/video-analysis/video-analysis.tsx"],"sourcesContent":["/* eslint-disable no-console */\nimport { memo, useCallback, useEffect, useRef, useState, type FC } from 'react';\n\nimport FlexView from '../../ui/layout/flex-view';\nimport Text from '../../ui/text/text';\nimport useVideoAnalysis from './hooks/use-video-analysis';\nimport VideoAnalysisOverlay from './video-analysis-overlay/video-analysis-overlay';\nimport * as Styled from './video-analysis-styled';\nimport type { IVideoAnalysisProps } from './video-analysis-types';\nimport Separator from '../../ui/separator/separator';\n\n/**\n * VideoAnalysis Component\n *\n * Real-time video quality analysis for online tutoring sessions using:\n * - MediaPipe FaceLandmarker (478 3D landmarks)\n * - TensorFlow.js emotion recognition\n *\n * Analyzes pre-recorded videos or live camera feeds to show visibility, camera angle,\n * lighting, sleep detection (teachers), and emotion recognition (students).\n *\n * Features:\n * - Face detection with 478 3D landmarks (better accuracy in low-light)\n * - Enhanced eye-closure detection (90%+ accuracy, even for frontal faces)\n * - 3D head pose tracking for camera angle analysis\n * - Visibility detection (optimal: 10-25%)\n * - Camera angle analysis (optimal/too-close/too-far/off-center/looking-away)\n * - Lighting quality (dark/good/bright)\n * - Sleep/drowsiness detection using Eye Aspect Ratio (EAR)\n * - Emotion recognition (happy/sad/neutral) with TensorFlow.js\n * - Region of Interest (ROI) cropping for screen recordings\n * - Away detection with configurable thresholds\n *\n * Performance Improvements (vs face-api.js):\n * - 50% faster (30-40 FPS vs 20-25 FPS)\n * - 20% better frontal face eye-closure detection (90% vs 70%)\n * - 42% better low-light detection (85% vs 60%)\n * - 23% better emotion accuracy (80% vs 65%)\n */\nconst VideoAnalysis: FC<IVideoAnalysisProps> = memo(function VideoAnalysis({\n videoUrl,\n detectEmotions = false,\n detectSleep = false,\n width = 640,\n height = 480,\n autoPlay = true,\n loop = true,\n cropRegion,\n awayDetectionEnabled = false,\n}) {\n const videoRef = useRef<HTMLVideoElement>(null);\n const croppedVideoRef = useRef<HTMLVideoElement>(null);\n const canvasRef = useRef<HTMLCanvasElement>(null);\n const [videoError, setVideoError] = useState<string | null>(null);\n const [isVideoReady, setIsVideoReady] = useState(false);\n const [canvasReady, setCanvasReady] = useState(false);\n const [overlayStyle, setOverlayStyle] = useState<{\n left: string;\n top: string;\n width: string;\n height: string;\n } | null>(null);\n\n // When cropRegion is provided, analyze the canvas directly (which shows cropped video)\n // Otherwise analyze the original video\n const analysisRef = cropRegion && canvasReady ? canvasRef : videoRef;\n\n // Video analysis hook V2 (MediaPipe + TensorFlow.js)\n const { metrics, isLoading, error } = useVideoAnalysis(analysisRef, {\n enabled: isVideoReady && (!cropRegion || canvasReady),\n detectEmotions,\n detectSleep,\n awayDetectionEnabled,\n });\n\n // Setup video source - use camera if no videoUrl provided\n useEffect(() => {\n const video = videoRef.current;\n\n const setupVideo = async () => {\n if (!video) return;\n\n setVideoError(null);\n setIsVideoReady(false);\n\n try {\n if (!videoUrl) {\n // No video URL provided - use camera\n const stream = await navigator.mediaDevices.getUserMedia({\n video: true,\n audio: false,\n });\n\n video.srcObject = stream;\n setIsVideoReady(true);\n } else {\n // Check for unsupported video sources\n if (\n videoUrl.includes('youtube.com') ||\n videoUrl.includes('youtu.be') ||\n videoUrl.includes('vimeo.com')\n ) {\n setVideoError(\n 'YouTube/Vimeo URLs are not supported. Please use direct video file URLs (MP4, WebM, OGG). Example: https://commondatastorage.googleapis.com/gtv-videos-bucket/sample/BigBuckBunny.mp4',\n );\n\n return;\n }\n\n // Use video URL\n video.src = videoUrl;\n video.load();\n }\n } catch (err) {\n const errorMessage = err instanceof Error ? err.message : 'Failed to setup video';\n\n setVideoError(errorMessage);\n console.error('Video setup error:', err);\n }\n };\n\n setupVideo();\n\n // Cleanup - capture video ref value\n return () => {\n if (video?.srcObject) {\n const stream = video.srcObject as MediaStream;\n\n stream.getTracks().forEach(track => track.stop());\n video.srcObject = null;\n }\n };\n }, [videoUrl]);\n\n // Crop video to region of interest\n useEffect(() => {\n if (!cropRegion || !isVideoReady) return;\n\n const video = videoRef.current;\n const canvas = canvasRef.current;\n const croppedVideo = croppedVideoRef.current;\n\n if (!video || !canvas || !croppedVideo) return;\n\n const ctx = canvas.getContext('2d');\n\n if (!ctx) return;\n\n let animationFrameId: number;\n let streamInitialized = false;\n let cropWidth = 0;\n let cropHeight = 0;\n let cropX = 0;\n let cropY = 0;\n let frameCount = 0;\n\n const updateCroppedFrame = () => {\n frameCount++;\n if (video.readyState === video.HAVE_ENOUGH_DATA) {\n const videoWidth = video.videoWidth;\n const videoHeight = video.videoHeight;\n\n if (videoWidth === 0 || videoHeight === 0) {\n animationFrameId = requestAnimationFrame(updateCroppedFrame);\n\n return;\n }\n\n // Calculate crop dimensions\n cropX = cropRegion.x * videoWidth;\n cropY = cropRegion.y * videoHeight;\n cropWidth = cropRegion.width * videoWidth;\n cropHeight = cropRegion.height * videoHeight;\n\n // Set canvas size to crop dimensions (only once)\n if (!streamInitialized) {\n canvas.width = cropWidth;\n canvas.height = cropHeight;\n\n console.log('[VideoAnalysis] Initializing crop region:', {\n cropX,\n cropY,\n cropWidth,\n cropHeight,\n originalWidth: videoWidth,\n originalHeight: videoHeight,\n videoCurrentTime: video.currentTime,\n });\n\n // Calculate the correct overlay position accounting for video display vs actual size\n const displayWidth = video.clientWidth;\n const displayHeight = video.clientHeight;\n const scaleX = displayWidth / videoWidth;\n const scaleY = displayHeight / videoHeight;\n\n console.log('[VideoAnalysis] Display vs actual dimensions:', {\n displayWidth,\n displayHeight,\n videoWidth,\n videoHeight,\n scaleX,\n scaleY,\n });\n\n // Update overlay style to match the actual cropped region on screen\n setOverlayStyle({\n left: `${cropX * scaleX}px`,\n top: `${cropY * scaleY}px`,\n width: `${cropWidth * scaleX}px`,\n height: `${cropHeight * scaleY}px`,\n });\n\n // Initialize stream with 30fps\n const stream = canvas.captureStream(30);\n\n croppedVideo.srcObject = stream;\n\n // Add event listeners to monitor cropped video state\n const logReadyState = () => {\n console.log(\n '[VideoAnalysis] Cropped video ready state changed:',\n croppedVideo.readyState,\n );\n };\n\n croppedVideo.addEventListener('loadedmetadata', () => {\n console.log('[VideoAnalysis] Cropped video metadata loaded');\n });\n croppedVideo.addEventListener('loadeddata', logReadyState);\n croppedVideo.addEventListener('canplay', logReadyState);\n croppedVideo.addEventListener('canplaythrough', logReadyState);\n\n // Ensure cropped video plays\n croppedVideo.play().catch(err => {\n console.error('[VideoAnalysis] Failed to play cropped video:', err);\n });\n\n streamInitialized = true;\n setCanvasReady(true); // Signal that canvas is ready for analysis\n\n console.log('[VideoAnalysis] Canvas ready for face analysis');\n }\n\n // CRITICAL: Draw cropped region to canvas on EVERY frame\n // This continuously updates the canvas with the latest cropped video frame\n ctx.drawImage(video, cropX, cropY, cropWidth, cropHeight, 0, 0, cropWidth, cropHeight);\n\n // Log every 60 frames (~1 second at 60fps) to verify continuous drawing\n if (frameCount % 60 === 0) {\n console.log('[VideoAnalysis] Canvas update:', {\n frameCount,\n videoTime: video.currentTime.toFixed(2),\n videoPlaying: !video.paused,\n });\n }\n } else {\n // Video not ready - log occasionally\n if (frameCount % 60 === 0) {\n console.log('[VideoAnalysis] Waiting for video, readyState:', video.readyState);\n }\n }\n\n // Continue animation loop regardless of video state\n animationFrameId = requestAnimationFrame(updateCroppedFrame);\n };\n\n // Start the animation loop\n updateCroppedFrame();\n\n return () => {\n if (animationFrameId) {\n cancelAnimationFrame(animationFrameId);\n }\n\n if (croppedVideo.srcObject) {\n const stream = croppedVideo.srcObject as MediaStream;\n\n stream.getTracks().forEach(track => track.stop());\n croppedVideo.srcObject = null;\n }\n };\n }, [cropRegion, isVideoReady]);\n\n // Handle video metadata loaded\n const handleLoadedMetadata = useCallback(() => {\n setIsVideoReady(true);\n }, []);\n\n const handleVideoError = useCallback((e: React.SyntheticEvent<HTMLVideoElement, Event>) => {\n const video = e.currentTarget;\n const errorMsg = video.error\n ? `Video error: ${video.error.message} (code: ${video.error.code})`\n : 'Unknown video error';\n\n setVideoError(errorMsg);\n console.error('Video error:', video.error);\n }, []);\n\n return (\n <FlexView $flexDirection=\"column\" $flexGapX={2} $alignItems=\"center\">\n <FlexView $position=\"relative\" $width={width} $height={height}>\n <Styled.Video\n ref={videoRef}\n width={width}\n height={height}\n autoPlay={autoPlay}\n loop={loop}\n muted\n playsInline\n controls\n crossOrigin=\"anonymous\"\n onLoadedMetadata={handleLoadedMetadata}\n onError={handleVideoError}\n />\n {cropRegion && overlayStyle && (\n <Styled.AnalysisRegionOverlay\n $position=\"absolute\"\n $left={overlayStyle.left}\n $top={overlayStyle.top}\n $width={overlayStyle.width}\n $height={overlayStyle.height}\n >\n <Styled.AnalysisRegionLabel\n $position=\"absolute\"\n $background=\"GREEN_4\"\n $borderRadius={3}\n $gap={2}\n $gutter={6}\n >\n <Text $renderAs=\"ub3\" $color=\"REAL_BLACK\">\n Analysis Region\n </Text>\n </Styled.AnalysisRegionLabel>\n </Styled.AnalysisRegionOverlay>\n )}\n <VideoAnalysisOverlay\n metrics={metrics}\n isLoading={isLoading}\n visible={isVideoReady}\n isLiveCamera={!videoUrl}\n />\n </FlexView>\n\n {/* Canvas for cropping - visible for debugging when cropRegion is active */}\n <Styled.DebugCanvas ref={canvasRef} $show={!!cropRegion} />\n <video ref={croppedVideoRef} style={{ display: 'none' }} autoPlay muted playsInline />\n\n {/* Debug info */}\n {cropRegion && canvasReady && (\n <FlexView $flexDirection=\"column\" $flexGapX={0.5}>\n <Text $renderAs=\"ab1-bold\">Debug: Canvas Being Analyzed</Text>\n <Text $renderAs=\"ub2\" $color=\"GREY_3\">\n This canvas shows the cropped region that face-api.js analyzes. If you don't see a\n clear face, adjust the cropRegion values.\n </Text>\n </FlexView>\n )}\n\n {videoError && (\n <FlexView\n $flexDirection=\"column\"\n $gutterX={1.5}\n $gapX={1}\n $background=\"RED_4\"\n $borderRadius={4}\n $width={width}\n >\n <Text $renderAs=\"ab1-bold\" $color=\"RED\">\n Error: {videoError}\n </Text>\n {(videoError.includes('YouTube') || videoError.includes('Vimeo')) && (\n <FlexView $flexDirection=\"column\" $flexGapX={0.5}>\n <Text $renderAs=\"ab2-bold\">How to get a direct video URL:</Text>\n <Text $renderAs=\"ub2\" as=\"ul\" style={{ paddingLeft: '20px' }}>\n <li>\n Use services like{' '}\n <a\n href=\"https://www.downloadhelper.net/\"\n target=\"_blank\"\n rel=\"noopener noreferrer\"\n >\n Video DownloadHelper\n </a>{' '}\n to download YouTube videos\n </li>\n <li>Host the downloaded MP4 file on your server or cloud storage</li>\n <li>\n Or use public test videos like:{' '}\n <code>\n https://commondatastorage.googleapis.com/gtv-videos-bucket/sample/BigBuckBunny.mp4\n </code>\n </li>\n </Text>\n </FlexView>\n )}\n </FlexView>\n )}\n\n {error && (\n <FlexView\n $flexDirection=\"column\"\n $gutterX={1.5}\n $gapX={1}\n $background=\"ORANGE_1\"\n $borderRadius={4}\n $width={width}\n >\n <Text $renderAs=\"ab1-bold\" $color=\"ORANGE_4\">\n Analysis Error: {error.message}\n </Text>\n </FlexView>\n )}\n\n <FlexView\n $flexDirection=\"column\"\n $gutterX={2}\n $gapX={1.5}\n $background=\"GREY_1\"\n $borderRadius={8}\n $width={width}\n >\n <Text $renderAs=\"ab1-bold\">Current Metrics:</Text>\n <Separator heightX={1} />\n <FlexView\n $flexDirection=\"row\"\n $flexWrap\n $flexGapX={1}\n style={{ display: 'grid', gridTemplateColumns: '1fr 1fr' }}\n >\n <Text $renderAs=\"ub2\">\n <strong>Face Detected:</strong> {metrics.faceDetected ? 'Yes' : 'No'}\n </Text>\n <Text $renderAs=\"ub2\">\n <strong>Visibility:</strong> {metrics.visibility.toFixed(1)}%\n </Text>\n <Text $renderAs=\"ub2\">\n <strong>Camera Angle:</strong> {metrics.cameraAngle}\n </Text>\n <Text $renderAs=\"ub2\">\n <strong>Lighting:</strong> {metrics.lighting}\n </Text>\n {detectSleep && (\n <>\n <Text $renderAs=\"ub2\">\n <strong>Awake:</strong> {metrics.isAwake ? 'Yes' : 'Drowsy'}\n </Text>\n <Text $renderAs=\"ub2\">\n <strong>EAR:</strong> {metrics.eyeAspectRatio.toFixed(3)}\n </Text>\n </>\n )}\n {detectEmotions && metrics.emotionConfidence > 0 && (\n <>\n <Text $renderAs=\"ub2\">\n <strong>Emotion:</strong> {metrics.emotion}\n </Text>\n <Text $renderAs=\"ub2\">\n <strong>Confidence:</strong> {(metrics.emotionConfidence * 100).toFixed(0)}%\n </Text>\n </>\n )}\n </FlexView>\n </FlexView>\n </FlexView>\n );\n});\n\nexport default VideoAnalysis;\n"],"names":["VideoAnalysis","memo","videoUrl","detectEmotions","detectSleep","width","height","autoPlay","loop","cropRegion","awayDetectionEnabled","videoRef","useRef","croppedVideoRef","canvasRef","videoError","setVideoError","useState","isVideoReady","setIsVideoReady","canvasReady","setCanvasReady","overlayStyle","setOverlayStyle","analysisRef","metrics","isLoading","error","useVideoAnalysis","useEffect","video","stream","err","errorMessage","track","canvas","croppedVideo","ctx","animationFrameId","streamInitialized","cropWidth","cropHeight","cropX","cropY","frameCount","updateCroppedFrame","videoWidth","videoHeight","displayWidth","displayHeight","scaleX","scaleY","logReadyState","handleLoadedMetadata","useCallback","handleVideoError","errorMsg","FlexView","jsxs","jsx","Styled.Video","Styled.AnalysisRegionOverlay","Styled.AnalysisRegionLabel","Text","VideoAnalysisOverlay","Styled.DebugCanvas","Separator","Fragment","VideoAnalysis$1"],"mappings":";;;;;;;;AAuCA,MAAMA,KAAyCC,GAAK,SAAuB;AAAA,EACzE,UAAAC;AAAA,EACA,gBAAAC,IAAiB;AAAA,EACjB,aAAAC,IAAc;AAAA,EACd,OAAAC,IAAQ;AAAA,EACR,QAAAC,IAAS;AAAA,EACT,UAAAC,IAAW;AAAA,EACX,MAAAC,IAAO;AAAA,EACP,YAAAC;AAAA,EACA,sBAAAC,IAAuB;AACzB,GAAG;AACK,QAAAC,IAAWC,EAAyB,IAAI,GACxCC,IAAkBD,EAAyB,IAAI,GAC/CE,IAAYF,EAA0B,IAAI,GAC1C,CAACG,GAAYC,CAAa,IAAIC,EAAwB,IAAI,GAC1D,CAACC,GAAcC,CAAe,IAAIF,EAAS,EAAK,GAChD,CAACG,GAAaC,CAAc,IAAIJ,EAAS,EAAK,GAC9C,CAACK,GAAcC,CAAe,IAAIN,EAK9B,IAAI,GAIRO,IAAcf,KAAcW,IAAcN,IAAYH,GAGtD,EAAE,SAAAc,GAAS,WAAAC,GAAW,OAAAC,EAAM,IAAIC,GAAiBJ,GAAa;AAAA,IAClE,SAASN,MAAiB,CAACT,KAAcW;AAAA,IACzC,gBAAAjB;AAAA,IACA,aAAAC;AAAA,IACA,sBAAAM;AAAA,EAAA,CACD;AAGD,EAAAmB,EAAU,MAAM;AACd,UAAMC,IAAQnB,EAAS;AA4CZ,YA1CQ,YAAY;AAC7B,UAAKmB,GAEL;AAAA,QAAAd,EAAc,IAAI,GAClBG,EAAgB,EAAK;AAEjB,YAAA;AACF,cAAKjB,GASE;AAGH,gBAAAA,EAAS,SAAS,aAAa,KAC/BA,EAAS,SAAS,UAAU,KAC5BA,EAAS,SAAS,WAAW,GAC7B;AACA,cAAAc;AAAA,gBACE;AAAA,cAAA;AAGF;AAAA,YACF;AAGA,YAAAc,EAAM,MAAM5B,GACZ4B,EAAM,KAAK;AAAA,UACb,OA1Be;AAEb,kBAAMC,IAAS,MAAM,UAAU,aAAa,aAAa;AAAA,cACvD,OAAO;AAAA,cACP,OAAO;AAAA,YAAA,CACR;AAED,YAAAD,EAAM,YAAYC,GAClBZ,EAAgB,EAAI;AAAA,UAAA;AAAA,iBAmBfa,GAAK;AACZ,gBAAMC,IAAeD,aAAe,QAAQA,EAAI,UAAU;AAE1D,UAAAhB,EAAciB,CAAY,GAClB,QAAA,MAAM,sBAAsBD,CAAG;AAAA,QACzC;AAAA;AAAA,IAAA,MAMK,MAAM;AACX,MAAIF,KAAA,QAAAA,EAAO,cACMA,EAAM,UAEd,YAAY,QAAQ,CAASI,MAAAA,EAAM,MAAM,GAChDJ,EAAM,YAAY;AAAA,IACpB;AAAA,EACF,GACC,CAAC5B,CAAQ,CAAC,GAGb2B,EAAU,MAAM;AACV,QAAA,CAACpB,KAAc,CAACS,EAAc;AAElC,UAAMY,IAAQnB,EAAS,SACjBwB,IAASrB,EAAU,SACnBsB,IAAevB,EAAgB;AAErC,QAAI,CAACiB,KAAS,CAACK,KAAU,CAACC,EAAc;AAElC,UAAAC,IAAMF,EAAO,WAAW,IAAI;AAElC,QAAI,CAACE,EAAK;AAEN,QAAAC,GACAC,IAAoB,IACpBC,IAAY,GACZC,IAAa,GACbC,IAAQ,GACRC,IAAQ,GACRC,IAAa;AAEjB,UAAMC,IAAqB,MAAM;AAE3B,UADJD,KACId,EAAM,eAAeA,EAAM,kBAAkB;AAC/C,cAAMgB,IAAahB,EAAM,YACnBiB,IAAcjB,EAAM;AAEtB,YAAAgB,MAAe,KAAKC,MAAgB,GAAG;AACzC,UAAAT,IAAmB,sBAAsBO,CAAkB;AAE3D;AAAA,QACF;AASA,YANAH,IAAQjC,EAAW,IAAIqC,GACvBH,IAAQlC,EAAW,IAAIsC,GACvBP,IAAY/B,EAAW,QAAQqC,GAC/BL,IAAahC,EAAW,SAASsC,GAG7B,CAACR,GAAmB;AACtB,UAAAJ,EAAO,QAAQK,GACfL,EAAO,SAASM,GAEhB,QAAQ,IAAI,6CAA6C;AAAA,YACvD,OAAAC;AAAA,YACA,OAAAC;AAAA,YACA,WAAAH;AAAA,YACA,YAAAC;AAAA,YACA,eAAeK;AAAA,YACf,gBAAgBC;AAAA,YAChB,kBAAkBjB,EAAM;AAAA,UAAA,CACzB;AAGD,gBAAMkB,IAAelB,EAAM,aACrBmB,IAAgBnB,EAAM,cACtBoB,IAASF,IAAeF,GACxBK,IAASF,IAAgBF;AAE/B,kBAAQ,IAAI,iDAAiD;AAAA,YAC3D,cAAAC;AAAA,YACA,eAAAC;AAAA,YACA,YAAAH;AAAA,YACA,aAAAC;AAAA,YACA,QAAAG;AAAA,YACA,QAAAC;AAAA,UAAA,CACD,GAGe5B,EAAA;AAAA,YACd,MAAM,GAAGmB,IAAQQ,CAAM;AAAA,YACvB,KAAK,GAAGP,IAAQQ,CAAM;AAAA,YACtB,OAAO,GAAGX,IAAYU,CAAM;AAAA,YAC5B,QAAQ,GAAGT,IAAaU,CAAM;AAAA,UAAA,CAC/B;AAGK,gBAAApB,KAASI,EAAO,cAAc,EAAE;AAEtC,UAAAC,EAAa,YAAYL;AAGzB,gBAAMqB,IAAgB,MAAM;AAClB,oBAAA;AAAA,cACN;AAAA,cACAhB,EAAa;AAAA,YAAA;AAAA,UACf;AAGW,UAAAA,EAAA,iBAAiB,kBAAkB,MAAM;AACpD,oBAAQ,IAAI,+CAA+C;AAAA,UAAA,CAC5D,GACYA,EAAA,iBAAiB,cAAcgB,CAAa,GAC5ChB,EAAA,iBAAiB,WAAWgB,CAAa,GACzChB,EAAA,iBAAiB,kBAAkBgB,CAAa,GAGhDhB,EAAA,KAAA,EAAO,MAAM,CAAOJ,OAAA;AACvB,oBAAA,MAAM,iDAAiDA,EAAG;AAAA,UAAA,CACnE,GAEmBO,IAAA,IACpBlB,EAAe,EAAI,GAEnB,QAAQ,IAAI,gDAAgD;AAAA,QAC9D;AAII,QAAAgB,EAAA,UAAUP,GAAOY,GAAOC,GAAOH,GAAWC,GAAY,GAAG,GAAGD,GAAWC,CAAU,GAGjFG,IAAa,OAAO,KACtB,QAAQ,IAAI,kCAAkC;AAAA,UAC5C,YAAAA;AAAA,UACA,WAAWd,EAAM,YAAY,QAAQ,CAAC;AAAA,UACtC,cAAc,CAACA,EAAM;AAAA,QAAA,CACtB;AAAA,MACH;AAGI,QAAAc,IAAa,OAAO,KACd,QAAA,IAAI,kDAAkDd,EAAM,UAAU;AAKlF,MAAAQ,IAAmB,sBAAsBO,CAAkB;AAAA,IAAA;AAI1C,WAAAA,KAEZ,MAAM;AACX,MAAIP,KACF,qBAAqBA,CAAgB,GAGnCF,EAAa,cACAA,EAAa,UAErB,YAAY,QAAQ,CAASF,MAAAA,EAAM,MAAM,GAChDE,EAAa,YAAY;AAAA,IAC3B;AAAA,EACF,GACC,CAAC3B,GAAYS,CAAY,CAAC;AAGvB,QAAAmC,IAAuBC,EAAY,MAAM;AAC7C,IAAAnC,EAAgB,EAAI;AAAA,EACtB,GAAG,CAAE,CAAA,GAECoC,IAAmBD,EAAY,CAAC,MAAqD;AACzF,UAAMxB,IAAQ,EAAE,eACV0B,IAAW1B,EAAM,QACnB,gBAAgBA,EAAM,MAAM,OAAO,WAAWA,EAAM,MAAM,IAAI,MAC9D;AAEJ,IAAAd,EAAcwC,CAAQ,GACd,QAAA,MAAM,gBAAgB1B,EAAM,KAAK;AAAA,EAC3C,GAAG,CAAE,CAAA;AAEL,2BACG2B,GAAS,EAAA,gBAAe,UAAS,WAAW,GAAG,aAAY,UAC1D,UAAA;AAAA,IAAA,gBAAAC,EAACD,KAAS,WAAU,YAAW,QAAQpD,GAAO,SAASC,GACrD,UAAA;AAAA,MAAA,gBAAAqD;AAAA,QAACC;AAAAA,QAAA;AAAA,UACC,KAAKjD;AAAA,UACL,OAAAN;AAAA,UACA,QAAAC;AAAA,UACA,UAAAC;AAAA,UACA,MAAAC;AAAA,UACA,OAAK;AAAA,UACL,aAAW;AAAA,UACX,UAAQ;AAAA,UACR,aAAY;AAAA,UACZ,kBAAkB6C;AAAA,UAClB,SAASE;AAAA,QAAA;AAAA,MACX;AAAA,MACC9C,KAAca,KACb,gBAAAqC;AAAA,QAACE;AAAAA,QAAA;AAAA,UACC,WAAU;AAAA,UACV,OAAOvC,EAAa;AAAA,UACpB,MAAMA,EAAa;AAAA,UACnB,QAAQA,EAAa;AAAA,UACrB,SAASA,EAAa;AAAA,UAEtB,UAAA,gBAAAqC;AAAA,YAACG;AAAAA,YAAA;AAAA,cACC,WAAU;AAAA,cACV,aAAY;AAAA,cACZ,eAAe;AAAA,cACf,MAAM;AAAA,cACN,SAAS;AAAA,cAET,4BAACC,GAAK,EAAA,WAAU,OAAM,QAAO,cAAa,UAE1C,mBAAA;AAAA,YAAA;AAAA,UACF;AAAA,QAAA;AAAA,MACF;AAAA,MAEF,gBAAAJ;AAAA,QAACK;AAAA,QAAA;AAAA,UACC,SAAAvC;AAAA,UACA,WAAAC;AAAA,UACA,SAASR;AAAA,UACT,cAAc,CAAChB;AAAA,QAAA;AAAA,MACjB;AAAA,IAAA,GACF;AAAA,IAGA,gBAAAyD,EAACM,IAAA,EAAmB,KAAKnD,GAAW,OAAO,CAAC,CAACL,GAAY;AAAA,IACxD,gBAAAkD,EAAA,SAAA,EAAM,KAAK9C,GAAiB,OAAO,EAAE,SAAS,OAAO,GAAG,UAAQ,IAAC,OAAK,IAAC,aAAW,IAAC;AAAA,IAGnFJ,KAAcW,KACb,gBAAAsC,EAACD,KAAS,gBAAe,UAAS,WAAW,KAC3C,UAAA;AAAA,MAAC,gBAAAE,EAAAI,GAAA,EAAK,WAAU,YAAW,UAA4B,gCAAA;AAAA,wBACtDA,GAAK,EAAA,WAAU,OAAM,QAAO,UAAS,UAGtC,gIAAA;AAAA,IAAA,GACF;AAAA,IAGDhD,KACC,gBAAA2C;AAAA,MAACD;AAAA,MAAA;AAAA,QACC,gBAAe;AAAA,QACf,UAAU;AAAA,QACV,OAAO;AAAA,QACP,aAAY;AAAA,QACZ,eAAe;AAAA,QACf,QAAQpD;AAAA,QAER,UAAA;AAAA,UAAA,gBAAAqD,EAACK,GAAK,EAAA,WAAU,YAAW,QAAO,OAAM,UAAA;AAAA,YAAA;AAAA,YAC9BhD;AAAA,UAAA,GACV;AAAA,WACEA,EAAW,SAAS,SAAS,KAAKA,EAAW,SAAS,OAAO,MAC5D,gBAAA2C,EAAAD,GAAA,EAAS,gBAAe,UAAS,WAAW,KAC3C,UAAA;AAAA,YAAC,gBAAAE,EAAAI,GAAA,EAAK,WAAU,YAAW,UAA8B,kCAAA;AAAA,YACzD,gBAAAL,EAACK,GAAK,EAAA,WAAU,OAAM,IAAG,MAAK,OAAO,EAAE,aAAa,OAAA,GAClD,UAAA;AAAA,cAAA,gBAAAL,EAAC,MAAG,EAAA,UAAA;AAAA,gBAAA;AAAA,gBACgB;AAAA,gBAClB,gBAAAC;AAAA,kBAAC;AAAA,kBAAA;AAAA,oBACC,MAAK;AAAA,oBACL,QAAO;AAAA,oBACP,KAAI;AAAA,oBACL,UAAA;AAAA,kBAAA;AAAA,gBAED;AAAA,gBAAK;AAAA,gBAAI;AAAA,cAAA,GAEX;AAAA,cACA,gBAAAA,EAAC,QAAG,UAA4D,+DAAA,CAAA;AAAA,gCAC/D,MAAG,EAAA,UAAA;AAAA,gBAAA;AAAA,gBAC8B;AAAA,gBAChC,gBAAAA,EAAC,UAAK,UAEN,qFAAA,CAAA;AAAA,cAAA,GACF;AAAA,YAAA,GACF;AAAA,UAAA,GACF;AAAA,QAAA;AAAA,MAAA;AAAA,IAEJ;AAAA,IAGDhC,KACC,gBAAAgC;AAAA,MAACF;AAAA,MAAA;AAAA,QACC,gBAAe;AAAA,QACf,UAAU;AAAA,QACV,OAAO;AAAA,QACP,aAAY;AAAA,QACZ,eAAe;AAAA,QACf,QAAQpD;AAAA,QAER,UAAC,gBAAAqD,EAAAK,GAAA,EAAK,WAAU,YAAW,QAAO,YAAW,UAAA;AAAA,UAAA;AAAA,UAC1BpC,EAAM;AAAA,QAAA,GACzB;AAAA,MAAA;AAAA,IACF;AAAA,IAGF,gBAAA+B;AAAA,MAACD;AAAA,MAAA;AAAA,QACC,gBAAe;AAAA,QACf,UAAU;AAAA,QACV,OAAO;AAAA,QACP,aAAY;AAAA,QACZ,eAAe;AAAA,QACf,QAAQpD;AAAA,QAER,UAAA;AAAA,UAAC,gBAAAsD,EAAAI,GAAA,EAAK,WAAU,YAAW,UAAgB,oBAAA;AAAA,UAC3C,gBAAAJ,EAACO,IAAU,EAAA,SAAS,EAAG,CAAA;AAAA,UACvB,gBAAAR;AAAA,YAACD;AAAA,YAAA;AAAA,cACC,gBAAe;AAAA,cACf,WAAS;AAAA,cACT,WAAW;AAAA,cACX,OAAO,EAAE,SAAS,QAAQ,qBAAqB,UAAU;AAAA,cAEzD,UAAA;AAAA,gBAAC,gBAAAC,EAAAK,GAAA,EAAK,WAAU,OACd,UAAA;AAAA,kBAAA,gBAAAJ,EAAC,YAAO,UAAc,iBAAA,CAAA;AAAA,kBAAS;AAAA,kBAAElC,EAAQ,eAAe,QAAQ;AAAA,gBAAA,GAClE;AAAA,gBACA,gBAAAiC,EAACK,GAAK,EAAA,WAAU,OACd,UAAA;AAAA,kBAAA,gBAAAJ,EAAC,YAAO,UAAW,cAAA,CAAA;AAAA,kBAAS;AAAA,kBAAElC,EAAQ,WAAW,QAAQ,CAAC;AAAA,kBAAE;AAAA,gBAAA,GAC9D;AAAA,gBACA,gBAAAiC,EAACK,GAAK,EAAA,WAAU,OACd,UAAA;AAAA,kBAAA,gBAAAJ,EAAC,YAAO,UAAa,gBAAA,CAAA;AAAA,kBAAS;AAAA,kBAAElC,EAAQ;AAAA,gBAAA,GAC1C;AAAA,gBACA,gBAAAiC,EAACK,GAAK,EAAA,WAAU,OACd,UAAA;AAAA,kBAAA,gBAAAJ,EAAC,YAAO,UAAS,YAAA,CAAA;AAAA,kBAAS;AAAA,kBAAElC,EAAQ;AAAA,gBAAA,GACtC;AAAA,gBACCrB,KAEG,gBAAAsD,EAAAS,GAAA,EAAA,UAAA;AAAA,kBAAC,gBAAAT,EAAAK,GAAA,EAAK,WAAU,OACd,UAAA;AAAA,oBAAA,gBAAAJ,EAAC,YAAO,UAAM,SAAA,CAAA;AAAA,oBAAS;AAAA,oBAAElC,EAAQ,UAAU,QAAQ;AAAA,kBAAA,GACrD;AAAA,kBACA,gBAAAiC,EAACK,GAAK,EAAA,WAAU,OACd,UAAA;AAAA,oBAAA,gBAAAJ,EAAC,YAAO,UAAI,OAAA,CAAA;AAAA,oBAAS;AAAA,oBAAElC,EAAQ,eAAe,QAAQ,CAAC;AAAA,kBAAA,GACzD;AAAA,gBAAA,GACF;AAAA,gBAEDtB,KAAkBsB,EAAQ,oBAAoB,KAE3C,gBAAAiC,EAAAS,GAAA,EAAA,UAAA;AAAA,kBAAC,gBAAAT,EAAAK,GAAA,EAAK,WAAU,OACd,UAAA;AAAA,oBAAA,gBAAAJ,EAAC,YAAO,UAAQ,WAAA,CAAA;AAAA,oBAAS;AAAA,oBAAElC,EAAQ;AAAA,kBAAA,GACrC;AAAA,kBACA,gBAAAiC,EAACK,GAAK,EAAA,WAAU,OACd,UAAA;AAAA,oBAAA,gBAAAJ,EAAC,YAAO,UAAW,cAAA,CAAA;AAAA,oBAAS;AAAA,qBAAGlC,EAAQ,oBAAoB,KAAK,QAAQ,CAAC;AAAA,oBAAE;AAAA,kBAAA,GAC7E;AAAA,gBAAA,GACF;AAAA,cAAA;AAAA,YAAA;AAAA,UAEJ;AAAA,QAAA;AAAA,MAAA;AAAA,IACF;AAAA,EACF,EAAA,CAAA;AAEJ,CAAC,GAED2C,KAAepE;"}
|
package/dist/index.d.ts
CHANGED
|
@@ -3465,14 +3465,6 @@ declare interface IOnChapterExitWarningProps {
|
|
|
3465
3465
|
onSuccess?: () => void;
|
|
3466
3466
|
}
|
|
3467
3467
|
|
|
3468
|
-
/**
|
|
3469
|
-
* Callback triggered when user is away from camera
|
|
3470
|
-
* @param awayDurationMs - How long the user has been away (in milliseconds)
|
|
3471
|
-
*/
|
|
3472
|
-
declare interface IOnUserAwayCallback {
|
|
3473
|
-
(awayDurationMs: number): void;
|
|
3474
|
-
}
|
|
3475
|
-
|
|
3476
3468
|
declare interface IOnViewSummaryParams extends IStudentProfileSummary {
|
|
3477
3469
|
userAttemptId?: string | null;
|
|
3478
3470
|
}
|
|
@@ -5292,6 +5284,7 @@ export declare interface IVideoAnalysisMetrics {
|
|
|
5292
5284
|
emotionConfidence: number;
|
|
5293
5285
|
isAway: boolean;
|
|
5294
5286
|
awayDurationMs: number;
|
|
5287
|
+
sleepDurationMs: number;
|
|
5295
5288
|
}
|
|
5296
5289
|
|
|
5297
5290
|
export declare interface IVideoAnalysisOptions {
|
|
@@ -5300,8 +5293,6 @@ export declare interface IVideoAnalysisOptions {
|
|
|
5300
5293
|
detectSleep?: boolean;
|
|
5301
5294
|
useCanvas?: boolean;
|
|
5302
5295
|
awayDetectionEnabled?: boolean;
|
|
5303
|
-
awayDurationThreshold?: number;
|
|
5304
|
-
onUserAway?: (awayDurationMs: number) => void;
|
|
5305
5296
|
}
|
|
5306
5297
|
|
|
5307
5298
|
declare interface IVideoAnalysisOverlayProps {
|
|
@@ -5321,8 +5312,6 @@ declare interface IVideoAnalysisProps {
|
|
|
5321
5312
|
loop?: boolean;
|
|
5322
5313
|
cropRegion?: ICropRegion;
|
|
5323
5314
|
awayDetectionEnabled?: boolean;
|
|
5324
|
-
awayDurationThreshold?: number;
|
|
5325
|
-
onUserAway?: IOnUserAwayCallback;
|
|
5326
5315
|
}
|
|
5327
5316
|
|
|
5328
5317
|
declare interface IViewport {
|