@wix/legends-platform-sdk 1.31.0 → 1.33.0

package/dist/esm/components/ChromaKey/ChromaKey.js

@@ -1,5 +1,7 @@
 import _objectWithoutPropertiesLoose from "@babel/runtime/helpers/objectWithoutPropertiesLoose";
 const _excluded = ["className", "style", "width", "height", "pip", "pipFallback", "onPiPEnter", "onPiPExit", "onPiPError"];
+import { useStartupAttempt } from '../../startup-telemetry/useStartupAttempt';
+import { recordCanvasFrame } from '../../startup-telemetry/frame';
 import React, { useEffect, useRef, useState } from 'react';
 import { chromaKey } from './chroma-key';
 import { pipActiveAtom, usePictureInPictureAuto } from '../../hooks/usePictureInPicture';
@@ -35,6 +37,8 @@ export function ChromaKey(props) {
   const _useState2 = useState(null),
     isWebGLAvailable = _useState2[0],
     setIsWebGLAvailable = _useState2[1];
+  const startupAttempt = useStartupAttempt();
+  const visibleAttempt = children.props.type === 'agent' ? startupAttempt : undefined;
   const $video = useRef(null);
   const $canvas = useRef(null);
   const isPiPActive = useAtomValue(pipActiveAtom);
@@ -59,7 +63,12 @@ export function ChromaKey(props) {
       spill,
       similarity,
       smoothness,
-      background
+      background,
+      onFrameDrawn: () => {
+        if ($canvas.current && $video.current) {
+          recordCanvasFrame(visibleAttempt, $canvas.current, $video.current);
+        }
+      }
     });
     setIsWebGLAvailable(process !== null);
     if (!process) {
@@ -86,7 +95,7 @@ export function ChromaKey(props) {
       isActive = false;
       cancelAnimationFrame(rafId);
     };
-  }, [color, spill, similarity, smoothness, isPlayable, debug, background]);
+  }, [color, spill, similarity, smoothness, isPlayable, debug, background, visibleAttempt]);
   return /*#__PURE__*/React.createElement("div", {
     className: className,
     style: {
@@ -111,6 +120,7 @@ export function ChromaKey(props) {
   }), !isPlayable && fallback, showPipFallback && pipFallback, /*#__PURE__*/React.cloneElement(children, {
     ...restVideoProps,
     // cloneElement retains original props; only ChromaKey should own PiP.
+    startupRenderSurface: useVideoFallback ? 'video' : 'canvas',
     pip: false,
     pipFallback: undefined,
     onPiPEnter: undefined,

package/dist/esm/components/ChromaKey/ChromaKey.js.map

@@ -1 +1 @@
-{"version":3,"names":["React","useEffect","useRef","useState","chromaKey","pipActiveAtom","usePictureInPictureAuto","useAtomValue","ChromaKey","props","_props$debug","debug","children","_props$color","color","_props$spill","spill","_props$smoothness","smoothness","_props$similarity","similarity","fallback","background","_children$props","className","style","width","height","pip","pipFallback","onPiPEnter","onPiPExit","onPiPError","restVideoProps","_objectWithoutPropertiesLoose","_excluded","_useState","isPlayable","setIsPlayable","_useState2","isWebGLAvailable","setIsWebGLAvailable","$video","$canvas","isPiPActive","useVideoFallback","pipElement","current","element","onEnter","onExit","onError","showPipFallback","console","warn","process","log","rafId","isActive","loop","requestAnimationFrame","cancelAnimationFrame","createElement","ref","display","position","opacity","pointerEvents","visibility","zIndex","cloneElement","undefined","onCanPlay","checkPlayable","onLoadedMetadata","event","readyState","currentTarget","HTMLMediaElement","HAVE_FUTURE_DATA"],"sources":["../../../../src/components/ChromaKey/ChromaKey.tsx"],"sourcesContent":["import React, { useEffect, useRef, useState } from 'react';\n\nimport { chromaKey } from './chroma-key';\nimport {\n  pipActiveAtom,\n  usePictureInPictureAuto,\n} from '../../hooks/usePictureInPicture';\nimport { useAtomValue } from 'jotai';\n\ninterface ChromaKeyProps extends React.HTMLAttributes<HTMLCanvasElement> {\n  debug?: boolean;\n  color?: string;\n  spill?: number;\n  similarity?: number;\n  smoothness?: number;\n  fallback?: React.ReactNode;\n  background?: string | null;\n  children: React.ReactElement<\n    React.VideoHTMLAttributes<HTMLVideoElement> & {\n      ref?: React.Ref<HTMLVideoElement>;\n      pip?: boolean;\n      pipFallback?: React.ReactNode;\n      onPiPEnter?: () => void;\n      onPiPExit?: () => void;\n      onPiPError?: (error: Error) => void;\n    }\n  >;\n}\n\nexport function ChromaKey(props: ChromaKeyProps) {\n  const {\n    debug = false,\n    children,\n    color = '#03ff9c',\n    spill = 0.1,\n    smoothness = 0.15,\n    similarity = 0.3,\n    fallback,\n    background,\n  } = props;\n\n  const {\n    className,\n    style,\n    width,\n    height,\n    pip,\n    pipFallback,\n    onPiPEnter,\n    onPiPExit,\n    onPiPError,\n    ...restVideoProps\n  } = children.props;\n  const [isPlayable, setIsPlayable] = useState(false);\n  const [isWebGLAvailable, setIsWebGLAvailable] = useState<boolean | null>(\n    null,\n  );\n\n  const $video = useRef<HTMLVideoElement>(null);\n  const $canvas = useRef<HTMLCanvasElement>(null);\n  const isPiPActive = useAtomValue(pipActiveAtom);\n  const useVideoFallback = isWebGLAvailable === false;\n  const pipElement =\n    isWebGLAvailable === null\n      ? null\n      : useVideoFallback\n      ? $video.current\n      : $canvas.current;\n\n  // Setup PiP - hook handles canvas → video conversion internally\n  usePictureInPictureAuto({\n    element: pip && isPlayable ? pipElement : null,\n    onEnter: onPiPEnter,\n    onExit: onPiPExit,\n    onError: onPiPError,\n  });\n\n  const showPipFallback = pip && isPiPActive;\n\n  useEffect(() => {\n    if (!$video.current || !$canvas.current || !isPlayable) {\n      debug && console.warn('Video or canvas is not initialized');\n\n      return;\n    }\n\n    const process = chromaKey($video.current, $canvas.current, {\n      color,\n      spill,\n      similarity,\n      smoothness,\n      background,\n    });\n    setIsWebGLAvailable(process !== null);\n    if (!process) {\n      return;\n    }\n\n    debug &&\n      console.log('Chroma key initialized', $video.current, $canvas.current, {\n        color,\n        spill,\n        similarity,\n        smoothness,\n      });\n\n    let rafId: number;\n    let isActive = true;\n\n    function loop() {\n      if (!isActive) {\n        return;\n      }\n      process?.();\n      rafId = requestAnimationFrame(loop);\n    }\n\n    rafId = requestAnimationFrame(loop);\n\n    return () => {\n      debug && console.log('Chroma key process cleanup');\n      isActive = false;\n      cancelAnimationFrame(rafId);\n    };\n  }, [color, spill, similarity, smoothness, isPlayable, debug, background]);\n\n  return (\n    <div\n      className={className}\n      style={{\n        ...style,\n        width,\n        height,\n      }}\n    >\n      <canvas\n        ref={$canvas}\n        style={{\n          width: '100%',\n          height: '100%',\n          display: useVideoFallback || showPipFallback ? 'none' : 'block',\n          ...(isPlayable\n            ? {}\n            : {\n                position: 'absolute',\n                opacity: 0,\n                pointerEvents: 'none',\n                visibility: 'hidden',\n                zIndex: -1,\n              }),\n        }}\n      />\n      {!isPlayable && fallback}\n      {showPipFallback && pipFallback}\n\n      {React.cloneElement(children, {\n        ...restVideoProps,\n        // cloneElement retains original props; only ChromaKey should own PiP.\n        pip: false,\n        pipFallback: undefined,\n        onPiPEnter: undefined,\n        onPiPExit: undefined,\n        onPiPError: undefined,\n        ref: $video,\n        style: useVideoFallback\n          ? {\n              width: '100%',\n              height: '100%',\n              ...style,\n              display: isPlayable && !showPipFallback ? style?.display : 'none',\n            }\n          : { display: 'none' },\n        onCanPlay: checkPlayable,\n        onLoadedMetadata: checkPlayable,\n      })}\n    </div>\n  );\n\n  function checkPlayable(event: React.SyntheticEvent<HTMLVideoElement>) {\n    const readyState = event.currentTarget.readyState;\n\n    setIsPlayable(readyState >= HTMLMediaElement.HAVE_FUTURE_DATA);\n  }\n}\n"],"mappings":";;AAAA,OAAOA,KAAK,IAAIC,SAAS,EAAEC,MAAM,EAAEC,QAAQ,QAAQ,OAAO;AAE1D,SAASC,SAAS,QAAQ,cAAc;AACxC,SACEC,aAAa,EACbC,uBAAuB,QAClB,iCAAiC;AACxC,SAASC,YAAY,QAAQ,OAAO;AAsBpC,OAAO,SAASC,SAASA,CAACC,KAAqB,EAAE;EAC/C,MAAAC,YAAA,GASID,KAAK,CARPE,KAAK;IAALA,KAAK,GAAAD,YAAA,cAAG,KAAK,GAAAA,YAAA;IACbE,QAAQ,GAONH,KAAK,CAPPG,QAAQ;IAAAC,YAAA,GAONJ,KAAK,CANPK,KAAK;IAALA,KAAK,GAAAD,YAAA,cAAG,SAAS,GAAAA,YAAA;IAAAE,YAAA,GAMfN,KAAK,CALPO,KAAK;IAALA,KAAK,GAAAD,YAAA,cAAG,GAAG,GAAAA,YAAA;IAAAE,iBAAA,GAKTR,KAAK,CAJPS,UAAU;IAAVA,UAAU,GAAAD,iBAAA,cAAG,IAAI,GAAAA,iBAAA;IAAAE,iBAAA,GAIfV,KAAK,CAHPW,UAAU;IAAVA,UAAU,GAAAD,iBAAA,cAAG,GAAG,GAAAA,iBAAA;IAChBE,QAAQ,GAENZ,KAAK,CAFPY,QAAQ;IACRC,UAAU,GACRb,KAAK,CADPa,UAAU;EAGZ,MAAAC,eAAA,GAWIX,QAAQ,CAACH,KAAK;IAVhBe,SAAS,GAAAD,eAAA,CAATC,SAAS;IACTC,KAAK,GAAAF,eAAA,CAALE,KAAK;IACLC,KAAK,GAAAH,eAAA,CAALG,KAAK;IACLC,MAAM,GAAAJ,eAAA,CAANI,MAAM;IACNC,GAAG,GAAAL,eAAA,CAAHK,GAAG;IACHC,WAAW,GAAAN,eAAA,CAAXM,WAAW;IACXC,UAAU,GAAAP,eAAA,CAAVO,UAAU;IACVC,SAAS,GAAAR,eAAA,CAATQ,SAAS;IACTC,UAAU,GAAAT,eAAA,CAAVS,UAAU;IACPC,cAAc,GAAAC,6BAAA,CAAAX,eAAA,EAAAY,SAAA;EAEnB,MAAAC,SAAA,GAAoCjC,QAAQ,CAAC,KAAK,CAAC;IAA5CkC,UAAU,GAAAD,SAAA;IAAEE,aAAa,GAAAF,SAAA;EAChC,MAAAG,UAAA,GAAgDpC,QAAQ,CACtD,IACF,CAAC;IAFMqC,gBAAgB,GAAAD,UAAA;IAAEE,mBAAmB,GAAAF,UAAA;EAI5C,MAAMG,MAAM,GAAGxC,MAAM,CAAmB,IAAI,CAAC;EAC7C,MAAMyC,OAAO,GAAGzC,MAAM,CAAoB,IAAI,CAAC;EAC/C,MAAM0C,WAAW,GAAGrC,YAAY,CAACF,aAAa,CAAC;EAC/C,MAAMwC,gBAAgB,GAAGL,gBAAgB,KAAK,KAAK;EACnD,MAAMM,UAAU,GACdN,gBAAgB,KAAK,IAAI,GACrB,IAAI,GACJK,gBAAgB,GAChBH,MAAM,CAACK,OAAO,GACdJ,OAAO,CAACI,OAAO;;EAErB;EACAzC,uBAAuB,CAAC;IACtB0C,OAAO,EAAEpB,GAAG,IAAIS,UAAU,GAAGS,UAAU,GAAG,IAAI;IAC9CG,OAAO,EAAEnB,UAAU;IACnBoB,MAAM,EAAEnB,SAAS;IACjBoB,OAAO,EAAEnB;EACX,CAAC,CAAC;EAEF,MAAMoB,eAAe,GAAGxB,GAAG,IAAIgB,WAAW;EAE1C3C,SAAS,CAAC,MAAM;IACd,IAAI,CAACyC,MAAM,CAACK,OAAO,IAAI,CAACJ,OAAO,CAACI,OAAO,IAAI,CAACV,UAAU,EAAE;MACtD1B,KAAK,IAAI0C,OAAO,CAACC,IAAI,CAAC,oCAAoC,CAAC;MAE3D;IACF;IAEA,MAAMC,OAAO,GAAGnD,SAAS,CAACsC,MAAM,CAACK,OAAO,EAAEJ,OAAO,CAACI,OAAO,EAAE;MACzDjC,KAAK;MACLE,KAAK;MACLI,UAAU;MACVF,UAAU;MACVI;IACF,CAAC,CAAC;IACFmB,mBAAmB,CAACc,OAAO,KAAK,IAAI,CAAC;IACrC,IAAI,CAACA,OAAO,EAAE;MACZ;IACF;IAEA5C,KAAK,IACH0C,OAAO,CAACG,GAAG,CAAC,wBAAwB,EAAEd,MAAM,CAACK,OAAO,EAAEJ,OAAO,CAACI,OAAO,EAAE;MACrEjC,KAAK;MACLE,KAAK;MACLI,UAAU;MACVF;IACF,CAAC,CAAC;IAEJ,IAAIuC,KAAa;IACjB,IAAIC,QAAQ,GAAG,IAAI;IAEnB,SAASC,IAAIA,CAAA,EAAG;MACd,IAAI,CAACD,QAAQ,EAAE;QACb;MACF;MACAH,OAAO,YAAPA,OAAO,CAAG,CAAC;MACXE,KAAK,GAAGG,qBAAqB,CAACD,IAAI,CAAC;IACrC;IAEAF,KAAK,GAAGG,qBAAqB,CAACD,IAAI,CAAC;IAEnC,OAAO,MAAM;MACXhD,KAAK,IAAI0C,OAAO,CAACG,GAAG,CAAC,4BAA4B,CAAC;MAClDE,QAAQ,GAAG,KAAK;MAChBG,oBAAoB,CAACJ,KAAK,CAAC;IAC7B,CAAC;EACH,CAAC,EAAE,CAAC3C,KAAK,EAAEE,KAAK,EAAEI,UAAU,EAAEF,UAAU,EAAEmB,UAAU,EAAE1B,KAAK,EAAEW,UAAU,CAAC,CAAC;EAEzE,oBACEtB,KAAA,CAAA8D,aAAA;IACEtC,SAAS,EAAEA,SAAU;IACrBC,KAAK,EAAE;MACL,GAAGA,KAAK;MACRC,KAAK;MACLC;IACF;EAAE,gBAEF3B,KAAA,CAAA8D,aAAA;IACEC,GAAG,EAAEpB,OAAQ;IACblB,KAAK,EAAE;MACLC,KAAK,EAAE,MAAM;MACbC,MAAM,EAAE,MAAM;MACdqC,OAAO,EAAEnB,gBAAgB,IAAIO,eAAe,GAAG,MAAM,GAAG,OAAO;MAC/D,IAAIf,UAAU,GACV,CAAC,CAAC,GACF;QACE4B,QAAQ,EAAE,UAAU;QACpBC,OAAO,EAAE,CAAC;QACVC,aAAa,EAAE,MAAM;QACrBC,UAAU,EAAE,QAAQ;QACpBC,MAAM,EAAE,CAAC;MACX,CAAC;IACP;EAAE,CACH,CAAC,EACD,CAAChC,UAAU,IAAIhB,QAAQ,EACvB+B,eAAe,IAAIvB,WAAW,eAE9B7B,KAAK,CAACsE,YAAY,CAAC1D,QAAQ,EAAE;IAC5B,GAAGqB,cAAc;IACjB;IACAL,GAAG,EAAE,KAAK;IACVC,WAAW,EAAE0C,SAAS;IACtBzC,UAAU,EAAEyC,SAAS;IACrBxC,SAAS,EAAEwC,SAAS;IACpBvC,UAAU,EAAEuC,SAAS;IACrBR,GAAG,EAAErB,MAAM;IACXjB,KAAK,EAAEoB,gBAAgB,GACnB;MACEnB,KAAK,EAAE,MAAM;MACbC,MAAM,EAAE,MAAM;MACd,GAAGF,KAAK;MACRuC,OAAO,EAAE3B,UAAU,IAAI,CAACe,eAAe,GAAG3B,KAAK,oBAALA,KAAK,CAAEuC,OAAO,GAAG;IAC7D,CAAC,GACD;MAAEA,OAAO,EAAE;IAAO,CAAC;IACvBQ,SAAS,EAAEC,aAAa;IACxBC,gBAAgB,EAAED;EACpB,CAAC,CACE,CAAC;EAGR,SAASA,aAAaA,CAACE,KAA6C,EAAE;IACpE,MAAMC,UAAU,GAAGD,KAAK,CAACE,aAAa,CAACD,UAAU;IAEjDtC,aAAa,CAACsC,UAAU,IAAIE,gBAAgB,CAACC,gBAAgB,CAAC;EAChE;AACF","ignoreList":[]}
+{"version":3,"names":["useStartupAttempt","recordCanvasFrame","React","useEffect","useRef","useState","chromaKey","pipActiveAtom","usePictureInPictureAuto","useAtomValue","ChromaKey","props","_props$debug","debug","children","_props$color","color","_props$spill","spill","_props$smoothness","smoothness","_props$similarity","similarity","fallback","background","_children$props","className","style","width","height","pip","pipFallback","onPiPEnter","onPiPExit","onPiPError","restVideoProps","_objectWithoutPropertiesLoose","_excluded","_useState","isPlayable","setIsPlayable","_useState2","isWebGLAvailable","setIsWebGLAvailable","startupAttempt","visibleAttempt","type","undefined","$video","$canvas","isPiPActive","useVideoFallback","pipElement","current","element","onEnter","onExit","onError","showPipFallback","console","warn","process","onFrameDrawn","log","rafId","isActive","loop","requestAnimationFrame","cancelAnimationFrame","createElement","ref","display","position","opacity","pointerEvents","visibility","zIndex","cloneElement","startupRenderSurface","onCanPlay","checkPlayable","onLoadedMetadata","event","readyState","currentTarget","HTMLMediaElement","HAVE_FUTURE_DATA"],"sources":["../../../../src/components/ChromaKey/ChromaKey.tsx"],"sourcesContent":["import { useStartupAttempt } from '../../startup-telemetry/useStartupAttempt';\nimport { recordCanvasFrame } from '../../startup-telemetry/frame';\nimport React, { useEffect, useRef, useState } from 'react';\n\nimport { chromaKey } from './chroma-key';\nimport {\n  pipActiveAtom,\n  usePictureInPictureAuto,\n} from '../../hooks/usePictureInPicture';\nimport { useAtomValue } from 'jotai';\n\ninterface ChromaKeyProps extends React.HTMLAttributes<HTMLCanvasElement> {\n  debug?: boolean;\n  color?: string;\n  spill?: number;\n  similarity?: number;\n  smoothness?: number;\n  fallback?: React.ReactNode;\n  background?: string | null;\n  children: React.ReactElement<\n    React.VideoHTMLAttributes<HTMLVideoElement> & {\n      ref?: React.Ref<HTMLVideoElement>;\n      startupRenderSurface?: 'video' | 'canvas';\n      type?: 'agent' | 'user' | 'screen';\n      pip?: boolean;\n      pipFallback?: React.ReactNode;\n      onPiPEnter?: () => void;\n      onPiPExit?: () => void;\n      onPiPError?: (error: Error) => void;\n    }\n  >;\n}\n\nexport function ChromaKey(props: ChromaKeyProps) {\n  const {\n    debug = false,\n    children,\n    color = '#03ff9c',\n    spill = 0.1,\n    smoothness = 0.15,\n    similarity = 0.3,\n    fallback,\n    background,\n  } = props;\n\n  const {\n    className,\n    style,\n    width,\n    height,\n    pip,\n    pipFallback,\n    onPiPEnter,\n    onPiPExit,\n    onPiPError,\n    ...restVideoProps\n  } = children.props;\n  const [isPlayable, setIsPlayable] = useState(false);\n  const [isWebGLAvailable, setIsWebGLAvailable] = useState<boolean | null>(\n    null,\n  );\n\n  const startupAttempt = useStartupAttempt();\n  const visibleAttempt =\n    children.props.type === 'agent' ? startupAttempt : undefined;\n  const $video = useRef<HTMLVideoElement>(null);\n  const $canvas = useRef<HTMLCanvasElement>(null);\n  const isPiPActive = useAtomValue(pipActiveAtom);\n  const useVideoFallback = isWebGLAvailable === false;\n  const pipElement =\n    isWebGLAvailable === null\n      ? null\n      : useVideoFallback\n      ? $video.current\n      : $canvas.current;\n\n  // Setup PiP - hook handles canvas → video conversion internally\n  usePictureInPictureAuto({\n    element: pip && isPlayable ? pipElement : null,\n    onEnter: onPiPEnter,\n    onExit: onPiPExit,\n    onError: onPiPError,\n  });\n\n  const showPipFallback = pip && isPiPActive;\n\n  useEffect(() => {\n    if (!$video.current || !$canvas.current || !isPlayable) {\n      debug && console.warn('Video or canvas is not initialized');\n\n      return;\n    }\n\n    const process = chromaKey($video.current, $canvas.current, {\n      color,\n      spill,\n      similarity,\n      smoothness,\n      background,\n      onFrameDrawn: () => {\n        if ($canvas.current && $video.current) {\n          recordCanvasFrame(visibleAttempt, $canvas.current, $video.current);\n        }\n      },\n    });\n    setIsWebGLAvailable(process !== null);\n    if (!process) {\n      return;\n    }\n\n    debug &&\n      console.log('Chroma key initialized', $video.current, $canvas.current, {\n        color,\n        spill,\n        similarity,\n        smoothness,\n      });\n\n    let rafId: number;\n    let isActive = true;\n\n    function loop() {\n      if (!isActive) {\n        return;\n      }\n      process?.();\n      rafId = requestAnimationFrame(loop);\n    }\n\n    rafId = requestAnimationFrame(loop);\n\n    return () => {\n      debug && console.log('Chroma key process cleanup');\n      isActive = false;\n      cancelAnimationFrame(rafId);\n    };\n  }, [\n    color,\n    spill,\n    similarity,\n    smoothness,\n    isPlayable,\n    debug,\n    background,\n    visibleAttempt,\n  ]);\n\n  return (\n    <div\n      className={className}\n      style={{\n        ...style,\n        width,\n        height,\n      }}\n    >\n      <canvas\n        ref={$canvas}\n        style={{\n          width: '100%',\n          height: '100%',\n          display: useVideoFallback || showPipFallback ? 'none' : 'block',\n          ...(isPlayable\n            ? {}\n            : {\n                position: 'absolute',\n                opacity: 0,\n                pointerEvents: 'none',\n                visibility: 'hidden',\n                zIndex: -1,\n              }),\n        }}\n      />\n      {!isPlayable && fallback}\n      {showPipFallback && pipFallback}\n\n      {React.cloneElement(children, {\n        ...restVideoProps,\n        // cloneElement retains original props; only ChromaKey should own PiP.\n        startupRenderSurface: useVideoFallback ? 'video' : 'canvas',\n        pip: false,\n        pipFallback: undefined,\n        onPiPEnter: undefined,\n        onPiPExit: undefined,\n        onPiPError: undefined,\n        ref: $video,\n        style: useVideoFallback\n          ? {\n              width: '100%',\n              height: '100%',\n              ...style,\n              display: isPlayable && !showPipFallback ? style?.display : 'none',\n            }\n          : { display: 'none' },\n        onCanPlay: checkPlayable,\n        onLoadedMetadata: checkPlayable,\n      })}\n    </div>\n  );\n\n  function checkPlayable(event: React.SyntheticEvent<HTMLVideoElement>) {\n    const readyState = event.currentTarget.readyState;\n\n    setIsPlayable(readyState >= HTMLMediaElement.HAVE_FUTURE_DATA);\n  }\n}\n"],"mappings":";;AAAA,SAASA,iBAAiB,QAAQ,2CAA2C;AAC7E,SAASC,iBAAiB,QAAQ,+BAA+B;AACjE,OAAOC,KAAK,IAAIC,SAAS,EAAEC,MAAM,EAAEC,QAAQ,QAAQ,OAAO;AAE1D,SAASC,SAAS,QAAQ,cAAc;AACxC,SACEC,aAAa,EACbC,uBAAuB,QAClB,iCAAiC;AACxC,SAASC,YAAY,QAAQ,OAAO;AAwBpC,OAAO,SAASC,SAASA,CAACC,KAAqB,EAAE;EAC/C,MAAAC,YAAA,GASID,KAAK,CARPE,KAAK;IAALA,KAAK,GAAAD,YAAA,cAAG,KAAK,GAAAA,YAAA;IACbE,QAAQ,GAONH,KAAK,CAPPG,QAAQ;IAAAC,YAAA,GAONJ,KAAK,CANPK,KAAK;IAALA,KAAK,GAAAD,YAAA,cAAG,SAAS,GAAAA,YAAA;IAAAE,YAAA,GAMfN,KAAK,CALPO,KAAK;IAALA,KAAK,GAAAD,YAAA,cAAG,GAAG,GAAAA,YAAA;IAAAE,iBAAA,GAKTR,KAAK,CAJPS,UAAU;IAAVA,UAAU,GAAAD,iBAAA,cAAG,IAAI,GAAAA,iBAAA;IAAAE,iBAAA,GAIfV,KAAK,CAHPW,UAAU;IAAVA,UAAU,GAAAD,iBAAA,cAAG,GAAG,GAAAA,iBAAA;IAChBE,QAAQ,GAENZ,KAAK,CAFPY,QAAQ;IACRC,UAAU,GACRb,KAAK,CADPa,UAAU;EAGZ,MAAAC,eAAA,GAWIX,QAAQ,CAACH,KAAK;IAVhBe,SAAS,GAAAD,eAAA,CAATC,SAAS;IACTC,KAAK,GAAAF,eAAA,CAALE,KAAK;IACLC,KAAK,GAAAH,eAAA,CAALG,KAAK;IACLC,MAAM,GAAAJ,eAAA,CAANI,MAAM;IACNC,GAAG,GAAAL,eAAA,CAAHK,GAAG;IACHC,WAAW,GAAAN,eAAA,CAAXM,WAAW;IACXC,UAAU,GAAAP,eAAA,CAAVO,UAAU;IACVC,SAAS,GAAAR,eAAA,CAATQ,SAAS;IACTC,UAAU,GAAAT,eAAA,CAAVS,UAAU;IACPC,cAAc,GAAAC,6BAAA,CAAAX,eAAA,EAAAY,SAAA;EAEnB,MAAAC,SAAA,GAAoCjC,QAAQ,CAAC,KAAK,CAAC;IAA5CkC,UAAU,GAAAD,SAAA;IAAEE,aAAa,GAAAF,SAAA;EAChC,MAAAG,UAAA,GAAgDpC,QAAQ,CACtD,IACF,CAAC;IAFMqC,gBAAgB,GAAAD,UAAA;IAAEE,mBAAmB,GAAAF,UAAA;EAI5C,MAAMG,cAAc,GAAG5C,iBAAiB,CAAC,CAAC;EAC1C,MAAM6C,cAAc,GAClB/B,QAAQ,CAACH,KAAK,CAACmC,IAAI,KAAK,OAAO,GAAGF,cAAc,GAAGG,SAAS;EAC9D,MAAMC,MAAM,GAAG5C,MAAM,CAAmB,IAAI,CAAC;EAC7C,MAAM6C,OAAO,GAAG7C,MAAM,CAAoB,IAAI,CAAC;EAC/C,MAAM8C,WAAW,GAAGzC,YAAY,CAACF,aAAa,CAAC;EAC/C,MAAM4C,gBAAgB,GAAGT,gBAAgB,KAAK,KAAK;EACnD,MAAMU,UAAU,GACdV,gBAAgB,KAAK,IAAI,GACrB,IAAI,GACJS,gBAAgB,GAChBH,MAAM,CAACK,OAAO,GACdJ,OAAO,CAACI,OAAO;;EAErB;EACA7C,uBAAuB,CAAC;IACtB8C,OAAO,EAAExB,GAAG,IAAIS,UAAU,GAAGa,UAAU,GAAG,IAAI;IAC9CG,OAAO,EAAEvB,UAAU;IACnBwB,MAAM,EAAEvB,SAAS;IACjBwB,OAAO,EAAEvB;EACX,CAAC,CAAC;EAEF,MAAMwB,eAAe,GAAG5B,GAAG,IAAIoB,WAAW;EAE1C/C,SAAS,CAAC,MAAM;IACd,IAAI,CAAC6C,MAAM,CAACK,OAAO,IAAI,CAACJ,OAAO,CAACI,OAAO,IAAI,CAACd,UAAU,EAAE;MACtD1B,KAAK,IAAI8C,OAAO,CAACC,IAAI,CAAC,oCAAoC,CAAC;MAE3D;IACF;IAEA,MAAMC,OAAO,GAAGvD,SAAS,CAAC0C,MAAM,CAACK,OAAO,EAAEJ,OAAO,CAACI,OAAO,EAAE;MACzDrC,KAAK;MACLE,KAAK;MACLI,UAAU;MACVF,UAAU;MACVI,UAAU;MACVsC,YAAY,EAAEA,CAAA,KAAM;QAClB,IAAIb,OAAO,CAACI,OAAO,IAAIL,MAAM,CAACK,OAAO,EAAE;UACrCpD,iBAAiB,CAAC4C,cAAc,EAAEI,OAAO,CAACI,OAAO,EAAEL,MAAM,CAACK,OAAO,CAAC;QACpE;MACF;IACF,CAAC,CAAC;IACFV,mBAAmB,CAACkB,OAAO,KAAK,IAAI,CAAC;IACrC,IAAI,CAACA,OAAO,EAAE;MACZ;IACF;IAEAhD,KAAK,IACH8C,OAAO,CAACI,GAAG,CAAC,wBAAwB,EAAEf,MAAM,CAACK,OAAO,EAAEJ,OAAO,CAACI,OAAO,EAAE;MACrErC,KAAK;MACLE,KAAK;MACLI,UAAU;MACVF;IACF,CAAC,CAAC;IAEJ,IAAI4C,KAAa;IACjB,IAAIC,QAAQ,GAAG,IAAI;IAEnB,SAASC,IAAIA,CAAA,EAAG;MACd,IAAI,CAACD,QAAQ,EAAE;QACb;MACF;MACAJ,OAAO,YAAPA,OAAO,CAAG,CAAC;MACXG,KAAK,GAAGG,qBAAqB,CAACD,IAAI,CAAC;IACrC;IAEAF,KAAK,GAAGG,qBAAqB,CAACD,IAAI,CAAC;IAEnC,OAAO,MAAM;MACXrD,KAAK,IAAI8C,OAAO,CAACI,GAAG,CAAC,4BAA4B,CAAC;MAClDE,QAAQ,GAAG,KAAK;MAChBG,oBAAoB,CAACJ,KAAK,CAAC;IAC7B,CAAC;EACH,CAAC,EAAE,CACDhD,KAAK,EACLE,KAAK,EACLI,UAAU,EACVF,UAAU,EACVmB,UAAU,EACV1B,KAAK,EACLW,UAAU,EACVqB,cAAc,CACf,CAAC;EAEF,oBACE3C,KAAA,CAAAmE,aAAA;IACE3C,SAAS,EAAEA,SAAU;IACrBC,KAAK,EAAE;MACL,GAAGA,KAAK;MACRC,KAAK;MACLC;IACF;EAAE,gBAEF3B,KAAA,CAAAmE,aAAA;IACEC,GAAG,EAAErB,OAAQ;IACbtB,KAAK,EAAE;MACLC,KAAK,EAAE,MAAM;MACbC,MAAM,EAAE,MAAM;MACd0C,OAAO,EAAEpB,gBAAgB,IAAIO,eAAe,GAAG,MAAM,GAAG,OAAO;MAC/D,IAAInB,UAAU,GACV,CAAC,CAAC,GACF;QACEiC,QAAQ,EAAE,UAAU;QACpBC,OAAO,EAAE,CAAC;QACVC,aAAa,EAAE,MAAM;QACrBC,UAAU,EAAE,QAAQ;QACpBC,MAAM,EAAE,CAAC;MACX,CAAC;IACP;EAAE,CACH,CAAC,EACD,CAACrC,UAAU,IAAIhB,QAAQ,EACvBmC,eAAe,IAAI3B,WAAW,eAE9B7B,KAAK,CAAC2E,YAAY,CAAC/D,QAAQ,EAAE;IAC5B,GAAGqB,cAAc;IACjB;IACA2C,oBAAoB,EAAE3B,gBAAgB,GAAG,OAAO,GAAG,QAAQ;IAC3DrB,GAAG,EAAE,KAAK;IACVC,WAAW,EAAEgB,SAAS;IACtBf,UAAU,EAAEe,SAAS;IACrBd,SAAS,EAAEc,SAAS;IACpBb,UAAU,EAAEa,SAAS;IACrBuB,GAAG,EAAEtB,MAAM;IACXrB,KAAK,EAAEwB,gBAAgB,GACnB;MACEvB,KAAK,EAAE,MAAM;MACbC,MAAM,EAAE,MAAM;MACd,GAAGF,KAAK;MACR4C,OAAO,EAAEhC,UAAU,IAAI,CAACmB,eAAe,GAAG/B,KAAK,oBAALA,KAAK,CAAE4C,OAAO,GAAG;IAC7D,CAAC,GACD;MAAEA,OAAO,EAAE;IAAO,CAAC;IACvBQ,SAAS,EAAEC,aAAa;IACxBC,gBAAgB,EAAED;EACpB,CAAC,CACE,CAAC;EAGR,SAASA,aAAaA,CAACE,KAA6C,EAAE;IACpE,MAAMC,UAAU,GAAGD,KAAK,CAACE,aAAa,CAACD,UAAU;IAEjD3C,aAAa,CAAC2C,UAAU,IAAIE,gBAAgB,CAACC,gBAAgB,CAAC;EAChE;AACF","ignoreList":[]}

package/dist/esm/components/ChromaKey/chroma-key.js

@@ -88,6 +88,9 @@ export function chromaKey($video, $canvas, options) {
     gl.uniform3f($color$, parseInt(color.substr(1, 2), 16) / 255, parseInt(color.substr(3, 2), 16) / 255, parseInt(color.substr(5, 2), 16) / 255);
     gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGB, gl.RGB, gl.UNSIGNED_BYTE, $video);
     gl.drawArrays(gl.TRIANGLE_FAN, 0, 4);
+    if (!gl.isContextLost()) {
+      options.onFrameDrawn == null || options.onFrameDrawn();
+    }
   };
 }
 //# sourceMappingURL=chroma-key.js.map

package/dist/esm/components/ChromaKey/chroma-key.js.map

@@ -1 +1 @@
-{"version":3,"names":["invariant","shaders","chromaKey","$video","$canvas","options","gl","getContext","alpha","premultipliedAlpha","preserveDrawingBuffer","BUFFER","createBuffer","TEXTURE","createTexture","program","createProgram","_VS_","createShader","VERTEX_SHADER","_FS_","FRAGMENT_SHADER","shaderSource","VertexAlign","ChromaKey","compileShader","getShaderParameter","COMPILE_STATUS","console","error","getShaderInfoLog","attachShader","linkProgram","useProgram","bindTexture","TEXTURE_2D","bindBuffer","ARRAY_BUFFER","bufferData","Float32Array","STATIC_DRAW","texParameteri","TEXTURE_MIN_FILTER","LINEAR","TEXTURE_WRAP_S","CLAMP_TO_EDGE","TEXTURE_WRAP_T","$coords$","getAttribLocation","$texture$","getUniformLocation","$width$","$height$","$color$","$similarity$","$smoothness$","$spill$","vertexAttribPointer","FLOAT","enableVertexAttribArray","activeTexture","TEXTURE0","enable","BLEND","blendFunc","SRC_ALPHA","ONE_MINUS_SRC_ALPHA","process","color","spill","similarity","smoothness","background","videoWidth","videoHeight","width","window","devicePixelRatio","height","viewport","r","parseInt","substring","g","b","clearColor","clear","COLOR_BUFFER_BIT","uniform1i","uniform1f","uniform3f","substr","texImage2D","RGB","UNSIGNED_BYTE","drawArrays","TRIANGLE_FAN"],"sources":["../../../../src/components/ChromaKey/chroma-key.ts"],"sourcesContent":["import invariant from 'invariant';\n\nimport * as shaders from './shaders';\n\nexport type ProcessCallback = () => void;\n\nexport function chromaKey(\n  $video: HTMLVideoElement,\n  $canvas: HTMLCanvasElement,\n  options: {\n    color: string;\n    spill: number;\n    similarity: number;\n    smoothness: number;\n    background?: string | null;\n  },\n): ProcessCallback | null {\n  const gl = $canvas.getContext('webgl', {\n    alpha: true,\n    premultipliedAlpha: false,\n    preserveDrawingBuffer: true,\n  });\n\n  if (!gl) {\n    return null;\n  }\n\n  const BUFFER = gl.createBuffer();\n  const TEXTURE = gl.createTexture();\n  const program = gl.createProgram();\n  const _VS_ = gl.createShader(gl.VERTEX_SHADER);\n  const _FS_ = gl.createShader(gl.FRAGMENT_SHADER);\n\n  invariant(program, 'Program is not initialized');\n  invariant(_VS_, 'Vertex shader is not initialized');\n  invariant(_FS_, 'Fragment shader is not initialized');\n\n  gl.shaderSource(_VS_, shaders.VertexAlign);\n  gl.shaderSource(_FS_, shaders.ChromaKey);\n\n  gl.compileShader(_VS_);\n  gl.compileShader(_FS_);\n\n  if (!gl.getShaderParameter(_FS_, gl.COMPILE_STATUS)) {\n    console.error(\n      'Fragment shader compilation failed',\n      gl.getShaderInfoLog(_FS_),\n    );\n  }\n\n  gl.attachShader(program, _VS_);\n  gl.attachShader(program, _FS_);\n\n  gl.linkProgram(program);\n  gl.useProgram(program);\n\n  gl.bindTexture(gl.TEXTURE_2D, TEXTURE);\n  gl.bindBuffer(gl.ARRAY_BUFFER, BUFFER);\n\n  gl.bufferData(\n    gl.ARRAY_BUFFER,\n    new Float32Array([-1, 1, -1, -1, 1, -1, 1, 1]),\n    gl.STATIC_DRAW,\n  );\n\n  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);\n  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);\n  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);\n\n  const $coords$ = gl.getAttribLocation(program, 'coords');\n  const $texture$ = gl.getUniformLocation(program, 'tex');\n  const $width$ = gl.getUniformLocation(program, 'texWidth');\n  const $height$ = gl.getUniformLocation(program, 'texHeight');\n  const $color$ = gl.getUniformLocation(program, 'keyColor');\n  const $similarity$ = gl.getUniformLocation(program, 'similarity');\n  const $smoothness$ = gl.getUniformLocation(program, 'smoothness');\n  const $spill$ = gl.getUniformLocation(program, 'spill');\n\n  gl.vertexAttribPointer($coords$, 2, gl.FLOAT, false, 0, 0);\n  gl.enableVertexAttribArray($coords$);\n\n  gl.activeTexture(gl.TEXTURE0);\n\n  // Enable blending for transparency (critical for Safari)\n  gl.enable(gl.BLEND);\n  gl.blendFunc(gl.SRC_ALPHA, gl.ONE_MINUS_SRC_ALPHA);\n\n  return function process() {\n    const { color, spill, similarity, smoothness, background } = options;\n\n    // Use video dimensions directly instead of metadata (for Safari compatibility)\n    const videoWidth = $video.videoWidth;\n    const videoHeight = $video.videoHeight;\n\n    if (!videoWidth || !videoHeight) {\n      return; // Video not ready yet\n    }\n\n    const width = videoWidth * window.devicePixelRatio;\n    const height = videoHeight * window.devicePixelRatio;\n\n    $canvas.width = width;\n    $canvas.height = height;\n\n    gl.viewport(0, 0, width, height);\n\n    // Clear with background color or transparent\n    if (background) {\n      const r = parseInt(background.substring(1, 3), 16) / 255;\n      const g = parseInt(background.substring(3, 5), 16) / 255;\n      const b = parseInt(background.substring(5, 7), 16) / 255;\n      gl.clearColor(r, g, b, 1);\n    } else {\n      gl.clearColor(0, 0, 0, 0);\n    }\n    gl.clear(gl.COLOR_BUFFER_BIT);\n\n    gl.useProgram(program);\n\n    gl.uniform1i($texture$, 0);\n    gl.uniform1f($width$, width);\n    gl.uniform1f($height$, height);\n    gl.uniform1f($spill$, spill);\n    gl.uniform1f($similarity$, similarity);\n    gl.uniform1f($smoothness$, smoothness);\n    gl.uniform3f(\n      $color$,\n      parseInt(color.substr(1, 2), 16) / 255,\n      parseInt(color.substr(3, 2), 16) / 255,\n      parseInt(color.substr(5, 2), 16) / 255,\n    );\n\n    gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGB, gl.RGB, gl.UNSIGNED_BYTE, $video);\n    gl.drawArrays(gl.TRIANGLE_FAN, 0, 4);\n  };\n}\n"],"mappings":"AAAA,OAAOA,SAAS,MAAM,WAAW;AAEjC,OAAO,KAAKC,OAAO,MAAM,WAAW;AAIpC,OAAO,SAASC,SAASA,CACvBC,MAAwB,EACxBC,OAA0B,EAC1BC,OAMC,EACuB;EACxB,MAAMC,EAAE,GAAGF,OAAO,CAACG,UAAU,CAAC,OAAO,EAAE;IACrCC,KAAK,EAAE,IAAI;IACXC,kBAAkB,EAAE,KAAK;IACzBC,qBAAqB,EAAE;EACzB,CAAC,CAAC;EAEF,IAAI,CAACJ,EAAE,EAAE;IACP,OAAO,IAAI;EACb;EAEA,MAAMK,MAAM,GAAGL,EAAE,CAACM,YAAY,CAAC,CAAC;EAChC,MAAMC,OAAO,GAAGP,EAAE,CAACQ,aAAa,CAAC,CAAC;EAClC,MAAMC,OAAO,GAAGT,EAAE,CAACU,aAAa,CAAC,CAAC;EAClC,MAAMC,IAAI,GAAGX,EAAE,CAACY,YAAY,CAACZ,EAAE,CAACa,aAAa,CAAC;EAC9C,MAAMC,IAAI,GAAGd,EAAE,CAACY,YAAY,CAACZ,EAAE,CAACe,eAAe,CAAC;EAEhDrB,SAAS,CAACe,OAAO,EAAE,4BAA4B,CAAC;EAChDf,SAAS,CAACiB,IAAI,EAAE,kCAAkC,CAAC;EACnDjB,SAAS,CAACoB,IAAI,EAAE,oCAAoC,CAAC;EAErDd,EAAE,CAACgB,YAAY,CAACL,IAAI,EAAEhB,OAAO,CAACsB,WAAW,CAAC;EAC1CjB,EAAE,CAACgB,YAAY,CAACF,IAAI,EAAEnB,OAAO,CAACuB,SAAS,CAAC;EAExClB,EAAE,CAACmB,aAAa,CAACR,IAAI,CAAC;EACtBX,EAAE,CAACmB,aAAa,CAACL,IAAI,CAAC;EAEtB,IAAI,CAACd,EAAE,CAACoB,kBAAkB,CAACN,IAAI,EAAEd,EAAE,CAACqB,cAAc,CAAC,EAAE;IACnDC,OAAO,CAACC,KAAK,CACX,oCAAoC,EACpCvB,EAAE,CAACwB,gBAAgB,CAACV,IAAI,CAC1B,CAAC;EACH;EAEAd,EAAE,CAACyB,YAAY,CAAChB,OAAO,EAAEE,IAAI,CAAC;EAC9BX,EAAE,CAACyB,YAAY,CAAChB,OAAO,EAAEK,IAAI,CAAC;EAE9Bd,EAAE,CAAC0B,WAAW,CAACjB,OAAO,CAAC;EACvBT,EAAE,CAAC2B,UAAU,CAAClB,OAAO,CAAC;EAEtBT,EAAE,CAAC4B,WAAW,CAAC5B,EAAE,CAAC6B,UAAU,EAAEtB,OAAO,CAAC;EACtCP,EAAE,CAAC8B,UAAU,CAAC9B,EAAE,CAAC+B,YAAY,EAAE1B,MAAM,CAAC;EAEtCL,EAAE,CAACgC,UAAU,CACXhC,EAAE,CAAC+B,YAAY,EACf,IAAIE,YAAY,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,EAC9CjC,EAAE,CAACkC,WACL,CAAC;EAEDlC,EAAE,CAACmC,aAAa,CAACnC,EAAE,CAAC6B,UAAU,EAAE7B,EAAE,CAACoC,kBAAkB,EAAEpC,EAAE,CAACqC,MAAM,CAAC;EACjErC,EAAE,CAACmC,aAAa,CAACnC,EAAE,CAAC6B,UAAU,EAAE7B,EAAE,CAACsC,cAAc,EAAEtC,EAAE,CAACuC,aAAa,CAAC;EACpEvC,EAAE,CAACmC,aAAa,CAACnC,EAAE,CAAC6B,UAAU,EAAE7B,EAAE,CAACwC,cAAc,EAAExC,EAAE,CAACuC,aAAa,CAAC;EAEpE,MAAME,QAAQ,GAAGzC,EAAE,CAAC0C,iBAAiB,CAACjC,OAAO,EAAE,QAAQ,CAAC;EACxD,MAAMkC,SAAS,GAAG3C,EAAE,CAAC4C,kBAAkB,CAACnC,OAAO,EAAE,KAAK,CAAC;EACvD,MAAMoC,OAAO,GAAG7C,EAAE,CAAC4C,kBAAkB,CAACnC,OAAO,EAAE,UAAU,CAAC;EAC1D,MAAMqC,QAAQ,GAAG9C,EAAE,CAAC4C,kBAAkB,CAACnC,OAAO,EAAE,WAAW,CAAC;EAC5D,MAAMsC,OAAO,GAAG/C,EAAE,CAAC4C,kBAAkB,CAACnC,OAAO,EAAE,UAAU,CAAC;EAC1D,MAAMuC,YAAY,GAAGhD,EAAE,CAAC4C,kBAAkB,CAACnC,OAAO,EAAE,YAAY,CAAC;EACjE,MAAMwC,YAAY,GAAGjD,EAAE,CAAC4C,kBAAkB,CAACnC,OAAO,EAAE,YAAY,CAAC;EACjE,MAAMyC,OAAO,GAAGlD,EAAE,CAAC4C,kBAAkB,CAACnC,OAAO,EAAE,OAAO,CAAC;EAEvDT,EAAE,CAACmD,mBAAmB,CAACV,QAAQ,EAAE,CAAC,EAAEzC,EAAE,CAACoD,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;EAC1DpD,EAAE,CAACqD,uBAAuB,CAACZ,QAAQ,CAAC;EAEpCzC,EAAE,CAACsD,aAAa,CAACtD,EAAE,CAACuD,QAAQ,CAAC;;EAE7B;EACAvD,EAAE,CAACwD,MAAM,CAACxD,EAAE,CAACyD,KAAK,CAAC;EACnBzD,EAAE,CAAC0D,SAAS,CAAC1D,EAAE,CAAC2D,SAAS,EAAE3D,EAAE,CAAC4D,mBAAmB,CAAC;EAElD,OAAO,SAASC,OAAOA,CAAA,EAAG;IACxB,MAAQC,KAAK,GAAgD/D,OAAO,CAA5D+D,KAAK;MAAEC,KAAK,GAAyChE,OAAO,CAArDgE,KAAK;MAAEC,UAAU,GAA6BjE,OAAO,CAA9CiE,UAAU;MAAEC,UAAU,GAAiBlE,OAAO,CAAlCkE,UAAU;MAAEC,UAAU,GAAKnE,OAAO,CAAtBmE,UAAU;;IAExD;IACA,MAAMC,UAAU,GAAGtE,MAAM,CAACsE,UAAU;IACpC,MAAMC,WAAW,GAAGvE,MAAM,CAACuE,WAAW;IAEtC,IAAI,CAACD,UAAU,IAAI,CAACC,WAAW,EAAE;MAC/B,OAAO,CAAC;IACV;IAEA,MAAMC,KAAK,GAAGF,UAAU,GAAGG,MAAM,CAACC,gBAAgB;IAClD,MAAMC,MAAM,GAAGJ,WAAW,GAAGE,MAAM,CAACC,gBAAgB;IAEpDzE,OAAO,CAACuE,KAAK,GAAGA,KAAK;IACrBvE,OAAO,CAAC0E,MAAM,GAAGA,MAAM;IAEvBxE,EAAE,CAACyE,QAAQ,CAAC,CAAC,EAAE,CAAC,EAAEJ,KAAK,EAAEG,MAAM,CAAC;;IAEhC;IACA,IAAIN,UAAU,EAAE;MACd,MAAMQ,CAAC,GAAGC,QAAQ,CAACT,UAAU,CAACU,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG;MACxD,MAAMC,CAAC,GAAGF,QAAQ,CAACT,UAAU,CAACU,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG;MACxD,MAAME,CAAC,GAAGH,QAAQ,CAACT,UAAU,CAACU,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG;MACxD5E,EAAE,CAAC+E,UAAU,CAACL,CAAC,EAAEG,CAAC,EAAEC,CAAC,EAAE,CAAC,CAAC;IAC3B,CAAC,MAAM;MACL9E,EAAE,CAAC+E,UAAU,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IAC3B;IACA/E,EAAE,CAACgF,KAAK,CAAChF,EAAE,CAACiF,gBAAgB,CAAC;IAE7BjF,EAAE,CAAC2B,UAAU,CAAClB,OAAO,CAAC;IAEtBT,EAAE,CAACkF,SAAS,CAACvC,SAAS,EAAE,CAAC,CAAC;IAC1B3C,EAAE,CAACmF,SAAS,CAACtC,OAAO,EAAEwB,KAAK,CAAC;IAC5BrE,EAAE,CAACmF,SAAS,CAACrC,QAAQ,EAAE0B,MAAM,CAAC;IAC9BxE,EAAE,CAACmF,SAAS,CAACjC,OAAO,EAAEa,KAAK,CAAC;IAC5B/D,EAAE,CAACmF,SAAS,CAACnC,YAAY,EAAEgB,UAAU,CAAC;IACtChE,EAAE,CAACmF,SAAS,CAAClC,YAAY,EAAEgB,UAAU,CAAC;IACtCjE,EAAE,CAACoF,SAAS,CACVrC,OAAO,EACP4B,QAAQ,CAACb,KAAK,CAACuB,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,EACtCV,QAAQ,CAACb,KAAK,CAACuB,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,EACtCV,QAAQ,CAACb,KAAK,CAACuB,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,GACrC,CAAC;IAEDrF,EAAE,CAACsF,UAAU,CAACtF,EAAE,CAAC6B,UAAU,EAAE,CAAC,EAAE7B,EAAE,CAACuF,GAAG,EAAEvF,EAAE,CAACuF,GAAG,EAAEvF,EAAE,CAACwF,aAAa,EAAE3F,MAAM,CAAC;IACzEG,EAAE,CAACyF,UAAU,CAACzF,EAAE,CAAC0F,YAAY,EAAE,CAAC,EAAE,CAAC,CAAC;EACtC,CAAC;AACH","ignoreList":[]}
+{"version":3,"names":["invariant","shaders","chromaKey","$video","$canvas","options","gl","getContext","alpha","premultipliedAlpha","preserveDrawingBuffer","BUFFER","createBuffer","TEXTURE","createTexture","program","createProgram","_VS_","createShader","VERTEX_SHADER","_FS_","FRAGMENT_SHADER","shaderSource","VertexAlign","ChromaKey","compileShader","getShaderParameter","COMPILE_STATUS","console","error","getShaderInfoLog","attachShader","linkProgram","useProgram","bindTexture","TEXTURE_2D","bindBuffer","ARRAY_BUFFER","bufferData","Float32Array","STATIC_DRAW","texParameteri","TEXTURE_MIN_FILTER","LINEAR","TEXTURE_WRAP_S","CLAMP_TO_EDGE","TEXTURE_WRAP_T","$coords$","getAttribLocation","$texture$","getUniformLocation","$width$","$height$","$color$","$similarity$","$smoothness$","$spill$","vertexAttribPointer","FLOAT","enableVertexAttribArray","activeTexture","TEXTURE0","enable","BLEND","blendFunc","SRC_ALPHA","ONE_MINUS_SRC_ALPHA","process","color","spill","similarity","smoothness","background","videoWidth","videoHeight","width","window","devicePixelRatio","height","viewport","r","parseInt","substring","g","b","clearColor","clear","COLOR_BUFFER_BIT","uniform1i","uniform1f","uniform3f","substr","texImage2D","RGB","UNSIGNED_BYTE","drawArrays","TRIANGLE_FAN","isContextLost","onFrameDrawn"],"sources":["../../../../src/components/ChromaKey/chroma-key.ts"],"sourcesContent":["import invariant from 'invariant';\n\nimport * as shaders from './shaders';\n\nexport type ProcessCallback = () => void;\n\nexport function chromaKey(\n  $video: HTMLVideoElement,\n  $canvas: HTMLCanvasElement,\n  options: {\n    onFrameDrawn?: () => void;\n    color: string;\n    spill: number;\n    similarity: number;\n    smoothness: number;\n    background?: string | null;\n  },\n): ProcessCallback | null {\n  const gl = $canvas.getContext('webgl', {\n    alpha: true,\n    premultipliedAlpha: false,\n    preserveDrawingBuffer: true,\n  });\n\n  if (!gl) {\n    return null;\n  }\n\n  const BUFFER = gl.createBuffer();\n  const TEXTURE = gl.createTexture();\n  const program = gl.createProgram();\n  const _VS_ = gl.createShader(gl.VERTEX_SHADER);\n  const _FS_ = gl.createShader(gl.FRAGMENT_SHADER);\n\n  invariant(program, 'Program is not initialized');\n  invariant(_VS_, 'Vertex shader is not initialized');\n  invariant(_FS_, 'Fragment shader is not initialized');\n\n  gl.shaderSource(_VS_, shaders.VertexAlign);\n  gl.shaderSource(_FS_, shaders.ChromaKey);\n\n  gl.compileShader(_VS_);\n  gl.compileShader(_FS_);\n\n  if (!gl.getShaderParameter(_FS_, gl.COMPILE_STATUS)) {\n    console.error(\n      'Fragment shader compilation failed',\n      gl.getShaderInfoLog(_FS_),\n    );\n  }\n\n  gl.attachShader(program, _VS_);\n  gl.attachShader(program, _FS_);\n\n  gl.linkProgram(program);\n  gl.useProgram(program);\n\n  gl.bindTexture(gl.TEXTURE_2D, TEXTURE);\n  gl.bindBuffer(gl.ARRAY_BUFFER, BUFFER);\n\n  gl.bufferData(\n    gl.ARRAY_BUFFER,\n    new Float32Array([-1, 1, -1, -1, 1, -1, 1, 1]),\n    gl.STATIC_DRAW,\n  );\n\n  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);\n  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);\n  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);\n\n  const $coords$ = gl.getAttribLocation(program, 'coords');\n  const $texture$ = gl.getUniformLocation(program, 'tex');\n  const $width$ = gl.getUniformLocation(program, 'texWidth');\n  const $height$ = gl.getUniformLocation(program, 'texHeight');\n  const $color$ = gl.getUniformLocation(program, 'keyColor');\n  const $similarity$ = gl.getUniformLocation(program, 'similarity');\n  const $smoothness$ = gl.getUniformLocation(program, 'smoothness');\n  const $spill$ = gl.getUniformLocation(program, 'spill');\n\n  gl.vertexAttribPointer($coords$, 2, gl.FLOAT, false, 0, 0);\n  gl.enableVertexAttribArray($coords$);\n\n  gl.activeTexture(gl.TEXTURE0);\n\n  // Enable blending for transparency (critical for Safari)\n  gl.enable(gl.BLEND);\n  gl.blendFunc(gl.SRC_ALPHA, gl.ONE_MINUS_SRC_ALPHA);\n\n  return function process() {\n    const { color, spill, similarity, smoothness, background } = options;\n\n    // Use video dimensions directly instead of metadata (for Safari compatibility)\n    const videoWidth = $video.videoWidth;\n    const videoHeight = $video.videoHeight;\n\n    if (!videoWidth || !videoHeight) {\n      return; // Video not ready yet\n    }\n\n    const width = videoWidth * window.devicePixelRatio;\n    const height = videoHeight * window.devicePixelRatio;\n\n    $canvas.width = width;\n    $canvas.height = height;\n\n    gl.viewport(0, 0, width, height);\n\n    // Clear with background color or transparent\n    if (background) {\n      const r = parseInt(background.substring(1, 3), 16) / 255;\n      const g = parseInt(background.substring(3, 5), 16) / 255;\n      const b = parseInt(background.substring(5, 7), 16) / 255;\n      gl.clearColor(r, g, b, 1);\n    } else {\n      gl.clearColor(0, 0, 0, 0);\n    }\n    gl.clear(gl.COLOR_BUFFER_BIT);\n\n    gl.useProgram(program);\n\n    gl.uniform1i($texture$, 0);\n    gl.uniform1f($width$, width);\n    gl.uniform1f($height$, height);\n    gl.uniform1f($spill$, spill);\n    gl.uniform1f($similarity$, similarity);\n    gl.uniform1f($smoothness$, smoothness);\n    gl.uniform3f(\n      $color$,\n      parseInt(color.substr(1, 2), 16) / 255,\n      parseInt(color.substr(3, 2), 16) / 255,\n      parseInt(color.substr(5, 2), 16) / 255,\n    );\n\n    gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGB, gl.RGB, gl.UNSIGNED_BYTE, $video);\n    gl.drawArrays(gl.TRIANGLE_FAN, 0, 4);\n    if (!gl.isContextLost()) {\n      options.onFrameDrawn?.();\n    }\n  };\n}\n"],"mappings":"AAAA,OAAOA,SAAS,MAAM,WAAW;AAEjC,OAAO,KAAKC,OAAO,MAAM,WAAW;AAIpC,OAAO,SAASC,SAASA,CACvBC,MAAwB,EACxBC,OAA0B,EAC1BC,OAOC,EACuB;EACxB,MAAMC,EAAE,GAAGF,OAAO,CAACG,UAAU,CAAC,OAAO,EAAE;IACrCC,KAAK,EAAE,IAAI;IACXC,kBAAkB,EAAE,KAAK;IACzBC,qBAAqB,EAAE;EACzB,CAAC,CAAC;EAEF,IAAI,CAACJ,EAAE,EAAE;IACP,OAAO,IAAI;EACb;EAEA,MAAMK,MAAM,GAAGL,EAAE,CAACM,YAAY,CAAC,CAAC;EAChC,MAAMC,OAAO,GAAGP,EAAE,CAACQ,aAAa,CAAC,CAAC;EAClC,MAAMC,OAAO,GAAGT,EAAE,CAACU,aAAa,CAAC,CAAC;EAClC,MAAMC,IAAI,GAAGX,EAAE,CAACY,YAAY,CAACZ,EAAE,CAACa,aAAa,CAAC;EAC9C,MAAMC,IAAI,GAAGd,EAAE,CAACY,YAAY,CAACZ,EAAE,CAACe,eAAe,CAAC;EAEhDrB,SAAS,CAACe,OAAO,EAAE,4BAA4B,CAAC;EAChDf,SAAS,CAACiB,IAAI,EAAE,kCAAkC,CAAC;EACnDjB,SAAS,CAACoB,IAAI,EAAE,oCAAoC,CAAC;EAErDd,EAAE,CAACgB,YAAY,CAACL,IAAI,EAAEhB,OAAO,CAACsB,WAAW,CAAC;EAC1CjB,EAAE,CAACgB,YAAY,CAACF,IAAI,EAAEnB,OAAO,CAACuB,SAAS,CAAC;EAExClB,EAAE,CAACmB,aAAa,CAACR,IAAI,CAAC;EACtBX,EAAE,CAACmB,aAAa,CAACL,IAAI,CAAC;EAEtB,IAAI,CAACd,EAAE,CAACoB,kBAAkB,CAACN,IAAI,EAAEd,EAAE,CAACqB,cAAc,CAAC,EAAE;IACnDC,OAAO,CAACC,KAAK,CACX,oCAAoC,EACpCvB,EAAE,CAACwB,gBAAgB,CAACV,IAAI,CAC1B,CAAC;EACH;EAEAd,EAAE,CAACyB,YAAY,CAAChB,OAAO,EAAEE,IAAI,CAAC;EAC9BX,EAAE,CAACyB,YAAY,CAAChB,OAAO,EAAEK,IAAI,CAAC;EAE9Bd,EAAE,CAAC0B,WAAW,CAACjB,OAAO,CAAC;EACvBT,EAAE,CAAC2B,UAAU,CAAClB,OAAO,CAAC;EAEtBT,EAAE,CAAC4B,WAAW,CAAC5B,EAAE,CAAC6B,UAAU,EAAEtB,OAAO,CAAC;EACtCP,EAAE,CAAC8B,UAAU,CAAC9B,EAAE,CAAC+B,YAAY,EAAE1B,MAAM,CAAC;EAEtCL,EAAE,CAACgC,UAAU,CACXhC,EAAE,CAAC+B,YAAY,EACf,IAAIE,YAAY,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,EAC9CjC,EAAE,CAACkC,WACL,CAAC;EAEDlC,EAAE,CAACmC,aAAa,CAACnC,EAAE,CAAC6B,UAAU,EAAE7B,EAAE,CAACoC,kBAAkB,EAAEpC,EAAE,CAACqC,MAAM,CAAC;EACjErC,EAAE,CAACmC,aAAa,CAACnC,EAAE,CAAC6B,UAAU,EAAE7B,EAAE,CAACsC,cAAc,EAAEtC,EAAE,CAACuC,aAAa,CAAC;EACpEvC,EAAE,CAACmC,aAAa,CAACnC,EAAE,CAAC6B,UAAU,EAAE7B,EAAE,CAACwC,cAAc,EAAExC,EAAE,CAACuC,aAAa,CAAC;EAEpE,MAAME,QAAQ,GAAGzC,EAAE,CAAC0C,iBAAiB,CAACjC,OAAO,EAAE,QAAQ,CAAC;EACxD,MAAMkC,SAAS,GAAG3C,EAAE,CAAC4C,kBAAkB,CAACnC,OAAO,EAAE,KAAK,CAAC;EACvD,MAAMoC,OAAO,GAAG7C,EAAE,CAAC4C,kBAAkB,CAACnC,OAAO,EAAE,UAAU,CAAC;EAC1D,MAAMqC,QAAQ,GAAG9C,EAAE,CAAC4C,kBAAkB,CAACnC,OAAO,EAAE,WAAW,CAAC;EAC5D,MAAMsC,OAAO,GAAG/C,EAAE,CAAC4C,kBAAkB,CAACnC,OAAO,EAAE,UAAU,CAAC;EAC1D,MAAMuC,YAAY,GAAGhD,EAAE,CAAC4C,kBAAkB,CAACnC,OAAO,EAAE,YAAY,CAAC;EACjE,MAAMwC,YAAY,GAAGjD,EAAE,CAAC4C,kBAAkB,CAACnC,OAAO,EAAE,YAAY,CAAC;EACjE,MAAMyC,OAAO,GAAGlD,EAAE,CAAC4C,kBAAkB,CAACnC,OAAO,EAAE,OAAO,CAAC;EAEvDT,EAAE,CAACmD,mBAAmB,CAACV,QAAQ,EAAE,CAAC,EAAEzC,EAAE,CAACoD,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;EAC1DpD,EAAE,CAACqD,uBAAuB,CAACZ,QAAQ,CAAC;EAEpCzC,EAAE,CAACsD,aAAa,CAACtD,EAAE,CAACuD,QAAQ,CAAC;;EAE7B;EACAvD,EAAE,CAACwD,MAAM,CAACxD,EAAE,CAACyD,KAAK,CAAC;EACnBzD,EAAE,CAAC0D,SAAS,CAAC1D,EAAE,CAAC2D,SAAS,EAAE3D,EAAE,CAAC4D,mBAAmB,CAAC;EAElD,OAAO,SAASC,OAAOA,CAAA,EAAG;IACxB,MAAQC,KAAK,GAAgD/D,OAAO,CAA5D+D,KAAK;MAAEC,KAAK,GAAyChE,OAAO,CAArDgE,KAAK;MAAEC,UAAU,GAA6BjE,OAAO,CAA9CiE,UAAU;MAAEC,UAAU,GAAiBlE,OAAO,CAAlCkE,UAAU;MAAEC,UAAU,GAAKnE,OAAO,CAAtBmE,UAAU;;IAExD;IACA,MAAMC,UAAU,GAAGtE,MAAM,CAACsE,UAAU;IACpC,MAAMC,WAAW,GAAGvE,MAAM,CAACuE,WAAW;IAEtC,IAAI,CAACD,UAAU,IAAI,CAACC,WAAW,EAAE;MAC/B,OAAO,CAAC;IACV;IAEA,MAAMC,KAAK,GAAGF,UAAU,GAAGG,MAAM,CAACC,gBAAgB;IAClD,MAAMC,MAAM,GAAGJ,WAAW,GAAGE,MAAM,CAACC,gBAAgB;IAEpDzE,OAAO,CAACuE,KAAK,GAAGA,KAAK;IACrBvE,OAAO,CAAC0E,MAAM,GAAGA,MAAM;IAEvBxE,EAAE,CAACyE,QAAQ,CAAC,CAAC,EAAE,CAAC,EAAEJ,KAAK,EAAEG,MAAM,CAAC;;IAEhC;IACA,IAAIN,UAAU,EAAE;MACd,MAAMQ,CAAC,GAAGC,QAAQ,CAACT,UAAU,CAACU,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG;MACxD,MAAMC,CAAC,GAAGF,QAAQ,CAACT,UAAU,CAACU,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG;MACxD,MAAME,CAAC,GAAGH,QAAQ,CAACT,UAAU,CAACU,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG;MACxD5E,EAAE,CAAC+E,UAAU,CAACL,CAAC,EAAEG,CAAC,EAAEC,CAAC,EAAE,CAAC,CAAC;IAC3B,CAAC,MAAM;MACL9E,EAAE,CAAC+E,UAAU,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IAC3B;IACA/E,EAAE,CAACgF,KAAK,CAAChF,EAAE,CAACiF,gBAAgB,CAAC;IAE7BjF,EAAE,CAAC2B,UAAU,CAAClB,OAAO,CAAC;IAEtBT,EAAE,CAACkF,SAAS,CAACvC,SAAS,EAAE,CAAC,CAAC;IAC1B3C,EAAE,CAACmF,SAAS,CAACtC,OAAO,EAAEwB,KAAK,CAAC;IAC5BrE,EAAE,CAACmF,SAAS,CAACrC,QAAQ,EAAE0B,MAAM,CAAC;IAC9BxE,EAAE,CAACmF,SAAS,CAACjC,OAAO,EAAEa,KAAK,CAAC;IAC5B/D,EAAE,CAACmF,SAAS,CAACnC,YAAY,EAAEgB,UAAU,CAAC;IACtChE,EAAE,CAACmF,SAAS,CAAClC,YAAY,EAAEgB,UAAU,CAAC;IACtCjE,EAAE,CAACoF,SAAS,CACVrC,OAAO,EACP4B,QAAQ,CAACb,KAAK,CAACuB,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,EACtCV,QAAQ,CAACb,KAAK,CAACuB,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,EACtCV,QAAQ,CAACb,KAAK,CAACuB,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,GACrC,CAAC;IAEDrF,EAAE,CAACsF,UAAU,CAACtF,EAAE,CAAC6B,UAAU,EAAE,CAAC,EAAE7B,EAAE,CAACuF,GAAG,EAAEvF,EAAE,CAACuF,GAAG,EAAEvF,EAAE,CAACwF,aAAa,EAAE3F,MAAM,CAAC;IACzEG,EAAE,CAACyF,UAAU,CAACzF,EAAE,CAAC0F,YAAY,EAAE,CAAC,EAAE,CAAC,CAAC;IACpC,IAAI,CAAC1F,EAAE,CAAC2F,aAAa,CAAC,CAAC,EAAE;MACvB5F,OAAO,CAAC6F,YAAY,YAApB7F,OAAO,CAAC6F,YAAY,CAAG,CAAC;IAC1B;EACF,CAAC;AACH","ignoreList":[]}

package/dist/esm/components/LegendsPlatform/LegendsPlatform.js

@@ -1,11 +1,13 @@
-import React, { useMemo } from 'react';
+import React, { useMemo, useRef } from 'react';
 import { Provider, createStore } from 'jotai';
 import { LiveKitRoom } from '@livekit/components-react';
 import { QueryClientProvider, QueryClient } from '@tanstack/react-query';
 import { Provider as BusProvider } from 'react-bus';
 import { FetchPersonaChatService } from '../../services/FetchPersonaChatService';
 import { SdkConsumerContext } from '../../sdk-consumer';
+import { StartupTelemetrySession } from '../../startup-telemetry/attempt';
 export function LegendsPlatform(props) {
+  var _props$startupTelemet, _props$startupTelemet2, _props$startupTelemet3;
   const chatId = props.chatId,
     namespace = props.namespace,
     children = props.children,
@@ -32,12 +34,35 @@ export function LegendsPlatform(props) {
     }
     return undefined;
   }, [personaChatService, apiKey]);
+  const telemetryRef = useRef(props.startupTelemetry);
+  telemetryRef.current = props.startupTelemetry;
+  const telemetryEnabled = (_props$startupTelemet = props.startupTelemetry) == null ? void 0 : _props$startupTelemet.enabled;
+  const intentId = (_props$startupTelemet2 = props.startupTelemetry) == null || (_props$startupTelemet2 = _props$startupTelemet2.intent) == null ? void 0 : _props$startupTelemet2.id;
+  const attemptId = (_props$startupTelemet3 = props.startupTelemetry) == null || (_props$startupTelemet3 = _props$startupTelemet3.attempt) == null ? void 0 : _props$startupTelemet3.id;
+  const startupTelemetrySession = useMemo(() => {
+    const config = telemetryRef.current;
+    if (!telemetryEnabled || !config) {
+      return undefined;
+    }
+    return new StartupTelemetrySession({
+      ...config,
+      report: event => {
+        var _telemetryRef$current;
+        if ((_telemetryRef$current = telemetryRef.current) != null && _telemetryRef$current.enabled) {
+          return telemetryRef.current.report == null ? void 0 : telemetryRef.current.report(event);
+        }
+        return undefined;
+      }
+    }, chatId);
+    // A new intent/attempt changes ownership; report callbacks stay live via the ref.
+  }, [chatId, telemetryEnabled, intentId, attemptId]);
   const consumer = useMemo(() => ({
     chatId,
     namespace,
     tools,
-    personaChatService: resolvedService
-  }), [chatId, namespace, tools, resolvedService]);
+    personaChatService: resolvedService,
+    startupTelemetrySession
+  }), [chatId, namespace, tools, resolvedService, startupTelemetrySession]);
   return /*#__PURE__*/React.createElement(SdkConsumerContext.Provider, {
     value: consumer
   }, /*#__PURE__*/React.createElement(QueryClientProvider, {

package/dist/esm/components/LegendsPlatform/LegendsPlatform.js.map

@@ -1 +1 @@
-{"version":3,"names":["React","useMemo","Provider","createStore","LiveKitRoom","QueryClientProvider","QueryClient","BusProvider","FetchPersonaChatService","SdkConsumerContext","LegendsPlatform","props","chatId","namespace","children","className","_props$tools","tools","apiKey","personaChatService","store","queryClient","defaultOptions","queries","refetchOnMount","refetchOnWindowFocus","resolvedService","undefined","consumer","createElement","value","client","serverUrl","token"],"sources":["../../../../src/components/LegendsPlatform/LegendsPlatform.tsx"],"sourcesContent":["import React, { useMemo } from 'react';\nimport { Provider, createStore } from 'jotai';\nimport { type Store } from 'jotai/vanilla/store';\nimport { LiveKitRoom } from '@livekit/components-react';\nimport { QueryClientProvider, QueryClient } from '@tanstack/react-query';\nimport { Provider as BusProvider } from 'react-bus';\n\nimport type { ToolsRegistry } from '../../hooks/useTools/model/types';\nimport type { IPersonaChatService } from '../../types/service';\nimport { FetchPersonaChatService } from '../../services/FetchPersonaChatService';\nimport { SdkConsumerContext } from '../../sdk-consumer';\n\nexport type LegendsPlatformProps = {\n  chatId: string;\n  namespace: string;\n  apiKey?: string;\n  personaChatService?: IPersonaChatService;\n  store?: Store;\n  tools?: ToolsRegistry;\n  children: React.ReactNode;\n  queryClient?: QueryClient;\n  className?: string;\n};\n\nexport function LegendsPlatform(props: LegendsPlatformProps) {\n  const {\n    chatId,\n    namespace,\n    children,\n    className,\n    tools = {},\n    apiKey,\n    personaChatService,\n  } = props;\n\n  const store = useMemo(() => props.store || createStore(), [props.store]);\n\n  const queryClient = useMemo(\n    () =>\n      props.queryClient ||\n      new QueryClient({\n        defaultOptions: {\n          queries: { refetchOnMount: false, refetchOnWindowFocus: false },\n        },\n      }),\n    [props.queryClient],\n  );\n\n  const resolvedService = useMemo<IPersonaChatService | undefined>(() => {\n    if (personaChatService) {\n      return personaChatService;\n    }\n    if (apiKey) {\n      return new FetchPersonaChatService(apiKey);\n    }\n    return undefined;\n  }, [personaChatService, apiKey]);\n\n  const consumer = useMemo(\n    () => ({ chatId, namespace, tools, personaChatService: resolvedService }),\n    [chatId, namespace, tools, resolvedService],\n  );\n\n  return (\n    <SdkConsumerContext.Provider value={consumer}>\n      <QueryClientProvider client={queryClient}>\n        <Provider store={store}>\n          <BusProvider>\n            <LiveKitRoom\n              className={className}\n              serverUrl={undefined}\n              token={undefined}\n            >\n              {children}\n            </LiveKitRoom>\n          </BusProvider>\n        </Provider>\n      </QueryClientProvider>\n    </SdkConsumerContext.Provider>\n  );\n}\n"],"mappings":"AAAA,OAAOA,KAAK,IAAIC,OAAO,QAAQ,OAAO;AACtC,SAASC,QAAQ,EAAEC,WAAW,QAAQ,OAAO;AAE7C,SAASC,WAAW,QAAQ,2BAA2B;AACvD,SAASC,mBAAmB,EAAEC,WAAW,QAAQ,uBAAuB;AACxE,SAASJ,QAAQ,IAAIK,WAAW,QAAQ,WAAW;AAInD,SAASC,uBAAuB,QAAQ,wCAAwC;AAChF,SAASC,kBAAkB,QAAQ,oBAAoB;AAcvD,OAAO,SAASC,eAAeA,CAACC,KAA2B,EAAE;EAC3D,MACEC,MAAM,GAOJD,KAAK,CAPPC,MAAM;IACNC,SAAS,GAMPF,KAAK,CANPE,SAAS;IACTC,QAAQ,GAKNH,KAAK,CALPG,QAAQ;IACRC,SAAS,GAIPJ,KAAK,CAJPI,SAAS;IAAAC,YAAA,GAIPL,KAAK,CAHPM,KAAK;IAALA,KAAK,GAAAD,YAAA,cAAG,CAAC,CAAC,GAAAA,YAAA;IACVE,MAAM,GAEJP,KAAK,CAFPO,MAAM;IACNC,kBAAkB,GAChBR,KAAK,CADPQ,kBAAkB;EAGpB,MAAMC,KAAK,GAAGnB,OAAO,CAAC,MAAMU,KAAK,CAACS,KAAK,IAAIjB,WAAW,CAAC,CAAC,EAAE,CAACQ,KAAK,CAACS,KAAK,CAAC,CAAC;EAExE,MAAMC,WAAW,GAAGpB,OAAO,CACzB,MACEU,KAAK,CAACU,WAAW,IACjB,IAAIf,WAAW,CAAC;IACdgB,cAAc,EAAE;MACdC,OAAO,EAAE;QAAEC,cAAc,EAAE,KAAK;QAAEC,oBAAoB,EAAE;MAAM;IAChE;EACF,CAAC,CAAC,EACJ,CAACd,KAAK,CAACU,WAAW,CACpB,CAAC;EAED,MAAMK,eAAe,GAAGzB,OAAO,CAAkC,MAAM;IACrE,IAAIkB,kBAAkB,EAAE;MACtB,OAAOA,kBAAkB;IAC3B;IACA,IAAID,MAAM,EAAE;MACV,OAAO,IAAIV,uBAAuB,CAACU,MAAM,CAAC;IAC5C;IACA,OAAOS,SAAS;EAClB,CAAC,EAAE,CAACR,kBAAkB,EAAED,MAAM,CAAC,CAAC;EAEhC,MAAMU,QAAQ,GAAG3B,OAAO,CACtB,OAAO;IAAEW,MAAM;IAAEC,SAAS;IAAEI,KAAK;IAAEE,kBAAkB,EAAEO;EAAgB,CAAC,CAAC,EACzE,CAACd,MAAM,EAAEC,SAAS,EAAEI,KAAK,EAAES,eAAe,CAC5C,CAAC;EAED,oBACE1B,KAAA,CAAA6B,aAAA,CAACpB,kBAAkB,CAACP,QAAQ;IAAC4B,KAAK,EAAEF;EAAS,gBAC3C5B,KAAA,CAAA6B,aAAA,CAACxB,mBAAmB;IAAC0B,MAAM,EAAEV;EAAY,gBACvCrB,KAAA,CAAA6B,aAAA,CAAC3B,QAAQ;IAACkB,KAAK,EAAEA;EAAM,gBACrBpB,KAAA,CAAA6B,aAAA,CAACtB,WAAW,qBACVP,KAAA,CAAA6B,aAAA,CAACzB,WAAW;IACVW,SAAS,EAAEA,SAAU;IACrBiB,SAAS,EAAEL,SAAU;IACrBM,KAAK,EAAEN;EAAU,GAEhBb,QACU,CACF,CACL,CACS,CACM,CAAC;AAElC","ignoreList":[]}
+{"version":3,"names":["React","useMemo","useRef","Provider","createStore","LiveKitRoom","QueryClientProvider","QueryClient","BusProvider","FetchPersonaChatService","SdkConsumerContext","StartupTelemetrySession","LegendsPlatform","props","_props$startupTelemet","_props$startupTelemet2","_props$startupTelemet3","chatId","namespace","children","className","_props$tools","tools","apiKey","personaChatService","store","queryClient","defaultOptions","queries","refetchOnMount","refetchOnWindowFocus","resolvedService","undefined","telemetryRef","startupTelemetry","current","telemetryEnabled","enabled","intentId","intent","id","attemptId","attempt","startupTelemetrySession","config","report","event","_telemetryRef$current","consumer","createElement","value","client","serverUrl","token"],"sources":["../../../../src/components/LegendsPlatform/LegendsPlatform.tsx"],"sourcesContent":["import React, { useMemo, useRef } from 'react';\nimport { Provider, createStore } from 'jotai';\nimport { type Store } from 'jotai/vanilla/store';\nimport { LiveKitRoom } from '@livekit/components-react';\nimport { QueryClientProvider, QueryClient } from '@tanstack/react-query';\nimport { Provider as BusProvider } from 'react-bus';\n\nimport type { ToolsRegistry } from '../../hooks/useTools/model/types';\nimport type { IPersonaChatService } from '../../types/service';\nimport { FetchPersonaChatService } from '../../services/FetchPersonaChatService';\nimport { SdkConsumerContext } from '../../sdk-consumer';\nimport {\n  StartupTelemetrySession,\n  type StartupTelemetryOptions,\n} from '../../startup-telemetry/attempt';\n\nexport type LegendsPlatformProps = {\n  startupTelemetry?: StartupTelemetryOptions;\n  chatId: string;\n  namespace: string;\n  apiKey?: string;\n  personaChatService?: IPersonaChatService;\n  store?: Store;\n  tools?: ToolsRegistry;\n  children: React.ReactNode;\n  queryClient?: QueryClient;\n  className?: string;\n};\n\nexport function LegendsPlatform(props: LegendsPlatformProps) {\n  const {\n    chatId,\n    namespace,\n    children,\n    className,\n    tools = {},\n    apiKey,\n    personaChatService,\n  } = props;\n\n  const store = useMemo(() => props.store || createStore(), [props.store]);\n\n  const queryClient = useMemo(\n    () =>\n      props.queryClient ||\n      new QueryClient({\n        defaultOptions: {\n          queries: { refetchOnMount: false, refetchOnWindowFocus: false },\n        },\n      }),\n    [props.queryClient],\n  );\n\n  const resolvedService = useMemo<IPersonaChatService | undefined>(() => {\n    if (personaChatService) {\n      return personaChatService;\n    }\n    if (apiKey) {\n      return new FetchPersonaChatService(apiKey);\n    }\n    return undefined;\n  }, [personaChatService, apiKey]);\n\n  const telemetryRef = useRef(props.startupTelemetry);\n  telemetryRef.current = props.startupTelemetry;\n  const telemetryEnabled = props.startupTelemetry?.enabled;\n  const intentId = props.startupTelemetry?.intent?.id;\n  const attemptId = props.startupTelemetry?.attempt?.id;\n  const startupTelemetrySession = useMemo(() => {\n    const config = telemetryRef.current;\n    if (!telemetryEnabled || !config) {\n      return undefined;\n    }\n    return new StartupTelemetrySession(\n      {\n        ...config,\n        report: (event) => {\n          if (telemetryRef.current?.enabled) {\n            return telemetryRef.current.report?.(event);\n          }\n          return undefined;\n        },\n      },\n      chatId,\n    );\n    // A new intent/attempt changes ownership; report callbacks stay live via the ref.\n  }, [chatId, telemetryEnabled, intentId, attemptId]);\n\n  const consumer = useMemo(\n    () => ({\n      chatId,\n      namespace,\n      tools,\n      personaChatService: resolvedService,\n      startupTelemetrySession,\n    }),\n    [chatId, namespace, tools, resolvedService, startupTelemetrySession],\n  );\n\n  return (\n    <SdkConsumerContext.Provider value={consumer}>\n      <QueryClientProvider client={queryClient}>\n        <Provider store={store}>\n          <BusProvider>\n            <LiveKitRoom\n              className={className}\n              serverUrl={undefined}\n              token={undefined}\n            >\n              {children}\n            </LiveKitRoom>\n          </BusProvider>\n        </Provider>\n      </QueryClientProvider>\n    </SdkConsumerContext.Provider>\n  );\n}\n"],"mappings":"AAAA,OAAOA,KAAK,IAAIC,OAAO,EAAEC,MAAM,QAAQ,OAAO;AAC9C,SAASC,QAAQ,EAAEC,WAAW,QAAQ,OAAO;AAE7C,SAASC,WAAW,QAAQ,2BAA2B;AACvD,SAASC,mBAAmB,EAAEC,WAAW,QAAQ,uBAAuB;AACxE,SAASJ,QAAQ,IAAIK,WAAW,QAAQ,WAAW;AAInD,SAASC,uBAAuB,QAAQ,wCAAwC;AAChF,SAASC,kBAAkB,QAAQ,oBAAoB;AACvD,SACEC,uBAAuB,QAElB,iCAAiC;AAexC,OAAO,SAASC,eAAeA,CAACC,KAA2B,EAAE;EAAA,IAAAC,qBAAA,EAAAC,sBAAA,EAAAC,sBAAA;EAC3D,MACEC,MAAM,GAOJJ,KAAK,CAPPI,MAAM;IACNC,SAAS,GAMPL,KAAK,CANPK,SAAS;IACTC,QAAQ,GAKNN,KAAK,CALPM,QAAQ;IACRC,SAAS,GAIPP,KAAK,CAJPO,SAAS;IAAAC,YAAA,GAIPR,KAAK,CAHPS,KAAK;IAALA,KAAK,GAAAD,YAAA,cAAG,CAAC,CAAC,GAAAA,YAAA;IACVE,MAAM,GAEJV,KAAK,CAFPU,MAAM;IACNC,kBAAkB,GAChBX,KAAK,CADPW,kBAAkB;EAGpB,MAAMC,KAAK,GAAGxB,OAAO,CAAC,MAAMY,KAAK,CAACY,KAAK,IAAIrB,WAAW,CAAC,CAAC,EAAE,CAACS,KAAK,CAACY,KAAK,CAAC,CAAC;EAExE,MAAMC,WAAW,GAAGzB,OAAO,CACzB,MACEY,KAAK,CAACa,WAAW,IACjB,IAAInB,WAAW,CAAC;IACdoB,cAAc,EAAE;MACdC,OAAO,EAAE;QAAEC,cAAc,EAAE,KAAK;QAAEC,oBAAoB,EAAE;MAAM;IAChE;EACF,CAAC,CAAC,EACJ,CAACjB,KAAK,CAACa,WAAW,CACpB,CAAC;EAED,MAAMK,eAAe,GAAG9B,OAAO,CAAkC,MAAM;IACrE,IAAIuB,kBAAkB,EAAE;MACtB,OAAOA,kBAAkB;IAC3B;IACA,IAAID,MAAM,EAAE;MACV,OAAO,IAAId,uBAAuB,CAACc,MAAM,CAAC;IAC5C;IACA,OAAOS,SAAS;EAClB,CAAC,EAAE,CAACR,kBAAkB,EAAED,MAAM,CAAC,CAAC;EAEhC,MAAMU,YAAY,GAAG/B,MAAM,CAACW,KAAK,CAACqB,gBAAgB,CAAC;EACnDD,YAAY,CAACE,OAAO,GAAGtB,KAAK,CAACqB,gBAAgB;EAC7C,MAAME,gBAAgB,IAAAtB,qBAAA,GAAGD,KAAK,CAACqB,gBAAgB,qBAAtBpB,qBAAA,CAAwBuB,OAAO;EACxD,MAAMC,QAAQ,IAAAvB,sBAAA,GAAGF,KAAK,CAACqB,gBAAgB,cAAAnB,sBAAA,GAAtBA,sBAAA,CAAwBwB,MAAM,qBAA9BxB,sBAAA,CAAgCyB,EAAE;EACnD,MAAMC,SAAS,IAAAzB,sBAAA,GAAGH,KAAK,CAACqB,gBAAgB,cAAAlB,sBAAA,GAAtBA,sBAAA,CAAwB0B,OAAO,qBAA/B1B,sBAAA,CAAiCwB,EAAE;EACrD,MAAMG,uBAAuB,GAAG1C,OAAO,CAAC,MAAM;IAC5C,MAAM2C,MAAM,GAAGX,YAAY,CAACE,OAAO;IACnC,IAAI,CAACC,gBAAgB,IAAI,CAACQ,MAAM,EAAE;MAChC,OAAOZ,SAAS;IAClB;IACA,OAAO,IAAIrB,uBAAuB,CAChC;MACE,GAAGiC,MAAM;MACTC,MAAM,EAAGC,KAAK,IAAK;QAAA,IAAAC,qBAAA;QACjB,KAAAA,qBAAA,GAAId,YAAY,CAACE,OAAO,aAApBY,qBAAA,CAAsBV,OAAO,EAAE;UACjC,OAAOJ,YAAY,CAACE,OAAO,CAACU,MAAM,oBAA3BZ,YAAY,CAACE,OAAO,CAACU,MAAM,CAAGC,KAAK,CAAC;QAC7C;QACA,OAAOd,SAAS;MAClB;IACF,CAAC,EACDf,MACF,CAAC;IACD;EACF,CAAC,EAAE,CAACA,MAAM,EAAEmB,gBAAgB,EAAEE,QAAQ,EAAEG,SAAS,CAAC,CAAC;EAEnD,MAAMO,QAAQ,GAAG/C,OAAO,CACtB,OAAO;IACLgB,MAAM;IACNC,SAAS;IACTI,KAAK;IACLE,kBAAkB,EAAEO,eAAe;IACnCY;EACF,CAAC,CAAC,EACF,CAAC1B,MAAM,EAAEC,SAAS,EAAEI,KAAK,EAAES,eAAe,EAAEY,uBAAuB,CACrE,CAAC;EAED,oBACE3C,KAAA,CAAAiD,aAAA,CAACvC,kBAAkB,CAACP,QAAQ;IAAC+C,KAAK,EAAEF;EAAS,gBAC3ChD,KAAA,CAAAiD,aAAA,CAAC3C,mBAAmB;IAAC6C,MAAM,EAAEzB;EAAY,gBACvC1B,KAAA,CAAAiD,aAAA,CAAC9C,QAAQ;IAACsB,KAAK,EAAEA;EAAM,gBACrBzB,KAAA,CAAAiD,aAAA,CAACzC,WAAW,qBACVR,KAAA,CAAAiD,aAAA,CAAC5C,WAAW;IACVe,SAAS,EAAEA,SAAU;IACrBgC,SAAS,EAAEpB,SAAU;IACrBqB,KAAK,EAAErB;EAAU,GAEhBb,QACU,CACF,CACL,CACS,CACM,CAAC;AAElC","ignoreList":[]}

package/dist/esm/components/Video/Video.js

@@ -1,6 +1,6 @@
 import _extends from "@babel/runtime/helpers/extends";
 import _objectWithoutPropertiesLoose from "@babel/runtime/helpers/objectWithoutPropertiesLoose";
-const _excluded = ["type", "fallback", "fit", "automirror", "pip", "onPiPEnter", "onPiPExit", "onPiPError", "pipFallback", "style"];
+const _excluded = ["type", "startupRenderSurface", "fallback", "fit", "automirror", "pip", "onPiPEnter", "onPiPExit", "onPiPError", "pipFallback", "style"];
 import React, { forwardRef, useEffect, useImperativeHandle, useState } from 'react';
 import invariant from 'invariant';
 import { useAtomValue } from 'jotai';
@@ -8,8 +8,12 @@ import { state } from '../../conversation';
 import { useAttachVideo } from './useAttachVideo';
 import { useEmitConversationEvent } from '../../hooks/useConversationEvents/useEmitConversationEvent';
 import { usePictureInPictureAuto, pipActiveAtom } from '../../hooks/usePictureInPicture';
+import { useStartupAttempt } from '../../startup-telemetry/useStartupAttempt';
+import { observeVideoFrame } from '../../startup-telemetry/frame';
 export const Video = /*#__PURE__*/forwardRef((props, ref) => {
   const type = props.type,
+    _props$startupRenderS = props.startupRenderSurface,
+    startupRenderSurface = _props$startupRenderS === void 0 ? 'video' : _props$startupRenderS,
     _props$fallback = props.fallback,
     Fallback = _props$fallback === void 0 ? () => null : _props$fallback,
     _props$fit = props.fit,
@@ -29,6 +33,7 @@ export const Video = /*#__PURE__*/forwardRef((props, ref) => {
     setMirror = _useState[1];
   const conversation = useAtomValue(state.conversation);
   const emitEvent = useEmitConversationEvent();
+  const startupAttempt = useStartupAttempt();
   invariant(conversation, 'Conversation is not initialized');
   invariant(type === 'screen' ? true : conversation.type === 'video', 'Conversation type is not video');
   const _useAttachVideo = useAttachVideo({
@@ -55,6 +60,16 @@ export const Video = /*#__PURE__*/forwardRef((props, ref) => {
       });
     }
   }, [isVideoPlayable, type]);
+  useEffect(() => {
+    if (!startupAttempt || type !== 'agent' || !isVideoPlayable || !videoRef.current) {
+      return;
+    }
+    startupAttempt.mark('track_subscribed');
+    if (startupRenderSurface === 'video') {
+      return observeVideoFrame(startupAttempt, videoRef.current);
+    }
+    return undefined;
+  }, [startupAttempt, type, isVideoPlayable, startupRenderSurface, videoRef]);
   return /*#__PURE__*/React.createElement(React.Fragment, null, /*#__PURE__*/React.createElement("video", _extends({
     ref: videoRef,
     muted: true,

package/dist/esm/components/Video/Video.js.map

@@ -1 +1 @@
-{"version":3,"names":["React","forwardRef","useEffect","useImperativeHandle","useState","invariant","useAtomValue","state","useAttachVideo","useEmitConversationEvent","usePictureInPictureAuto","pipActiveAtom","Video","props","ref","type","_props$fallback","fallback","Fallback","_props$fit","fit","_props$automirror","automirror","_props$pip","pip","onPiPEnter","onPiPExit","onPiPError","pipFallback","style","videoProps","_objectWithoutPropertiesLoose","_excluded","_useState","mirror","setMirror","conversation","emitEvent","_useAttachVideo","videoRef","isVideoPlayable","element","current","onEnter","onExit","onError","isPiPActive","showPipFallback","createElement","Fragment","_extends","muted","playsInline","autoPlay","objectFit","display","transform"],"sources":["../../../../src/components/Video/Video.tsx"],"sourcesContent":["import React, {\n  forwardRef,\n  useEffect,\n  useImperativeHandle,\n  useState,\n} from 'react';\nimport invariant from 'invariant';\nimport { useAtomValue } from 'jotai';\nimport { state } from '../../conversation';\nimport { useAttachVideo } from './useAttachVideo';\nimport { useEmitConversationEvent } from '../../hooks/useConversationEvents/useEmitConversationEvent';\nimport {\n  usePictureInPictureAuto,\n  pipActiveAtom,\n} from '../../hooks/usePictureInPicture';\nimport { type VideoProps } from './types';\n\nexport const Video = forwardRef<HTMLVideoElement, VideoProps>((props, ref) => {\n  const {\n    type,\n    fallback: Fallback = () => null,\n    fit = 'cover',\n    automirror = true,\n    pip = false,\n    onPiPEnter,\n    onPiPExit,\n    onPiPError,\n    pipFallback,\n    style,\n    ...videoProps\n  } = props;\n\n  const [mirror, setMirror] = useState(props.mirror);\n\n  const conversation = useAtomValue(state.conversation);\n  const emitEvent = useEmitConversationEvent();\n\n  invariant(conversation, 'Conversation is not initialized');\n  invariant(\n    type === 'screen' ? true : conversation.type === 'video',\n    'Conversation type is not video',\n  );\n\n  const { videoRef, isVideoPlayable } = useAttachVideo({ type });\n\n  usePictureInPictureAuto({\n    element: pip && isVideoPlayable ? videoRef.current : null,\n    onEnter: onPiPEnter,\n    onExit: onPiPExit,\n    onError: onPiPError,\n  });\n\n  const isPiPActive = useAtomValue(pipActiveAtom);\n\n  useImperativeHandle(ref, () => videoRef.current!);\n\n  const showPipFallback = pip && isPiPActive;\n\n  useEffect(() => {\n    setMirror(props.mirror ?? (automirror && type === 'user'));\n  }, [automirror, type, props.mirror]);\n\n  useEffect(() => {\n    if (isVideoPlayable && type === 'agent') {\n      emitEvent('agentVideoLoaded', {\n        videoRef,\n      });\n    }\n  }, [isVideoPlayable, type]);\n\n  return (\n    <>\n      <video\n        ref={videoRef}\n        muted\n        playsInline\n        autoPlay\n        style={{\n          objectFit: fit,\n          display: isVideoPlayable && !showPipFallback ? 'block' : 'none',\n          transform: mirror ? 'scaleX(-1)' : 'none',\n          ...style,\n        }}\n        {...videoProps}\n      />\n      {!isVideoPlayable && <Fallback style={style} />}\n      {showPipFallback && pipFallback}\n    </>\n  );\n});\n"],"mappings":";;;AAAA,OAAOA,KAAK,IACVC,UAAU,EACVC,SAAS,EACTC,mBAAmB,EACnBC,QAAQ,QACH,OAAO;AACd,OAAOC,SAAS,MAAM,WAAW;AACjC,SAASC,YAAY,QAAQ,OAAO;AACpC,SAASC,KAAK,QAAQ,oBAAoB;AAC1C,SAASC,cAAc,QAAQ,kBAAkB;AACjD,SAASC,wBAAwB,QAAQ,4DAA4D;AACrG,SACEC,uBAAuB,EACvBC,aAAa,QACR,iCAAiC;AAGxC,OAAO,MAAMC,KAAK,gBAAGX,UAAU,CAA+B,CAACY,KAAK,EAAEC,GAAG,KAAK;EAC5E,MACEC,IAAI,GAWFF,KAAK,CAXPE,IAAI;IAAAC,eAAA,GAWFH,KAAK,CAVPI,QAAQ;IAAEC,QAAQ,GAAAF,eAAA,cAAG,MAAM,IAAI,GAAAA,eAAA;IAAAG,UAAA,GAU7BN,KAAK,CATPO,GAAG;IAAHA,GAAG,GAAAD,UAAA,cAAG,OAAO,GAAAA,UAAA;IAAAE,iBAAA,GASXR,KAAK,CARPS,UAAU;IAAVA,UAAU,GAAAD,iBAAA,cAAG,IAAI,GAAAA,iBAAA;IAAAE,UAAA,GAQfV,KAAK,CAPPW,GAAG;IAAHA,GAAG,GAAAD,UAAA,cAAG,KAAK,GAAAA,UAAA;IACXE,UAAU,GAMRZ,KAAK,CANPY,UAAU;IACVC,SAAS,GAKPb,KAAK,CALPa,SAAS;IACTC,UAAU,GAIRd,KAAK,CAJPc,UAAU;IACVC,WAAW,GAGTf,KAAK,CAHPe,WAAW;IACXC,KAAK,GAEHhB,KAAK,CAFPgB,KAAK;IACFC,UAAU,GAAAC,6BAAA,CACXlB,KAAK,EAAAmB,SAAA;EAET,MAAAC,SAAA,GAA4B7B,QAAQ,CAACS,KAAK,CAACqB,MAAM,CAAC;IAA3CA,MAAM,GAAAD,SAAA;IAAEE,SAAS,GAAAF,SAAA;EAExB,MAAMG,YAAY,GAAG9B,YAAY,CAACC,KAAK,CAAC6B,YAAY,CAAC;EACrD,MAAMC,SAAS,GAAG5B,wBAAwB,CAAC,CAAC;EAE5CJ,SAAS,CAAC+B,YAAY,EAAE,iCAAiC,CAAC;EAC1D/B,SAAS,CACPU,IAAI,KAAK,QAAQ,GAAG,IAAI,GAAGqB,YAAY,CAACrB,IAAI,KAAK,OAAO,EACxD,gCACF,CAAC;EAED,MAAAuB,eAAA,GAAsC9B,cAAc,CAAC;MAAEO;IAAK,CAAC,CAAC;IAAtDwB,QAAQ,GAAAD,eAAA,CAARC,QAAQ;IAAEC,eAAe,GAAAF,eAAA,CAAfE,eAAe;EAEjC9B,uBAAuB,CAAC;IACtB+B,OAAO,EAAEjB,GAAG,IAAIgB,eAAe,GAAGD,QAAQ,CAACG,OAAO,GAAG,IAAI;IACzDC,OAAO,EAAElB,UAAU;IACnBmB,MAAM,EAAElB,SAAS;IACjBmB,OAAO,EAAElB;EACX,CAAC,CAAC;EAEF,MAAMmB,WAAW,GAAGxC,YAAY,CAACK,aAAa,CAAC;EAE/CR,mBAAmB,CAACW,GAAG,EAAE,MAAMyB,QAAQ,CAACG,OAAQ,CAAC;EAEjD,MAAMK,eAAe,GAAGvB,GAAG,IAAIsB,WAAW;EAE1C5C,SAAS,CAAC,MAAM;IACdiC,SAAS,CAACtB,KAAK,CAACqB,MAAM,KAAKZ,UAAU,IAAIP,IAAI,KAAK,MAAM,CAAC,CAAC;EAC5D,CAAC,EAAE,CAACO,UAAU,EAAEP,IAAI,EAAEF,KAAK,CAACqB,MAAM,CAAC,CAAC;EAEpChC,SAAS,CAAC,MAAM;IACd,IAAIsC,eAAe,IAAIzB,IAAI,KAAK,OAAO,EAAE;MACvCsB,SAAS,CAAC,kBAAkB,EAAE;QAC5BE;MACF,CAAC,CAAC;IACJ;EACF,CAAC,EAAE,CAACC,eAAe,EAAEzB,IAAI,CAAC,CAAC;EAE3B,oBACEf,KAAA,CAAAgD,aAAA,CAAAhD,KAAA,CAAAiD,QAAA,qBACEjD,KAAA,CAAAgD,aAAA,UAAAE,QAAA;IACEpC,GAAG,EAAEyB,QAAS;IACdY,KAAK;IACLC,WAAW;IACXC,QAAQ;IACRxB,KAAK,EAAE;MACLyB,SAAS,EAAElC,GAAG;MACdmC,OAAO,EAAEf,eAAe,IAAI,CAACO,eAAe,GAAG,OAAO,GAAG,MAAM;MAC/DS,SAAS,EAAEtB,MAAM,GAAG,YAAY,GAAG,MAAM;MACzC,GAAGL;IACL;EAAE,GACEC,UAAU,CACf,CAAC,EACD,CAACU,eAAe,iBAAIxC,KAAA,CAAAgD,aAAA,CAAC9B,QAAQ;IAACW,KAAK,EAAEA;EAAM,CAAE,CAAC,EAC9CkB,eAAe,IAAInB,WACpB,CAAC;AAEP,CAAC,CAAC","ignoreList":[]}
+{"version":3,"names":["React","forwardRef","useEffect","useImperativeHandle","useState","invariant","useAtomValue","state","useAttachVideo","useEmitConversationEvent","usePictureInPictureAuto","pipActiveAtom","useStartupAttempt","observeVideoFrame","Video","props","ref","type","_props$startupRenderS","startupRenderSurface","_props$fallback","fallback","Fallback","_props$fit","fit","_props$automirror","automirror","_props$pip","pip","onPiPEnter","onPiPExit","onPiPError","pipFallback","style","videoProps","_objectWithoutPropertiesLoose","_excluded","_useState","mirror","setMirror","conversation","emitEvent","startupAttempt","_useAttachVideo","videoRef","isVideoPlayable","element","current","onEnter","onExit","onError","isPiPActive","showPipFallback","mark","undefined","createElement","Fragment","_extends","muted","playsInline","autoPlay","objectFit","display","transform"],"sources":["../../../../src/components/Video/Video.tsx"],"sourcesContent":["import React, {\n  forwardRef,\n  useEffect,\n  useImperativeHandle,\n  useState,\n} from 'react';\nimport invariant from 'invariant';\nimport { useAtomValue } from 'jotai';\nimport { state } from '../../conversation';\nimport { useAttachVideo } from './useAttachVideo';\nimport { useEmitConversationEvent } from '../../hooks/useConversationEvents/useEmitConversationEvent';\nimport {\n  usePictureInPictureAuto,\n  pipActiveAtom,\n} from '../../hooks/usePictureInPicture';\nimport { type VideoProps } from './types';\nimport { useStartupAttempt } from '../../startup-telemetry/useStartupAttempt';\nimport { observeVideoFrame } from '../../startup-telemetry/frame';\n\nexport const Video = forwardRef<HTMLVideoElement, VideoProps>((props, ref) => {\n  const {\n    type,\n    startupRenderSurface = 'video',\n    fallback: Fallback = () => null,\n    fit = 'cover',\n    automirror = true,\n    pip = false,\n    onPiPEnter,\n    onPiPExit,\n    onPiPError,\n    pipFallback,\n    style,\n    ...videoProps\n  } = props;\n\n  const [mirror, setMirror] = useState(props.mirror);\n\n  const conversation = useAtomValue(state.conversation);\n  const emitEvent = useEmitConversationEvent();\n  const startupAttempt = useStartupAttempt();\n\n  invariant(conversation, 'Conversation is not initialized');\n  invariant(\n    type === 'screen' ? true : conversation.type === 'video',\n    'Conversation type is not video',\n  );\n\n  const { videoRef, isVideoPlayable } = useAttachVideo({ type });\n\n  usePictureInPictureAuto({\n    element: pip && isVideoPlayable ? videoRef.current : null,\n    onEnter: onPiPEnter,\n    onExit: onPiPExit,\n    onError: onPiPError,\n  });\n\n  const isPiPActive = useAtomValue(pipActiveAtom);\n\n  useImperativeHandle(ref, () => videoRef.current!);\n\n  const showPipFallback = pip && isPiPActive;\n\n  useEffect(() => {\n    setMirror(props.mirror ?? (automirror && type === 'user'));\n  }, [automirror, type, props.mirror]);\n\n  useEffect(() => {\n    if (isVideoPlayable && type === 'agent') {\n      emitEvent('agentVideoLoaded', {\n        videoRef,\n      });\n    }\n  }, [isVideoPlayable, type]);\n\n  useEffect(() => {\n    if (\n      !startupAttempt ||\n      type !== 'agent' ||\n      !isVideoPlayable ||\n      !videoRef.current\n    ) {\n      return;\n    }\n    startupAttempt.mark('track_subscribed');\n    if (startupRenderSurface === 'video') {\n      return observeVideoFrame(startupAttempt, videoRef.current);\n    }\n    return undefined;\n  }, [startupAttempt, type, isVideoPlayable, startupRenderSurface, videoRef]);\n\n  return (\n    <>\n      <video\n        ref={videoRef}\n        muted\n        playsInline\n        autoPlay\n        style={{\n          objectFit: fit,\n          display: isVideoPlayable && !showPipFallback ? 'block' : 'none',\n          transform: mirror ? 'scaleX(-1)' : 'none',\n          ...style,\n        }}\n        {...videoProps}\n      />\n      {!isVideoPlayable && <Fallback style={style} />}\n      {showPipFallback && pipFallback}\n    </>\n  );\n});\n"],"mappings":";;;AAAA,OAAOA,KAAK,IACVC,UAAU,EACVC,SAAS,EACTC,mBAAmB,EACnBC,QAAQ,QACH,OAAO;AACd,OAAOC,SAAS,MAAM,WAAW;AACjC,SAASC,YAAY,QAAQ,OAAO;AACpC,SAASC,KAAK,QAAQ,oBAAoB;AAC1C,SAASC,cAAc,QAAQ,kBAAkB;AACjD,SAASC,wBAAwB,QAAQ,4DAA4D;AACrG,SACEC,uBAAuB,EACvBC,aAAa,QACR,iCAAiC;AAExC,SAASC,iBAAiB,QAAQ,2CAA2C;AAC7E,SAASC,iBAAiB,QAAQ,+BAA+B;AAEjE,OAAO,MAAMC,KAAK,gBAAGb,UAAU,CAA+B,CAACc,KAAK,EAAEC,GAAG,KAAK;EAC5E,MACEC,IAAI,GAYFF,KAAK,CAZPE,IAAI;IAAAC,qBAAA,GAYFH,KAAK,CAXPI,oBAAoB;IAApBA,oBAAoB,GAAAD,qBAAA,cAAG,OAAO,GAAAA,qBAAA;IAAAE,eAAA,GAW5BL,KAAK,CAVPM,QAAQ;IAAEC,QAAQ,GAAAF,eAAA,cAAG,MAAM,IAAI,GAAAA,eAAA;IAAAG,UAAA,GAU7BR,KAAK,CATPS,GAAG;IAAHA,GAAG,GAAAD,UAAA,cAAG,OAAO,GAAAA,UAAA;IAAAE,iBAAA,GASXV,KAAK,CARPW,UAAU;IAAVA,UAAU,GAAAD,iBAAA,cAAG,IAAI,GAAAA,iBAAA;IAAAE,UAAA,GAQfZ,KAAK,CAPPa,GAAG;IAAHA,GAAG,GAAAD,UAAA,cAAG,KAAK,GAAAA,UAAA;IACXE,UAAU,GAMRd,KAAK,CANPc,UAAU;IACVC,SAAS,GAKPf,KAAK,CALPe,SAAS;IACTC,UAAU,GAIRhB,KAAK,CAJPgB,UAAU;IACVC,WAAW,GAGTjB,KAAK,CAHPiB,WAAW;IACXC,KAAK,GAEHlB,KAAK,CAFPkB,KAAK;IACFC,UAAU,GAAAC,6BAAA,CACXpB,KAAK,EAAAqB,SAAA;EAET,MAAAC,SAAA,GAA4BjC,QAAQ,CAACW,KAAK,CAACuB,MAAM,CAAC;IAA3CA,MAAM,GAAAD,SAAA;IAAEE,SAAS,GAAAF,SAAA;EAExB,MAAMG,YAAY,GAAGlC,YAAY,CAACC,KAAK,CAACiC,YAAY,CAAC;EACrD,MAAMC,SAAS,GAAGhC,wBAAwB,CAAC,CAAC;EAC5C,MAAMiC,cAAc,GAAG9B,iBAAiB,CAAC,CAAC;EAE1CP,SAAS,CAACmC,YAAY,EAAE,iCAAiC,CAAC;EAC1DnC,SAAS,CACPY,IAAI,KAAK,QAAQ,GAAG,IAAI,GAAGuB,YAAY,CAACvB,IAAI,KAAK,OAAO,EACxD,gCACF,CAAC;EAED,MAAA0B,eAAA,GAAsCnC,cAAc,CAAC;MAAES;IAAK,CAAC,CAAC;IAAtD2B,QAAQ,GAAAD,eAAA,CAARC,QAAQ;IAAEC,eAAe,GAAAF,eAAA,CAAfE,eAAe;EAEjCnC,uBAAuB,CAAC;IACtBoC,OAAO,EAAElB,GAAG,IAAIiB,eAAe,GAAGD,QAAQ,CAACG,OAAO,GAAG,IAAI;IACzDC,OAAO,EAAEnB,UAAU;IACnBoB,MAAM,EAAEnB,SAAS;IACjBoB,OAAO,EAAEnB;EACX,CAAC,CAAC;EAEF,MAAMoB,WAAW,GAAG7C,YAAY,CAACK,aAAa,CAAC;EAE/CR,mBAAmB,CAACa,GAAG,EAAE,MAAM4B,QAAQ,CAACG,OAAQ,CAAC;EAEjD,MAAMK,eAAe,GAAGxB,GAAG,IAAIuB,WAAW;EAE1CjD,SAAS,CAAC,MAAM;IACdqC,SAAS,CAACxB,KAAK,CAACuB,MAAM,KAAKZ,UAAU,IAAIT,IAAI,KAAK,MAAM,CAAC,CAAC;EAC5D,CAAC,EAAE,CAACS,UAAU,EAAET,IAAI,EAAEF,KAAK,CAACuB,MAAM,CAAC,CAAC;EAEpCpC,SAAS,CAAC,MAAM;IACd,IAAI2C,eAAe,IAAI5B,IAAI,KAAK,OAAO,EAAE;MACvCwB,SAAS,CAAC,kBAAkB,EAAE;QAC5BG;MACF,CAAC,CAAC;IACJ;EACF,CAAC,EAAE,CAACC,eAAe,EAAE5B,IAAI,CAAC,CAAC;EAE3Bf,SAAS,CAAC,MAAM;IACd,IACE,CAACwC,cAAc,IACfzB,IAAI,KAAK,OAAO,IAChB,CAAC4B,eAAe,IAChB,CAACD,QAAQ,CAACG,OAAO,EACjB;MACA;IACF;IACAL,cAAc,CAACW,IAAI,CAAC,kBAAkB,CAAC;IACvC,IAAIlC,oBAAoB,KAAK,OAAO,EAAE;MACpC,OAAON,iBAAiB,CAAC6B,cAAc,EAAEE,QAAQ,CAACG,OAAO,CAAC;IAC5D;IACA,OAAOO,SAAS;EAClB,CAAC,EAAE,CAACZ,cAAc,EAAEzB,IAAI,EAAE4B,eAAe,EAAE1B,oBAAoB,EAAEyB,QAAQ,CAAC,CAAC;EAE3E,oBACE5C,KAAA,CAAAuD,aAAA,CAAAvD,KAAA,CAAAwD,QAAA,qBACExD,KAAA,CAAAuD,aAAA,UAAAE,QAAA;IACEzC,GAAG,EAAE4B,QAAS;IACdc,KAAK;IACLC,WAAW;IACXC,QAAQ;IACR3B,KAAK,EAAE;MACL4B,SAAS,EAAErC,GAAG;MACdsC,OAAO,EAAEjB,eAAe,IAAI,CAACO,eAAe,GAAG,OAAO,GAAG,MAAM;MAC/DW,SAAS,EAAEzB,MAAM,GAAG,YAAY,GAAG,MAAM;MACzC,GAAGL;IACL;EAAE,GACEC,UAAU,CACf,CAAC,EACD,CAACW,eAAe,iBAAI7C,KAAA,CAAAuD,aAAA,CAACjC,QAAQ;IAACW,KAAK,EAAEA;EAAM,CAAE,CAAC,EAC9CmB,eAAe,IAAIpB,WACpB,CAAC;AAEP,CAAC,CAAC","ignoreList":[]}

package/dist/esm/components/Video/types.js.map

@@ -1 +1 @@
-{"version":3,"names":[],"sources":["../../../../src/components/Video/types.ts"],"sourcesContent":["import React from 'react';\n\nexport type FallbackProps = {\n  style?: React.CSSProperties;\n};\n\nexport type VideoProps = {\n  mirror?: boolean;\n  automirror?: boolean;\n  type: 'user' | 'agent' | 'screen';\n  fit?: 'contain' | 'cover';\n  fallback?: React.ComponentType<FallbackProps>;\n  /** Enable Picture-in-Picture mode */\n  pip?: boolean;\n  /** Callback when entering PiP */\n  onPiPEnter?: () => void;\n  /** Callback when exiting PiP */\n  onPiPExit?: () => void;\n  /** Callback when PiP error occurs */\n  onPiPError?: (error: Error) => void;\n  /** Content to render when PiP is active */\n  pipFallback?: React.ReactNode;\n} & Omit<React.VideoHTMLAttributes<HTMLVideoElement>, 'onResize'>;\n"],"mappings":"","ignoreList":[]}
+{"version":3,"names":[],"sources":["../../../../src/components/Video/types.ts"],"sourcesContent":["import React from 'react';\n\nexport type FallbackProps = {\n  style?: React.CSSProperties;\n};\n\nexport type VideoProps = {\n  /** Internal: canvas wrappers own the final presentation milestone. */\n  startupRenderSurface?: 'video' | 'canvas';\n  mirror?: boolean;\n  automirror?: boolean;\n  type: 'user' | 'agent' | 'screen';\n  fit?: 'contain' | 'cover';\n  fallback?: React.ComponentType<FallbackProps>;\n  /** Enable Picture-in-Picture mode */\n  pip?: boolean;\n  /** Callback when entering PiP */\n  onPiPEnter?: () => void;\n  /** Callback when exiting PiP */\n  onPiPExit?: () => void;\n  /** Callback when PiP error occurs */\n  onPiPError?: (error: Error) => void;\n  /** Content to render when PiP is active */\n  pipFallback?: React.ReactNode;\n} & Omit<React.VideoHTMLAttributes<HTMLVideoElement>, 'onResize'>;\n"],"mappings":"","ignoreList":[]}

package/dist/esm/components/VideoPostProcessor/VideoPostProcessor.js

@@ -1,9 +1,16 @@
+import { useStartupAttempt } from '../../startup-telemetry/useStartupAttempt';
+import { recordCanvasFrame } from '../../startup-telemetry/frame';
 import React, { useEffect, useRef } from 'react';
 import { transparentBackground, render } from './effects';
 import invariant from 'invariant';
 export function VideoPostProcessor(_ref) {
   let effects = _ref.effects,
     children = _ref.children;
+  const startupAttempt = useStartupAttempt();
+  const visibleAttempt = children.props.type === 'agent' ? startupAttempt : undefined;
+  const effectsKey = effects.join(',');
+  const effectsRef = useRef(effects);
+  effectsRef.current = effects;
   const $video = useRef(null);
   const $canvas = useRef(null);
   useEffect(() => {
@@ -22,22 +29,24 @@ export function VideoPostProcessor(_ref) {
       invariant($video.current, 'Video is not initialized');
       invariant($canvas.current, 'Canvas is not initialized');
       render($video.current, $canvas.current);
-      for (const effect of effects) {
+      for (const effect of effectsRef.current) {
         switch (effect) {
           case 'transparent-background':
             transparentBackground($canvas.current);
             break;
         }
       }
+      recordCanvasFrame(visibleAttempt, $canvas.current, $video.current);
       id = requestAnimationFrame(process);
     }
-  }, [effects.join(',')]);
+  }, [effectsKey, visibleAttempt]);
   return /*#__PURE__*/React.createElement(React.Fragment, null, /*#__PURE__*/React.createElement("canvas", {
     ref: $canvas,
     style: children.props.style,
     className: children.props.className
   }), /*#__PURE__*/React.cloneElement(children, {
     ...children.props,
+    startupRenderSurface: 'canvas',
     ref: $video,
     style: {
       display: 'none'

package/dist/esm/components/VideoPostProcessor/VideoPostProcessor.js.map

@@ -1 +1 @@
-{"version":3,"names":["React","useEffect","useRef","transparentBackground","render","invariant","VideoPostProcessor","_ref","effects","children","$video","$canvas","id","requestAnimationFrame","init","cancelAnimationFrame","current","videoWidth","videoHeight","process","effect","join","createElement","Fragment","ref","style","props","className","cloneElement","display"],"sources":["../../../../src/components/VideoPostProcessor/VideoPostProcessor.tsx"],"sourcesContent":["import React, { useEffect, useRef } from 'react';\n\nimport { transparentBackground, render } from './effects';\nimport invariant from 'invariant';\n\ntype VideoPostProcessorEffect = 'transparent-background';\n\ninterface VideoPostProcessorProps {\n  effects: VideoPostProcessorEffect[];\n\n  children: React.ReactElement<\n    React.VideoHTMLAttributes<HTMLVideoElement> & {\n      ref: React.RefObject<HTMLVideoElement>;\n    }\n  >;\n}\n\nexport function VideoPostProcessor({\n  effects,\n  children,\n}: VideoPostProcessorProps) {\n  const $video = useRef<HTMLVideoElement>(null);\n  const $canvas = useRef<HTMLCanvasElement>(null);\n\n  useEffect(() => {\n    let id = requestAnimationFrame(init);\n\n    return () => {\n      cancelAnimationFrame(id);\n    };\n\n    function init() {\n      if (\n        !$video.current ||\n        !$canvas.current ||\n        !$video.current.videoWidth ||\n        !$video.current.videoHeight\n      ) {\n        id = requestAnimationFrame(init);\n        return;\n      }\n\n      id = requestAnimationFrame(process);\n    }\n\n    function process() {\n      invariant($video.current, 'Video is not initialized');\n      invariant($canvas.current, 'Canvas is not initialized');\n\n      render($video.current, $canvas.current);\n\n      for (const effect of effects) {\n        switch (effect) {\n          case 'transparent-background':\n            transparentBackground($canvas.current);\n            break;\n        }\n      }\n\n      id = requestAnimationFrame(process);\n    }\n  }, [effects.join(',')]);\n\n  return (\n    <>\n      <canvas\n        ref={$canvas}\n        style={children.props.style}\n        className={children.props.className}\n      />\n\n      {React.cloneElement(children, {\n        ...children.props,\n        ref: $video,\n        style: { display: 'none' },\n      })}\n    </>\n  );\n}\n"],"mappings":"AAAA,OAAOA,KAAK,IAAIC,SAAS,EAAEC,MAAM,QAAQ,OAAO;AAEhD,SAASC,qBAAqB,EAAEC,MAAM,QAAQ,WAAW;AACzD,OAAOC,SAAS,MAAM,WAAW;AAcjC,OAAO,SAASC,kBAAkBA,CAAAC,IAAA,EAGN;EAAA,IAF1BC,OAAO,GAAAD,IAAA,CAAPC,OAAO;IACPC,QAAQ,GAAAF,IAAA,CAARE,QAAQ;EAER,MAAMC,MAAM,GAAGR,MAAM,CAAmB,IAAI,CAAC;EAC7C,MAAMS,OAAO,GAAGT,MAAM,CAAoB,IAAI,CAAC;EAE/CD,SAAS,CAAC,MAAM;IACd,IAAIW,EAAE,GAAGC,qBAAqB,CAACC,IAAI,CAAC;IAEpC,OAAO,MAAM;MACXC,oBAAoB,CAACH,EAAE,CAAC;IAC1B,CAAC;IAED,SAASE,IAAIA,CAAA,EAAG;MACd,IACE,CAACJ,MAAM,CAACM,OAAO,IACf,CAACL,OAAO,CAACK,OAAO,IAChB,CAACN,MAAM,CAACM,OAAO,CAACC,UAAU,IAC1B,CAACP,MAAM,CAACM,OAAO,CAACE,WAAW,EAC3B;QACAN,EAAE,GAAGC,qBAAqB,CAACC,IAAI,CAAC;QAChC;MACF;MAEAF,EAAE,GAAGC,qBAAqB,CAACM,OAAO,CAAC;IACrC;IAEA,SAASA,OAAOA,CAAA,EAAG;MACjBd,SAAS,CAACK,MAAM,CAACM,OAAO,EAAE,0BAA0B,CAAC;MACrDX,SAAS,CAACM,OAAO,CAACK,OAAO,EAAE,2BAA2B,CAAC;MAEvDZ,MAAM,CAACM,MAAM,CAACM,OAAO,EAAEL,OAAO,CAACK,OAAO,CAAC;MAEvC,KAAK,MAAMI,MAAM,IAAIZ,OAAO,EAAE;QAC5B,QAAQY,MAAM;UACZ,KAAK,wBAAwB;YAC3BjB,qBAAqB,CAACQ,OAAO,CAACK,OAAO,CAAC;YACtC;QACJ;MACF;MAEAJ,EAAE,GAAGC,qBAAqB,CAACM,OAAO,CAAC;IACrC;EACF,CAAC,EAAE,CAACX,OAAO,CAACa,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;EAEvB,oBACErB,KAAA,CAAAsB,aAAA,CAAAtB,KAAA,CAAAuB,QAAA,qBACEvB,KAAA,CAAAsB,aAAA;IACEE,GAAG,EAAEb,OAAQ;IACbc,KAAK,EAAEhB,QAAQ,CAACiB,KAAK,CAACD,KAAM;IAC5BE,SAAS,EAAElB,QAAQ,CAACiB,KAAK,CAACC;EAAU,CACrC,CAAC,eAED3B,KAAK,CAAC4B,YAAY,CAACnB,QAAQ,EAAE;IAC5B,GAAGA,QAAQ,CAACiB,KAAK;IACjBF,GAAG,EAAEd,MAAM;IACXe,KAAK,EAAE;MAAEI,OAAO,EAAE;IAAO;EAC3B,CAAC,CACD,CAAC;AAEP","ignoreList":[]}
+{"version":3,"names":["useStartupAttempt","recordCanvasFrame","React","useEffect","useRef","transparentBackground","render","invariant","VideoPostProcessor","_ref","effects","children","startupAttempt","visibleAttempt","props","type","undefined","effectsKey","join","effectsRef","current","$video","$canvas","id","requestAnimationFrame","init","cancelAnimationFrame","videoWidth","videoHeight","process","effect","createElement","Fragment","ref","style","className","cloneElement","startupRenderSurface","display"],"sources":["../../../../src/components/VideoPostProcessor/VideoPostProcessor.tsx"],"sourcesContent":["import { useStartupAttempt } from '../../startup-telemetry/useStartupAttempt';\nimport { recordCanvasFrame } from '../../startup-telemetry/frame';\nimport React, { useEffect, useRef } from 'react';\n\nimport { transparentBackground, render } from './effects';\nimport invariant from 'invariant';\n\ntype VideoPostProcessorEffect = 'transparent-background';\n\ninterface VideoPostProcessorProps {\n  effects: VideoPostProcessorEffect[];\n\n  children: React.ReactElement<\n    React.VideoHTMLAttributes<HTMLVideoElement> & {\n      ref: React.RefObject<HTMLVideoElement>;\n      startupRenderSurface?: 'video' | 'canvas';\n      type?: 'agent' | 'user' | 'screen';\n    }\n  >;\n}\n\nexport function VideoPostProcessor({\n  effects,\n  children,\n}: VideoPostProcessorProps) {\n  const startupAttempt = useStartupAttempt();\n  const visibleAttempt =\n    children.props.type === 'agent' ? startupAttempt : undefined;\n  const effectsKey = effects.join(',');\n  const effectsRef = useRef(effects);\n  effectsRef.current = effects;\n  const $video = useRef<HTMLVideoElement>(null);\n  const $canvas = useRef<HTMLCanvasElement>(null);\n\n  useEffect(() => {\n    let id = requestAnimationFrame(init);\n\n    return () => {\n      cancelAnimationFrame(id);\n    };\n\n    function init() {\n      if (\n        !$video.current ||\n        !$canvas.current ||\n        !$video.current.videoWidth ||\n        !$video.current.videoHeight\n      ) {\n        id = requestAnimationFrame(init);\n        return;\n      }\n\n      id = requestAnimationFrame(process);\n    }\n\n    function process() {\n      invariant($video.current, 'Video is not initialized');\n      invariant($canvas.current, 'Canvas is not initialized');\n\n      render($video.current, $canvas.current);\n\n      for (const effect of effectsRef.current) {\n        switch (effect) {\n          case 'transparent-background':\n            transparentBackground($canvas.current);\n            break;\n        }\n      }\n\n      recordCanvasFrame(visibleAttempt, $canvas.current, $video.current);\n      id = requestAnimationFrame(process);\n    }\n  }, [effectsKey, visibleAttempt]);\n\n  return (\n    <>\n      <canvas\n        ref={$canvas}\n        style={children.props.style}\n        className={children.props.className}\n      />\n\n      {React.cloneElement(children, {\n        ...children.props,\n        startupRenderSurface: 'canvas',\n        ref: $video,\n        style: { display: 'none' },\n      })}\n    </>\n  );\n}\n"],"mappings":"AAAA,SAASA,iBAAiB,QAAQ,2CAA2C;AAC7E,SAASC,iBAAiB,QAAQ,+BAA+B;AACjE,OAAOC,KAAK,IAAIC,SAAS,EAAEC,MAAM,QAAQ,OAAO;AAEhD,SAASC,qBAAqB,EAAEC,MAAM,QAAQ,WAAW;AACzD,OAAOC,SAAS,MAAM,WAAW;AAgBjC,OAAO,SAASC,kBAAkBA,CAAAC,IAAA,EAGN;EAAA,IAF1BC,OAAO,GAAAD,IAAA,CAAPC,OAAO;IACPC,QAAQ,GAAAF,IAAA,CAARE,QAAQ;EAER,MAAMC,cAAc,GAAGZ,iBAAiB,CAAC,CAAC;EAC1C,MAAMa,cAAc,GAClBF,QAAQ,CAACG,KAAK,CAACC,IAAI,KAAK,OAAO,GAAGH,cAAc,GAAGI,SAAS;EAC9D,MAAMC,UAAU,GAAGP,OAAO,CAACQ,IAAI,CAAC,GAAG,CAAC;EACpC,MAAMC,UAAU,GAAGf,MAAM,CAACM,OAAO,CAAC;EAClCS,UAAU,CAACC,OAAO,GAAGV,OAAO;EAC5B,MAAMW,MAAM,GAAGjB,MAAM,CAAmB,IAAI,CAAC;EAC7C,MAAMkB,OAAO,GAAGlB,MAAM,CAAoB,IAAI,CAAC;EAE/CD,SAAS,CAAC,MAAM;IACd,IAAIoB,EAAE,GAAGC,qBAAqB,CAACC,IAAI,CAAC;IAEpC,OAAO,MAAM;MACXC,oBAAoB,CAACH,EAAE,CAAC;IAC1B,CAAC;IAED,SAASE,IAAIA,CAAA,EAAG;MACd,IACE,CAACJ,MAAM,CAACD,OAAO,IACf,CAACE,OAAO,CAACF,OAAO,IAChB,CAACC,MAAM,CAACD,OAAO,CAACO,UAAU,IAC1B,CAACN,MAAM,CAACD,OAAO,CAACQ,WAAW,EAC3B;QACAL,EAAE,GAAGC,qBAAqB,CAACC,IAAI,CAAC;QAChC;MACF;MAEAF,EAAE,GAAGC,qBAAqB,CAACK,OAAO,CAAC;IACrC;IAEA,SAASA,OAAOA,CAAA,EAAG;MACjBtB,SAAS,CAACc,MAAM,CAACD,OAAO,EAAE,0BAA0B,CAAC;MACrDb,SAAS,CAACe,OAAO,CAACF,OAAO,EAAE,2BAA2B,CAAC;MAEvDd,MAAM,CAACe,MAAM,CAACD,OAAO,EAAEE,OAAO,CAACF,OAAO,CAAC;MAEvC,KAAK,MAAMU,MAAM,IAAIX,UAAU,CAACC,OAAO,EAAE;QACvC,QAAQU,MAAM;UACZ,KAAK,wBAAwB;YAC3BzB,qBAAqB,CAACiB,OAAO,CAACF,OAAO,CAAC;YACtC;QACJ;MACF;MAEAnB,iBAAiB,CAACY,cAAc,EAAES,OAAO,CAACF,OAAO,EAAEC,MAAM,CAACD,OAAO,CAAC;MAClEG,EAAE,GAAGC,qBAAqB,CAACK,OAAO,CAAC;IACrC;EACF,CAAC,EAAE,CAACZ,UAAU,EAAEJ,cAAc,CAAC,CAAC;EAEhC,oBACEX,KAAA,CAAA6B,aAAA,CAAA7B,KAAA,CAAA8B,QAAA,qBACE9B,KAAA,CAAA6B,aAAA;IACEE,GAAG,EAAEX,OAAQ;IACbY,KAAK,EAAEvB,QAAQ,CAACG,KAAK,CAACoB,KAAM;IAC5BC,SAAS,EAAExB,QAAQ,CAACG,KAAK,CAACqB;EAAU,CACrC,CAAC,eAEDjC,KAAK,CAACkC,YAAY,CAACzB,QAAQ,EAAE;IAC5B,GAAGA,QAAQ,CAACG,KAAK;IACjBuB,oBAAoB,EAAE,QAAQ;IAC9BJ,GAAG,EAAEZ,MAAM;IACXa,KAAK,EAAE;MAAEI,OAAO,EAAE;IAAO;EAC3B,CAAC,CACD,CAAC;AAEP","ignoreList":[]}

package/dist/esm/hooks/useConversation/livekit/useConversation.js

@@ -26,13 +26,15 @@ export function useConversation(_ref) {
     _ref$startWithCamera = _ref.startWithCamera,
     startWithCamera = _ref$startWithCamera === void 0 ? false : _ref$startWithCamera,
     firstConversation = _ref.firstConversation,
-    lastConversationTopic = _ref.lastConversationTopic;
+    lastConversationTopic = _ref.lastConversationTopic,
+    conversationLanguage = _ref.conversationLanguage;
   setLogLevel(LogLevel.silent);
   const conversation = useCreateConversation({
     type,
     config,
     firstConversation,
-    lastConversationTopic
+    lastConversationTopic,
+    conversationLanguage
   });
   const room = useEnsureRoom();
   const speaker = useAtomValue(state.speaker);
@@ -44,9 +46,11 @@ export function useConversation(_ref) {
     setCamera = _useAtom2[1];
   const _useSdkConsumer = useSdkConsumer(),
     _useSdkConsumer$tools = _useSdkConsumer.tools,
-    toolsRegistry = _useSdkConsumer$tools === void 0 ? {} : _useSdkConsumer$tools;
+    toolsRegistry = _useSdkConsumer$tools === void 0 ? {} : _useSdkConsumer$tools,
+    startupTelemetrySession = _useSdkConsumer.startupTelemetrySession;
   const toolRuntime = useToolCallingRuntime();
   const toolsRegistryRef = useLatest(toolsRegistry);
+  const startupSessionRef = useLatest(startupTelemetrySession);
 
   // for use in register/unregister room events
   const conversationRef = useLatest(conversation.data);
@@ -160,8 +164,11 @@ export function useConversation(_ref) {
     const url = new URL((conversation == null || (_conversation$data3 = conversation.data) == null ? void 0 : _conversation$data3.url) || '');
     const accessToken = url.searchParams.get('token');
     try {
+      var _startupTelemetrySess, _startupTelemetrySess2;
+      startupTelemetrySession == null || (_startupTelemetrySess = startupTelemetrySession.get(conversation.data.id)) == null || _startupTelemetrySess.mark('join_requested');
       setUserConnectionState('joining');
       await (room == null ? void 0 : room.connect(conversation.data.url, accessToken, {}));
+      startupTelemetrySession == null || (_startupTelemetrySess2 = startupTelemetrySession.get(conversation.data.id)) == null || _startupTelemetrySess2.mark('room_connected');
 
       // sometimes the agent is already joined before the user joins the conversation
       const remoteParticipant = findJoinedAgentParticipant(room, type);
@@ -170,6 +177,8 @@ export function useConversation(_ref) {
       }
       enableMediaDevices();
     } catch (error) {
+      var _startupTelemetrySess3;
+      startupTelemetrySession == null || (_startupTelemetrySess3 = startupTelemetrySession.get(conversation.data.id)) == null || _startupTelemetrySess3.finish('error', 'join_failed');
       console.error('Failed to join', error);
       setUserConnectionState('join-error');
     }
@@ -403,6 +412,8 @@ export function useConversation(_ref) {
     setConversationStatus('error');
   }
   function handleDisconnectedEvent(reason) {
+    var _startupSessionRef$cu, _conversationRef$curr0;
+    (_startupSessionRef$cu = startupSessionRef.current) == null || (_startupSessionRef$cu = _startupSessionRef$cu.get((_conversationRef$curr0 = conversationRef.current) == null ? void 0 : _conversationRef$curr0.id)) == null || _startupSessionRef$cu.finish(reason === DisconnectReason.CLIENT_INITIATED ? 'cancelled' : 'error', 'disconnected_before_frame');
     isConnectedRef.current = false;
     acousticPhaseRef.current = 'idle';
     acousticT0Ref.current = null;
@@ -451,18 +462,19 @@ export function useConversation(_ref) {
     }
   }
   function handleAgentJoin(participant) {
-    var _conversationRef$curr0;
+    var _startupSessionRef$cu2, _conversationRef$curr1, _conversationRef$curr10;
+    (_startupSessionRef$cu2 = startupSessionRef.current) == null || (_startupSessionRef$cu2 = _startupSessionRef$cu2.get((_conversationRef$curr1 = conversationRef.current) == null ? void 0 : _conversationRef$curr1.id)) == null || _startupSessionRef$cu2.mark('agent_joined');
     setAgentConnectionState('joined');
     emitEvent('agentJoined', {
-      conversationId: (_conversationRef$curr0 = conversationRef.current) == null ? void 0 : _conversationRef$curr0.id,
+      conversationId: (_conversationRef$curr10 = conversationRef.current) == null ? void 0 : _conversationRef$curr10.id,
       participant
     });
   }
   function handleParticipantDisconnectedEvent(participant) {
-    var _conversationRef$curr1;
+    var _conversationRef$curr11;
     setAgentConnectionState('left');
     emitEvent('agentLeft', {
-      conversationId: (_conversationRef$curr1 = conversationRef.current) == null ? void 0 : _conversationRef$curr1.id,
+      conversationId: (_conversationRef$curr11 = conversationRef.current) == null ? void 0 : _conversationRef$curr11.id,
       participant
     });
   }
@@ -471,13 +483,13 @@ export function useConversation(_ref) {
       return;
     }
     try {
-      var _conversationRef$curr10;
+      var _conversationRef$curr12;
       setUserConnectionState('leaving');
       await (room == null ? void 0 : room.disconnect(true));
       setConversationStatus('destroyed');
       setJoinedAt(undefined);
       emitEvent('callEnded', {
-        conversationId: (_conversationRef$curr10 = conversationRef.current) == null ? void 0 : _conversationRef$curr10.id
+        conversationId: (_conversationRef$curr12 = conversationRef.current) == null ? void 0 : _conversationRef$curr12.id
       });
     } catch (error) {
       console.error('Failed to leave', error);
@@ -485,22 +497,22 @@ export function useConversation(_ref) {
     }
   }
   async function handleExpire() {
-    var _conversationRef$curr11, _conversationRef$curr12;
+    var _conversationRef$curr13, _conversationRef$curr14;
     // The duration timer keeps running after the call ends.
     if (conversationStatus === 'credit-exhausted' || conversationStatus === 'conversation-limit-reached' || conversationStatus === 'plan-minutes-exhausted') {
       return;
     }
     emitEvent('timeExceeded', {
-      conversationId: (_conversationRef$curr11 = conversationRef.current) == null ? void 0 : _conversationRef$curr11.id,
-      limit: (_conversationRef$curr12 = conversationRef.current) == null ? void 0 : _conversationRef$curr12.limit
+      conversationId: (_conversationRef$curr13 = conversationRef.current) == null ? void 0 : _conversationRef$curr13.id,
+      limit: (_conversationRef$curr14 = conversationRef.current) == null ? void 0 : _conversationRef$curr14.limit
     });
     await leave();
     setConversationStatus('exceeded');
   }
   async function interrupt() {
-    var _conversationRef$curr13;
+    var _conversationRef$curr15;
     emitEvent('interrupt', {
-      conversationId: (_conversationRef$curr13 = conversationRef.current) == null ? void 0 : _conversationRef$curr13.id
+      conversationId: (_conversationRef$curr15 = conversationRef.current) == null ? void 0 : _conversationRef$curr15.id
     });
     await publishConversationDataMessage(room, {}, 'interrupt');
   }
@@ -510,7 +522,7 @@ export function useConversation(_ref) {
     }, 'inactivity_dismiss');
   }
   async function send(message, attachments) {
-    var _conversationRef$curr14;
+    var _conversationRef$curr16;
     if (attachments === void 0) {
       attachments = [];
     }
@@ -521,7 +533,7 @@ export function useConversation(_ref) {
       }
     }));
     emitEvent('userSendMessage', {
-      conversationId: (_conversationRef$curr14 = conversationRef.current) == null ? void 0 : _conversationRef$curr14.id,
+      conversationId: (_conversationRef$curr16 = conversationRef.current) == null ? void 0 : _conversationRef$curr16.id,
       message
     });
   }