@vanillaskyai/video 0.10.9 → 0.10.10

package/CHANGELOG.md

@@ -4,6 +4,12 @@ VanillaSky follows semantic versioning. This changelog begins with the 0.1 beta.
 
 ## Unreleased
 
+## 0.10.10
+
+- Accept multiline streamed chat JSON objects without waiting for the entire answer, while retaining bounded parsing and strict scene validation.
+
+- Wait for a presented frame with playable future data before starting narration, retaining bounded cold startup and quick recovery for footage that stalls after playback was available.
+
 ## 0.10.9
 
 - Use essential subject hints and explicit exclusions when selecting starter Pexels footage, and isolate cached selections by those hints. Metadata-free results remain unverified provider-ranked fallbacks.

package/dist/check-runtime.js

@@ -1,6 +1,6 @@
 import {
   VideoFrame
-} from "./chunk-KUCEOG2L.js";
+} from "./chunk-FFKAHNYD.js";
 import "./chunk-5JBMYQP6.js";
 import "./chunk-224QNWRA.js";
 import "./chunk-SPVTJH3F.js";

package/dist/{chunk-Q3LPOPHL.js → chunk-3S5FXE6G.js}

@@ -1,7 +1,7 @@
 import {
   SceneBackground,
   getMediaBackgroundProps
-} from "./chunk-J7EGPURK.js";
+} from "./chunk-YERSNTFL.js";
 
 // src/visual-system/scene-templates/cinema-media.tsx
 import { jsx } from "react/jsx-runtime";

package/dist/{chunk-KUCEOG2L.js → chunk-FFKAHNYD.js}

@@ -61,14 +61,18 @@ function MountedSceneReadiness({ scene, playing, fallback = false, onFailure })
         }
         if (isVideo) {
           const video = layer.querySelector("video");
-          if (video && video.getAttribute("src") === mediaUrl && video.currentSrc === video.src && video.readyState >= 2) {
+          if (video && video.getAttribute("src") === mediaUrl && video.currentSrc === video.src && video.readyState >= HTMLMediaElement.HAVE_FUTURE_DATA) {
             if (presented === video) {
               finish(void 0, true);
               return;
             }
             observed = video;
             if (video.requestVideoFrameCallback) {
-              callback = video.requestVideoFrameCallback(() => finish(void 0, true));
+              callback = video.requestVideoFrameCallback(() => {
+                callback = void 0;
+                presented = video;
+                check();
+              });
               return;
             }
             finish(void 0, true);

package/dist/{chunk-R5ZX2LJV.js → chunk-XL3H42K5.js}

@@ -1,7 +1,7 @@
 import {
   SceneBackground,
   getMediaBackgroundProps
-} from "./chunk-J7EGPURK.js";
+} from "./chunk-YERSNTFL.js";
 import {
   editorialFont
 } from "./chunk-4YM2M62S.js";

package/dist/{chunk-J7EGPURK.js → chunk-YERSNTFL.js}

@@ -257,7 +257,7 @@ var SceneVideoBackdrop = ({
   const [waitingKey, setWaitingKey] = useState();
   const [exhaustedKey, setExhaustedKey] = useState();
   const videoRef = useRef(null);
-  const presentedVideoUrl = useRef(void 0);
+  const playableVideoUrl = useRef(void 0);
   const startedVideoUrl = useRef(void 0);
   const startedPlaybackId = useRef(void 0);
   const videoPresentationKey = `${playbackId}\0${mediaUrl}`;
@@ -278,7 +278,7 @@ var SceneVideoBackdrop = ({
     const video = videoRef.current;
     if (!video) return;
     const expectedSource = video.getAttribute("src") === mediaUrl ? video.src : void 0;
-    let awaitingFirstFrame = presentedVideoUrl.current !== mediaUrl || video.currentSrc !== expectedSource;
+    let awaitingPlayback = playableVideoUrl.current !== mediaUrl || video.currentSrc !== expectedSource;
     let previousTime = video.currentTime;
     let forwardFrames = 0;
     let stopped = false;
@@ -287,8 +287,9 @@ var SceneVideoBackdrop = ({
     const observe = (_now, metadata) => {
       if (stopped) return;
       const currentSource = video.currentSrc === expectedSource;
-      if (currentSource && awaitingFirstFrame && (metadata || presentedVideoUrl.current === mediaUrl)) {
-        awaitingFirstFrame = false;
+      if (currentSource && awaitingPlayback && video.readyState >= HTMLMediaElement.HAVE_FUTURE_DATA) {
+        playableVideoUrl.current = mediaUrl;
+        awaitingPlayback = false;
         clearTimeout(deadline);
         deadline = setTimeout(fail, 1e3);
       }
@@ -308,10 +309,10 @@ var SceneVideoBackdrop = ({
     const fail = () => {
       if (!stopped) {
         stopped = true;
-        unavailable(awaitingFirstFrame ? "frame-readiness-timeout" : "stalled-media");
+        unavailable(awaitingPlayback ? "frame-readiness-timeout" : "stalled-media");
       }
     };
-    let deadline = setTimeout(fail, awaitingFirstFrame ? 8e3 : 1e3);
+    let deadline = setTimeout(fail, awaitingPlayback ? 8e3 : 1e3);
     observe();
     return () => {
       stopped = true;
@@ -329,7 +330,7 @@ var SceneVideoBackdrop = ({
     let frame;
     const markPresented = () => {
       if (stopped || !video.isConnected || presentationRef.current.key !== videoPresentationKey || video.getAttribute("src") !== mediaUrl || video.currentSrc !== video.src) return false;
-      presentedVideoUrl.current = mediaUrl;
+      if (video.readyState >= HTMLMediaElement.HAVE_FUTURE_DATA) playableVideoUrl.current = mediaUrl;
       video.dispatchEvent(new Event("vanillasky:video-frame-presented", { bubbles: true }));
       onReadyRef.current?.();
       setDecodedVideoUrl(mediaUrl);
@@ -449,7 +450,10 @@ var SceneVideoBackdrop = ({
         onLoadedMetadata: (event) => fitDuration(event.currentTarget),
         onEnded: (event) => continueMotion(event.currentTarget),
         onPlay: enforceRequestedPause,
-        onPlaying: enforceRequestedPause,
+        onPlaying: (event) => {
+          if (event.currentTarget.currentSrc === event.currentTarget.src && event.currentTarget.getAttribute("src") === mediaUrl && event.currentTarget.readyState >= HTMLMediaElement.HAVE_FUTURE_DATA) playableVideoUrl.current = mediaUrl;
+          enforceRequestedPause(event);
+        },
         onWaiting: () => {
           if (isPlaying) setWaitingKey(videoPresentationKey);
         },

package/dist/{cinema-media-WUS7CKPC.js → cinema-media-IAK3LC2J.js}

@@ -1,8 +1,8 @@
 import {
   MediaScene,
   MediaSceneTemplate
-} from "./chunk-Q3LPOPHL.js";
-import "./chunk-J7EGPURK.js";
+} from "./chunk-3S5FXE6G.js";
+import "./chunk-YERSNTFL.js";
 import "./chunk-5JBMYQP6.js";
 import "./chunk-224QNWRA.js";
 export {

package/dist/{comparison-T4OYCOH3.js → comparison-MCTN376M.js}

@@ -1,7 +1,7 @@
 import {
   EditorialSurface
-} from "./chunk-R5ZX2LJV.js";
-import "./chunk-J7EGPURK.js";
+} from "./chunk-XL3H42K5.js";
+import "./chunk-YERSNTFL.js";
 import {
   editorialLabel,
   fade

package/dist/{editorial-timeline-JHQZIPNE.js → editorial-timeline-435UCJ45.js}

@@ -1,7 +1,7 @@
 import {
   EditorialSurface
-} from "./chunk-R5ZX2LJV.js";
-import "./chunk-J7EGPURK.js";
+} from "./chunk-XL3H42K5.js";
+import "./chunk-YERSNTFL.js";
 import {
   fade
 } from "./chunk-4YM2M62S.js";

package/dist/{key-figure-YWN2OBQU.js → key-figure-BSOQ3TYN.js}

@@ -1,7 +1,7 @@
 import {
   EditorialSurface
-} from "./chunk-R5ZX2LJV.js";
-import "./chunk-J7EGPURK.js";
+} from "./chunk-XL3H42K5.js";
+import "./chunk-YERSNTFL.js";
 import {
   editorialLabel,
   fade

package/dist/{mobile-message-CDKOERRZ.js → mobile-message-GVFA56QP.js}

@@ -1,9 +1,9 @@
 import {
   MediaScene
-} from "./chunk-Q3LPOPHL.js";
+} from "./chunk-3S5FXE6G.js";
 import {
   spring
-} from "./chunk-J7EGPURK.js";
+} from "./chunk-YERSNTFL.js";
 import {
   editorialFont
 } from "./chunk-4YM2M62S.js";

package/dist/{quote-JKABOMIG.js → quote-H2X4XGPB.js}

@@ -1,7 +1,7 @@
 import {
   EditorialSurface
-} from "./chunk-R5ZX2LJV.js";
-import "./chunk-J7EGPURK.js";
+} from "./chunk-XL3H42K5.js";
+import "./chunk-YERSNTFL.js";
 import {
   fade
 } from "./chunk-4YM2M62S.js";

package/dist/react.js

@@ -40,7 +40,7 @@ import {
   VideoFrame,
   getDimensions,
   sceneReadinessKey
-} from "./chunk-KUCEOG2L.js";
+} from "./chunk-FFKAHNYD.js";
 import "./chunk-5JBMYQP6.js";
 import "./chunk-224QNWRA.js";
 import {
@@ -94,13 +94,13 @@ import { createElement } from "react";
 
 // src/visual-system/catalog/builtin-loaders.generated.ts
 var GENERATED_BUILTIN_TEMPLATE_LOADERS = {
-  "cinemaMedia": () => import("./cinema-media-WUS7CKPC.js").then((module) => ({ default: module.MediaSceneTemplate })),
+  "cinemaMedia": () => import("./cinema-media-IAK3LC2J.js").then((module) => ({ default: module.MediaSceneTemplate })),
   "chapterTitle": () => import("./chapter-title-2JVDU62E.js").then((module) => ({ default: module.TitleSceneTemplate })),
-  "editorialTimeline": () => import("./editorial-timeline-JHQZIPNE.js").then((module) => ({ default: module.TimelineSceneTemplate })),
-  "mobileMessage": () => import("./mobile-message-CDKOERRZ.js").then((module) => ({ default: module.NotificationSceneTemplate })),
-  "comparison": () => import("./comparison-T4OYCOH3.js").then((module) => ({ default: module.ComparisonSceneTemplate })),
-  "quote": () => import("./quote-JKABOMIG.js").then((module) => ({ default: module.QuoteSceneTemplate })),
-  "keyFigure": () => import("./key-figure-YWN2OBQU.js").then((module) => ({ default: module.KeyFigureSceneTemplate }))
+  "editorialTimeline": () => import("./editorial-timeline-435UCJ45.js").then((module) => ({ default: module.TimelineSceneTemplate })),
+  "mobileMessage": () => import("./mobile-message-GVFA56QP.js").then((module) => ({ default: module.NotificationSceneTemplate })),
+  "comparison": () => import("./comparison-MCTN376M.js").then((module) => ({ default: module.ComparisonSceneTemplate })),
+  "quote": () => import("./quote-H2X4XGPB.js").then((module) => ({ default: module.QuoteSceneTemplate })),
+  "keyFigure": () => import("./key-figure-BSOQ3TYN.js").then((module) => ({ default: module.KeyFigureSceneTemplate }))
 };
 
 // src/visual-system/catalog/builtin-player.generated.ts

package/dist/server.js

@@ -1017,23 +1017,53 @@ function createChatShotPlanner(options) {
           bodyDuration += shot.durationSec;
           return scenePart(shot);
         };
+        let cursor = 0, depth = 0, quoted = false, escaped = false;
+        const takeFrame = () => {
+          if (cursor === 0) {
+            buffer = buffer.trimStart();
+            if (!buffer) return;
+            if (buffer[0] !== "{" && buffer[0] !== "[") {
+              const newline = buffer.indexOf("\n");
+              if (newline < 0) return;
+              const raw = buffer.slice(0, newline);
+              buffer = buffer.slice(newline + 1);
+              return raw;
+            }
+          }
+          for (; cursor < buffer.length; cursor++) {
+            const character = buffer[cursor];
+            if (quoted) {
+              if (escaped) escaped = false;
+              else if (character === "\\") escaped = true;
+              else if (character === '"') quoted = false;
+            } else if (character === '"') quoted = true;
+            else if (character === "{" || character === "[") depth++;
+            else if (character === "}" || character === "]") {
+              depth--;
+              if (depth === 0) {
+                const raw = buffer.slice(0, cursor + 1);
+                buffer = buffer.slice(cursor + 1);
+                cursor = 0;
+                return raw;
+              }
+            }
+          }
+        };
         try {
           for await (const delta of upstream) {
             context.signal.throwIfAborted();
             if (typeof delta !== "string") throw new Error("The LLM adapter returned a non-text delta");
             buffer += delta;
             if (buffer.length > 32768) throw new Error("Chat plan line exceeds the bounded stream limit");
-            let newline = buffer.indexOf("\n");
-            while (newline >= 0) {
-              const raw = buffer.slice(0, newline);
-              buffer = buffer.slice(newline + 1);
+            let raw = takeFrame();
+            while (raw !== void 0) {
               try {
                 const part = line(raw);
                 if (part) yield JSON.stringify(part) + "\n";
               } catch (cause) {
                 reject(cause);
               }
-              newline = buffer.indexOf("\n");
+              raw = takeFrame();
             }
           }
           if (buffer.trim()) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vanillaskyai/video",
-  "version": "0.10.9",
+  "version": "0.10.10",
   "description": "Open-source voice-and-video chat SDK for AI applications.",
   "keywords": [
     "video-chat",

package/registry/items/backgrounds.json

@@ -40,7 +40,7 @@
       "path": "src/visual-system/scene-templates/scene-video-backdrop.tsx",
       "type": "registry:lib",
       "target": "vanillasky/scene-templates/scene-video-backdrop.tsx",
-      "content": "import React, { useCallback, useEffect, useRef, useState } from \"react\";\nimport { getBackgroundTransform } from \"../backgrounds\";\nimport { type MediaRecoveryReason, useMediaAudio, useMediaFailure, useNarrationPreroll } from \"./external-video-backdrop\";\nimport { resolveMediaPosition } from \"./media-position\";\n\nexport interface SceneVideoBackdropProps {\n  mediaUrl: string;\n  mediaPoster?: string;\n  mediaPosition?: string;\n  backgroundEffect?: string;\n  progress: number;\n  /** Narration-led visible duration; muted or pitch-preserving footage may be gently retimed. */\n  sceneDuration?: number;\n  /** Internal player-owned decoder priming, distinct from viewer pause. */\n  preparingNarration?: boolean;\n  beatIntensity?: number;\n  isPlaying: boolean;\n  muted?: boolean;\n  volume?: number;\n  playbackId?: string;\n  onReady?: () => void;\n  onError?: () => void;\n}\n\nexport const SceneVideoBackdrop: React.FC<SceneVideoBackdropProps> = ({\n  mediaUrl,\n  mediaPoster,\n  mediaPosition = \"center\",\n  backgroundEffect,\n  progress,\n  sceneDuration,\n  preparingNarration = false,\n  beatIntensity = 0,\n  isPlaying,\n  muted,\n  volume,\n  playbackId = mediaUrl,\n  onReady,\n  onError,\n}) => {\n  const inheritedAudio = useMediaAudio();\n  const reportMediaFailure = useMediaFailure();\n  const inheritedPreroll = useNarrationPreroll();\n  const rewindPreroll = preparingNarration || inheritedPreroll;\n  const resolvedMuted = muted ?? inheritedAudio.muted;\n  const resolvedVolume = volume ?? inheritedAudio.volume;\n  const resolvedPosition = resolveMediaPosition(mediaPosition);\n  const bgTransform = getBackgroundTransform(backgroundEffect, progress, beatIntensity);\n  const [decodedVideoUrl, setDecodedVideoUrl] = useState<string>();\n  const [waitingKey, setWaitingKey] = useState<string>();\n  const [exhaustedKey, setExhaustedKey] = useState<string>();\n\n  const videoRef = useRef<HTMLVideoElement>(null);\n  const presentedVideoUrl = useRef<string | undefined>(undefined);\n  const startedVideoUrl = useRef<string | undefined>(undefined);\n  const startedPlaybackId = useRef<string | undefined>(undefined);\n  const videoPresentationKey = `${playbackId}\\0${mediaUrl}`;\n\n  const presentationRef = useRef({ key: videoPresentationKey, playing: isPlaying });\n  presentationRef.current = { key: videoPresentationKey, playing: isPlaying };\n  const unavailable = (reason: MediaRecoveryReason = \"playback-error\") => {\n    if (presentationRef.current.key === videoPresentationKey && presentationRef.current.playing) {\n      setExhaustedKey(videoPresentationKey);\n      onError?.();\n      reportMediaFailure?.(reason);\n    }\n  };\n  useEffect(() => {\n    if (!isPlaying || waitingKey !== videoPresentationKey) {\n      if (waitingKey) setWaitingKey(undefined);\n      return;\n    }\n    const video = videoRef.current;\n    if (!video) return;\n    const expectedSource = video.getAttribute(\"src\") === mediaUrl ? video.src : undefined;\n    let awaitingFirstFrame = presentedVideoUrl.current !== mediaUrl || video.currentSrc !== expectedSource;\n    let previousTime = video.currentTime;\n    let forwardFrames = 0;\n    let stopped = false;\n    let frame: number | undefined;\n    let poll: ReturnType<typeof setTimeout> | undefined;\n    const observe = (_now?: number, metadata?: VideoFrameCallbackMetadata) => {\n      if (stopped) return;\n      const currentSource = video.currentSrc === expectedSource;\n      if (currentSource && awaitingFirstFrame && (metadata || presentedVideoUrl.current === mediaUrl)) {\n        awaitingFirstFrame = false;\n        clearTimeout(deadline);\n        deadline = setTimeout(fail, 1000);\n      }\n      const time = metadata?.mediaTime ?? video.currentTime;\n      if (!currentSource || video.seeking || time < previousTime) forwardFrames = 0;\n      else if (time > previousTime + .001) forwardFrames++;\n      previousTime = time;\n      // One seek frame is not resumed motion. Require consecutive forward\n      // observations before releasing the original bounded stall deadline.\n      if (forwardFrames >= 2) {\n        stopped = true;\n        clearTimeout(deadline);\n        setWaitingKey(undefined);\n        return;\n      }\n      if (video.requestVideoFrameCallback) frame = video.requestVideoFrameCallback(observe);\n      else poll = setTimeout(observe, 50);\n    };\n    // A seek can emit waiting without another playing event, even while frames\n    // resume. Keep the decoder visible and observe motion directly. A real\n    // stall gets the player's authored chapter instead of an endless spinner.\n    const fail = () => { if (!stopped) { stopped = true; unavailable(awaitingFirstFrame ? \"frame-readiness-timeout\" : \"stalled-media\"); } };\n    // Initial network/decode work has the same bound as mounted readiness.\n    // Only a source that has presented a frame can be judged as stalled motion.\n    let deadline = setTimeout(fail, awaitingFirstFrame ? 8000 : 1000);\n    observe();\n    return () => {\n      stopped = true;\n      clearTimeout(deadline);\n      clearTimeout(poll);\n      if (frame !== undefined) video.cancelVideoFrameCallback?.(frame);\n    };\n  }, [waitingKey, videoPresentationKey, isPlaying]);\n\n  const onReadyRef = useRef(onReady);\n  onReadyRef.current = onReady;\n  useEffect(() => {\n    const video = videoRef.current;\n    if (!video) return;\n    let stopped = false;\n    let frame: number | undefined;\n    const markPresented = () => {\n      if (stopped || !video.isConnected || presentationRef.current.key !== videoPresentationKey\n        || video.getAttribute(\"src\") !== mediaUrl || video.currentSrc !== video.src) return false;\n      presentedVideoUrl.current = mediaUrl;\n      video.dispatchEvent(new Event(\"vanillasky:video-frame-presented\", { bubbles: true }));\n      onReadyRef.current?.();\n      setDecodedVideoUrl(mediaUrl);\n      stopped = true;\n      return true;\n    };\n    const observe = () => {\n      if (stopped || frame !== undefined) return;\n      if (video.requestVideoFrameCallback) {\n        frame = video.requestVideoFrameCallback(() => {\n          frame = undefined;\n          if (!markPresented()) observe();\n        });\n      } else if (video.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA) markPresented();\n    };\n    // A cached resource can finish loading while its Suspense tree is still\n    // detached. Observe the mounted frame even if loadeddata was missed.\n    video.addEventListener(\"loadeddata\", observe);\n    observe();\n    return () => {\n      stopped = true;\n      video.removeEventListener(\"loadeddata\", observe);\n      if (frame !== undefined) video.cancelVideoFrameCallback?.(frame);\n    };\n  }, [mediaUrl, videoPresentationKey]);\n\n  const fitDuration = useCallback((video: HTMLVideoElement) => {\n    // Allow a small decode-to-speech onset margin without changing narration.\n    video.playbackRate = (resolvedMuted || video.preservesPitch === true) && sceneDuration && Number.isFinite(video.duration) && video.duration > 0\n      ? Math.max(.75, Math.min(1, video.duration / (sceneDuration + .2))) : 1;\n  }, [resolvedMuted, sceneDuration]);\n  useEffect(() => {\n    if (videoRef.current) fitDuration(videoRef.current);\n  }, [fitDuration]);\n  const continueMotion = (video: HTMLVideoElement) => {\n    if (!isPlaying) return;\n    // The finite scene clock bounds silent coverage. Speech may outlast a\n    // short clip; repeat motion until the scene ends, never audible dialogue.\n    if (!resolvedMuted) {\n      unavailable();\n      return;\n    }\n    video.currentTime = 0;\n    void video.play().catch(() => unavailable());\n  };\n\n  useEffect(() => {\n    const video = videoRef.current;\n    if (!video) return;\n    // React Strict Mode rehearses setup → cleanup → setup in development.\n    // The cleanup deliberately releases the decoder, so the repeated setup\n    // must restore the declarative source before the playback effect runs.\n    if (video.getAttribute(\"src\") !== mediaUrl) {\n      video.setAttribute(\"src\", mediaUrl);\n      video.load();\n    }\n  }, [mediaUrl]);\n\n  useEffect(() => {\n    const video = videoRef.current;\n    if (!video) return;\n    // A source change already starts a native load via React's src update.\n    // Tear down the decoder only on unmount, never cancel that new request.\n    return () => {\n      video.pause();\n      video.removeAttribute(\"src\");\n      video.load();\n      startedVideoUrl.current = undefined;\n      startedPlaybackId.current = undefined;\n    };\n  }, []);\n\n  useEffect(() => {\n    const video = videoRef.current;\n    if (video) video.volume = resolvedVolume;\n  }, [resolvedVolume]);\n\n  useEffect(() => {\n    const video = videoRef.current;\n    if (!video) return;\n    if (!isPlaying) {\n      video.pause();\n      if (rewindPreroll && video.currentTime > 0) video.currentTime = 0;\n      return;\n    }\n    if (startedPlaybackId.current === playbackId) {\n      if (video.ended) continueMotion(video);\n      else void video.play().catch(() => unavailable());\n      return;\n    }\n    const changingSource = startedVideoUrl.current !== undefined && startedVideoUrl.current !== mediaUrl;\n    fitDuration(video);\n    if (!changingSource && video.currentTime > 0) video.currentTime = 0;\n    video.play().catch(() => unavailable());\n    startedVideoUrl.current = mediaUrl;\n    startedPlaybackId.current = playbackId;\n  }, [isPlaying, mediaUrl, playbackId, rewindPreroll]);\n\n  const enforceRequestedPause = (event: React.SyntheticEvent<HTMLVideoElement>) => {\n    // WebKit may enter playback without a playing event while seeking or\n    // waiting. Both native start events must honor the latest requested hold.\n    if (presentationRef.current.playing) return;\n    event.currentTarget.pause();\n    if (rewindPreroll && event.currentTarget.currentTime > 0) event.currentTarget.currentTime = 0;\n  };\n\n  const mediaStyle: React.CSSProperties = {\n    position: \"absolute\",\n    inset: 0,\n    width: \"100%\",\n    height: \"100%\",\n    objectFit: \"cover\",\n    objectPosition: resolvedPosition,\n    transform: bgTransform.transform,\n    transformOrigin: bgTransform.transformOrigin,\n  };\n  return (\n    <>\n      {exhaustedKey === videoPresentationKey && <div\n        role=\"status\" data-media-continuity=\"exhausted\"\n        style={{ position: \"absolute\", inset: 0, zIndex: 3, background: \"#000\", color: \"#bbb\", display: \"grid\", placeContent: \"center\", font: \"14px system-ui\" }}\n      >Visual unavailable</div>}\n      <video\n        ref={videoRef}\n        src={mediaUrl}\n        poster={decodedVideoUrl !== mediaUrl ? mediaPoster || undefined : undefined}\n        muted={resolvedMuted}\n        loop={false}\n        playsInline\n        preload=\"auto\"\n        onLoadedMetadata={event => fitDuration(event.currentTarget)}\n        onEnded={event => continueMotion(event.currentTarget)}\n        onPlay={enforceRequestedPause}\n        onPlaying={enforceRequestedPause}\n        onWaiting={() => { if (isPlaying) setWaitingKey(videoPresentationKey); }}\n        onError={onError}\n        data-media-position={mediaPosition}\n        data-video-backdrop=\"scene\"\n        style={{ ...mediaStyle, visibility: exhaustedKey === videoPresentationKey ? \"hidden\" : undefined }}\n      />\n    </>\n  );\n};\n"
+      "content": "import React, { useCallback, useEffect, useRef, useState } from \"react\";\nimport { getBackgroundTransform } from \"../backgrounds\";\nimport { type MediaRecoveryReason, useMediaAudio, useMediaFailure, useNarrationPreroll } from \"./external-video-backdrop\";\nimport { resolveMediaPosition } from \"./media-position\";\n\nexport interface SceneVideoBackdropProps {\n  mediaUrl: string;\n  mediaPoster?: string;\n  mediaPosition?: string;\n  backgroundEffect?: string;\n  progress: number;\n  /** Narration-led visible duration; muted or pitch-preserving footage may be gently retimed. */\n  sceneDuration?: number;\n  /** Internal player-owned decoder priming, distinct from viewer pause. */\n  preparingNarration?: boolean;\n  beatIntensity?: number;\n  isPlaying: boolean;\n  muted?: boolean;\n  volume?: number;\n  playbackId?: string;\n  onReady?: () => void;\n  onError?: () => void;\n}\n\nexport const SceneVideoBackdrop: React.FC<SceneVideoBackdropProps> = ({\n  mediaUrl,\n  mediaPoster,\n  mediaPosition = \"center\",\n  backgroundEffect,\n  progress,\n  sceneDuration,\n  preparingNarration = false,\n  beatIntensity = 0,\n  isPlaying,\n  muted,\n  volume,\n  playbackId = mediaUrl,\n  onReady,\n  onError,\n}) => {\n  const inheritedAudio = useMediaAudio();\n  const reportMediaFailure = useMediaFailure();\n  const inheritedPreroll = useNarrationPreroll();\n  const rewindPreroll = preparingNarration || inheritedPreroll;\n  const resolvedMuted = muted ?? inheritedAudio.muted;\n  const resolvedVolume = volume ?? inheritedAudio.volume;\n  const resolvedPosition = resolveMediaPosition(mediaPosition);\n  const bgTransform = getBackgroundTransform(backgroundEffect, progress, beatIntensity);\n  const [decodedVideoUrl, setDecodedVideoUrl] = useState<string>();\n  const [waitingKey, setWaitingKey] = useState<string>();\n  const [exhaustedKey, setExhaustedKey] = useState<string>();\n\n  const videoRef = useRef<HTMLVideoElement>(null);\n  const playableVideoUrl = useRef<string | undefined>(undefined);\n  const startedVideoUrl = useRef<string | undefined>(undefined);\n  const startedPlaybackId = useRef<string | undefined>(undefined);\n  const videoPresentationKey = `${playbackId}\\0${mediaUrl}`;\n\n  const presentationRef = useRef({ key: videoPresentationKey, playing: isPlaying });\n  presentationRef.current = { key: videoPresentationKey, playing: isPlaying };\n  const unavailable = (reason: MediaRecoveryReason = \"playback-error\") => {\n    if (presentationRef.current.key === videoPresentationKey && presentationRef.current.playing) {\n      setExhaustedKey(videoPresentationKey);\n      onError?.();\n      reportMediaFailure?.(reason);\n    }\n  };\n  useEffect(() => {\n    if (!isPlaying || waitingKey !== videoPresentationKey) {\n      if (waitingKey) setWaitingKey(undefined);\n      return;\n    }\n    const video = videoRef.current;\n    if (!video) return;\n    const expectedSource = video.getAttribute(\"src\") === mediaUrl ? video.src : undefined;\n    let awaitingPlayback = playableVideoUrl.current !== mediaUrl || video.currentSrc !== expectedSource;\n    let previousTime = video.currentTime;\n    let forwardFrames = 0;\n    let stopped = false;\n    let frame: number | undefined;\n    let poll: ReturnType<typeof setTimeout> | undefined;\n    const observe = (_now?: number, metadata?: VideoFrameCallbackMetadata) => {\n      if (stopped) return;\n      const currentSource = video.currentSrc === expectedSource;\n      if (currentSource && awaitingPlayback && video.readyState >= HTMLMediaElement.HAVE_FUTURE_DATA) {\n        playableVideoUrl.current = mediaUrl;\n        awaitingPlayback = false;\n        clearTimeout(deadline);\n        deadline = setTimeout(fail, 1000);\n      }\n      const time = metadata?.mediaTime ?? video.currentTime;\n      if (!currentSource || video.seeking || time < previousTime) forwardFrames = 0;\n      else if (time > previousTime + .001) forwardFrames++;\n      previousTime = time;\n      // One seek frame is not resumed motion. Require consecutive forward\n      // observations before releasing the original bounded stall deadline.\n      if (forwardFrames >= 2) {\n        stopped = true;\n        clearTimeout(deadline);\n        setWaitingKey(undefined);\n        return;\n      }\n      if (video.requestVideoFrameCallback) frame = video.requestVideoFrameCallback(observe);\n      else poll = setTimeout(observe, 50);\n    };\n    // A seek can emit waiting without another playing event, even while frames\n    // resume. Keep the decoder visible and observe motion directly. A real\n    // stall gets the player's authored chapter instead of an endless spinner.\n    const fail = () => { if (!stopped) { stopped = true; unavailable(awaitingPlayback ? \"frame-readiness-timeout\" : \"stalled-media\"); } };\n    // Initial network/decode work has the same bound as mounted readiness.\n    // A decoded still with no future data is still cold, even after its first\n    // frame callback. Keep the short bound only after playback was available.\n    let deadline = setTimeout(fail, awaitingPlayback ? 8000 : 1000);\n    observe();\n    return () => {\n      stopped = true;\n      clearTimeout(deadline);\n      clearTimeout(poll);\n      if (frame !== undefined) video.cancelVideoFrameCallback?.(frame);\n    };\n  }, [waitingKey, videoPresentationKey, isPlaying]);\n\n  const onReadyRef = useRef(onReady);\n  onReadyRef.current = onReady;\n  useEffect(() => {\n    const video = videoRef.current;\n    if (!video) return;\n    let stopped = false;\n    let frame: number | undefined;\n    const markPresented = () => {\n      if (stopped || !video.isConnected || presentationRef.current.key !== videoPresentationKey\n        || video.getAttribute(\"src\") !== mediaUrl || video.currentSrc !== video.src) return false;\n      if (video.readyState >= HTMLMediaElement.HAVE_FUTURE_DATA) playableVideoUrl.current = mediaUrl;\n      video.dispatchEvent(new Event(\"vanillasky:video-frame-presented\", { bubbles: true }));\n      onReadyRef.current?.();\n      setDecodedVideoUrl(mediaUrl);\n      stopped = true;\n      return true;\n    };\n    const observe = () => {\n      if (stopped || frame !== undefined) return;\n      if (video.requestVideoFrameCallback) {\n        frame = video.requestVideoFrameCallback(() => {\n          frame = undefined;\n          if (!markPresented()) observe();\n        });\n      } else if (video.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA) markPresented();\n    };\n    // A cached resource can finish loading while its Suspense tree is still\n    // detached. Observe the mounted frame even if loadeddata was missed.\n    video.addEventListener(\"loadeddata\", observe);\n    observe();\n    return () => {\n      stopped = true;\n      video.removeEventListener(\"loadeddata\", observe);\n      if (frame !== undefined) video.cancelVideoFrameCallback?.(frame);\n    };\n  }, [mediaUrl, videoPresentationKey]);\n\n  const fitDuration = useCallback((video: HTMLVideoElement) => {\n    // Allow a small decode-to-speech onset margin without changing narration.\n    video.playbackRate = (resolvedMuted || video.preservesPitch === true) && sceneDuration && Number.isFinite(video.duration) && video.duration > 0\n      ? Math.max(.75, Math.min(1, video.duration / (sceneDuration + .2))) : 1;\n  }, [resolvedMuted, sceneDuration]);\n  useEffect(() => {\n    if (videoRef.current) fitDuration(videoRef.current);\n  }, [fitDuration]);\n  const continueMotion = (video: HTMLVideoElement) => {\n    if (!isPlaying) return;\n    // The finite scene clock bounds silent coverage. Speech may outlast a\n    // short clip; repeat motion until the scene ends, never audible dialogue.\n    if (!resolvedMuted) {\n      unavailable();\n      return;\n    }\n    video.currentTime = 0;\n    void video.play().catch(() => unavailable());\n  };\n\n  useEffect(() => {\n    const video = videoRef.current;\n    if (!video) return;\n    // React Strict Mode rehearses setup → cleanup → setup in development.\n    // The cleanup deliberately releases the decoder, so the repeated setup\n    // must restore the declarative source before the playback effect runs.\n    if (video.getAttribute(\"src\") !== mediaUrl) {\n      video.setAttribute(\"src\", mediaUrl);\n      video.load();\n    }\n  }, [mediaUrl]);\n\n  useEffect(() => {\n    const video = videoRef.current;\n    if (!video) return;\n    // A source change already starts a native load via React's src update.\n    // Tear down the decoder only on unmount, never cancel that new request.\n    return () => {\n      video.pause();\n      video.removeAttribute(\"src\");\n      video.load();\n      startedVideoUrl.current = undefined;\n      startedPlaybackId.current = undefined;\n    };\n  }, []);\n\n  useEffect(() => {\n    const video = videoRef.current;\n    if (video) video.volume = resolvedVolume;\n  }, [resolvedVolume]);\n\n  useEffect(() => {\n    const video = videoRef.current;\n    if (!video) return;\n    if (!isPlaying) {\n      video.pause();\n      if (rewindPreroll && video.currentTime > 0) video.currentTime = 0;\n      return;\n    }\n    if (startedPlaybackId.current === playbackId) {\n      if (video.ended) continueMotion(video);\n      else void video.play().catch(() => unavailable());\n      return;\n    }\n    const changingSource = startedVideoUrl.current !== undefined && startedVideoUrl.current !== mediaUrl;\n    fitDuration(video);\n    if (!changingSource && video.currentTime > 0) video.currentTime = 0;\n    video.play().catch(() => unavailable());\n    startedVideoUrl.current = mediaUrl;\n    startedPlaybackId.current = playbackId;\n  }, [isPlaying, mediaUrl, playbackId, rewindPreroll]);\n\n  const enforceRequestedPause = (event: React.SyntheticEvent<HTMLVideoElement>) => {\n    // WebKit may enter playback without a playing event while seeking or\n    // waiting. Both native start events must honor the latest requested hold.\n    if (presentationRef.current.playing) return;\n    event.currentTarget.pause();\n    if (rewindPreroll && event.currentTarget.currentTime > 0) event.currentTarget.currentTime = 0;\n  };\n\n  const mediaStyle: React.CSSProperties = {\n    position: \"absolute\",\n    inset: 0,\n    width: \"100%\",\n    height: \"100%\",\n    objectFit: \"cover\",\n    objectPosition: resolvedPosition,\n    transform: bgTransform.transform,\n    transformOrigin: bgTransform.transformOrigin,\n  };\n  return (\n    <>\n      {exhaustedKey === videoPresentationKey && <div\n        role=\"status\" data-media-continuity=\"exhausted\"\n        style={{ position: \"absolute\", inset: 0, zIndex: 3, background: \"#000\", color: \"#bbb\", display: \"grid\", placeContent: \"center\", font: \"14px system-ui\" }}\n      >Visual unavailable</div>}\n      <video\n        ref={videoRef}\n        src={mediaUrl}\n        poster={decodedVideoUrl !== mediaUrl ? mediaPoster || undefined : undefined}\n        muted={resolvedMuted}\n        loop={false}\n        playsInline\n        preload=\"auto\"\n        onLoadedMetadata={event => fitDuration(event.currentTarget)}\n        onEnded={event => continueMotion(event.currentTarget)}\n        onPlay={enforceRequestedPause}\n        onPlaying={event => {\n          if (event.currentTarget.currentSrc === event.currentTarget.src\n            && event.currentTarget.getAttribute(\"src\") === mediaUrl\n            && event.currentTarget.readyState >= HTMLMediaElement.HAVE_FUTURE_DATA) playableVideoUrl.current = mediaUrl;\n          enforceRequestedPause(event);\n        }}\n        onWaiting={() => { if (isPlaying) setWaitingKey(videoPresentationKey); }}\n        onError={onError}\n        data-media-position={mediaPosition}\n        data-video-backdrop=\"scene\"\n        style={{ ...mediaStyle, visibility: exhaustedKey === videoPresentationKey ? \"hidden\" : undefined }}\n      />\n    </>\n  );\n};\n"
     },
     {
       "path": "src/visual-system/scene-templates/color-utils.ts",

package/starters/video-chat/package.json

@@ -9,7 +9,7 @@
     "preview": "vite preview"
   },
   "dependencies": {
-    "@vanillaskyai/video": "0.10.9",
+    "@vanillaskyai/video": "0.10.10",
     "react": "^19.2.8",
     "react-dom": "^19.2.8",
     "@ai-sdk/anthropic": "^3.0.0",