@vanillaskyai/video 0.10.1 → 0.10.2

package/CHANGELOG.md

@@ -4,6 +4,12 @@ VanillaSky follows semantic versioning. This changelog begins with the 0.1 beta.
 
 ## Unreleased
 
+## 0.10.2
+
+- Keep generated narration playing in Safari after a delayed opening-to-body handoff by reusing the permitted audio element; isolate late playback failures from subsequent lines.
+- In drop mode, a media miss without usable authored fallback skips only that scene and preserves later valid scenes and the ending. Strict mode still fails. The skipped beat is reported; its content cannot be reconstructed without a usable asset or fallback.
+- Guide first-shot narration to fit its five-second footage budget and develop the answer in later scenes, without imposing a fixed sentence length on the rest of the video.
+
 ## 0.10.1
 
 - Continue narration through failed built-in media using the authored fallback or the existing graphic on black. Keep the recovery anchor after URL resolution, and observe actual mounted images for photo readiness.

package/dist/react.js

@@ -1550,6 +1550,8 @@ function createVideoChatVoice(options = {}) {
   const lines = /* @__PURE__ */ new Map();
   const pendingLoads = /* @__PURE__ */ new Set();
   let sounding;
+  let generatedElement;
+  let stopGenerated;
   let browserFinish;
   let held = false;
   let silent = false;
@@ -1674,7 +1676,12 @@ function createVideoChatVoice(options = {}) {
     },
     resume() {
       held = false;
-      if (sounding) void sounding.play().catch(() => playbackFailure?.());
+      if (sounding) {
+        const fail = playbackFailure;
+        void sounding.play().catch(() => {
+          if (!held) fail?.();
+        });
+      }
       if (!silent) globalThis.speechSynthesis?.resume();
     },
     setMuted(muted) {
@@ -1753,9 +1760,11 @@ function createVideoChatVoice(options = {}) {
       }
       let playbackFailed = false;
       try {
-        const element = new Audio(line.src);
+        stopGenerated?.();
+        const element = generatedElement ??= new Audio();
+        element.src = line.src;
         element.muted = silent;
-        if (offsetSeconds !== void 0) element.currentTime = offsetSeconds;
+        element.currentTime = offsetSeconds ?? 0;
         sounding = element;
         await new Promise((resolve) => {
           let finished = false;
@@ -1771,6 +1780,7 @@ function createVideoChatVoice(options = {}) {
             element.onplaying = null;
             element.onended = null;
             element.onerror = null;
+            if (stopGenerated === stop) stopGenerated = void 0;
             if (sounding === element) {
               sounding = void 0;
               playbackFailure = void 0;
@@ -1778,13 +1788,17 @@ function createVideoChatVoice(options = {}) {
             resolve();
           };
           const stop = () => {
+            if (finished) return;
             element.pause();
             finish();
           };
           const fail = () => {
-            playbackFailed = true;
-            stop();
+            if (!finished) {
+              playbackFailed = true;
+              stop();
+            }
           };
+          stopGenerated = stop;
           const clearWatchdog = watchSpeech(Math.max(line.seconds, estimatedBrowserSeconds(text)), fail);
           speechStops.add(stop);
           playbackFailure = fail;
@@ -1799,7 +1813,9 @@ function createVideoChatVoice(options = {}) {
           element.onerror = fail;
           signal.addEventListener("abort", stop, { once: true });
           try {
-            if (!held) void element.play().catch(fail);
+            if (!held) void element.play().catch(() => {
+              if (!held) fail();
+            });
           } catch {
             fail();
           }
@@ -1825,6 +1841,11 @@ function createVideoChatVoice(options = {}) {
       speechStops.clear();
       sounding?.pause();
       sounding = void 0;
+      generatedElement?.pause();
+      generatedElement?.removeAttribute?.("src");
+      generatedElement?.load?.();
+      generatedElement = void 0;
+      stopGenerated = void 0;
       stopBrowser();
       for (const line of lines.values()) {
         if (line.source === "generated") URL.revokeObjectURL(line.src);

package/dist/server.js

@@ -670,27 +670,38 @@ function createMediaResolvingPlanner(options) {
         return false;
       }
     };
-    const resolveOne = (part, openingReady2) => resolvePartVariables({
-      part,
-      requestId: context.request.requestId,
-      templateId: part.type === "scene.add" ? part.scene.templateId : void 0,
-      input: context.request.input,
-      signal: context.signal,
-      templates: options.templates,
-      resolveMedia: options.resolveMedia,
-      approveUrl: options.approveUrl,
-      openingReady: openingReady2 ?? options.isOpeningReady(context.request.input),
-      claimBudget,
-      onFallback: () => getGenerationLifecycleSink(context)?.reportWarning?.({
-        code: "provider_warning",
-        category: "provider",
-        message: "Some visuals use a simple background.",
-        recoverable: true
-      })
-    });
+    const resolveOne = async (part, openingReady2) => {
+      try {
+        return await resolvePartVariables({
+          part,
+          requestId: context.request.requestId,
+          templateId: part.type === "scene.add" ? part.scene.templateId : void 0,
+          input: context.request.input,
+          signal: context.signal,
+          templates: options.templates,
+          resolveMedia: options.resolveMedia,
+          approveUrl: options.approveUrl,
+          openingReady: openingReady2 ?? options.isOpeningReady(context.request.input),
+          claimBudget,
+          onFallback: () => getGenerationLifecycleSink(context)?.reportWarning?.({
+            code: "provider_warning",
+            category: "provider",
+            message: "Some visuals use a simple background.",
+            recoverable: true
+          })
+        });
+      } catch (cause) {
+        if (context.signal.aborted || part.type !== "scene.add") throw cause;
+        const error = cause instanceof Error ? cause : new Error(String(cause));
+        if (!getGenerationLifecycleSink(context)?.rejectPart?.(error)) throw error;
+        return void 0;
+      }
+    };
     if (limit === 1) {
       for await (const part of options.planner(context)) {
-        if (preflight(part)) yield await resolveOne(part);
+        if (!preflight(part)) continue;
+        const resolved = await resolveOne(part);
+        if (resolved) yield resolved;
       }
       return;
     }
@@ -748,7 +759,7 @@ function createMediaResolvingPlanner(options) {
         pending.shift();
         notifyProducer();
         if ("cause" in result) throw result.cause;
-        yield result.part;
+        if (result.part) yield result.part;
       }
       await producer;
       if (producerError !== void 0) throw producerError;
@@ -931,6 +942,7 @@ function createVideoChatResponseInstructions(generatedVideoAvailable, openingAlr
     generatedVideoAvailable ? 'First emit one host-consumed opening JSON object: {"type":"video-chat.opening","spokenHook":"6-9 words","mediaKeyword":"literal subject","firstShot":{"text":"short grounded fallback","narration":"a natural spoken beat advancing the hook","mediaKeyword":"2-8 concrete words","shotDirection":"optional framing and action"}}. Include firstShot only when a distinctive illustrative shot serves the story; omit it for an abstract explanation best opened with graphics. When present, the host inserts it once and begins generation immediately; do not emit it again.' : openingAlreadyProvided ? "Start directly with scene.add; the opening has already been supplied." : 'First emit {"type":"video-chat.opening","spokenHook":"6-9 words","mediaKeyword":"literal subject"}. This is host-consumed, not a scene.',
     openingAlreadyProvided ? "Preserve the supplied opening exactly; continue it without repeating its words or claim." : void 0,
     "The spokenHook is already heard before the first scene. Start firstShot.narration with the next fact, cause, action or consequence; never restate the hook. The first ordinary scene must then advance beyond firstShot rather than repeat its narration or opening claim. If firstShot is omitted, that first scene must advance the hook directly. firstShot.text, fallbackText and chapter titles must also express that new beat, not reuse the hook or prior shot copy; write a concise visual anchor of roughly 2\u20136 words within the schema budget, never a full narration sentence, clipped statistic or qualifying clause.",
+    "The optional firstShot has five seconds of footage: give its narration one brief sentence of roughly 8\u201310 spoken words that fits that budget. Carry further explanation, actions or dialogue in later scenes instead of squeezing the answer into the first shot. This limit applies only to firstShot; pace later narration to its own scene.",
     "Continue with only as many scenes as the story and duration need. Never pad to a fixed count. Every emitted scene carries narration beside variables and timing; no additional narration request should be needed.",
     "Make most of the film relevant footage: use full-bleed media for action, subjects and atmosphere, with graphics only to clarify a specific relationship or piece of evidence. Maintain consistent setting, lighting and subject while varying shot scale. Avoid unrelated cinematic montages.",
     "Use the installed catalog: ordered events, comparison, exact quote, one key figure, chapter or a message only when its narrative job fits. For an ordinary explanation, roughly one graphic-led explanatory beat per 30 seconds is a starting point, plus a brief chapter opening when needed. Let other intents use the visual form they need. Do not place graphic beats consecutively unless no honest relevant media is available. Give media-capable graphics a relevant background when it supports the meaning; keep black when no matching media exists. Do not force every template into a video or use lists of bullets.",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vanillaskyai/video",
-  "version": "0.10.1",
+  "version": "0.10.2",
   "description": "Open-source voice-and-video chat SDK for AI applications.",
   "keywords": [
     "video-chat",

package/starters/video-chat/package.json

@@ -9,7 +9,7 @@
     "preview": "vite preview"
   },
   "dependencies": {
-    "@vanillaskyai/video": "0.10.1",
+    "@vanillaskyai/video": "0.10.2",
     "react": "^19.2.8",
     "react-dom": "^19.2.8",
     "@ai-sdk/anthropic": "^3.0.0",