@vanillaskyai/video 0.5.7 → 0.6.0

package/examples/server-integrations/src/ai-sdk-media.ts

@@ -0,0 +1,90 @@
+// App-owned media generation for VanillaSky scenes.
+//
+// VanillaSky never generates media and never selects a provider. It calls
+// `resolveMedia` with a bounded query and the scene being resolved; the
+// application decides what to generate, where to store it, and what it costs.
+//
+// This adapter is provider-neutral: it takes AI SDK model instances rather
+// than naming a vendor, so the same code runs against any image or video
+// model the AI SDK supports. Choose the provider where you build the models,
+// not here.
+import { experimental_generateVideo, generateImage, type ImageModel } from "ai";
+import type { MediaResolver, MediaResolverContext, ResolvedMedia } from "@vanillaskyai/video/server";
+
+// `ai` declares its video model and provider-option types internally without
+// exporting them, so derive them from the call signatures instead of naming
+// them. This keeps the adapter correct across AI SDK releases.
+type VideoModel = Parameters<typeof experimental_generateVideo>[0]["model"];
+type ProviderOptions = Parameters<typeof generateImage>[0]["providerOptions"];
+
+/** Persist generated bytes and return a URL the player can load. */
+export type StoreGeneratedMedia = (media: {
+  data: Uint8Array;
+  mediaType: string;
+  kind: "image" | "video";
+  /** Stable per scene, so a retried request reuses the stored object. */
+  idempotencyKey: string;
+}) => Promise<{ url: string }>;
+
+export interface GeneratedMediaOptions {
+  imageModel: ImageModel;
+  /** Omit to resolve every scene as an image. */
+  videoModel?: VideoModel;
+  store: StoreGeneratedMedia;
+  /** Provider-specific settings, passed straight through to the AI SDK. */
+  providerOptions?: ProviderOptions;
+}
+
+function aspectRatio(context: MediaResolverContext): "16:9" | "9:16" {
+  return context.input.orientation === "portrait" ? "9:16" : "16:9";
+}
+
+// One generated asset per scene, keyed so a retry of the same request cannot
+// be billed twice.
+function idempotencyKey(context: MediaResolverContext): string {
+  return `${context.requestId}:${context.scene.id}`;
+}
+
+export function createGeneratedMediaResolver(options: GeneratedMediaOptions): MediaResolver {
+  return async (query, context): Promise<ResolvedMedia | null> => {
+    const wantsVideo = context.preferredType === "video" && options.videoModel !== undefined;
+
+    if (wantsVideo && options.videoModel) {
+      const result = await experimental_generateVideo({
+        model: options.videoModel,
+        prompt: query,
+        aspectRatio: aspectRatio(context),
+        // Generated media is billable. A silent retry can produce a second
+        // charge for one scene, so retries stay an explicit, budget-aware
+        // decision in the application rather than an SDK default.
+        maxRetries: 0,
+        abortSignal: context.signal,
+        providerOptions: options.providerOptions,
+      });
+      const stored = await options.store({
+        data: result.video.uint8Array,
+        mediaType: result.video.mediaType,
+        kind: "video",
+        idempotencyKey: idempotencyKey(context),
+      });
+      return { url: stored.url, type: "video" };
+    }
+
+    const result = await generateImage({
+      model: options.imageModel,
+      prompt: query,
+      aspectRatio: aspectRatio(context),
+      n: 1,
+      maxRetries: 0,
+      abortSignal: context.signal,
+      providerOptions: options.providerOptions,
+    });
+    const stored = await options.store({
+      data: result.image.uint8Array,
+      mediaType: result.image.mediaType,
+      kind: "image",
+      idempotencyKey: idempotencyKey(context),
+    });
+    return { url: stored.url, type: "image" };
+  };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vanillaskyai/video",
-  "version": "0.5.7",
+  "version": "0.6.0",
   "description": "Open-source video response SDK for personalized AI applications.",
   "keywords": [
     "generative-video",
@@ -35,6 +35,7 @@
     "examples/nextjs-quickstart/package.json",
     "examples/nextjs-quickstart/src",
     "examples/nextjs-quickstart/tsconfig.json",
+    "examples/server-integrations/src/ai-sdk-media.ts",
     "README.md",
     "LICENSE",
     "SECURITY.md",

package/registry/items/backgrounds.json

@@ -28,7 +28,7 @@
       "path": "src/visual-system/scene-templates/external-video-backdrop.tsx",
       "type": "registry:lib",
       "target": "vanillasky/scene-templates/external-video-backdrop.tsx",
-      "content": "import React from \"react\";\n\n// Source-owned templates may live in a consumer's tree while VideoFrame comes\n// from the package. Both copies must observe the same internal context or the\n// consumer template would mount a second video over the player-owned plane.\nexport type ExternalVideoBackdropMode = false | \"pending\" | \"ready\" | \"fallback\";\n\nconst sharedContext = globalThis as typeof globalThis & {\n  __vanillaskyExternalVideoBackdropContext?: React.Context<ExternalVideoBackdropMode>;\n};\nconst ExternalVideoBackdropContext = sharedContext.__vanillaskyExternalVideoBackdropContext\n  ??= React.createContext<ExternalVideoBackdropMode>(false);\n\nexport function ExternalVideoBackdropProvider({\n  mode,\n  children,\n}: {\n  mode: ExternalVideoBackdropMode;\n  children: React.ReactNode;\n}) {\n  return (\n    <ExternalVideoBackdropContext.Provider value={mode}>\n      {children}\n    </ExternalVideoBackdropContext.Provider>\n  );\n}\n\nexport function useExternalVideoBackdrop(): ExternalVideoBackdropMode {\n  return React.useContext(ExternalVideoBackdropContext);\n}\n"
+      "content": "import React from \"react\";\n\n// Source-owned templates may live in a consumer's tree while VideoFrame comes\n// from the package. Both copies must observe the same internal context or the\n// consumer template would mount a second video over the player-owned plane,\n// and would never inherit the player's native media audio state.\nexport type ExternalVideoBackdropMode = false | \"pending\" | \"ready\" | \"fallback\";\n\ninterface BackdropContextValue {\n  mode: ExternalVideoBackdropMode;\n  audioMuted: boolean;\n  audioVolume: number;\n}\n\nconst DEFAULT: BackdropContextValue = { mode: false, audioMuted: true, audioVolume: 1 };\n\nconst sharedContext = globalThis as typeof globalThis & {\n  __vanillaskyVideoBackdropContext?: React.Context<BackdropContextValue>;\n};\nconst BackdropContext = sharedContext.__vanillaskyVideoBackdropContext\n  ??= React.createContext<BackdropContextValue>(DEFAULT);\n\nexport function ExternalVideoBackdropProvider({\n  mode,\n  audioMuted = true,\n  audioVolume = 1,\n  children,\n}: {\n  mode: ExternalVideoBackdropMode;\n  audioMuted?: boolean;\n  audioVolume?: number;\n  children: React.ReactNode;\n}) {\n  const value = React.useMemo(\n    () => ({ mode, audioMuted, audioVolume }),\n    [mode, audioMuted, audioVolume],\n  );\n  return (\n    <BackdropContext.Provider value={value}>\n      {children}\n    </BackdropContext.Provider>\n  );\n}\n\nexport function useExternalVideoBackdrop(): ExternalVideoBackdropMode {\n  return React.useContext(BackdropContext).mode;\n}\n\nexport function useMediaAudio(): { muted: boolean; volume: number } {\n  const { audioMuted, audioVolume } = React.useContext(BackdropContext);\n  return { muted: audioMuted, volume: audioVolume };\n}\n"
     },
     {
       "path": "src/visual-system/scene-templates/media-position.ts",
@@ -40,7 +40,7 @@
       "path": "src/visual-system/scene-templates/scene-video-backdrop.tsx",
       "type": "registry:lib",
       "target": "vanillasky/scene-templates/scene-video-backdrop.tsx",
-      "content": "import React, { useEffect, useRef, useState } from \"react\";\nimport { getBackgroundTransform } from \"../backgrounds\";\nimport { resolveMediaPosition } from \"./media-position\";\n\nexport interface SceneVideoBackdropProps {\n  mediaUrl: string;\n  mediaPoster?: string;\n  mediaPosition?: string;\n  backgroundEffect?: string;\n  progress: number;\n  beatIntensity?: number;\n  isPlaying: boolean;\n  playbackId?: string;\n  retainPoster?: boolean;\n  persistent?: boolean;\n  preparedPoster?: {\n    presentationKey: string;\n    mediaPoster: string;\n    mediaPosition: string;\n    backgroundEffect?: string;\n    /** Existing global transition progress. On decoder-constrained Safari,\n     * this fades the decoded incoming still above the outgoing video before\n     * the single video element changes source. */\n    opacity?: number;\n  };\n  onReady?: () => void;\n  onError?: () => void;\n}\n\nexport const SceneVideoBackdrop: React.FC<SceneVideoBackdropProps> = ({\n  mediaUrl,\n  mediaPoster,\n  mediaPosition = \"center\",\n  backgroundEffect,\n  progress,\n  beatIntensity = 0,\n  isPlaying,\n  playbackId = mediaUrl,\n  retainPoster = false,\n  persistent = false,\n  preparedPoster,\n  onReady,\n  onError,\n}) => {\n  const resolvedPosition = resolveMediaPosition(mediaPosition);\n  const bgTransform = getBackgroundTransform(backgroundEffect, progress, beatIntensity);\n  const [decodedVideoUrl, setDecodedVideoUrl] = useState<string>();\n  const videoRef = useRef<HTMLVideoElement>(null);\n  const startedVideoUrl = useRef<string | undefined>(undefined);\n  const startedPlaybackId = useRef<string | undefined>(undefined);\n  const videoPresentationKey = `${playbackId}\\0${mediaUrl}`;\n\n  useEffect(() => {\n    const video = videoRef.current;\n    if (!video) return;\n    if (!isPlaying) {\n      video.pause();\n      startedVideoUrl.current = undefined;\n      startedPlaybackId.current = undefined;\n      return;\n    }\n    if (startedPlaybackId.current === playbackId) return;\n    const changingSource = startedVideoUrl.current !== undefined && startedVideoUrl.current !== mediaUrl;\n    video.playbackRate = 1;\n    if (!changingSource && video.currentTime > 0) video.currentTime = 0;\n    video.play().catch(() => {});\n    startedVideoUrl.current = mediaUrl;\n    startedPlaybackId.current = playbackId;\n  }, [isPlaying, mediaUrl, playbackId]);\n\n  useEffect(() => {\n    const video = videoRef.current;\n    return () => {\n      if (!video) return;\n      video.pause();\n      video.removeAttribute(\"src\");\n      video.load();\n    };\n  }, []);\n\n  const mediaStyle: React.CSSProperties = {\n    position: \"absolute\",\n    inset: 0,\n    width: \"100%\",\n    height: \"100%\",\n    objectFit: \"cover\",\n    objectPosition: resolvedPosition,\n    transform: bgTransform.transform,\n    transformOrigin: bgTransform.transformOrigin,\n    zIndex: persistent ? 1 : undefined,\n  };\n  const preparedPosition = preparedPoster\n    ? resolveMediaPosition(preparedPoster.mediaPosition)\n    : resolvedPosition;\n  const preparedTransform = getBackgroundTransform(preparedPoster?.backgroundEffect, 0, 0);\n  const posterPlanes = [\n    ...(persistent && mediaPoster ? [{\n      presentationKey: videoPresentationKey,\n      mediaPoster,\n      mediaPosition: resolvedPosition,\n      transform: bgTransform.transform,\n      transformOrigin: bgTransform.transformOrigin,\n      opacity: 1,\n      zIndex: 0,\n      role: \"current\",\n    }] : []),\n    ...(preparedPoster && preparedPoster.presentationKey !== videoPresentationKey ? [{\n      presentationKey: preparedPoster.presentationKey,\n      mediaPoster: preparedPoster.mediaPoster,\n      mediaPosition: preparedPosition,\n      transform: preparedTransform.transform,\n      transformOrigin: preparedTransform.transformOrigin,\n      opacity: preparedPoster.opacity ?? 0,\n      zIndex: 2,\n      role: \"prepared\",\n    }] : []),\n  ];\n\n  return (\n    <>\n      {posterPlanes.map((posterPlane) => (\n        <img\n          key={posterPlane.presentationKey}\n          src={posterPlane.mediaPoster}\n          alt=\"\"\n          aria-hidden=\"true\"\n          draggable={false}\n          data-video-poster-plane={posterPlane.role}\n          data-video-poster-visible={posterPlane.opacity > 0 ? \"true\" : \"false\"}\n          style={{\n            position: \"absolute\",\n            inset: 0,\n            width: \"100%\",\n            height: \"100%\",\n            objectFit: \"cover\",\n            objectPosition: posterPlane.mediaPosition,\n            transform: posterPlane.transform,\n            transformOrigin: posterPlane.transformOrigin,\n            zIndex: posterPlane.zIndex,\n            opacity: posterPlane.opacity,\n            pointerEvents: \"none\",\n          }}\n        />\n      ))}\n      <video\n        ref={videoRef}\n        src={mediaUrl}\n        poster={retainPoster || decodedVideoUrl !== mediaUrl ? mediaPoster || undefined : undefined}\n        muted\n        loop\n        playsInline\n        preload=\"auto\"\n        onLoadedData={(event) => {\n          const video = event.currentTarget;\n          const markPresented = () => {\n            if (!video.isConnected) return;\n            onReady?.();\n            if (!retainPoster) setDecodedVideoUrl(mediaUrl);\n          };\n          if (video.requestVideoFrameCallback) {\n            video.requestVideoFrameCallback(markPresented);\n            return;\n          }\n          markPresented();\n        }}\n        onError={onError}\n        data-media-position={mediaPosition}\n        data-video-backdrop={persistent ? \"persistent\" : \"scene\"}\n        style={mediaStyle}\n      />\n    </>\n  );\n};\n"
+      "content": "import React, { useEffect, useRef, useState } from \"react\";\nimport { getBackgroundTransform } from \"../backgrounds\";\nimport { useMediaAudio } from \"./external-video-backdrop\";\nimport { resolveMediaPosition } from \"./media-position\";\n\nexport interface SceneVideoBackdropProps {\n  mediaUrl: string;\n  mediaPoster?: string;\n  mediaPosition?: string;\n  backgroundEffect?: string;\n  progress: number;\n  beatIntensity?: number;\n  isPlaying: boolean;\n  muted?: boolean;\n  volume?: number;\n  playbackId?: string;\n  retainPoster?: boolean;\n  persistent?: boolean;\n  preparedPoster?: {\n    presentationKey: string;\n    mediaPoster: string;\n    mediaPosition: string;\n    backgroundEffect?: string;\n    /** Existing global transition progress. On decoder-constrained Safari,\n     * this fades the decoded incoming still above the outgoing video before\n     * the single video element changes source. */\n    opacity?: number;\n  };\n  onReady?: () => void;\n  onError?: () => void;\n}\n\nexport const SceneVideoBackdrop: React.FC<SceneVideoBackdropProps> = ({\n  mediaUrl,\n  mediaPoster,\n  mediaPosition = \"center\",\n  backgroundEffect,\n  progress,\n  beatIntensity = 0,\n  isPlaying,\n  muted,\n  volume,\n  playbackId = mediaUrl,\n  retainPoster = false,\n  persistent = false,\n  preparedPoster,\n  onReady,\n  onError,\n}) => {\n  const inheritedAudio = useMediaAudio();\n  const resolvedMuted = muted ?? inheritedAudio.muted;\n  const resolvedVolume = volume ?? inheritedAudio.volume;\n  const resolvedPosition = resolveMediaPosition(mediaPosition);\n  const bgTransform = getBackgroundTransform(backgroundEffect, progress, beatIntensity);\n  const [decodedVideoUrl, setDecodedVideoUrl] = useState<string>();\n  const videoRef = useRef<HTMLVideoElement>(null);\n  const startedVideoUrl = useRef<string | undefined>(undefined);\n  const startedPlaybackId = useRef<string | undefined>(undefined);\n  const videoPresentationKey = `${playbackId}\\0${mediaUrl}`;\n\n  useEffect(() => {\n    const video = videoRef.current;\n    if (!video) return;\n    // React Strict Mode rehearses setup → cleanup → setup in development.\n    // The cleanup deliberately releases the decoder, so the repeated setup\n    // must restore the declarative source before the playback effect runs.\n    if (video.getAttribute(\"src\") !== mediaUrl) {\n      video.setAttribute(\"src\", mediaUrl);\n      video.load();\n    }\n    return () => {\n      video.pause();\n      video.removeAttribute(\"src\");\n      video.load();\n      startedVideoUrl.current = undefined;\n      startedPlaybackId.current = undefined;\n    };\n  }, [mediaUrl]);\n\n  useEffect(() => {\n    const video = videoRef.current;\n    if (video) video.volume = resolvedVolume;\n  }, [resolvedVolume]);\n\n  useEffect(() => {\n    const video = videoRef.current;\n    if (!video) return;\n    if (!isPlaying) {\n      video.pause();\n      startedVideoUrl.current = undefined;\n      startedPlaybackId.current = undefined;\n      return;\n    }\n    if (startedPlaybackId.current === playbackId) return;\n    const changingSource = startedVideoUrl.current !== undefined && startedVideoUrl.current !== mediaUrl;\n    video.playbackRate = 1;\n    if (!changingSource && video.currentTime > 0) video.currentTime = 0;\n    video.play().catch(() => {});\n    startedVideoUrl.current = mediaUrl;\n    startedPlaybackId.current = playbackId;\n  }, [isPlaying, mediaUrl, playbackId]);\n\n  const mediaStyle: React.CSSProperties = {\n    position: \"absolute\",\n    inset: 0,\n    width: \"100%\",\n    height: \"100%\",\n    objectFit: \"cover\",\n    objectPosition: resolvedPosition,\n    transform: bgTransform.transform,\n    transformOrigin: bgTransform.transformOrigin,\n    zIndex: persistent ? 1 : undefined,\n  };\n  const preparedPosition = preparedPoster\n    ? resolveMediaPosition(preparedPoster.mediaPosition)\n    : resolvedPosition;\n  const preparedTransform = getBackgroundTransform(preparedPoster?.backgroundEffect, 0, 0);\n  const posterPlanes = [\n    ...(persistent && mediaPoster ? [{\n      presentationKey: videoPresentationKey,\n      mediaPoster,\n      mediaPosition: resolvedPosition,\n      transform: bgTransform.transform,\n      transformOrigin: bgTransform.transformOrigin,\n      opacity: 1,\n      zIndex: 0,\n      role: \"current\",\n    }] : []),\n    ...(preparedPoster && preparedPoster.presentationKey !== videoPresentationKey ? [{\n      presentationKey: preparedPoster.presentationKey,\n      mediaPoster: preparedPoster.mediaPoster,\n      mediaPosition: preparedPosition,\n      transform: preparedTransform.transform,\n      transformOrigin: preparedTransform.transformOrigin,\n      opacity: preparedPoster.opacity ?? 0,\n      zIndex: 2,\n      role: \"prepared\",\n    }] : []),\n  ];\n\n  return (\n    <>\n      {posterPlanes.map((posterPlane) => (\n        <img\n          key={posterPlane.presentationKey}\n          src={posterPlane.mediaPoster}\n          alt=\"\"\n          aria-hidden=\"true\"\n          draggable={false}\n          data-video-poster-plane={posterPlane.role}\n          data-video-poster-visible={posterPlane.opacity > 0 ? \"true\" : \"false\"}\n          style={{\n            position: \"absolute\",\n            inset: 0,\n            width: \"100%\",\n            height: \"100%\",\n            objectFit: \"cover\",\n            objectPosition: posterPlane.mediaPosition,\n            transform: posterPlane.transform,\n            transformOrigin: posterPlane.transformOrigin,\n            zIndex: posterPlane.zIndex,\n            opacity: posterPlane.opacity,\n            pointerEvents: \"none\",\n          }}\n        />\n      ))}\n      <video\n        ref={videoRef}\n        src={mediaUrl}\n        poster={retainPoster || decodedVideoUrl !== mediaUrl ? mediaPoster || undefined : undefined}\n        muted={resolvedMuted}\n        loop\n        playsInline\n        preload=\"auto\"\n        onLoadedData={(event) => {\n          const video = event.currentTarget;\n          const markPresented = () => {\n            if (!video.isConnected) return;\n            onReady?.();\n            if (!retainPoster) setDecodedVideoUrl(mediaUrl);\n          };\n          if (video.requestVideoFrameCallback) {\n            video.requestVideoFrameCallback(markPresented);\n            return;\n          }\n          markPresented();\n        }}\n        onError={onError}\n        data-media-position={mediaPosition}\n        data-video-backdrop={persistent ? \"persistent\" : \"scene\"}\n        style={mediaStyle}\n      />\n    </>\n  );\n};\n"
     },
     {
       "path": "src/visual-system/scene-templates/color-utils.ts",

package/dist/chunk-C6WVCZRW.js

@@ -1,19 +0,0 @@
-// src/visual-system/scene-templates/external-video-backdrop.tsx
-import React from "react";
-import { jsx } from "react/jsx-runtime";
-var sharedContext = globalThis;
-var ExternalVideoBackdropContext = sharedContext.__vanillaskyExternalVideoBackdropContext ??= React.createContext(false);
-function ExternalVideoBackdropProvider({
-  mode,
-  children
-}) {
-  return /* @__PURE__ */ jsx(ExternalVideoBackdropContext.Provider, { value: mode, children });
-}
-function useExternalVideoBackdrop() {
-  return React.useContext(ExternalVideoBackdropContext);
-}
-
-export {
-  ExternalVideoBackdropProvider,
-  useExternalVideoBackdrop
-};