@vanillaskyai/video 0.7.1 → 0.8.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. package/CHANGELOG.md +63 -0
  2. package/PUBLIC-API.md +83 -79
  3. package/README.md +20 -15
  4. package/dist/assets/vanillasky-logo.svg +151 -0
  5. package/dist/{builtin-server-KB7FKF6A.js → builtin-server-CUYPIDE7.js} +2 -2
  6. package/dist/check-runtime.js +1 -1
  7. package/dist/{chunk-2PYO6VAC.js → chunk-2DOLF4QN.js} +1 -43
  8. package/dist/{chunk-GAIOHGNR.js → chunk-2OKB27TG.js} +42 -45
  9. package/dist/{chunk-K746NUIP.js → chunk-2USVB3CY.js} +80 -29
  10. package/dist/{chunk-RE4IMWJR.js → chunk-3VRMKFM7.js} +87 -14
  11. package/dist/{chunk-JNN3EYVP.js → chunk-4UNKZEAT.js} +46 -46
  12. package/dist/{chunk-2XT4MZ76.js → chunk-APQKY6P3.js} +36 -3
  13. package/dist/{chunk-LLPMVDJZ.js → chunk-L7QGGKJ2.js} +69 -38
  14. package/dist/{chunk-RGF452LL.js → chunk-OWBT45EN.js} +1 -1
  15. package/dist/{chunk-LKBZX7GV.js → chunk-PP3MCPN4.js} +4 -4
  16. package/dist/chunk-ZD2VTUYR.js +28 -0
  17. package/dist/cli.js +131 -24
  18. package/dist/{compose-video-PD6LKKRK.js → compose-video-2NFH4DZY.js} +3 -3
  19. package/dist/{events-B6qS1Lsb.d.ts → events-BQ6z264n.d.ts} +1 -1
  20. package/dist/index.d.ts +5 -67
  21. package/dist/index.js +2 -4
  22. package/dist/{kit-8g46H2RZ.d.ts → kit-tQnCPMOM.d.ts} +1 -1
  23. package/dist/react.d.ts +33 -53
  24. package/dist/react.js +982 -648
  25. package/dist/server.d.ts +11 -13
  26. package/dist/server.js +170 -81
  27. package/dist/template-catalog.js +1 -1
  28. package/dist/templates.d.ts +3 -3
  29. package/dist/test.d.ts +2 -2
  30. package/dist/test.js +3 -3
  31. package/dist/{text-stream-ROTNGSD4.js → text-stream-54LXABJW.js} +1 -1
  32. package/dist/{types-2wHBqtg8.d.ts → types-CEh9eUFu.d.ts} +1 -1
  33. package/dist/{types-DrABlRa7.d.ts → types-DWFO6qca.d.ts} +1 -1
  34. package/docs/agent-integration.md +9 -3
  35. package/docs/architecture.md +8 -10
  36. package/docs/concepts.md +16 -40
  37. package/docs/custom-templates.md +27 -23
  38. package/docs/customization.md +23 -55
  39. package/docs/errors.md +58 -68
  40. package/docs/getting-started.md +21 -14
  41. package/docs/immersive-interface.md +87 -0
  42. package/docs/media-and-audio.md +2 -2
  43. package/docs/performance.md +69 -0
  44. package/docs/persistence.md +12 -27
  45. package/docs/production.md +2 -2
  46. package/docs/provider-integration.md +11 -5
  47. package/docs/reference/design-system.html +115 -0
  48. package/docs/reference/provider-adapters.md +37 -89
  49. package/docs/security.md +1 -1
  50. package/docs/streaming-protocol.md +2 -2
  51. package/examples/custom-template/README.md +6 -6
  52. package/package.json +6 -7
  53. package/registry/items/barChart.json +2 -2
  54. package/registry/items/bigNumber.json +2 -2
  55. package/registry/items/brandMessage.json +2 -2
  56. package/registry/items/cardList.json +2 -2
  57. package/registry/items/codeEditor.json +2 -2
  58. package/registry/items/ctaLogo.json +2 -2
  59. package/registry/items/ctaMedia.json +2 -2
  60. package/registry/items/incomingCall.json +2 -2
  61. package/registry/items/media.json +2 -2
  62. package/registry/items/milestone.json +2 -2
  63. package/registry/items/notification.json +2 -2
  64. package/registry/items/phoneMockup.json +2 -2
  65. package/registry/items/problemSolution.json +2 -2
  66. package/registry/items/progressRing.json +2 -2
  67. package/registry/items/promptInput.json +2 -2
  68. package/registry/items/reaction.json +2 -2
  69. package/registry/items/reviewStack.json +2 -2
  70. package/registry/items/steps.json +2 -2
  71. package/registry/items/terminal.json +2 -2
  72. package/registry/items/testimonial.json +2 -2
  73. package/registry/items/tripleStats.json +2 -2
  74. package/registry/items/tweet.json +2 -2
  75. package/registry/items/webMockup.json +2 -2
  76. package/starters/video-chat/.env.example +4 -2
  77. package/starters/video-chat/README.md +17 -5
  78. package/starters/video-chat/index.html +2 -1
  79. package/starters/video-chat/package.json +2 -4
  80. package/starters/video-chat/providers/speech.ts +20 -0
  81. package/starters/video-chat/providers/video.ts +66 -0
  82. package/starters/video-chat/providers.ts +3 -0
  83. package/starters/video-chat/server.ts +4 -80
  84. package/starters/video-chat/stock.ts +114 -99
  85. package/styles/video-chat.css +650 -246
  86. package/dist/chunk-SKRGRKHY.js +0 -142
  87. package/dist/state-DZcKuS32.d.ts +0 -3
  88. package/docs/branding-and-personalization.md +0 -87
  89. package/examples/custom-template/supplied-media.tsx +0 -92
@@ -137,7 +137,7 @@
137
137
  "mediaKeyword": {
138
138
  "type": "string",
139
139
  "title": "Background search keyword",
140
- "description": "2-4 word English term for Pexels stock-footage search (auto-fills mediaUrl).",
140
+ "description": "2-4 word English term for stock-media search (auto-fills mediaUrl).",
141
141
  "format": "stock-media-keyword",
142
142
  "default": ""
143
143
  },
@@ -156,7 +156,7 @@
156
156
  "mediaPoster": {
157
157
  "type": "string",
158
158
  "title": "Background poster image",
159
- "description": "Still image URL shown while a video backdrop is decoding its first frame. Auto-filled from Pexels' thumbnail when fillPexelsUrls sets a video mediaUrl. Hides the gradient flash that would otherwise appear in the ~50–400ms gap between a <video> mounting and decoding its first frame.",
159
+ "description": "Still image URL shown while a video backdrop is decoding its first frame. A media resolver can supply a poster alongside a video mediaUrl. Hides the gradient flash that would otherwise appear in the ~50–400ms gap between a <video> mounting and decoding its first frame.",
160
160
  "format": "uri",
161
161
  "default": ""
162
162
  },
@@ -93,7 +93,7 @@
93
93
  "mediaKeyword": {
94
94
  "type": "string",
95
95
  "title": "Background search keyword",
96
- "description": "2-4 word English term for Pexels stock-footage search (auto-fills mediaUrl).",
96
+ "description": "2-4 word English term for stock-media search (auto-fills mediaUrl).",
97
97
  "format": "stock-media-keyword",
98
98
  "default": ""
99
99
  },
@@ -112,7 +112,7 @@
112
112
  "mediaPoster": {
113
113
  "type": "string",
114
114
  "title": "Background poster image",
115
- "description": "Still image URL shown while a video backdrop is decoding its first frame. Auto-filled from Pexels' thumbnail when fillPexelsUrls sets a video mediaUrl. Hides the gradient flash that would otherwise appear in the ~50–400ms gap between a <video> mounting and decoding its first frame.",
115
+ "description": "Still image URL shown while a video backdrop is decoding its first frame. A media resolver can supply a poster alongside a video mediaUrl. Hides the gradient flash that would otherwise appear in the ~50–400ms gap between a <video> mounting and decoding its first frame.",
116
116
  "format": "uri",
117
117
  "default": ""
118
118
  },
@@ -146,7 +146,7 @@
146
146
  "mediaKeyword": {
147
147
  "type": "string",
148
148
  "title": "Background search keyword",
149
- "description": "2-4 word English term for Pexels stock-footage search (auto-fills mediaUrl).",
149
+ "description": "2-4 word English term for stock-media search (auto-fills mediaUrl).",
150
150
  "format": "stock-media-keyword",
151
151
  "default": ""
152
152
  },
@@ -165,7 +165,7 @@
165
165
  "mediaPoster": {
166
166
  "type": "string",
167
167
  "title": "Background poster image",
168
- "description": "Still image URL shown while a video backdrop is decoding its first frame. Auto-filled from Pexels' thumbnail when fillPexelsUrls sets a video mediaUrl. Hides the gradient flash that would otherwise appear in the ~50–400ms gap between a <video> mounting and decoding its first frame.",
168
+ "description": "Still image URL shown while a video backdrop is decoding its first frame. A media resolver can supply a poster alongside a video mediaUrl. Hides the gradient flash that would otherwise appear in the ~50–400ms gap between a <video> mounting and decoding its first frame.",
169
169
  "format": "uri",
170
170
  "default": ""
171
171
  },
@@ -129,7 +129,7 @@
129
129
  "mediaKeyword": {
130
130
  "type": "string",
131
131
  "title": "Background search keyword",
132
- "description": "2-4 word English term for Pexels stock-footage search (auto-fills mediaUrl).",
132
+ "description": "2-4 word English term for stock-media search (auto-fills mediaUrl).",
133
133
  "format": "stock-media-keyword",
134
134
  "default": ""
135
135
  },
@@ -148,7 +148,7 @@
148
148
  "mediaPoster": {
149
149
  "type": "string",
150
150
  "title": "Background poster image",
151
- "description": "Still image URL shown while a video backdrop is decoding its first frame. Auto-filled from Pexels' thumbnail when fillPexelsUrls sets a video mediaUrl. Hides the gradient flash that would otherwise appear in the ~50–400ms gap between a <video> mounting and decoding its first frame.",
151
+ "description": "Still image URL shown while a video backdrop is decoding its first frame. A media resolver can supply a poster alongside a video mediaUrl. Hides the gradient flash that would otherwise appear in the ~50–400ms gap between a <video> mounting and decoding its first frame.",
152
152
  "format": "uri",
153
153
  "default": ""
154
154
  },
@@ -204,7 +204,7 @@
204
204
  "mediaKeyword": {
205
205
  "type": "string",
206
206
  "title": "Background search keyword",
207
- "description": "2-4 word English term for Pexels stock-footage search (auto-fills mediaUrl).",
207
+ "description": "2-4 word English term for stock-media search (auto-fills mediaUrl).",
208
208
  "default": ""
209
209
  },
210
210
  "mediaType": {
@@ -222,7 +222,7 @@
222
222
  "mediaPoster": {
223
223
  "type": "string",
224
224
  "title": "Background poster image",
225
- "description": "Still image URL shown while a video backdrop is decoding its first frame. Auto-filled from Pexels' thumbnail when fillPexelsUrls sets a video mediaUrl. Hides the gradient flash that would otherwise appear in the ~50–400ms gap between a <video> mounting and decoding its first frame.",
225
+ "description": "Still image URL shown while a video backdrop is decoding its first frame. A media resolver can supply a poster alongside a video mediaUrl. Hides the gradient flash that would otherwise appear in the ~50–400ms gap between a <video> mounting and decoding its first frame.",
226
226
  "format": "uri",
227
227
  "default": ""
228
228
  },
@@ -1,10 +1,12 @@
1
1
  # Required: plans the response, including its opening and narration, and suggestions.
2
2
  ANTHROPIC_API_KEY=
3
3
 
4
- # Optional: generated speech. The browser voice is used when this is empty.
4
+ # Optional: run `npx vanillasky providers add speech` first.
5
+ # Generated speech; browser voice remains the fallback.
5
6
  XAI_API_KEY=
6
7
 
7
- # Optional: generated video and server-side speech transcription.
8
+ # Optional: run `npx vanillasky providers add video` first.
9
+ # Generated video and server-side speech transcription.
8
10
  FAL_KEY=
9
11
 
10
12
  # Optional: stock video and photography for templates and suggestion cards.
@@ -20,11 +20,23 @@ ANTHROPIC_API_KEY=...
20
20
  ```
21
21
 
22
22
  That is enough for rendered templates and the browser's built-in voice. The
23
- optional keys progressively improve the same experience:
23
+ starter installs only the text provider. Enable optional adapters when needed:
24
24
 
25
- - `XAI_API_KEY` replaces the browser voice with generated speech.
26
- - `FAL_KEY` enables generated-video modes and server transcription.
27
- - `PEXELS_API_KEY` adds stock footage to rendered responses and suggestion cards.
25
+ ```bash
26
+ npx vanillasky providers add speech
27
+ npx vanillasky providers add video
28
+ ```
29
+
30
+ Each command installs its provider package and connects the adapter through
31
+ `providers.ts`. Run both commands to enable both upgrades.
32
+
33
+ - Speech: add `XAI_API_KEY` to replace browser voice with generated speech.
34
+ - Video: add `FAL_KEY` for generated video and server transcription.
35
+ - Stock: add `PEXELS_API_KEY` for footage and suggestion images; no installation
36
+ is needed.
37
+
38
+ The adapters live in `providers/` and remain application-owned. Running an
39
+ upgrade again can finish an interrupted dependency installation.
28
40
 
29
41
  Restart the development server after changing keys. Open
30
42
  <http://localhost:5173> and prompt anything.
@@ -42,7 +54,7 @@ Restart the development server after changing keys. Open
42
54
  4. The opening footage loops or holds until the first planned scene and its
43
55
  voice are ready, then the response timeline takes over without a blank gap.
44
56
  5. The SDK holds every planned scene for its measured or estimated speech time
45
- and prepares four possible next prompts while playback continues.
57
+ and loads possible next prompts independently after the response completes.
46
58
 
47
59
  The text model is the only required provider. Generated speech, transcription,
48
60
  stock media, and generated video are independent upgrades and never prevent a
@@ -2,8 +2,9 @@
2
2
  <html lang="en">
3
3
  <head>
4
4
  <meta charset="UTF-8" />
5
- <meta name="viewport" content="width=device-width, initial-scale=1.0" />
5
+ <meta name="viewport" content="width=device-width, initial-scale=1.0, viewport-fit=cover" />
6
6
  <title>VanillaSky video chat</title>
7
+ <style>html, body { margin: 0; min-height: 100%; background: #080b16; }</style>
7
8
  </head>
8
9
  <body>
9
10
  <div id="root"></div>
@@ -9,13 +9,11 @@
9
9
  "preview": "vite preview"
10
10
  },
11
11
  "dependencies": {
12
- "@vanillaskyai/video": "0.7.1",
12
+ "@vanillaskyai/video": "0.8.1",
13
13
  "react": "^19.2.8",
14
14
  "react-dom": "^19.2.8",
15
15
  "@ai-sdk/anthropic": "^3.0.0",
16
- "ai": "^7.0.77",
17
- "@ai-sdk/xai": "^4.0.52",
18
- "@fal-ai/client": "1.10.1"
16
+ "ai": "^7.0.77"
19
17
  },
20
18
  "devDependencies": {
21
19
  "@types/react": "^19.2.18",
@@ -0,0 +1,20 @@
1
+ import { createXai } from "@ai-sdk/xai";
2
+ import { experimental_generateSpeech as synthesizeSpeech } from "ai";
3
+ import type { VideoChatHandlerOptions } from "@vanillaskyai/video/server";
4
+
5
+ /** Optional generated speech; the SDK supplies the browser-voice fallback. */
6
+ export const speechProvider: Pick<VideoChatHandlerOptions, "generateSpeech"> = {
7
+ generateSpeech: process.env.XAI_API_KEY
8
+ ? async ({ text, signal }) => {
9
+ const speech = await synthesizeSpeech({
10
+ model: createXai({ apiKey: process.env.XAI_API_KEY }).speech(),
11
+ text,
12
+ abortSignal: signal,
13
+ });
14
+ return {
15
+ audio: speech.audio.uint8Array,
16
+ mediaType: speech.audio.mediaType || "audio/mpeg",
17
+ };
18
+ }
19
+ : undefined,
20
+ };
@@ -0,0 +1,66 @@
1
+ import { fal } from "@fal-ai/client";
2
+ import type { VideoChatHandlerOptions } from "@vanillaskyai/video/server";
3
+
4
+ const VIDEO_MODEL = process.env.FAL_VIDEO_MODEL ?? "minimax/h3-max/text-to-video";
5
+
6
+ async function filmScene(
7
+ subject: string,
8
+ generatedLook: string | undefined,
9
+ orientation: "portrait" | "landscape",
10
+ signal: AbortSignal,
11
+ ) {
12
+ fal.config({ credentials: process.env.FAL_KEY });
13
+ const started = Date.now();
14
+ console.log(`[video-chat] filming: ${subject.slice(0, 70)}`);
15
+ try {
16
+ const result = await fal.subscribe(VIDEO_MODEL, {
17
+ input: {
18
+ prompt: [
19
+ `Locked-off shot, ${orientation === "portrait" ? "9:16 vertical" : "16:9"}. ${subject}.`,
20
+ "One slow continuous camera move. Physically plausible motion.",
21
+ "No on-screen text, captions, subtitles, watermarks or logos.",
22
+ "Diegetic sound only. No music, no voiceover.",
23
+ generatedLook,
24
+ ].filter(Boolean).join("\n\n"),
25
+ duration: 5,
26
+ resolution: "480P",
27
+ aspect_ratio: orientation === "portrait" ? "9:16" : "16:9",
28
+ },
29
+ abortSignal: signal,
30
+ });
31
+ const url = result?.data?.video?.url;
32
+ if (typeof url !== "string") throw new Error("the video provider returned no url");
33
+ console.log(`[video-chat] filmed in ${Date.now() - started}ms`);
34
+ return url;
35
+ } catch (cause) {
36
+ const status = (cause as { status?: unknown })?.status;
37
+ console.error(
38
+ `[video-chat] filming failed after ${Date.now() - started}ms${typeof status === "number" ? ` (status ${status})` : ""}`,
39
+ );
40
+ throw cause;
41
+ }
42
+ }
43
+
44
+ /** Optional generated video and transcription; credentials remain server-only. */
45
+ export const videoProvider: Pick<VideoChatHandlerOptions, "generateVideo" | "transcribe"> = {
46
+ generateVideo: process.env.FAL_KEY
47
+ ? async (query, { generatedLook, orientation, signal }) => ({
48
+ url: await filmScene(query, generatedLook, orientation, signal),
49
+ type: "video" as const,
50
+ })
51
+ : undefined,
52
+ transcribe: process.env.FAL_KEY
53
+ ? async ({ audio, mediaType, signal }) => {
54
+ signal.throwIfAborted();
55
+ fal.config({ credentials: process.env.FAL_KEY });
56
+ const bytes = Uint8Array.from(audio);
57
+ const url = await fal.storage.upload(new Blob([bytes.buffer], { type: mediaType }));
58
+ signal.throwIfAborted();
59
+ const result = await fal.subscribe(process.env.FAL_TRANSCRIBE_MODEL ?? "fal-ai/whisper", {
60
+ input: { audio_url: url, task: "transcribe" },
61
+ abortSignal: AbortSignal.any([signal, AbortSignal.timeout(60_000)]),
62
+ });
63
+ return String((result?.data as { text?: unknown })?.text ?? "");
64
+ }
65
+ : undefined,
66
+ };
@@ -0,0 +1,3 @@
1
+ import type { VideoChatHandlerOptions } from "@vanillaskyai/video/server";
2
+
3
+ export const providers: Pick<VideoChatHandlerOptions, "generateSpeech" | "generateVideo" | "transcribe"> = {};
@@ -1,57 +1,11 @@
1
1
  import { anthropic } from "@ai-sdk/anthropic";
2
- import { fal } from "@fal-ai/client";
3
- import { createXai } from "@ai-sdk/xai";
4
- import {
5
- experimental_generateSpeech as synthesizeSpeech,
6
- generateText,
7
- streamText,
8
- } from "ai";
2
+ import { generateText, streamText } from "ai";
9
3
  import { createVideoChatHandler } from "@vanillaskyai/video/server";
10
4
  import { findStockFootage } from "./stock";
5
+ import { providers } from "./providers";
11
6
 
12
7
  const PLANNER_MODEL = process.env.ANTHROPIC_PLANNER_MODEL ?? "claude-sonnet-5";
13
8
  const NARRATION_MODEL = process.env.ANTHROPIC_NARRATION_MODEL ?? "claude-haiku-4-5";
14
- const VIDEO_MODEL = process.env.FAL_VIDEO_MODEL ?? "minimax/h3-max/text-to-video";
15
- const SHOT_DEADLINE_MS = 180_000;
16
-
17
- async function filmScene(
18
- subject: string,
19
- generatedLook: string | undefined,
20
- orientation: "portrait" | "landscape",
21
- signal: AbortSignal,
22
- ) {
23
- fal.config({ credentials: process.env.FAL_KEY });
24
- const started = Date.now();
25
- console.log(`[video-chat] filming: ${subject.slice(0, 70)}`);
26
- try {
27
- const result = await fal.subscribe(VIDEO_MODEL, {
28
- input: {
29
- prompt: [
30
- `Locked-off shot, ${orientation === "portrait" ? "9:16 vertical" : "16:9"}. ${subject}.`,
31
- "One slow continuous camera move. Physically plausible motion.",
32
- "No on-screen text, captions, subtitles, watermarks or logos.",
33
- "Diegetic sound only. No music, no voiceover.",
34
- generatedLook,
35
- ].filter(Boolean).join("\n\n"),
36
- duration: 5,
37
- resolution: "480P",
38
- aspect_ratio: orientation === "portrait" ? "9:16" : "16:9",
39
- },
40
- abortSignal: AbortSignal.any([signal, AbortSignal.timeout(SHOT_DEADLINE_MS)]),
41
- });
42
- const url = result?.data?.video?.url;
43
- if (typeof url !== "string") throw new Error("the video provider returned no url");
44
- console.log(`[video-chat] filmed in ${Date.now() - started}ms`);
45
- return url;
46
- } catch (cause) {
47
- const status = (cause as { status?: unknown })?.status;
48
- console.error(
49
- `[video-chat] filming failed after ${Date.now() - started}ms${typeof status === "number" ? ` (status ${status})` : ""}`,
50
- );
51
- throw cause;
52
- }
53
- }
54
-
55
9
  /**
56
10
  * The application chooses providers and keeps their credentials here. The SDK
57
11
  * owns the video-chat protocol, prompts, capability negotiation, spend limits,
@@ -87,40 +41,10 @@ export const handleVideoChat = createVideoChatHandler({
87
41
  });
88
42
  return text;
89
43
  },
90
- generateSpeech: process.env.XAI_API_KEY
91
- ? async ({ text, signal }) => {
92
- const speech = await synthesizeSpeech({
93
- model: createXai({ apiKey: process.env.XAI_API_KEY }).speech(),
94
- text,
95
- abortSignal: signal,
96
- });
97
- return {
98
- audio: speech.audio.uint8Array,
99
- mediaType: speech.audio.mediaType || "audio/mpeg",
100
- };
101
- }
102
- : undefined,
103
- transcribe: process.env.FAL_KEY
104
- ? async ({ audio, mediaType, signal }) => {
105
- fal.config({ credentials: process.env.FAL_KEY });
106
- const bytes = Uint8Array.from(audio);
107
- const url = await fal.storage.upload(new Blob([bytes.buffer], { type: mediaType }));
108
- const result = await fal.subscribe(process.env.FAL_TRANSCRIBE_MODEL ?? "fal-ai/whisper", {
109
- input: { audio_url: url, task: "transcribe" },
110
- abortSignal: AbortSignal.any([signal, AbortSignal.timeout(60_000)]),
111
- });
112
- return String((result?.data as { text?: unknown })?.text ?? "");
113
- }
114
- : undefined,
115
44
  searchMedia: process.env.PEXELS_API_KEY
116
- ? (query, { orientation, signal }) => findStockFootage(query, orientation, signal)
117
- : undefined,
118
- generateVideo: process.env.FAL_KEY
119
- ? async (query, { generatedLook, orientation, signal }) => ({
120
- url: await filmScene(query, generatedLook, orientation, signal),
121
- type: "video" as const,
122
- })
45
+ ? (query, { orientation, signal, fallbackQuery }) => findStockFootage(query, orientation, signal, fallbackQuery)
123
46
  : undefined,
47
+ ...providers,
124
48
  welcome: {
125
49
  heroQuery: process.env.VIDEO_CHAT_WELCOME_KEYWORD ?? "underwater ocean sunlight",
126
50
  prompts: [
@@ -1,26 +1,10 @@
1
- /**
2
- * Stock footage for the modes that do not generate any.
3
- *
4
- * A rendered card on a brand gradient is the least a template can look like,
5
- * and templates-only mode was every scene that way. A stock search costs a few
6
- * hundred milliseconds and nothing per request, so unlike a generated clip it
7
- * never lands on the critical path - the difference between cards on gradients
8
- * and cards over real footage is close to free.
9
- *
10
- * Ported from the site's playground, trimmed to what a video chat needs. Every URL
11
- * that comes back is checked against the host it must come from: the planner
12
- * asks for a subject, and what a search returns is not something to trust into
13
- * a page unexamined.
14
- */
15
1
  import type { VideoOrientation } from "@vanillaskyai/video";
16
2
 
17
- const VIDEO_SEARCH = "https://api.pexels.com/videos/search";
3
+ const VIDEO_SEARCH = "https://api.pexels.com/v1/videos/search";
18
4
  const PHOTO_SEARCH = "https://api.pexels.com/v1/search";
19
5
  const MAX_KEYWORD_LENGTH = 80;
20
- // Shorter than the shot deadline by two orders of magnitude, deliberately: a
21
- // scene with no footage is a scene on a gradient, and that is a better outcome
22
- // than a response that waits.
23
- const LOOKUP_TIMEOUT_MS = 4_000;
6
+ // One budget covers specific/broader video searches and photo recovery.
7
+ const LOOKUP_TIMEOUT_MS = 3_000;
24
8
 
25
9
  interface StockMedia {
26
10
  url: string;
@@ -28,12 +12,6 @@ interface StockMedia {
28
12
  posterUrl?: string;
29
13
  }
30
14
 
31
- interface VideoFile {
32
- link?: unknown;
33
- width?: number;
34
- height?: number;
35
- }
36
-
37
15
  /** One hour, keyed by query. The same response asked twice searches once. */
38
16
  const cache = new Map<string, { payload: unknown; expiresAt: number }>();
39
17
 
@@ -45,95 +23,132 @@ function normalizeKeyword(value: string): string {
45
23
  .slice(0, MAX_KEYWORD_LENGTH);
46
24
  }
47
25
 
48
- function trustedUrl(value: unknown, host: string): string {
49
- const url = new URL(String(value));
50
- if (url.protocol !== "https:" || url.hostname !== host) {
51
- throw new Error(`Stock search returned a URL outside ${host}`);
26
+ function record(value: unknown): Record<string, unknown> {
27
+ return value !== null && typeof value === "object" && !Array.isArray(value)
28
+ ? value as Record<string, unknown>
29
+ : {};
30
+ }
31
+
32
+ function entries(value: unknown): unknown[] {
33
+ return Array.isArray(value) ? value : [];
34
+ }
35
+
36
+ function trustedUrl(value: unknown, host: string): string | undefined {
37
+ if (typeof value !== "string") return undefined;
38
+ try {
39
+ const url = new URL(value);
40
+ if (url.protocol !== "https:" || url.hostname !== host || url.username || url.password || url.port) return undefined;
41
+ return url.toString();
42
+ } catch {
43
+ return undefined;
52
44
  }
53
- return url.toString();
54
45
  }
55
46
 
56
- function pickVideoFile(files: VideoFile[], orientation: VideoOrientation): VideoFile & { link: string } | null {
57
- const usable = files
58
- .filter((file): file is VideoFile & { link: string } => typeof file.link === "string")
59
- // One file on an unexpected host is one file to skip, not a reason to give
60
- // up the search: throwing here lost every other rendition of the same clip.
61
- .flatMap((file) => {
62
- try {
63
- return [{ ...file, link: trustedUrl(file.link, "videos.pexels.com") }];
64
- } catch {
65
- return [];
66
- }
67
- })
68
- .filter((file) => file.width && file.height);
69
- // Matched to the stage, because a clip cropped to the other shape loses
70
- // whatever the search was actually for - a landscape river in a portrait
71
- // frame is a column of water with both banks cut off.
72
- const wanted = orientation === "portrait"
73
- ? (file: VideoFile) => (file.height ?? 0) >= (file.width ?? 0)
74
- : (file: VideoFile) => (file.width ?? 0) >= (file.height ?? 0);
75
- return usable.find(wanted) ?? usable[0] ?? null;
47
+ function pickVideoFile(files: unknown, orientation: VideoOrientation): string | undefined {
48
+ const usable = entries(files).flatMap((value) => {
49
+ const file = record(value);
50
+ const direct = trustedUrl(file.link, "videos.pexels.com");
51
+ const external = trustedUrl(file.link, "player.vimeo.com");
52
+ const link = direct ?? (external && /^\/external\/[a-zA-Z0-9._-]+\.mp4$/.test(new URL(external).pathname) ? external : undefined);
53
+ const { width, height } = file;
54
+ if (!link || typeof width !== "number" || typeof height !== "number"
55
+ || !Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) return [];
56
+ return [{ link, width, height }];
57
+ });
58
+ // Prefer the stage's shape without discarding other safe renditions.
59
+ return (usable.find((file) => orientation === "portrait"
60
+ ? file.height >= file.width
61
+ : file.width >= file.height) ?? usable[0])?.link;
76
62
  }
77
63
 
78
64
  async function search(url: string, apiKey: string, signal: AbortSignal): Promise<unknown> {
79
65
  const cached = cache.get(url);
80
66
  if (cached && cached.expiresAt > Date.now()) return cached.payload;
81
67
 
82
- const response = await fetch(url, {
83
- headers: { Authorization: apiKey },
84
- signal: AbortSignal.any([signal, AbortSignal.timeout(LOOKUP_TIMEOUT_MS)]),
85
- });
86
- // Only a real response is worth keeping. Caching the failure would let one
87
- // rate-limited minute leave every response about the Moon on a gradient for
88
- // the hour that followed.
89
- if (!response.ok) return null;
90
- const payload = await response.json();
91
- cache.set(url, { payload, expiresAt: Date.now() + 60 * 60 * 1_000 });
92
- if (cache.size > 100) cache.delete(cache.keys().next().value!);
93
- return payload;
68
+ if (signal.aborted) return null;
69
+ try {
70
+ const response = await fetch(url, {
71
+ headers: { Authorization: apiKey },
72
+ signal,
73
+ });
74
+ if (!response.ok || signal.aborted) return null;
75
+ const payload: unknown = await response.json();
76
+ if (signal.aborted) return null;
77
+ cache.set(url, { payload, expiresAt: Date.now() + 60 * 60 * 1_000 });
78
+ if (cache.size > 100) cache.delete(cache.keys().next().value!);
79
+ return payload;
80
+ } catch {
81
+ // A failed video search must still allow the independent photo fallback.
82
+ if (!signal.aborted) console.warn("[video-chat] Stock lookup unavailable; trying a fallback.");
83
+ return null;
84
+ }
85
+ }
86
+
87
+ /** Metadata is a weak mismatch filter, never visual verification. Numeric URLs
88
+ * and missing descriptions are unknown and retain the provider's search ranking. */
89
+ function matchesMetadata(item: Record<string, unknown>, keyword: string): boolean {
90
+ let description = typeof item.alt === "string" ? item.alt : "";
91
+ if (!description) {
92
+ const page = trustedUrl(item.url, "www.pexels.com");
93
+ if (page) description = new URL(page).pathname.replace(/^\/(?:video|photo)\//, "");
94
+ }
95
+ const words = (value: string) => normalizeKeyword(value).toLowerCase().split(/[\s-]+/)
96
+ .filter(word => /\p{L}/u.test(word) && !["a", "an", "the", "of", "in", "on", "with", "and", "video", "photo", "free", "footage"].includes(word));
97
+ const labels = words(description);
98
+ const requested = words(keyword);
99
+ return labels.length === 0 || requested.length === 0 || requested.some(word => labels.includes(word));
94
100
  }
95
101
 
96
- /**
97
- * Find footage for one beat. Video first, a photograph if there is none.
98
- *
99
- * Returns null rather than throwing when there is nothing to show: the scene
100
- * falls back to the brand gradient, which is what it looked like before.
101
- */
102
- export async function findStockFootage(subject: string, orientation: VideoOrientation, signal: AbortSignal): Promise<StockMedia | null> {
102
+ /** Video queries precede photos. All work shares one deadline, including body
103
+ * parsing, even if a custom fetch implementation ignores its abort signal. */
104
+ export async function findStockFootage(subject: string, orientation: VideoOrientation, signal: AbortSignal, fallbackSubject?: string): Promise<StockMedia | null> {
103
105
  const apiKey = process.env.PEXELS_API_KEY;
104
106
  const keyword = normalizeKeyword(subject);
105
- if (!apiKey || !keyword) return null;
106
-
107
- const params = new URLSearchParams({ query: keyword, orientation, per_page: "3", page: "1" });
108
- const started = Date.now();
109
- try {
110
- const videos = await search(`${VIDEO_SEARCH}?${params}`, apiKey, signal) as
111
- { videos?: Array<{ image?: unknown; video_files?: VideoFile[] }> } | null;
112
- for (const video of videos?.videos ?? []) {
113
- const file = pickVideoFile(video.video_files ?? [], orientation);
114
- if (!file) continue;
115
- console.log(`[video-chat] stock video in ${Date.now() - started}ms: ${keyword}`);
116
- return {
117
- url: file.link,
118
- type: "video",
119
- // The still shown while the video decodes its first frame, which is
120
- // otherwise a flash of gradient where the footage should be.
121
- ...(video.image ? { posterUrl: trustedUrl(video.image, "images.pexels.com") } : {}),
122
- };
107
+ if (!apiKey || !keyword || signal.aborted) return null;
108
+ const keywords = [keyword];
109
+ const fallback = normalizeKeyword(fallbackSubject ?? "");
110
+ if (fallback && fallback.toLowerCase() !== keyword.toLowerCase()) keywords.push(fallback);
111
+ const controller = new AbortController();
112
+ let finish!: (value: null) => void;
113
+ const stopped = new Promise<null>(resolve => { finish = resolve; });
114
+ const abort = () => { controller.abort(); finish(null); };
115
+ signal.addEventListener("abort", abort, { once: true });
116
+ const timer = setTimeout(abort, LOOKUP_TIMEOUT_MS);
117
+ const active = controller.signal;
118
+ const lookup = async (): Promise<StockMedia | null> => {
119
+ for (const query of keywords) {
120
+ const params = new URLSearchParams({ query, orientation, per_page: "3", page: "1" });
121
+ const videos = record(await search(`${VIDEO_SEARCH}?${params}`, apiKey, active));
122
+ if (active.aborted) return null;
123
+ for (const value of entries(videos.videos)) {
124
+ const video = record(value);
125
+ if (!matchesMetadata(video, query)) continue;
126
+ const url = pickVideoFile(video.video_files, orientation);
127
+ if (!url) continue;
128
+ const posterUrl = trustedUrl(video.image, "images.pexels.com");
129
+ return { url, type: "video", ...(posterUrl ? { posterUrl } : {}) };
130
+ }
123
131
  }
124
-
125
- const photos = await search(`${PHOTO_SEARCH}?${params}`, apiKey, signal) as
126
- { photos?: Array<{ src?: { large2x?: unknown; large?: unknown } }> } | null;
127
- const photo = photos?.photos?.[0]?.src?.large2x ?? photos?.photos?.[0]?.src?.large;
128
- if (!photo) return null;
129
- console.log(`[video-chat] stock photo in ${Date.now() - started}ms: ${keyword}`);
130
- return { url: trustedUrl(photo, "images.pexels.com"), type: "image" };
131
- } catch (cause) {
132
- // A timeout, an abort, or a search that returned something unexpected. The
133
- // beat keeps its copy on the brand gradient.
134
- if (!signal.aborted) {
135
- console.warn(`[video-chat] stock search failed after ${Date.now() - started}ms:`, cause instanceof Error ? cause.message : cause);
132
+ for (const query of keywords) {
133
+ const params = new URLSearchParams({ query, orientation, per_page: "3", page: "1" });
134
+ const photos = record(await search(`${PHOTO_SEARCH}?${params}`, apiKey, active));
135
+ if (active.aborted) return null;
136
+ for (const value of entries(photos.photos)) {
137
+ const photo = record(value);
138
+ if (!matchesMetadata(photo, query)) continue;
139
+ const source = record(photo.src);
140
+ for (const candidate of [source.large2x, source.large]) {
141
+ const url = trustedUrl(candidate, "images.pexels.com");
142
+ if (url) return { url, type: "image" };
143
+ }
144
+ }
136
145
  }
137
146
  return null;
147
+ };
148
+ try {
149
+ return await Promise.race([lookup(), stopped]);
150
+ } finally {
151
+ clearTimeout(timer);
152
+ signal.removeEventListener("abort", abort);
138
153
  }
139
154
  }