@vanillaskyai/video 0.7.1 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +63 -0
- package/PUBLIC-API.md +83 -79
- package/README.md +20 -15
- package/dist/assets/vanillasky-logo.svg +151 -0
- package/dist/{builtin-server-KB7FKF6A.js → builtin-server-CUYPIDE7.js} +2 -2
- package/dist/check-runtime.js +1 -1
- package/dist/{chunk-2PYO6VAC.js → chunk-2DOLF4QN.js} +1 -43
- package/dist/{chunk-GAIOHGNR.js → chunk-2OKB27TG.js} +42 -45
- package/dist/{chunk-K746NUIP.js → chunk-2USVB3CY.js} +80 -29
- package/dist/{chunk-RE4IMWJR.js → chunk-3VRMKFM7.js} +87 -14
- package/dist/{chunk-JNN3EYVP.js → chunk-4UNKZEAT.js} +46 -46
- package/dist/{chunk-2XT4MZ76.js → chunk-APQKY6P3.js} +36 -3
- package/dist/{chunk-LLPMVDJZ.js → chunk-L7QGGKJ2.js} +69 -38
- package/dist/{chunk-RGF452LL.js → chunk-OWBT45EN.js} +1 -1
- package/dist/{chunk-LKBZX7GV.js → chunk-PP3MCPN4.js} +4 -4
- package/dist/chunk-ZD2VTUYR.js +28 -0
- package/dist/cli.js +131 -24
- package/dist/{compose-video-PD6LKKRK.js → compose-video-2NFH4DZY.js} +3 -3
- package/dist/{events-B6qS1Lsb.d.ts → events-BQ6z264n.d.ts} +1 -1
- package/dist/index.d.ts +5 -67
- package/dist/index.js +2 -4
- package/dist/{kit-8g46H2RZ.d.ts → kit-tQnCPMOM.d.ts} +1 -1
- package/dist/react.d.ts +33 -53
- package/dist/react.js +982 -648
- package/dist/server.d.ts +11 -13
- package/dist/server.js +170 -81
- package/dist/template-catalog.js +1 -1
- package/dist/templates.d.ts +3 -3
- package/dist/test.d.ts +2 -2
- package/dist/test.js +3 -3
- package/dist/{text-stream-ROTNGSD4.js → text-stream-54LXABJW.js} +1 -1
- package/dist/{types-2wHBqtg8.d.ts → types-CEh9eUFu.d.ts} +1 -1
- package/dist/{types-DrABlRa7.d.ts → types-DWFO6qca.d.ts} +1 -1
- package/docs/agent-integration.md +9 -3
- package/docs/architecture.md +8 -10
- package/docs/concepts.md +16 -40
- package/docs/custom-templates.md +27 -23
- package/docs/customization.md +23 -55
- package/docs/errors.md +58 -68
- package/docs/getting-started.md +21 -14
- package/docs/immersive-interface.md +87 -0
- package/docs/media-and-audio.md +2 -2
- package/docs/performance.md +69 -0
- package/docs/persistence.md +12 -27
- package/docs/production.md +2 -2
- package/docs/provider-integration.md +11 -5
- package/docs/reference/design-system.html +115 -0
- package/docs/reference/provider-adapters.md +37 -89
- package/docs/security.md +1 -1
- package/docs/streaming-protocol.md +2 -2
- package/examples/custom-template/README.md +6 -6
- package/package.json +6 -7
- package/registry/items/barChart.json +2 -2
- package/registry/items/bigNumber.json +2 -2
- package/registry/items/brandMessage.json +2 -2
- package/registry/items/cardList.json +2 -2
- package/registry/items/codeEditor.json +2 -2
- package/registry/items/ctaLogo.json +2 -2
- package/registry/items/ctaMedia.json +2 -2
- package/registry/items/incomingCall.json +2 -2
- package/registry/items/media.json +2 -2
- package/registry/items/milestone.json +2 -2
- package/registry/items/notification.json +2 -2
- package/registry/items/phoneMockup.json +2 -2
- package/registry/items/problemSolution.json +2 -2
- package/registry/items/progressRing.json +2 -2
- package/registry/items/promptInput.json +2 -2
- package/registry/items/reaction.json +2 -2
- package/registry/items/reviewStack.json +2 -2
- package/registry/items/steps.json +2 -2
- package/registry/items/terminal.json +2 -2
- package/registry/items/testimonial.json +2 -2
- package/registry/items/tripleStats.json +2 -2
- package/registry/items/tweet.json +2 -2
- package/registry/items/webMockup.json +2 -2
- package/starters/video-chat/.env.example +4 -2
- package/starters/video-chat/README.md +17 -5
- package/starters/video-chat/index.html +2 -1
- package/starters/video-chat/package.json +2 -4
- package/starters/video-chat/providers/speech.ts +20 -0
- package/starters/video-chat/providers/video.ts +66 -0
- package/starters/video-chat/providers.ts +3 -0
- package/starters/video-chat/server.ts +4 -80
- package/starters/video-chat/stock.ts +114 -99
- package/styles/video-chat.css +650 -246
- package/dist/chunk-SKRGRKHY.js +0 -142
- package/dist/state-DZcKuS32.d.ts +0 -3
- package/docs/branding-and-personalization.md +0 -87
- package/examples/custom-template/supplied-media.tsx +0 -92
|
@@ -137,7 +137,7 @@
|
|
|
137
137
|
"mediaKeyword": {
|
|
138
138
|
"type": "string",
|
|
139
139
|
"title": "Background search keyword",
|
|
140
|
-
"description": "2-4 word English term for
|
|
140
|
+
"description": "2-4 word English term for stock-media search (auto-fills mediaUrl).",
|
|
141
141
|
"format": "stock-media-keyword",
|
|
142
142
|
"default": ""
|
|
143
143
|
},
|
|
@@ -156,7 +156,7 @@
|
|
|
156
156
|
"mediaPoster": {
|
|
157
157
|
"type": "string",
|
|
158
158
|
"title": "Background poster image",
|
|
159
|
-
"description": "Still image URL shown while a video backdrop is decoding its first frame.
|
|
159
|
+
"description": "Still image URL shown while a video backdrop is decoding its first frame. A media resolver can supply a poster alongside a video mediaUrl. Hides the gradient flash that would otherwise appear in the ~50–400ms gap between a <video> mounting and decoding its first frame.",
|
|
160
160
|
"format": "uri",
|
|
161
161
|
"default": ""
|
|
162
162
|
},
|
|
@@ -93,7 +93,7 @@
|
|
|
93
93
|
"mediaKeyword": {
|
|
94
94
|
"type": "string",
|
|
95
95
|
"title": "Background search keyword",
|
|
96
|
-
"description": "2-4 word English term for
|
|
96
|
+
"description": "2-4 word English term for stock-media search (auto-fills mediaUrl).",
|
|
97
97
|
"format": "stock-media-keyword",
|
|
98
98
|
"default": ""
|
|
99
99
|
},
|
|
@@ -112,7 +112,7 @@
|
|
|
112
112
|
"mediaPoster": {
|
|
113
113
|
"type": "string",
|
|
114
114
|
"title": "Background poster image",
|
|
115
|
-
"description": "Still image URL shown while a video backdrop is decoding its first frame.
|
|
115
|
+
"description": "Still image URL shown while a video backdrop is decoding its first frame. A media resolver can supply a poster alongside a video mediaUrl. Hides the gradient flash that would otherwise appear in the ~50–400ms gap between a <video> mounting and decoding its first frame.",
|
|
116
116
|
"format": "uri",
|
|
117
117
|
"default": ""
|
|
118
118
|
},
|
|
@@ -146,7 +146,7 @@
|
|
|
146
146
|
"mediaKeyword": {
|
|
147
147
|
"type": "string",
|
|
148
148
|
"title": "Background search keyword",
|
|
149
|
-
"description": "2-4 word English term for
|
|
149
|
+
"description": "2-4 word English term for stock-media search (auto-fills mediaUrl).",
|
|
150
150
|
"format": "stock-media-keyword",
|
|
151
151
|
"default": ""
|
|
152
152
|
},
|
|
@@ -165,7 +165,7 @@
|
|
|
165
165
|
"mediaPoster": {
|
|
166
166
|
"type": "string",
|
|
167
167
|
"title": "Background poster image",
|
|
168
|
-
"description": "Still image URL shown while a video backdrop is decoding its first frame.
|
|
168
|
+
"description": "Still image URL shown while a video backdrop is decoding its first frame. A media resolver can supply a poster alongside a video mediaUrl. Hides the gradient flash that would otherwise appear in the ~50–400ms gap between a <video> mounting and decoding its first frame.",
|
|
169
169
|
"format": "uri",
|
|
170
170
|
"default": ""
|
|
171
171
|
},
|
|
@@ -129,7 +129,7 @@
|
|
|
129
129
|
"mediaKeyword": {
|
|
130
130
|
"type": "string",
|
|
131
131
|
"title": "Background search keyword",
|
|
132
|
-
"description": "2-4 word English term for
|
|
132
|
+
"description": "2-4 word English term for stock-media search (auto-fills mediaUrl).",
|
|
133
133
|
"format": "stock-media-keyword",
|
|
134
134
|
"default": ""
|
|
135
135
|
},
|
|
@@ -148,7 +148,7 @@
|
|
|
148
148
|
"mediaPoster": {
|
|
149
149
|
"type": "string",
|
|
150
150
|
"title": "Background poster image",
|
|
151
|
-
"description": "Still image URL shown while a video backdrop is decoding its first frame.
|
|
151
|
+
"description": "Still image URL shown while a video backdrop is decoding its first frame. A media resolver can supply a poster alongside a video mediaUrl. Hides the gradient flash that would otherwise appear in the ~50–400ms gap between a <video> mounting and decoding its first frame.",
|
|
152
152
|
"format": "uri",
|
|
153
153
|
"default": ""
|
|
154
154
|
},
|
|
@@ -204,7 +204,7 @@
|
|
|
204
204
|
"mediaKeyword": {
|
|
205
205
|
"type": "string",
|
|
206
206
|
"title": "Background search keyword",
|
|
207
|
-
"description": "2-4 word English term for
|
|
207
|
+
"description": "2-4 word English term for stock-media search (auto-fills mediaUrl).",
|
|
208
208
|
"default": ""
|
|
209
209
|
},
|
|
210
210
|
"mediaType": {
|
|
@@ -222,7 +222,7 @@
|
|
|
222
222
|
"mediaPoster": {
|
|
223
223
|
"type": "string",
|
|
224
224
|
"title": "Background poster image",
|
|
225
|
-
"description": "Still image URL shown while a video backdrop is decoding its first frame.
|
|
225
|
+
"description": "Still image URL shown while a video backdrop is decoding its first frame. A media resolver can supply a poster alongside a video mediaUrl. Hides the gradient flash that would otherwise appear in the ~50–400ms gap between a <video> mounting and decoding its first frame.",
|
|
226
226
|
"format": "uri",
|
|
227
227
|
"default": ""
|
|
228
228
|
},
|
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
# Required: plans the response, including its opening and narration, and suggestions.
|
|
2
2
|
ANTHROPIC_API_KEY=
|
|
3
3
|
|
|
4
|
-
# Optional:
|
|
4
|
+
# Optional: run `npx vanillasky providers add speech` first.
|
|
5
|
+
# Generated speech; browser voice remains the fallback.
|
|
5
6
|
XAI_API_KEY=
|
|
6
7
|
|
|
7
|
-
# Optional:
|
|
8
|
+
# Optional: run `npx vanillasky providers add video` first.
|
|
9
|
+
# Generated video and server-side speech transcription.
|
|
8
10
|
FAL_KEY=
|
|
9
11
|
|
|
10
12
|
# Optional: stock video and photography for templates and suggestion cards.
|
|
@@ -20,11 +20,23 @@ ANTHROPIC_API_KEY=...
|
|
|
20
20
|
```
|
|
21
21
|
|
|
22
22
|
That is enough for rendered templates and the browser's built-in voice. The
|
|
23
|
-
|
|
23
|
+
starter installs only the text provider. Enable optional adapters when needed:
|
|
24
24
|
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
25
|
+
```bash
|
|
26
|
+
npx vanillasky providers add speech
|
|
27
|
+
npx vanillasky providers add video
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
Each command installs its provider package and connects the adapter through
|
|
31
|
+
`providers.ts`. Run both commands to enable both upgrades.
|
|
32
|
+
|
|
33
|
+
- Speech: add `XAI_API_KEY` to replace browser voice with generated speech.
|
|
34
|
+
- Video: add `FAL_KEY` for generated video and server transcription.
|
|
35
|
+
- Stock: add `PEXELS_API_KEY` for footage and suggestion images; no installation
|
|
36
|
+
is needed.
|
|
37
|
+
|
|
38
|
+
The adapters live in `providers/` and remain application-owned. Running an
|
|
39
|
+
upgrade again can finish an interrupted dependency installation.
|
|
28
40
|
|
|
29
41
|
Restart the development server after changing keys. Open
|
|
30
42
|
<http://localhost:5173> and prompt anything.
|
|
@@ -42,7 +54,7 @@ Restart the development server after changing keys. Open
|
|
|
42
54
|
4. The opening footage loops or holds until the first planned scene and its
|
|
43
55
|
voice are ready, then the response timeline takes over without a blank gap.
|
|
44
56
|
5. The SDK holds every planned scene for its measured or estimated speech time
|
|
45
|
-
and
|
|
57
|
+
and loads possible next prompts independently after the response completes.
|
|
46
58
|
|
|
47
59
|
The text model is the only required provider. Generated speech, transcription,
|
|
48
60
|
stock media, and generated video are independent upgrades and never prevent a
|
|
@@ -2,8 +2,9 @@
|
|
|
2
2
|
<html lang="en">
|
|
3
3
|
<head>
|
|
4
4
|
<meta charset="UTF-8" />
|
|
5
|
-
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
5
|
+
<meta name="viewport" content="width=device-width, initial-scale=1.0, viewport-fit=cover" />
|
|
6
6
|
<title>VanillaSky video chat</title>
|
|
7
|
+
<style>html, body { margin: 0; min-height: 100%; background: #080b16; }</style>
|
|
7
8
|
</head>
|
|
8
9
|
<body>
|
|
9
10
|
<div id="root"></div>
|
|
@@ -9,13 +9,11 @@
|
|
|
9
9
|
"preview": "vite preview"
|
|
10
10
|
},
|
|
11
11
|
"dependencies": {
|
|
12
|
-
"@vanillaskyai/video": "0.
|
|
12
|
+
"@vanillaskyai/video": "0.8.1",
|
|
13
13
|
"react": "^19.2.8",
|
|
14
14
|
"react-dom": "^19.2.8",
|
|
15
15
|
"@ai-sdk/anthropic": "^3.0.0",
|
|
16
|
-
"ai": "^7.0.77"
|
|
17
|
-
"@ai-sdk/xai": "^4.0.52",
|
|
18
|
-
"@fal-ai/client": "1.10.1"
|
|
16
|
+
"ai": "^7.0.77"
|
|
19
17
|
},
|
|
20
18
|
"devDependencies": {
|
|
21
19
|
"@types/react": "^19.2.18",
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { createXai } from "@ai-sdk/xai";
|
|
2
|
+
import { experimental_generateSpeech as synthesizeSpeech } from "ai";
|
|
3
|
+
import type { VideoChatHandlerOptions } from "@vanillaskyai/video/server";
|
|
4
|
+
|
|
5
|
+
/** Optional generated speech; the SDK supplies the browser-voice fallback. */
|
|
6
|
+
export const speechProvider: Pick<VideoChatHandlerOptions, "generateSpeech"> = {
|
|
7
|
+
generateSpeech: process.env.XAI_API_KEY
|
|
8
|
+
? async ({ text, signal }) => {
|
|
9
|
+
const speech = await synthesizeSpeech({
|
|
10
|
+
model: createXai({ apiKey: process.env.XAI_API_KEY }).speech(),
|
|
11
|
+
text,
|
|
12
|
+
abortSignal: signal,
|
|
13
|
+
});
|
|
14
|
+
return {
|
|
15
|
+
audio: speech.audio.uint8Array,
|
|
16
|
+
mediaType: speech.audio.mediaType || "audio/mpeg",
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
: undefined,
|
|
20
|
+
};
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import { fal } from "@fal-ai/client";
|
|
2
|
+
import type { VideoChatHandlerOptions } from "@vanillaskyai/video/server";
|
|
3
|
+
|
|
4
|
+
const VIDEO_MODEL = process.env.FAL_VIDEO_MODEL ?? "minimax/h3-max/text-to-video";
|
|
5
|
+
|
|
6
|
+
async function filmScene(
|
|
7
|
+
subject: string,
|
|
8
|
+
generatedLook: string | undefined,
|
|
9
|
+
orientation: "portrait" | "landscape",
|
|
10
|
+
signal: AbortSignal,
|
|
11
|
+
) {
|
|
12
|
+
fal.config({ credentials: process.env.FAL_KEY });
|
|
13
|
+
const started = Date.now();
|
|
14
|
+
console.log(`[video-chat] filming: ${subject.slice(0, 70)}`);
|
|
15
|
+
try {
|
|
16
|
+
const result = await fal.subscribe(VIDEO_MODEL, {
|
|
17
|
+
input: {
|
|
18
|
+
prompt: [
|
|
19
|
+
`Locked-off shot, ${orientation === "portrait" ? "9:16 vertical" : "16:9"}. ${subject}.`,
|
|
20
|
+
"One slow continuous camera move. Physically plausible motion.",
|
|
21
|
+
"No on-screen text, captions, subtitles, watermarks or logos.",
|
|
22
|
+
"Diegetic sound only. No music, no voiceover.",
|
|
23
|
+
generatedLook,
|
|
24
|
+
].filter(Boolean).join("\n\n"),
|
|
25
|
+
duration: 5,
|
|
26
|
+
resolution: "480P",
|
|
27
|
+
aspect_ratio: orientation === "portrait" ? "9:16" : "16:9",
|
|
28
|
+
},
|
|
29
|
+
abortSignal: signal,
|
|
30
|
+
});
|
|
31
|
+
const url = result?.data?.video?.url;
|
|
32
|
+
if (typeof url !== "string") throw new Error("the video provider returned no url");
|
|
33
|
+
console.log(`[video-chat] filmed in ${Date.now() - started}ms`);
|
|
34
|
+
return url;
|
|
35
|
+
} catch (cause) {
|
|
36
|
+
const status = (cause as { status?: unknown })?.status;
|
|
37
|
+
console.error(
|
|
38
|
+
`[video-chat] filming failed after ${Date.now() - started}ms${typeof status === "number" ? ` (status ${status})` : ""}`,
|
|
39
|
+
);
|
|
40
|
+
throw cause;
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/** Optional generated video and transcription; credentials remain server-only. */
|
|
45
|
+
export const videoProvider: Pick<VideoChatHandlerOptions, "generateVideo" | "transcribe"> = {
|
|
46
|
+
generateVideo: process.env.FAL_KEY
|
|
47
|
+
? async (query, { generatedLook, orientation, signal }) => ({
|
|
48
|
+
url: await filmScene(query, generatedLook, orientation, signal),
|
|
49
|
+
type: "video" as const,
|
|
50
|
+
})
|
|
51
|
+
: undefined,
|
|
52
|
+
transcribe: process.env.FAL_KEY
|
|
53
|
+
? async ({ audio, mediaType, signal }) => {
|
|
54
|
+
signal.throwIfAborted();
|
|
55
|
+
fal.config({ credentials: process.env.FAL_KEY });
|
|
56
|
+
const bytes = Uint8Array.from(audio);
|
|
57
|
+
const url = await fal.storage.upload(new Blob([bytes.buffer], { type: mediaType }));
|
|
58
|
+
signal.throwIfAborted();
|
|
59
|
+
const result = await fal.subscribe(process.env.FAL_TRANSCRIBE_MODEL ?? "fal-ai/whisper", {
|
|
60
|
+
input: { audio_url: url, task: "transcribe" },
|
|
61
|
+
abortSignal: AbortSignal.any([signal, AbortSignal.timeout(60_000)]),
|
|
62
|
+
});
|
|
63
|
+
return String((result?.data as { text?: unknown })?.text ?? "");
|
|
64
|
+
}
|
|
65
|
+
: undefined,
|
|
66
|
+
};
|
|
@@ -1,57 +1,11 @@
|
|
|
1
1
|
import { anthropic } from "@ai-sdk/anthropic";
|
|
2
|
-
import {
|
|
3
|
-
import { createXai } from "@ai-sdk/xai";
|
|
4
|
-
import {
|
|
5
|
-
experimental_generateSpeech as synthesizeSpeech,
|
|
6
|
-
generateText,
|
|
7
|
-
streamText,
|
|
8
|
-
} from "ai";
|
|
2
|
+
import { generateText, streamText } from "ai";
|
|
9
3
|
import { createVideoChatHandler } from "@vanillaskyai/video/server";
|
|
10
4
|
import { findStockFootage } from "./stock";
|
|
5
|
+
import { providers } from "./providers";
|
|
11
6
|
|
|
12
7
|
const PLANNER_MODEL = process.env.ANTHROPIC_PLANNER_MODEL ?? "claude-sonnet-5";
|
|
13
8
|
const NARRATION_MODEL = process.env.ANTHROPIC_NARRATION_MODEL ?? "claude-haiku-4-5";
|
|
14
|
-
const VIDEO_MODEL = process.env.FAL_VIDEO_MODEL ?? "minimax/h3-max/text-to-video";
|
|
15
|
-
const SHOT_DEADLINE_MS = 180_000;
|
|
16
|
-
|
|
17
|
-
async function filmScene(
|
|
18
|
-
subject: string,
|
|
19
|
-
generatedLook: string | undefined,
|
|
20
|
-
orientation: "portrait" | "landscape",
|
|
21
|
-
signal: AbortSignal,
|
|
22
|
-
) {
|
|
23
|
-
fal.config({ credentials: process.env.FAL_KEY });
|
|
24
|
-
const started = Date.now();
|
|
25
|
-
console.log(`[video-chat] filming: ${subject.slice(0, 70)}`);
|
|
26
|
-
try {
|
|
27
|
-
const result = await fal.subscribe(VIDEO_MODEL, {
|
|
28
|
-
input: {
|
|
29
|
-
prompt: [
|
|
30
|
-
`Locked-off shot, ${orientation === "portrait" ? "9:16 vertical" : "16:9"}. ${subject}.`,
|
|
31
|
-
"One slow continuous camera move. Physically plausible motion.",
|
|
32
|
-
"No on-screen text, captions, subtitles, watermarks or logos.",
|
|
33
|
-
"Diegetic sound only. No music, no voiceover.",
|
|
34
|
-
generatedLook,
|
|
35
|
-
].filter(Boolean).join("\n\n"),
|
|
36
|
-
duration: 5,
|
|
37
|
-
resolution: "480P",
|
|
38
|
-
aspect_ratio: orientation === "portrait" ? "9:16" : "16:9",
|
|
39
|
-
},
|
|
40
|
-
abortSignal: AbortSignal.any([signal, AbortSignal.timeout(SHOT_DEADLINE_MS)]),
|
|
41
|
-
});
|
|
42
|
-
const url = result?.data?.video?.url;
|
|
43
|
-
if (typeof url !== "string") throw new Error("the video provider returned no url");
|
|
44
|
-
console.log(`[video-chat] filmed in ${Date.now() - started}ms`);
|
|
45
|
-
return url;
|
|
46
|
-
} catch (cause) {
|
|
47
|
-
const status = (cause as { status?: unknown })?.status;
|
|
48
|
-
console.error(
|
|
49
|
-
`[video-chat] filming failed after ${Date.now() - started}ms${typeof status === "number" ? ` (status ${status})` : ""}`,
|
|
50
|
-
);
|
|
51
|
-
throw cause;
|
|
52
|
-
}
|
|
53
|
-
}
|
|
54
|
-
|
|
55
9
|
/**
|
|
56
10
|
* The application chooses providers and keeps their credentials here. The SDK
|
|
57
11
|
* owns the video-chat protocol, prompts, capability negotiation, spend limits,
|
|
@@ -87,40 +41,10 @@ export const handleVideoChat = createVideoChatHandler({
|
|
|
87
41
|
});
|
|
88
42
|
return text;
|
|
89
43
|
},
|
|
90
|
-
generateSpeech: process.env.XAI_API_KEY
|
|
91
|
-
? async ({ text, signal }) => {
|
|
92
|
-
const speech = await synthesizeSpeech({
|
|
93
|
-
model: createXai({ apiKey: process.env.XAI_API_KEY }).speech(),
|
|
94
|
-
text,
|
|
95
|
-
abortSignal: signal,
|
|
96
|
-
});
|
|
97
|
-
return {
|
|
98
|
-
audio: speech.audio.uint8Array,
|
|
99
|
-
mediaType: speech.audio.mediaType || "audio/mpeg",
|
|
100
|
-
};
|
|
101
|
-
}
|
|
102
|
-
: undefined,
|
|
103
|
-
transcribe: process.env.FAL_KEY
|
|
104
|
-
? async ({ audio, mediaType, signal }) => {
|
|
105
|
-
fal.config({ credentials: process.env.FAL_KEY });
|
|
106
|
-
const bytes = Uint8Array.from(audio);
|
|
107
|
-
const url = await fal.storage.upload(new Blob([bytes.buffer], { type: mediaType }));
|
|
108
|
-
const result = await fal.subscribe(process.env.FAL_TRANSCRIBE_MODEL ?? "fal-ai/whisper", {
|
|
109
|
-
input: { audio_url: url, task: "transcribe" },
|
|
110
|
-
abortSignal: AbortSignal.any([signal, AbortSignal.timeout(60_000)]),
|
|
111
|
-
});
|
|
112
|
-
return String((result?.data as { text?: unknown })?.text ?? "");
|
|
113
|
-
}
|
|
114
|
-
: undefined,
|
|
115
44
|
searchMedia: process.env.PEXELS_API_KEY
|
|
116
|
-
? (query, { orientation, signal }) => findStockFootage(query, orientation, signal)
|
|
117
|
-
: undefined,
|
|
118
|
-
generateVideo: process.env.FAL_KEY
|
|
119
|
-
? async (query, { generatedLook, orientation, signal }) => ({
|
|
120
|
-
url: await filmScene(query, generatedLook, orientation, signal),
|
|
121
|
-
type: "video" as const,
|
|
122
|
-
})
|
|
45
|
+
? (query, { orientation, signal, fallbackQuery }) => findStockFootage(query, orientation, signal, fallbackQuery)
|
|
123
46
|
: undefined,
|
|
47
|
+
...providers,
|
|
124
48
|
welcome: {
|
|
125
49
|
heroQuery: process.env.VIDEO_CHAT_WELCOME_KEYWORD ?? "underwater ocean sunlight",
|
|
126
50
|
prompts: [
|
|
@@ -1,26 +1,10 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Stock footage for the modes that do not generate any.
|
|
3
|
-
*
|
|
4
|
-
* A rendered card on a brand gradient is the least a template can look like,
|
|
5
|
-
* and templates-only mode was every scene that way. A stock search costs a few
|
|
6
|
-
* hundred milliseconds and nothing per request, so unlike a generated clip it
|
|
7
|
-
* never lands on the critical path - the difference between cards on gradients
|
|
8
|
-
* and cards over real footage is close to free.
|
|
9
|
-
*
|
|
10
|
-
* Ported from the site's playground, trimmed to what a video chat needs. Every URL
|
|
11
|
-
* that comes back is checked against the host it must come from: the planner
|
|
12
|
-
* asks for a subject, and what a search returns is not something to trust into
|
|
13
|
-
* a page unexamined.
|
|
14
|
-
*/
|
|
15
1
|
import type { VideoOrientation } from "@vanillaskyai/video";
|
|
16
2
|
|
|
17
|
-
const VIDEO_SEARCH = "https://api.pexels.com/videos/search";
|
|
3
|
+
const VIDEO_SEARCH = "https://api.pexels.com/v1/videos/search";
|
|
18
4
|
const PHOTO_SEARCH = "https://api.pexels.com/v1/search";
|
|
19
5
|
const MAX_KEYWORD_LENGTH = 80;
|
|
20
|
-
//
|
|
21
|
-
|
|
22
|
-
// than a response that waits.
|
|
23
|
-
const LOOKUP_TIMEOUT_MS = 4_000;
|
|
6
|
+
// One budget covers specific/broader video searches and photo recovery.
|
|
7
|
+
const LOOKUP_TIMEOUT_MS = 3_000;
|
|
24
8
|
|
|
25
9
|
interface StockMedia {
|
|
26
10
|
url: string;
|
|
@@ -28,12 +12,6 @@ interface StockMedia {
|
|
|
28
12
|
posterUrl?: string;
|
|
29
13
|
}
|
|
30
14
|
|
|
31
|
-
interface VideoFile {
|
|
32
|
-
link?: unknown;
|
|
33
|
-
width?: number;
|
|
34
|
-
height?: number;
|
|
35
|
-
}
|
|
36
|
-
|
|
37
15
|
/** One hour, keyed by query. The same response asked twice searches once. */
|
|
38
16
|
const cache = new Map<string, { payload: unknown; expiresAt: number }>();
|
|
39
17
|
|
|
@@ -45,95 +23,132 @@ function normalizeKeyword(value: string): string {
|
|
|
45
23
|
.slice(0, MAX_KEYWORD_LENGTH);
|
|
46
24
|
}
|
|
47
25
|
|
|
48
|
-
function
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
26
|
+
function record(value: unknown): Record<string, unknown> {
|
|
27
|
+
return value !== null && typeof value === "object" && !Array.isArray(value)
|
|
28
|
+
? value as Record<string, unknown>
|
|
29
|
+
: {};
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function entries(value: unknown): unknown[] {
|
|
33
|
+
return Array.isArray(value) ? value : [];
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function trustedUrl(value: unknown, host: string): string | undefined {
|
|
37
|
+
if (typeof value !== "string") return undefined;
|
|
38
|
+
try {
|
|
39
|
+
const url = new URL(value);
|
|
40
|
+
if (url.protocol !== "https:" || url.hostname !== host || url.username || url.password || url.port) return undefined;
|
|
41
|
+
return url.toString();
|
|
42
|
+
} catch {
|
|
43
|
+
return undefined;
|
|
52
44
|
}
|
|
53
|
-
return url.toString();
|
|
54
45
|
}
|
|
55
46
|
|
|
56
|
-
function pickVideoFile(files:
|
|
57
|
-
const usable = files
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
.
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
// frame is a column of water with both banks cut off.
|
|
72
|
-
const wanted = orientation === "portrait"
|
|
73
|
-
? (file: VideoFile) => (file.height ?? 0) >= (file.width ?? 0)
|
|
74
|
-
: (file: VideoFile) => (file.width ?? 0) >= (file.height ?? 0);
|
|
75
|
-
return usable.find(wanted) ?? usable[0] ?? null;
|
|
47
|
+
function pickVideoFile(files: unknown, orientation: VideoOrientation): string | undefined {
|
|
48
|
+
const usable = entries(files).flatMap((value) => {
|
|
49
|
+
const file = record(value);
|
|
50
|
+
const direct = trustedUrl(file.link, "videos.pexels.com");
|
|
51
|
+
const external = trustedUrl(file.link, "player.vimeo.com");
|
|
52
|
+
const link = direct ?? (external && /^\/external\/[a-zA-Z0-9._-]+\.mp4$/.test(new URL(external).pathname) ? external : undefined);
|
|
53
|
+
const { width, height } = file;
|
|
54
|
+
if (!link || typeof width !== "number" || typeof height !== "number"
|
|
55
|
+
|| !Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) return [];
|
|
56
|
+
return [{ link, width, height }];
|
|
57
|
+
});
|
|
58
|
+
// Prefer the stage's shape without discarding other safe renditions.
|
|
59
|
+
return (usable.find((file) => orientation === "portrait"
|
|
60
|
+
? file.height >= file.width
|
|
61
|
+
: file.width >= file.height) ?? usable[0])?.link;
|
|
76
62
|
}
|
|
77
63
|
|
|
78
64
|
async function search(url: string, apiKey: string, signal: AbortSignal): Promise<unknown> {
|
|
79
65
|
const cached = cache.get(url);
|
|
80
66
|
if (cached && cached.expiresAt > Date.now()) return cached.payload;
|
|
81
67
|
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
68
|
+
if (signal.aborted) return null;
|
|
69
|
+
try {
|
|
70
|
+
const response = await fetch(url, {
|
|
71
|
+
headers: { Authorization: apiKey },
|
|
72
|
+
signal,
|
|
73
|
+
});
|
|
74
|
+
if (!response.ok || signal.aborted) return null;
|
|
75
|
+
const payload: unknown = await response.json();
|
|
76
|
+
if (signal.aborted) return null;
|
|
77
|
+
cache.set(url, { payload, expiresAt: Date.now() + 60 * 60 * 1_000 });
|
|
78
|
+
if (cache.size > 100) cache.delete(cache.keys().next().value!);
|
|
79
|
+
return payload;
|
|
80
|
+
} catch {
|
|
81
|
+
// A failed video search must still allow the independent photo fallback.
|
|
82
|
+
if (!signal.aborted) console.warn("[video-chat] Stock lookup unavailable; trying a fallback.");
|
|
83
|
+
return null;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/** Metadata is a weak mismatch filter, never visual verification. Numeric URLs
|
|
88
|
+
* and missing descriptions are unknown and retain the provider's search ranking. */
|
|
89
|
+
function matchesMetadata(item: Record<string, unknown>, keyword: string): boolean {
|
|
90
|
+
let description = typeof item.alt === "string" ? item.alt : "";
|
|
91
|
+
if (!description) {
|
|
92
|
+
const page = trustedUrl(item.url, "www.pexels.com");
|
|
93
|
+
if (page) description = new URL(page).pathname.replace(/^\/(?:video|photo)\//, "");
|
|
94
|
+
}
|
|
95
|
+
const words = (value: string) => normalizeKeyword(value).toLowerCase().split(/[\s-]+/)
|
|
96
|
+
.filter(word => /\p{L}/u.test(word) && !["a", "an", "the", "of", "in", "on", "with", "and", "video", "photo", "free", "footage"].includes(word));
|
|
97
|
+
const labels = words(description);
|
|
98
|
+
const requested = words(keyword);
|
|
99
|
+
return labels.length === 0 || requested.length === 0 || requested.some(word => labels.includes(word));
|
|
94
100
|
}
|
|
95
101
|
|
|
96
|
-
/**
|
|
97
|
-
*
|
|
98
|
-
|
|
99
|
-
* Returns null rather than throwing when there is nothing to show: the scene
|
|
100
|
-
* falls back to the brand gradient, which is what it looked like before.
|
|
101
|
-
*/
|
|
102
|
-
export async function findStockFootage(subject: string, orientation: VideoOrientation, signal: AbortSignal): Promise<StockMedia | null> {
|
|
102
|
+
/** Video queries precede photos. All work shares one deadline, including body
|
|
103
|
+
* parsing, even if a custom fetch implementation ignores its abort signal. */
|
|
104
|
+
export async function findStockFootage(subject: string, orientation: VideoOrientation, signal: AbortSignal, fallbackSubject?: string): Promise<StockMedia | null> {
|
|
103
105
|
const apiKey = process.env.PEXELS_API_KEY;
|
|
104
106
|
const keyword = normalizeKeyword(subject);
|
|
105
|
-
if (!apiKey || !keyword) return null;
|
|
106
|
-
|
|
107
|
-
const
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
107
|
+
if (!apiKey || !keyword || signal.aborted) return null;
|
|
108
|
+
const keywords = [keyword];
|
|
109
|
+
const fallback = normalizeKeyword(fallbackSubject ?? "");
|
|
110
|
+
if (fallback && fallback.toLowerCase() !== keyword.toLowerCase()) keywords.push(fallback);
|
|
111
|
+
const controller = new AbortController();
|
|
112
|
+
let finish!: (value: null) => void;
|
|
113
|
+
const stopped = new Promise<null>(resolve => { finish = resolve; });
|
|
114
|
+
const abort = () => { controller.abort(); finish(null); };
|
|
115
|
+
signal.addEventListener("abort", abort, { once: true });
|
|
116
|
+
const timer = setTimeout(abort, LOOKUP_TIMEOUT_MS);
|
|
117
|
+
const active = controller.signal;
|
|
118
|
+
const lookup = async (): Promise<StockMedia | null> => {
|
|
119
|
+
for (const query of keywords) {
|
|
120
|
+
const params = new URLSearchParams({ query, orientation, per_page: "3", page: "1" });
|
|
121
|
+
const videos = record(await search(`${VIDEO_SEARCH}?${params}`, apiKey, active));
|
|
122
|
+
if (active.aborted) return null;
|
|
123
|
+
for (const value of entries(videos.videos)) {
|
|
124
|
+
const video = record(value);
|
|
125
|
+
if (!matchesMetadata(video, query)) continue;
|
|
126
|
+
const url = pickVideoFile(video.video_files, orientation);
|
|
127
|
+
if (!url) continue;
|
|
128
|
+
const posterUrl = trustedUrl(video.image, "images.pexels.com");
|
|
129
|
+
return { url, type: "video", ...(posterUrl ? { posterUrl } : {}) };
|
|
130
|
+
}
|
|
123
131
|
}
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
132
|
+
for (const query of keywords) {
|
|
133
|
+
const params = new URLSearchParams({ query, orientation, per_page: "3", page: "1" });
|
|
134
|
+
const photos = record(await search(`${PHOTO_SEARCH}?${params}`, apiKey, active));
|
|
135
|
+
if (active.aborted) return null;
|
|
136
|
+
for (const value of entries(photos.photos)) {
|
|
137
|
+
const photo = record(value);
|
|
138
|
+
if (!matchesMetadata(photo, query)) continue;
|
|
139
|
+
const source = record(photo.src);
|
|
140
|
+
for (const candidate of [source.large2x, source.large]) {
|
|
141
|
+
const url = trustedUrl(candidate, "images.pexels.com");
|
|
142
|
+
if (url) return { url, type: "image" };
|
|
143
|
+
}
|
|
144
|
+
}
|
|
136
145
|
}
|
|
137
146
|
return null;
|
|
147
|
+
};
|
|
148
|
+
try {
|
|
149
|
+
return await Promise.race([lookup(), stopped]);
|
|
150
|
+
} finally {
|
|
151
|
+
clearTimeout(timer);
|
|
152
|
+
signal.removeEventListener("abort", abort);
|
|
138
153
|
}
|
|
139
154
|
}
|