@vanillaskyai/video 0.5.8 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +178 -0
- package/PUBLIC-API.md +129 -9
- package/README.md +70 -108
- package/dist/{bg-confetti-SHQI7ATB.js → bg-confetti-HITCGLGD.js} +1 -1
- package/dist/{bg-emoji-XYFRA63Q.js → bg-emoji-VU4UZYZN.js} +1 -1
- package/dist/{bg-media-FEBATBT2.js → bg-media-L34PDQXJ.js} +4 -4
- package/dist/{brand-message-P56WEJPG.js → brand-message-FUJ4STFO.js} +3 -3
- package/dist/{builtin-server-YHEZ2JRF.js → builtin-server-KB7FKF6A.js} +2 -2
- package/dist/{chart-bar-ZB45ZRKH.js → chart-bar-EGPHJATL.js} +4 -4
- package/dist/{chart-counter-HFYNE4HT.js → chart-counter-5FFWQNZO.js} +4 -4
- package/dist/{chart-progress-ring-ZKMLRGSB.js → chart-progress-ring-JMXQLMLC.js} +4 -4
- package/dist/check-runtime.js +2 -2
- package/dist/{chunk-66MNRUCR.js → chunk-2PYO6VAC.js} +51 -3
- package/dist/{chunk-ZQQQAAKP.js → chunk-2XT4MZ76.js} +33 -2
- package/dist/chunk-5CDAM24P.js +31 -0
- package/dist/{chunk-PNS52FL4.js → chunk-5SLENAJW.js} +19 -6
- package/dist/{chunk-EFL34TXF.js → chunk-AITKH6QT.js} +24 -8
- package/dist/{chunk-HT6BGERF.js → chunk-FRN6WKHA.js} +28 -10
- package/dist/{chunk-FZHMQFG3.js → chunk-GAIOHGNR.js} +5 -21
- package/dist/{chunk-3RV4YKB3.js → chunk-JNN3EYVP.js} +51 -26
- package/dist/{chunk-NGY3TNND.js → chunk-K746NUIP.js} +70 -50
- package/dist/{chunk-HVCPEAQF.js → chunk-LKBZX7GV.js} +9 -6
- package/dist/{chunk-G66Z5CWR.js → chunk-MMUXVA47.js} +27 -3
- package/dist/{chunk-6M6DATLW.js → chunk-MXZSDGZQ.js} +6 -5
- package/dist/{chunk-3OU7HIEB.js → chunk-RE4IMWJR.js} +15 -31
- package/dist/{chunk-FVTMYS6U.js → chunk-RGF452LL.js} +1 -1
- package/dist/{chunk-34O5BY6X.js → chunk-RXHW4EP4.js} +21 -4
- package/dist/chunk-SKRGRKHY.js +142 -0
- package/dist/{chunk-TCEHK2BW.js → chunk-YJJC4N4D.js} +2 -1
- package/dist/cli.js +438 -30
- package/dist/{compose-video-JWIEDTEJ.js → compose-video-PD6LKKRK.js} +4 -4
- package/dist/{cta-logo-S4OMTNXD.js → cta-logo-YR7LYUR5.js} +3 -3
- package/dist/{cta-media-EROTHDRR.js → cta-media-DQJBKUO2.js} +4 -4
- package/dist/{events-BBP30j3c.d.ts → events-B6qS1Lsb.d.ts} +2 -2
- package/dist/{incoming-call-5IWJ2TAS.js → incoming-call-54KQ7O5A.js} +3 -3
- package/dist/index.d.ts +89 -3
- package/dist/index.js +12 -1
- package/dist/{infographic-before-after-2I7YZVDD.js → infographic-before-after-ROS52GOW.js} +1 -1
- package/dist/{infographic-feature-list-7SK65U3K.js → infographic-feature-list-E7MGUDAN.js} +4 -4
- package/dist/{infographic-problem-solution-IW35S6BR.js → infographic-problem-solution-3S6NGO5N.js} +4 -4
- package/dist/{infographic-stat-row-XHUZZPDM.js → infographic-stat-row-SYB7AB7H.js} +4 -4
- package/dist/{infographic-steps-JDDEDI7X.js → infographic-steps-KCE6JWE2.js} +4 -4
- package/dist/{kit-DEG3fcaL.d.ts → kit-8g46H2RZ.d.ts} +1 -1
- package/dist/{prompt-input-6ZWXEL3V.js → prompt-input-MHX4O42G.js} +3 -3
- package/dist/react.d.ts +200 -6
- package/dist/react.js +1859 -43
- package/dist/{reaction-Z3JIM3MQ.js → reaction-SLEGR3BE.js} +4 -4
- package/dist/{scene-video-backdrop-IL4F2SVD.js → scene-video-backdrop-XFPX4O3I.js} +2 -1
- package/dist/server.d.ts +104 -4
- package/dist/server.js +976 -26
- package/dist/{showcase-code-NWQAAJIA.js → showcase-code-2HLAVKBM.js} +4 -4
- package/dist/{showcase-phone-5UW45ATT.js → showcase-phone-Q4HEO4LH.js} +4 -4
- package/dist/{showcase-terminal-UVOD25UF.js → showcase-terminal-DL45LUFG.js} +4 -4
- package/dist/{showcase-web-3AW6VHIG.js → showcase-web-2QZO73FZ.js} +4 -4
- package/dist/{social-milestone-KPKHFOB5.js → social-milestone-GEFUQSBF.js} +3 -3
- package/dist/{social-notification-EDOU6TZB.js → social-notification-FJ5VHVLK.js} +3 -3
- package/dist/{social-review-stack-EZOO33SV.js → social-review-stack-KXQCKFRP.js} +3 -3
- package/dist/{social-testimonial-DLOINMMY.js → social-testimonial-JZ7CHC7S.js} +3 -3
- package/dist/{social-tweet-6JUM323G.js → social-tweet-UCDXM25F.js} +3 -3
- package/dist/{system-prompt-4I6Z5HK3.js → system-prompt-RRXIWDDD.js} +5 -3
- package/dist/template-catalog.js +1 -1
- package/dist/templates.d.ts +3 -3
- package/dist/test.d.ts +2 -2
- package/dist/test.js +3 -3
- package/dist/{text-stream-J6EDDJP4.js → text-stream-FW4BLBAL.js} +2 -2
- package/dist/{types-BV9IqExh.d.ts → types-2wHBqtg8.d.ts} +29 -1
- package/dist/types-DrABlRa7.d.ts +46 -0
- package/docs/agent-integration.md +49 -25
- package/docs/architecture.md +37 -22
- package/docs/branding-and-personalization.md +1 -1
- package/docs/concepts.md +14 -0
- package/docs/custom-templates.md +9 -9
- package/docs/customization.md +41 -5
- package/docs/getting-started.md +81 -84
- package/docs/media-and-audio.md +113 -117
- package/docs/persistence.md +1 -1
- package/docs/production.md +81 -92
- package/docs/prompt-and-input.md +71 -161
- package/docs/provider-integration.md +91 -49
- package/docs/responsive-orientation.md +1 -1
- package/docs/security.md +6 -5
- package/docs/streaming-protocol.md +1 -1
- package/docs/testing.md +51 -43
- package/examples/custom-template/README.md +1 -1
- package/package.json +20 -21
- package/registry/items/backgrounds.json +2 -2
- package/registry/items/barChart.json +6 -4
- package/registry/items/beforeAfter.json +1 -1
- package/registry/items/bigNumber.json +4 -3
- package/registry/items/brandMessage.json +3 -2
- package/registry/items/cardList.json +6 -4
- package/registry/items/codeEditor.json +4 -3
- package/registry/items/confetti.json +1 -1
- package/registry/items/ctaLogo.json +3 -2
- package/registry/items/ctaMedia.json +4 -3
- package/registry/items/emojiBurst.json +1 -1
- package/registry/items/incomingCall.json +3 -2
- package/registry/items/media.json +4 -3
- package/registry/items/milestone.json +3 -2
- package/registry/items/notification.json +3 -2
- package/registry/items/phoneMockup.json +4 -3
- package/registry/items/problemSolution.json +4 -3
- package/registry/items/progressRing.json +4 -3
- package/registry/items/promptInput.json +3 -2
- package/registry/items/reaction.json +2 -2
- package/registry/items/reviewStack.json +3 -2
- package/registry/items/steps.json +6 -4
- package/registry/items/terminal.json +5 -4
- package/registry/items/testimonial.json +3 -2
- package/registry/items/tripleStats.json +4 -3
- package/registry/items/tweet.json +3 -2
- package/registry/items/webMockup.json +4 -3
- package/starters/video-chat/.env.example +16 -0
- package/starters/video-chat/README.md +63 -0
- package/starters/video-chat/index.html +12 -0
- package/starters/video-chat/package.json +28 -0
- package/starters/video-chat/server.ts +152 -0
- package/starters/video-chat/src/main.tsx +8 -0
- package/starters/video-chat/stock.ts +139 -0
- package/starters/video-chat/tsconfig.json +20 -0
- package/starters/video-chat/vite.config.ts +81 -0
- package/styles/video-chat.css +713 -0
- package/dist/chunk-C6WVCZRW.js +0 -19
- package/docs/input-and-first-scene.md +0 -64
- package/docs/integrate-nextjs.md +0 -86
- package/docs/live-channels.md +0 -149
- package/docs/use-cases.md +0 -59
- package/examples/nextjs-quickstart/.env.example +0 -2
- package/examples/nextjs-quickstart/README.md +0 -27
- package/examples/nextjs-quickstart/next-env.d.ts +0 -4
- package/examples/nextjs-quickstart/package.json +0 -25
- package/examples/nextjs-quickstart/src/app/api/video/route.ts +0 -24
- package/examples/nextjs-quickstart/src/app/layout.tsx +0 -5
- package/examples/nextjs-quickstart/src/app/page.tsx +0 -31
- package/examples/nextjs-quickstart/tsconfig.json +0 -26
package/CHANGELOG.md
CHANGED
|
@@ -4,6 +4,184 @@ VanillaSky follows semantic versioning. This changelog begins with the 0.1 beta.
|
|
|
4
4
|
|
|
5
5
|
## Unreleased
|
|
6
6
|
|
|
7
|
+
## 0.7.0
|
|
8
|
+
|
|
9
|
+
- Makes blank-folder onboarding one safe scoped command:
|
|
10
|
+
`npx @vanillaskyai/video init`. Init now pins registry installs to the exact
|
|
11
|
+
SDK version that ran it and preserves an exact local tarball during candidate
|
|
12
|
+
verification instead of silently replacing it with an older npm artifact.
|
|
13
|
+
|
|
14
|
+
- Keeps video-chat responses running when a text provider labels an ordinary
|
|
15
|
+
scene with an unsupported placement hint. The provider adapter drops that
|
|
16
|
+
harmless hint while preserving `placement: "closer"` and strict protocol
|
|
17
|
+
validation.
|
|
18
|
+
|
|
19
|
+
- Keeps `VideoChat` hydration stable when voice input exists only in the
|
|
20
|
+
browser, and preserves provider warnings, usage, finish reasons, and model
|
|
21
|
+
IDs while the chat opening is intercepted from the planner stream.
|
|
22
|
+
|
|
23
|
+
- Removes the duplicate one-shot React, Next.js, and provider examples plus
|
|
24
|
+
their obsolete product guides. The generated video-chat app is now the one
|
|
25
|
+
public setup path, while the internal Next.js provider matrix exercises the
|
|
26
|
+
same chat handler and interface.
|
|
27
|
+
|
|
28
|
+
- Streams each video-chat opening and scene plan from one model call. The first
|
|
29
|
+
NDJSON object carries a bounded 6-9 word hook and stock keyword; Full AI also
|
|
30
|
+
carries its exact first shot so generation starts while the remaining scenes
|
|
31
|
+
arrive. Welcome suggestions can include a prewritten hook that speaks
|
|
32
|
+
immediately with their loaded media, and `onFirstFrame` reports the measured
|
|
33
|
+
handoff to the first real scene.
|
|
34
|
+
|
|
35
|
+
- Speeds up both video-chat paths. Concurrent media resolution now releases the
|
|
36
|
+
first finished scene immediately instead of waiting for its queue to fill,
|
|
37
|
+
and the main planner writes narration on each scene instead of paying for a
|
|
38
|
+
second text-model round trip. Breaking (pre-1.0): removes the mixed `some` mode;
|
|
39
|
+
use `templates` for rendered responses or `full` for generated footage.
|
|
40
|
+
|
|
41
|
+
- Starts each video-chat answer with a short spoken hook over relevant stock
|
|
42
|
+
footage, then holds that opening until the first planned scene is ready.
|
|
43
|
+
Welcome and follow-up cards reuse their already-loaded media immediately;
|
|
44
|
+
typed prompts resolve footage from the hook's bounded media keyword. The
|
|
45
|
+
planner receives the exact spoken hook so the response continues without
|
|
46
|
+
repeating or contradicting it.
|
|
47
|
+
|
|
48
|
+
- Keeps key-only video chat clean in real browsers: planner catalogs now hide
|
|
49
|
+
templates whose required media cannot be supplied, and unavailable generated
|
|
50
|
+
speech switches to browser voice without repeated 404 responses.
|
|
51
|
+
|
|
52
|
+
- Removes the obsolete source-owned template copy and transcript probes from
|
|
53
|
+
the video-chat starter. The package, lower-level examples, and remaining
|
|
54
|
+
documentation now consistently present voice-and-video chat as the primary
|
|
55
|
+
path, with packed onboarding as its single clean-room browser gate.
|
|
56
|
+
|
|
57
|
+
- Makes the complete voice-and-video chat the primary README, getting-started,
|
|
58
|
+
agent, provider, concepts, and architecture path. The one-shot composition
|
|
59
|
+
APIs remain documented as an advanced non-chat integration.
|
|
60
|
+
|
|
61
|
+
- Adds `vanillasky init` and `vanillasky doctor`. Init creates the canonical
|
|
62
|
+
thin video-chat app, installs its app-owned provider packages, starts with
|
|
63
|
+
packaged templates and browser voice, and requests one server-only text key.
|
|
64
|
+
Doctor checks the generated shell and reports optional speech, media,
|
|
65
|
+
transcription, and generated-video capabilities without calling providers or
|
|
66
|
+
printing secrets. Template ownership commands now live under the breaking
|
|
67
|
+
pre-1.0 `vanillasky templates` namespace.
|
|
68
|
+
|
|
69
|
+
- Adds the complete default `VideoChat` interface and its explicitly imported,
|
|
70
|
+
fully scoped `video-chat.css`. Applications can now mount the same polished
|
|
71
|
+
voice-and-video chat in one component while retaining `useVideoChat` for
|
|
72
|
+
custom interfaces. The canonical starter no longer duplicates the UI,
|
|
73
|
+
interaction, accessibility, appearance, or voice-input code.
|
|
74
|
+
|
|
75
|
+
- Adds `useVideoChat` and `createVideoChatVoice` to the React entry. The SDK
|
|
76
|
+
now owns video-chat turns, conversation context, cancellation, retry,
|
|
77
|
+
opening speech, narration pacing, suggestions, replay, history selection,
|
|
78
|
+
and the browser-voice fallback behind one provider-neutral endpoint.
|
|
79
|
+
|
|
80
|
+
- Adds `VideoPlayer.onPlaybackEnd`, which fires when the visible playhead
|
|
81
|
+
actually reaches the end without changing the existing stream-composition
|
|
82
|
+
meaning of `onComplete`.
|
|
83
|
+
|
|
84
|
+
- Adds `createVideoChatHandler`, one provider-neutral server endpoint for the
|
|
85
|
+
general-purpose video-chat experience. The SDK now owns the response,
|
|
86
|
+
opening, narration, suggestion, speech, transcription, welcome, capability,
|
|
87
|
+
and server-held generated-video budget contracts; applications supply model
|
|
88
|
+
and media callbacks without exposing providers or keys to the browser.
|
|
89
|
+
|
|
90
|
+
- Reframes the canonical interactive demo as a general-purpose video chat
|
|
91
|
+
starter for explanations, stories, recommendations, and creative prompts.
|
|
92
|
+
It now runs with one text-model key, falls back to browser speech, and unlocks
|
|
93
|
+
generated speech, stock media, transcription, and generated video only when
|
|
94
|
+
their application-owned provider keys are configured.
|
|
95
|
+
|
|
96
|
+
- Adds `paused` to `VideoPlayer`: hold the playhead where it is, and release it
|
|
97
|
+
from the same frame. It completes `controls={false}`. Turning the player's own
|
|
98
|
+
controls off hands playback to the application, and until now there was no
|
|
99
|
+
lever to drive it with - a narrated answer could silence its voice but not
|
|
100
|
+
stop its picture, and the two came apart. Leaving the prop undefined keeps the
|
|
101
|
+
player's own behaviour untouched, and a video already at its end is not
|
|
102
|
+
resumed by it, since starting again is a replay rather than a continuation.
|
|
103
|
+
|
|
104
|
+
- Adds `maxResolvedMedia` to `createVideoHandler`: how many scenes in one
|
|
105
|
+
request may resolve media at all. Unbounded by default, which is right when
|
|
106
|
+
media is searched for and wrong when it is generated - the planner decides the
|
|
107
|
+
scene count and every scene is then a paid clip. Past the ceiling a scene keeps
|
|
108
|
+
its copy on the brand gradient, and a `media_budget_reached` warning reaches
|
|
109
|
+
`onWarning` once, since a spend policy is the application's business rather
|
|
110
|
+
than the browser's.
|
|
111
|
+
|
|
112
|
+
- Adds `useNarration` to the React entry: say a video's narration as it plays.
|
|
113
|
+
The line belongs to the scene, so it begins when that scene does, stops when
|
|
114
|
+
the picture moves on, and can be interrupted. The provider stays with the
|
|
115
|
+
application - it supplies anything that can `speak(text, { signal })`, whether
|
|
116
|
+
a realtime session, a speech model, or the browser's own synthesiser - so the
|
|
117
|
+
package gains no dependency and the choice of voice sits where the choice of
|
|
118
|
+
model already does.
|
|
119
|
+
|
|
120
|
+
- Adds `getSceneDuration`, `getSceneDurationBounds` and `getSpokenDuration`.
|
|
121
|
+
Templates already declare `minDuration`, `preferredDuration` and which fields
|
|
122
|
+
hold their content, and the runtime already computed a content-aware readable
|
|
123
|
+
duration from them - but internally, so applications reached for `minDuration`
|
|
124
|
+
instead. That is a compression bound, the least a template survives being
|
|
125
|
+
squeezed to when a video must fit a fixed length: 1 second for `media`, 1.5
|
|
126
|
+
for `bigNumber`. A narrated response built on it flashes past. `getSceneDuration`
|
|
127
|
+
answers the real question, taking a scene's `narration` into account, since
|
|
128
|
+
speech is slower than reading.
|
|
129
|
+
|
|
130
|
+
- Adds `generatedLook` to the video style and to `VideoInput.style`: the visual
|
|
131
|
+
language generated media is produced in. A style has two halves once media can
|
|
132
|
+
be generated - the brand decides how captions are drawn, this decides what the
|
|
133
|
+
footage behind them looks like - and they have to travel together, or a pale
|
|
134
|
+
illustrated ground ends up under dark documentary footage. It reaches
|
|
135
|
+
`resolveMedia` on the resolver context, so a provider prompt no longer has to
|
|
136
|
+
be threaded with it by hand, and it is stored with the video so a replay keeps
|
|
137
|
+
its look. Nothing is rendered from it.
|
|
138
|
+
|
|
139
|
+
- Adds `mediaConcurrency` to `createVideoHandler`: how many scenes may resolve
|
|
140
|
+
media at once, defaulting to one. Media resolution ran strictly in turn, which
|
|
141
|
+
is invisible for a stock search and costly for generated video - five clips at
|
|
142
|
+
a few seconds each is half a minute of nothing, and the only way around it was
|
|
143
|
+
to plan the shots and generate them outside the plan stream. Scenes are still
|
|
144
|
+
emitted in the order they were planned; only the waiting overlaps.
|
|
145
|
+
|
|
146
|
+
- Adds `createSceneTimeline`: compose a playable video from scenes the
|
|
147
|
+
application builds itself, appended through the player's `stream` prop.
|
|
148
|
+
`VideoInput.opening` holds a single line of copy, and replacing the `video`
|
|
149
|
+
prop restarts playback, so an application with its own opening scenes
|
|
150
|
+
previously had to emit protocol envelopes by hand - where the sequence, event
|
|
151
|
+
id, scene position and completion snapshot must all be exact and any mistake
|
|
152
|
+
rejects the whole stream silently. `awaitAudio` covers the related trap that
|
|
153
|
+
`audio.set` is only valid before the first scene, by holding openings until
|
|
154
|
+
the soundtrack is known.
|
|
155
|
+
|
|
156
|
+
- Adds optional `narration` to `VideoScene`: the line spoken aloud while that
|
|
157
|
+
scene is showing. A narrated video previously had to be carried as scenes plus
|
|
158
|
+
a separate script kept in step by index, which drifts; holding the line on the
|
|
159
|
+
scene keeps the two together through planning, playback, storage and replay,
|
|
160
|
+
so a stored video can be spoken again without the model that wrote it. The
|
|
161
|
+
renderer never draws it. Additive and optional, so existing videos parse
|
|
162
|
+
unchanged.
|
|
163
|
+
|
|
164
|
+
## 0.6.0
|
|
165
|
+
|
|
166
|
+
- Adds opt-in native scene-video audio to `VideoPlayer`, mixed with the existing
|
|
167
|
+
continuous soundtrack layer under one master mute control. Applications set
|
|
168
|
+
the embedded clip level with `nativeMediaAudio.volume`; the serialized
|
|
169
|
+
soundtrack retains its independent `audio.volume`.
|
|
170
|
+
- Passes `requestId` and the resolving `scene` to the server `resolveMedia`
|
|
171
|
+
callback, so applications can generate, attribute, and cache media per scene.
|
|
172
|
+
- Documents a provider-neutral app-owned media generation adapter in
|
|
173
|
+
`examples/server-integrations/src/ai-sdk-media.ts`. Generation stays outside
|
|
174
|
+
the SDK install: the application supplies the AI SDK model and storage.
|
|
175
|
+
- Extends the isolated rich-media consumer POC with an adaptive `/channel`
|
|
176
|
+
route: structured scene intent, deterministic stock/image/H3 Max routing,
|
|
177
|
+
manual overrides, explicit character/keyframe continuity, factual-safe
|
|
178
|
+
fallbacks, cancellation, deadline-aware generation, and a bounded
|
|
179
|
+
current-plus-next segment queue without changing the SDK API.
|
|
180
|
+
- Adds an isolated AI scene-director proof of concept where VanillaSky's trusted
|
|
181
|
+
planner chooses and explains app-owned generated imagery, varied delay-aware
|
|
182
|
+
GIF stickers, and varied progress-driven Lottie motion without changing the
|
|
183
|
+
stable SDK contract.
|
|
184
|
+
|
|
7
185
|
## 0.5.8
|
|
8
186
|
|
|
9
187
|
- Moved the public source to a fresh repository at
|
package/PUBLIC-API.md
CHANGED
|
@@ -92,11 +92,18 @@ provider-neutral text-delta escape hatch.
|
|
|
92
92
|
|
|
93
93
|
### Values
|
|
94
94
|
|
|
95
|
+
- `createVideoChatHandler(options)`
|
|
95
96
|
- `createVideoHandler(options)`
|
|
96
97
|
- `createServerTemplateRegistry(options)`
|
|
97
98
|
|
|
98
99
|
### Types
|
|
99
100
|
|
|
101
|
+
- `VideoChatCapabilities`
|
|
102
|
+
- `VideoChatConversationTurn`
|
|
103
|
+
- `VideoChatHandlerOptions`
|
|
104
|
+
- `VideoChatMode`
|
|
105
|
+
- `VideoChatWelcomeOptions`
|
|
106
|
+
- `VideoChatWelcomePrompt`
|
|
100
107
|
- `VideoHandlerOptions`
|
|
101
108
|
- `MediaResolver`
|
|
102
109
|
- `MediaResolverContext`
|
|
@@ -111,6 +118,38 @@ provider-neutral text-delta escape hatch.
|
|
|
111
118
|
|
|
112
119
|
### Handler contract
|
|
113
120
|
|
|
121
|
+
`createVideoChatHandler` is the opinionated general-purpose video-chat route.
|
|
122
|
+
Mount it once and use its bounded `action` query parameter for capabilities,
|
|
123
|
+
responses, opening media, narration, suggestions, speech,
|
|
124
|
+
transcription, and the welcome screen. It owns the general response prompts,
|
|
125
|
+
fixed visual-mode spend limits, capability fallbacks, and auxiliary response
|
|
126
|
+
shapes. The application supplies provider-neutral `streamText`, `generateText`,
|
|
127
|
+
`generateSpeech`, `transcribe`, `searchMedia`, and `generateVideo` callbacks.
|
|
128
|
+
Only the two text callbacks are required. Missing optional callbacks remove
|
|
129
|
+
their capability; templates and browser speech remain available.
|
|
130
|
+
|
|
131
|
+
- A response accepts `prompt`, `mode`, `orientation`, optional bounded
|
|
132
|
+
`conversation`, `opening`, `brand`, and `style`. `opening` is an optional
|
|
133
|
+
prewritten hook from a selected suggestion. The response returns protocol
|
|
134
|
+
`0.5` SSE and negotiates `data.video-chat-opening`, which carries the bounded
|
|
135
|
+
6-9 word hook and optional stock-search keyword before the first scene.
|
|
136
|
+
- The planner produces that opening as the first line of the same model stream
|
|
137
|
+
that produces the scenes. The separate opening-media action resolves its
|
|
138
|
+
keyword through the application-owned `searchMedia` callback, so stock lookup
|
|
139
|
+
never delays speech or planning. In `full` mode the first line also directs
|
|
140
|
+
the exact first generated scene; the handler consumes that private direction
|
|
141
|
+
and starts the clip while the model continues with scenes two through five.
|
|
142
|
+
- `templates` and `full` map to server-owned generated-media budgets of zero
|
|
143
|
+
and five. Without `generateVideo`, only `templates` is exposed
|
|
144
|
+
and forged generated-mode requests degrade to it.
|
|
145
|
+
- The response planner writes narration on each scene. The narration action is
|
|
146
|
+
retained as a compatibility fallback for missing lines, not used in the
|
|
147
|
+
normal path.
|
|
148
|
+
- Every action applies the same authorization, origin, request-size,
|
|
149
|
+
cancellation, safe-error, and server-only-provider boundaries.
|
|
150
|
+
|
|
151
|
+
`createVideoHandler` remains the lower-level video composition route:
|
|
152
|
+
|
|
114
153
|
- `authorize` is required for HTTP handlers. Use `authorize: "none"` only for
|
|
115
154
|
an intentionally non-public in-process/test handler.
|
|
116
155
|
- `streamText` receives the generated system prompt, grounded user prompt, and
|
|
@@ -145,18 +184,84 @@ provider-neutral text-delta escape hatch.
|
|
|
145
184
|
|
|
146
185
|
### Values
|
|
147
186
|
|
|
187
|
+
- `VideoChat(props)`
|
|
188
|
+
- `useVideoChat(options?)`
|
|
189
|
+
- `createVideoChatVoice(options?)`
|
|
148
190
|
- `useVideo(options?)`
|
|
149
191
|
- `VideoPlayer`
|
|
150
192
|
- `VideoError`
|
|
151
193
|
|
|
152
194
|
### Types
|
|
153
195
|
|
|
196
|
+
- `VideoChatProps`
|
|
197
|
+
- `UseVideoChatOptions`
|
|
198
|
+
- `UseVideoChatResult`
|
|
199
|
+
- `VideoChatAskOptions`
|
|
200
|
+
- `VideoChatFirstFrameMetric`
|
|
201
|
+
- `VideoChatTurn`
|
|
202
|
+
- `VideoChatStatus`
|
|
203
|
+
- `VideoChatMode`
|
|
204
|
+
- `VideoChatCapabilities`
|
|
205
|
+
- `VideoChatWelcome`
|
|
206
|
+
- `VideoChatSuggestion`
|
|
207
|
+
- `VideoChatMedia`
|
|
208
|
+
- `VideoChatVoice`
|
|
209
|
+
- `VideoChatPreparedSpeech`
|
|
210
|
+
- `CreateVideoChatVoiceOptions`
|
|
154
211
|
- `UseVideoOptions`
|
|
155
212
|
- `UseVideoResult`
|
|
156
213
|
- `VideoPlaybackMode`
|
|
157
214
|
- `VideoPlayerProps`
|
|
158
215
|
- `VideoErrorOptions`
|
|
159
216
|
|
|
217
|
+
`VideoChat` is the complete default interface. Import its scoped stylesheet
|
|
218
|
+
explicitly so the host application keeps control over CSS loading:
|
|
219
|
+
|
|
220
|
+
```tsx
|
|
221
|
+
import { VideoChat } from "@vanillaskyai/video/react";
|
|
222
|
+
import "@vanillaskyai/video/video-chat.css";
|
|
223
|
+
|
|
224
|
+
export function App() {
|
|
225
|
+
return <VideoChat />;
|
|
226
|
+
}
|
|
227
|
+
```
|
|
228
|
+
|
|
229
|
+
Pass the same session configuration through `options`, for example
|
|
230
|
+
`<VideoChat options={{ endpoint, headers, templates, brand }} />`. `className`
|
|
231
|
+
and `welcomeTitle` are the only shell-level customizations. The stylesheet is
|
|
232
|
+
scoped under `.vanillasky-video-chat` and does not style the host document.
|
|
233
|
+
|
|
234
|
+
`useVideoChat` is the headless client for `createVideoChatHandler`. It owns
|
|
235
|
+
the conversation lifecycle and returns UI-neutral state plus a spread-ready
|
|
236
|
+
player binding:
|
|
237
|
+
|
|
238
|
+
```tsx
|
|
239
|
+
const chat = useVideoChat();
|
|
240
|
+
|
|
241
|
+
await chat.ask("Tell me a tiny mystery set on a night train");
|
|
242
|
+
|
|
243
|
+
return chat.playerProps
|
|
244
|
+
? <VideoPlayer key={chat.playerKey} {...chat.playerProps} />
|
|
245
|
+
: null;
|
|
246
|
+
```
|
|
247
|
+
|
|
248
|
+
The hook loads capabilities and welcome suggestions, sends completed turns as
|
|
249
|
+
bounded context, cancels replaced prompts, retries once only before playback,
|
|
250
|
+
paces each scene to its prepared speech, and keeps pause, mute, replay, history,
|
|
251
|
+
captions, and actual playback completion synchronized. The default voice tries
|
|
252
|
+
the handler's generated-speech action and falls back to browser speech. Pass a
|
|
253
|
+
`VideoChatVoice` to replace it without rebuilding session orchestration.
|
|
254
|
+
`ask(prompt, { opening, openingMedia })` lets a custom interface start a
|
|
255
|
+
prewritten suggestion hook immediately and reuse its image or video without
|
|
256
|
+
another model or media lookup. The hook otherwise reads the opening from the
|
|
257
|
+
response stream, resolves its media keyword, holds that media while it is
|
|
258
|
+
spoken, and starts the planned timeline only when both speech and the first
|
|
259
|
+
scene are ready. `UseVideoChatOptions.onFirstFrame` receives a
|
|
260
|
+
`VideoChatFirstFrameMetric` once when a fresh response displays its first real
|
|
261
|
+
scene. Each `VideoChatTurn` exposes `openingMedia` and `completed`, so
|
|
262
|
+
custom interfaces can render the same handoff while partial or cancelled
|
|
263
|
+
responses remain visible without being mistaken for conversation context.
|
|
264
|
+
|
|
160
265
|
`UseVideoResult` has this conceptual shape:
|
|
161
266
|
|
|
162
267
|
```ts
|
|
@@ -191,6 +296,12 @@ mounted player autoplay with sound after the first successful viewer start.
|
|
|
191
296
|
`manual`, `muted-autoplay`, and `autoplay-with-sound` cover the other browser
|
|
192
297
|
startup policies.
|
|
193
298
|
|
|
299
|
+
Set `nativeMediaAudio={{ volume: 0.85 }}` when scene video files contain an
|
|
300
|
+
embedded audio track. The active clip's audio becomes a second layer alongside
|
|
301
|
+
the video's continuous `audio` soundtrack. Both follow the player's master
|
|
302
|
+
mute control; `nativeMediaAudio.volume` and serialized `audio.volume` set their
|
|
303
|
+
independent mix levels. Incoming preroll videos remain muted until active.
|
|
304
|
+
|
|
194
305
|
`VideoInput.opening` accepts custom copy, uses the deterministic fallback when
|
|
195
306
|
omitted, and accepts `false` when the application owns transient loading UI and
|
|
196
307
|
wants the completed video to begin with the first generated scene.
|
|
@@ -202,7 +313,9 @@ the internal reducer state are not public types.
|
|
|
202
313
|
completed video from the beginning instead of showing the replay affordance;
|
|
203
314
|
`onSceneChange(scene, index)` fires whenever the scene under the playhead
|
|
204
315
|
changes, including when a loop wraps back to the first scene. Streaming
|
|
205
|
-
playback is unaffected by either.
|
|
316
|
+
playback is unaffected by either. `onComplete(video)` reports that a streamed
|
|
317
|
+
response finished composing; `onPlaybackEnd(video)` reports that the visible
|
|
318
|
+
playhead actually reached the end for either a stream or saved replay.
|
|
206
319
|
|
|
207
320
|
`resolveVideoBrand` fills a partial brand with the documented defaults and
|
|
208
321
|
preset backgrounds, producing the fully resolved `VideoBrand` that `parseVideo`
|
|
@@ -279,14 +392,21 @@ invalid scenes, provider failure, content filtering, abort, and timeout.
|
|
|
279
392
|
|
|
280
393
|
The `vanillasky` binary supports:
|
|
281
394
|
|
|
282
|
-
- `
|
|
283
|
-
- `
|
|
284
|
-
- `
|
|
285
|
-
- `
|
|
286
|
-
- `
|
|
287
|
-
- `
|
|
288
|
-
- `
|
|
289
|
-
- `
|
|
395
|
+
- `init`
|
|
396
|
+
- `doctor`
|
|
397
|
+
- `templates list`
|
|
398
|
+
- `templates describe`
|
|
399
|
+
- `templates create`
|
|
400
|
+
- `templates add`
|
|
401
|
+
- `templates sync`
|
|
402
|
+
- `templates check`
|
|
403
|
+
- `templates add --dry-run`
|
|
404
|
+
- `templates add --diff`
|
|
405
|
+
|
|
406
|
+
`init` writes the canonical app-owned provider and configuration shell while
|
|
407
|
+
using the SDK's packaged interface, session, handler, and built-in templates.
|
|
408
|
+
`doctor` is read-only and offline. The old top-level template commands are not
|
|
409
|
+
aliases; this is an intentional pre-1.0 namespace change.
|
|
290
410
|
|
|
291
411
|
Generated customer files import only the public entry points in this document.
|
|
292
412
|
Browser registries import `createTemplateRegistry` from `/templates`. Server
|
package/README.md
CHANGED
|
@@ -1,156 +1,118 @@
|
|
|
1
|
-
# Give your AI a
|
|
1
|
+
# Give your AI a voice and a face
|
|
2
2
|
|
|
3
|
-
**VanillaSky is the open-source video
|
|
4
|
-
|
|
5
|
-
|
|
3
|
+
**VanillaSky is the open-source voice-and-video chat layer.** Add a polished,
|
|
4
|
+
general-purpose AI conversation that speaks and starts playing visual answers
|
|
5
|
+
while they are still being composed.
|
|
6
6
|
|
|
7
7
|
> **Status: Beta.** VanillaSky is pre-1.0 and its public API may change as we
|
|
8
8
|
> test it in real applications. Pin an exact version before production use.
|
|
9
9
|
|
|
10
|
-
Your application owns the
|
|
11
|
-
|
|
10
|
+
Your application owns the providers, keys, authentication, persistence,
|
|
11
|
+
branding, and product copy. VanillaSky owns the chat flow, planning prompts,
|
|
12
|
+
trusted templates, validation, streaming, voice timing, and player.
|
|
12
13
|
|
|
13
|
-
## Start
|
|
14
|
+
## Start on localhost
|
|
14
15
|
|
|
15
|
-
|
|
16
|
+
```bash
|
|
17
|
+
npx @vanillaskyai/video init
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
Add `ANTHROPIC_API_KEY` to the generated, ignored `.env.local`, then run:
|
|
16
21
|
|
|
17
22
|
```bash
|
|
18
|
-
|
|
23
|
+
npx vanillasky doctor
|
|
24
|
+
npm run dev
|
|
19
25
|
```
|
|
20
26
|
|
|
27
|
+
Open the reported localhost URL. One text key gives you the complete chat with
|
|
28
|
+
packaged templates and browser voice. No template setup is required.
|
|
29
|
+
|
|
30
|
+
Optional server-only keys progressively add capabilities without changing the
|
|
31
|
+
client:
|
|
32
|
+
|
|
33
|
+
| Key | Adds |
|
|
34
|
+
| --- | --- |
|
|
35
|
+
| `XAI_API_KEY` | Generated speech |
|
|
36
|
+
| `FAL_KEY` | Generated video and voice transcription |
|
|
37
|
+
| `PEXELS_API_KEY` | Stock media, including opening backgrounds |
|
|
38
|
+
|
|
39
|
+
`npx vanillasky doctor` reports readiness by key name and never prints values.
|
|
40
|
+
Provider SDKs remain dependencies of the generated application, not the core
|
|
41
|
+
package.
|
|
42
|
+
|
|
21
43
|
For coding agents:
|
|
22
44
|
|
|
23
45
|
```bash
|
|
24
46
|
npx skills add VanillaSkyAi/video@vanillasky
|
|
25
47
|
```
|
|
26
48
|
|
|
27
|
-
Then prompt: `Use $vanillasky to
|
|
28
|
-
|
|
29
|
-
## Connect your LLM
|
|
30
|
-
|
|
31
|
-
VanillaSky never chooses a provider or model. Configure that separately on the
|
|
32
|
-
server, then connect the model your application already owns. The route below
|
|
33
|
-
assumes an AI SDK `LanguageModel` exported as `videoModel`; if you do not have
|
|
34
|
-
one yet, choose a provider and current model in [Provider integration](docs/provider-integration.md).
|
|
35
|
-
|
|
36
|
-
Create one authenticated server route:
|
|
37
|
-
|
|
38
|
-
```ts
|
|
39
|
-
// app/api/video/route.ts
|
|
40
|
-
import { streamText } from "ai";
|
|
41
|
-
import { createVideoHandler } from "@vanillaskyai/video/server";
|
|
42
|
-
import { videoModel } from "@/lib/video-model";
|
|
43
|
-
|
|
44
|
-
const handle = createVideoHandler({
|
|
45
|
-
// Local development only. Replace with your session check before deploying.
|
|
46
|
-
authorize: (request) => {
|
|
47
|
-
if (process.env.VANILLASKY_LOCAL_DEMO !== "1") return false;
|
|
48
|
-
const hostname = new URL(request.url).hostname;
|
|
49
|
-
return hostname === "localhost" || hostname === "127.0.0.1";
|
|
50
|
-
},
|
|
51
|
-
streamText: ({ systemPrompt, userPrompt, signal }) => streamText({
|
|
52
|
-
model: videoModel,
|
|
53
|
-
system: systemPrompt,
|
|
54
|
-
prompt: userPrompt,
|
|
55
|
-
abortSignal: signal,
|
|
56
|
-
}),
|
|
57
|
-
});
|
|
58
|
-
|
|
59
|
-
export const POST = handle;
|
|
60
|
-
export const OPTIONS = handle;
|
|
61
|
-
```
|
|
49
|
+
Then prompt: `Use $vanillasky to set up and verify a general-purpose video chat in this project.`
|
|
62
50
|
|
|
63
|
-
|
|
64
|
-
so production builds and `next start` deny every request. Replace it before
|
|
65
|
-
deploying. The model can come from Anthropic, OpenAI, an AI SDK registry or
|
|
66
|
-
gateway, or any compatible streaming adapter. The
|
|
67
|
-
[Next.js quickstart](examples/nextjs-quickstart) shows the later, explicit
|
|
68
|
-
provider-selection step with a quality-oriented Claude Sonnet model.
|
|
51
|
+
## What init creates
|
|
69
52
|
|
|
70
|
-
|
|
71
|
-
server-only `resolveMedia` callback described in
|
|
72
|
-
[Media and soundtrack audio](docs/media-and-audio.md#media-providers).
|
|
53
|
+
The generated application is a thin, editable shell:
|
|
73
54
|
|
|
74
|
-
|
|
55
|
+
- `src/main.tsx` mounts the complete SDK-owned `<VideoChat />` interface;
|
|
56
|
+
- `server.ts` connects app-owned text, speech, transcription, stock, and video
|
|
57
|
+
providers through one `createVideoChatHandler`;
|
|
58
|
+
- `vite.config.ts` serves the UI and the single `/api/video-chat` endpoint;
|
|
59
|
+
- `.env.local` holds server-only keys and is ignored by Git.
|
|
75
60
|
|
|
76
|
-
|
|
61
|
+
The UI stays this small:
|
|
77
62
|
|
|
78
63
|
```tsx
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
import { VideoPlayer, useVideo } from "@vanillaskyai/video/react";
|
|
82
|
-
|
|
83
|
-
export function VideoResponse() {
|
|
84
|
-
const video = useVideo();
|
|
64
|
+
import { VideoChat } from "@vanillaskyai/video/react";
|
|
65
|
+
import "@vanillaskyai/video/video-chat.css";
|
|
85
66
|
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
input: "Activation increased from 41% to 58% after guided onboarding.",
|
|
89
|
-
personalization: { firstName: "Maya" },
|
|
90
|
-
}); }}>
|
|
91
|
-
Generate video
|
|
92
|
-
</button>
|
|
93
|
-
|
|
94
|
-
{video.error && <p role="alert">Video generation failed.</p>}
|
|
95
|
-
<VideoPlayer {...video.playerProps} />
|
|
96
|
-
</>;
|
|
67
|
+
export function App() {
|
|
68
|
+
return <VideoChat />;
|
|
97
69
|
}
|
|
98
70
|
```
|
|
99
71
|
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
72
|
+
Use `useVideoChat` when you want a custom interface while keeping the SDK-owned
|
|
73
|
+
conversation and playback lifecycle. Edit the generated server when you want a
|
|
74
|
+
different provider. The [provider guide](docs/provider-integration.md) explains
|
|
75
|
+
both boundaries.
|
|
103
76
|
|
|
104
|
-
|
|
77
|
+
## Templates are the built-in fallback
|
|
105
78
|
|
|
106
|
-
|
|
79
|
+
The packaged visual templates are the fast, inexpensive default and require no
|
|
80
|
+
copied source tree. Add a video provider when you want generated footage; the
|
|
81
|
+
same conversation can mix both modes.
|
|
107
82
|
|
|
108
|
-
|
|
83
|
+
Copy template source only when you want to own and edit it:
|
|
109
84
|
|
|
110
|
-
```
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
period: "Q2",
|
|
114
|
-
activation: { previous: 41, current: 58 },
|
|
115
|
-
cause: "guided onboarding",
|
|
116
|
-
}),
|
|
117
|
-
instructions: "Lead with the improvement, then explain what changed.",
|
|
118
|
-
personalization: { firstName: "Maya", plan: "Pro" },
|
|
119
|
-
});
|
|
85
|
+
```bash
|
|
86
|
+
npm install --save-dev tsx
|
|
87
|
+
npx vanillasky templates add bigNumber
|
|
120
88
|
```
|
|
121
89
|
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
or choose `knowledgeMode: "general"` when the model should answer a question
|
|
125
|
-
or develop content with stable general knowledge.
|
|
126
|
-
- Put presentation direction in `instructions`.
|
|
127
|
-
- Put viewer or account context in `personalization`.
|
|
128
|
-
- Add brand, approved media, soundtrack audio, or a smaller template set only
|
|
129
|
-
when the experience needs them.
|
|
90
|
+
That compiler is needed only for source-owned templates. See
|
|
91
|
+
[Custom templates](docs/custom-templates.md).
|
|
130
92
|
|
|
131
|
-
|
|
132
|
-
or speech synchronization. MP4/WebM encoding and export are application-owned.
|
|
133
|
-
Completed videos can be stored as JSON and replayed without calling the LLM.
|
|
93
|
+
## Go deeper
|
|
134
94
|
|
|
135
|
-
|
|
136
|
-
|
|
95
|
+
Completed chat responses are deterministic JSON and can be stored and replayed.
|
|
96
|
+
MP4/WebM export remains application-owned.
|
|
137
97
|
|
|
138
98
|
## Documentation
|
|
139
99
|
|
|
140
100
|
| Goal | Guide |
|
|
141
101
|
| --- | --- |
|
|
142
|
-
|
|
|
143
|
-
|
|
|
144
|
-
|
|
|
145
|
-
|
|
|
146
|
-
| Understand
|
|
102
|
+
| Run the complete chat | [Getting started](docs/getting-started.md) |
|
|
103
|
+
| Set it up with a coding agent | [Agent integration guide](docs/agent-integration.md) |
|
|
104
|
+
| Change or add providers | [Provider integration](docs/provider-integration.md) |
|
|
105
|
+
| Customize the interface | [Customization](docs/customization.md) |
|
|
106
|
+
| Understand prompts and grounding | [Prompt and input](docs/prompt-and-input.md) |
|
|
147
107
|
| Add brand or viewer context | [Branding and personalization](docs/branding-and-personalization.md) |
|
|
148
108
|
| Add media or soundtrack audio | [Media and soundtrack audio](docs/media-and-audio.md) |
|
|
149
109
|
| Persist and replay results | [Persistence and replay](docs/persistence.md) |
|
|
150
|
-
| Run a looping, self-refreshing channel | [Live channels](docs/live-channels.md) |
|
|
151
|
-
| Create source-owned templates | [Custom templates](docs/custom-templates.md) |
|
|
152
110
|
| Test routes and streams | [Test integrations](docs/testing.md) |
|
|
153
111
|
| Deploy securely | [Production](docs/production.md) · [Security](docs/security.md) |
|
|
154
112
|
| Inspect the API contract | [Public API](PUBLIC-API.md) · [Protocol](docs/reference/protocol.md) |
|
|
155
113
|
|
|
114
|
+
Try a keyless template response in the
|
|
115
|
+
[playground](https://vanillasky.ai/playground/), or visit
|
|
116
|
+
[vanillasky.ai](https://vanillasky.ai/).
|
|
117
|
+
|
|
156
118
|
Apache-2.0
|
|
@@ -5,18 +5,18 @@ import "./chunk-QAAO7OAY.js";
|
|
|
5
5
|
import {
|
|
6
6
|
SceneBackground,
|
|
7
7
|
getMediaBackgroundProps
|
|
8
|
-
} from "./chunk-
|
|
8
|
+
} from "./chunk-MXZSDGZQ.js";
|
|
9
9
|
import {
|
|
10
10
|
TemplateText
|
|
11
|
-
} from "./chunk-
|
|
11
|
+
} from "./chunk-YJJC4N4D.js";
|
|
12
12
|
import "./chunk-L5BTTCZV.js";
|
|
13
|
-
import "./chunk-
|
|
13
|
+
import "./chunk-FRN6WKHA.js";
|
|
14
14
|
import "./chunk-W3VDIBGR.js";
|
|
15
15
|
import "./chunk-XTOXFMLM.js";
|
|
16
|
-
import "./chunk-C6WVCZRW.js";
|
|
17
16
|
import {
|
|
18
17
|
hasSceneMedia
|
|
19
18
|
} from "./chunk-224QNWRA.js";
|
|
19
|
+
import "./chunk-5CDAM24P.js";
|
|
20
20
|
import {
|
|
21
21
|
resolveTokens
|
|
22
22
|
} from "./chunk-XWWLKRNU.js";
|