@vanillaskyai/video 0.6.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (130) hide show
  1. package/CHANGELOG.md +157 -0
  2. package/PUBLIC-API.md +123 -9
  3. package/README.md +69 -115
  4. package/dist/{bg-confetti-SHQI7ATB.js → bg-confetti-HITCGLGD.js} +1 -1
  5. package/dist/{bg-emoji-XYFRA63Q.js → bg-emoji-VU4UZYZN.js} +1 -1
  6. package/dist/{bg-media-G4XOQEAS.js → bg-media-L34PDQXJ.js} +2 -2
  7. package/dist/{brand-message-XHONEDNW.js → brand-message-FUJ4STFO.js} +1 -1
  8. package/dist/{builtin-server-YHEZ2JRF.js → builtin-server-KB7FKF6A.js} +2 -2
  9. package/dist/{chart-bar-HRHD57ML.js → chart-bar-EGPHJATL.js} +2 -2
  10. package/dist/{chart-counter-KFPV44RB.js → chart-counter-5FFWQNZO.js} +2 -2
  11. package/dist/{chart-progress-ring-OR2VOIHV.js → chart-progress-ring-JMXQLMLC.js} +2 -2
  12. package/dist/{chunk-66MNRUCR.js → chunk-2PYO6VAC.js} +51 -3
  13. package/dist/{chunk-ZQQQAAKP.js → chunk-2XT4MZ76.js} +33 -2
  14. package/dist/{chunk-PNS52FL4.js → chunk-5SLENAJW.js} +19 -6
  15. package/dist/{chunk-EFL34TXF.js → chunk-AITKH6QT.js} +24 -8
  16. package/dist/{chunk-FZHMQFG3.js → chunk-GAIOHGNR.js} +5 -21
  17. package/dist/{chunk-3RV4YKB3.js → chunk-JNN3EYVP.js} +51 -26
  18. package/dist/{chunk-HVCPEAQF.js → chunk-LKBZX7GV.js} +9 -6
  19. package/dist/{chunk-G66Z5CWR.js → chunk-MMUXVA47.js} +27 -3
  20. package/dist/{chunk-7BGAU6C3.js → chunk-MXZSDGZQ.js} +2 -1
  21. package/dist/{chunk-3OU7HIEB.js → chunk-RE4IMWJR.js} +15 -31
  22. package/dist/{chunk-FVTMYS6U.js → chunk-RGF452LL.js} +1 -1
  23. package/dist/{chunk-34O5BY6X.js → chunk-RXHW4EP4.js} +21 -4
  24. package/dist/chunk-SKRGRKHY.js +142 -0
  25. package/dist/{chunk-TCEHK2BW.js → chunk-YJJC4N4D.js} +2 -1
  26. package/dist/cli.js +438 -30
  27. package/dist/{compose-video-JWIEDTEJ.js → compose-video-PD6LKKRK.js} +4 -4
  28. package/dist/{cta-logo-JJOGK4UC.js → cta-logo-YR7LYUR5.js} +1 -1
  29. package/dist/{cta-media-IQ3MAWEH.js → cta-media-DQJBKUO2.js} +2 -2
  30. package/dist/{events-BBP30j3c.d.ts → events-B6qS1Lsb.d.ts} +2 -2
  31. package/dist/{incoming-call-KXF67OT2.js → incoming-call-54KQ7O5A.js} +1 -1
  32. package/dist/index.d.ts +89 -3
  33. package/dist/index.js +12 -1
  34. package/dist/{infographic-before-after-2I7YZVDD.js → infographic-before-after-ROS52GOW.js} +1 -1
  35. package/dist/{infographic-feature-list-QDM4I7XE.js → infographic-feature-list-E7MGUDAN.js} +2 -2
  36. package/dist/{infographic-problem-solution-FMLJCPG6.js → infographic-problem-solution-3S6NGO5N.js} +2 -2
  37. package/dist/{infographic-stat-row-VFTIXS5F.js → infographic-stat-row-SYB7AB7H.js} +2 -2
  38. package/dist/{infographic-steps-L7W24Z35.js → infographic-steps-KCE6JWE2.js} +2 -2
  39. package/dist/{kit-DEG3fcaL.d.ts → kit-8g46H2RZ.d.ts} +1 -1
  40. package/dist/{prompt-input-F3CYU2XH.js → prompt-input-MHX4O42G.js} +1 -1
  41. package/dist/react.d.ts +194 -6
  42. package/dist/react.js +1849 -40
  43. package/dist/{reaction-2XCLOJOK.js → reaction-SLEGR3BE.js} +2 -2
  44. package/dist/server.d.ts +102 -4
  45. package/dist/server.js +972 -27
  46. package/dist/{showcase-code-KHSMIPG6.js → showcase-code-2HLAVKBM.js} +2 -2
  47. package/dist/{showcase-phone-M4CEANIW.js → showcase-phone-Q4HEO4LH.js} +2 -2
  48. package/dist/{showcase-terminal-26I3DYF7.js → showcase-terminal-DL45LUFG.js} +2 -2
  49. package/dist/{showcase-web-TY6CZV2Z.js → showcase-web-2QZO73FZ.js} +2 -2
  50. package/dist/{social-milestone-TF6MX7ET.js → social-milestone-GEFUQSBF.js} +1 -1
  51. package/dist/{social-notification-2RJDAZSF.js → social-notification-FJ5VHVLK.js} +1 -1
  52. package/dist/{social-review-stack-2ICKJQG7.js → social-review-stack-KXQCKFRP.js} +1 -1
  53. package/dist/{social-testimonial-X7OGADA3.js → social-testimonial-JZ7CHC7S.js} +1 -1
  54. package/dist/{social-tweet-TK7SH2F6.js → social-tweet-UCDXM25F.js} +1 -1
  55. package/dist/{system-prompt-4I6Z5HK3.js → system-prompt-RRXIWDDD.js} +5 -3
  56. package/dist/template-catalog.js +1 -1
  57. package/dist/templates.d.ts +3 -3
  58. package/dist/test.d.ts +2 -2
  59. package/dist/test.js +3 -3
  60. package/dist/{text-stream-J6EDDJP4.js → text-stream-FW4BLBAL.js} +2 -2
  61. package/dist/{types-BV9IqExh.d.ts → types-2wHBqtg8.d.ts} +29 -1
  62. package/dist/types-DrABlRa7.d.ts +46 -0
  63. package/docs/agent-integration.md +49 -25
  64. package/docs/architecture.md +37 -22
  65. package/docs/branding-and-personalization.md +1 -1
  66. package/docs/concepts.md +14 -0
  67. package/docs/custom-templates.md +9 -9
  68. package/docs/customization.md +41 -5
  69. package/docs/getting-started.md +81 -84
  70. package/docs/media-and-audio.md +103 -166
  71. package/docs/persistence.md +1 -1
  72. package/docs/production.md +81 -92
  73. package/docs/prompt-and-input.md +71 -161
  74. package/docs/provider-integration.md +91 -49
  75. package/docs/responsive-orientation.md +1 -1
  76. package/docs/security.md +6 -5
  77. package/docs/streaming-protocol.md +1 -1
  78. package/docs/testing.md +51 -43
  79. package/examples/custom-template/README.md +1 -1
  80. package/package.json +20 -22
  81. package/registry/items/barChart.json +6 -4
  82. package/registry/items/beforeAfter.json +1 -1
  83. package/registry/items/bigNumber.json +4 -3
  84. package/registry/items/brandMessage.json +3 -2
  85. package/registry/items/cardList.json +6 -4
  86. package/registry/items/codeEditor.json +4 -3
  87. package/registry/items/confetti.json +1 -1
  88. package/registry/items/ctaLogo.json +3 -2
  89. package/registry/items/ctaMedia.json +4 -3
  90. package/registry/items/emojiBurst.json +1 -1
  91. package/registry/items/incomingCall.json +3 -2
  92. package/registry/items/media.json +4 -3
  93. package/registry/items/milestone.json +3 -2
  94. package/registry/items/notification.json +3 -2
  95. package/registry/items/phoneMockup.json +4 -3
  96. package/registry/items/problemSolution.json +4 -3
  97. package/registry/items/progressRing.json +4 -3
  98. package/registry/items/promptInput.json +3 -2
  99. package/registry/items/reaction.json +2 -2
  100. package/registry/items/reviewStack.json +3 -2
  101. package/registry/items/steps.json +6 -4
  102. package/registry/items/terminal.json +5 -4
  103. package/registry/items/testimonial.json +3 -2
  104. package/registry/items/tripleStats.json +4 -3
  105. package/registry/items/tweet.json +3 -2
  106. package/registry/items/webMockup.json +4 -3
  107. package/starters/video-chat/.env.example +16 -0
  108. package/starters/video-chat/README.md +63 -0
  109. package/starters/video-chat/index.html +12 -0
  110. package/starters/video-chat/package.json +28 -0
  111. package/starters/video-chat/server.ts +152 -0
  112. package/starters/video-chat/src/main.tsx +8 -0
  113. package/starters/video-chat/stock.ts +139 -0
  114. package/starters/video-chat/tsconfig.json +20 -0
  115. package/starters/video-chat/vite.config.ts +81 -0
  116. package/styles/video-chat.css +713 -0
  117. package/docs/input-and-first-scene.md +0 -64
  118. package/docs/integrate-nextjs.md +0 -86
  119. package/docs/live-channels.md +0 -149
  120. package/docs/use-cases.md +0 -59
  121. package/examples/nextjs-quickstart/.env.example +0 -2
  122. package/examples/nextjs-quickstart/README.md +0 -27
  123. package/examples/nextjs-quickstart/next-env.d.ts +0 -4
  124. package/examples/nextjs-quickstart/package.json +0 -25
  125. package/examples/nextjs-quickstart/src/app/api/video/route.ts +0 -33
  126. package/examples/nextjs-quickstart/src/app/layout.tsx +0 -5
  127. package/examples/nextjs-quickstart/src/app/page.tsx +0 -31
  128. package/examples/nextjs-quickstart/tsconfig.json +0 -26
  129. package/examples/server-integrations/README.md +0 -20
  130. package/examples/server-integrations/src/ai-sdk-media.ts +0 -90
package/CHANGELOG.md CHANGED
@@ -4,6 +4,163 @@ VanillaSky follows semantic versioning. This changelog begins with the 0.1 beta.
4
4
 
5
5
  ## Unreleased
6
6
 
7
+ ## 0.7.0
8
+
9
+ - Makes blank-folder onboarding one safe scoped command:
10
+ `npx @vanillaskyai/video init`. Init now pins registry installs to the exact
11
+ SDK version that ran it and preserves an exact local tarball during candidate
12
+ verification instead of silently replacing it with an older npm artifact.
13
+
14
+ - Keeps video-chat responses running when a text provider labels an ordinary
15
+ scene with an unsupported placement hint. The provider adapter drops that
16
+ harmless hint while preserving `placement: "closer"` and strict protocol
17
+ validation.
18
+
19
+ - Keeps `VideoChat` hydration stable when voice input exists only in the
20
+ browser, and preserves provider warnings, usage, finish reasons, and model
21
+ IDs while the chat opening is intercepted from the planner stream.
22
+
23
+ - Removes the duplicate one-shot React, Next.js, and provider examples plus
24
+ their obsolete product guides. The generated video-chat app is now the one
25
+ public setup path, while the internal Next.js provider matrix exercises the
26
+ same chat handler and interface.
27
+
28
+ - Streams each video-chat opening and scene plan from one model call. The first
29
+ NDJSON object carries a bounded 6-9 word hook and stock keyword; Full AI also
30
+ carries its exact first shot so generation starts while the remaining scenes
31
+ arrive. Welcome suggestions can include a prewritten hook that speaks
32
+ immediately with their loaded media, and `onFirstFrame` reports the measured
33
+ handoff to the first real scene.
34
+
35
+ - Speeds up both video-chat paths. Concurrent media resolution now releases the
36
+ first finished scene immediately instead of waiting for its queue to fill,
37
+ and the main planner writes narration on each scene instead of paying for a
38
+ second text-model round trip. Breaking (pre-1.0): removes the mixed `some` mode;
39
+ use `templates` for rendered responses or `full` for generated footage.
40
+
41
+ - Starts each video-chat answer with a short spoken hook over relevant stock
42
+ footage, then holds that opening until the first planned scene is ready.
43
+ Welcome and follow-up cards reuse their already-loaded media immediately;
44
+ typed prompts resolve footage from the hook's bounded media keyword. The
45
+ planner receives the exact spoken hook so the response continues without
46
+ repeating or contradicting it.
47
+
48
+ - Keeps key-only video chat clean in real browsers: planner catalogs now hide
49
+ templates whose required media cannot be supplied, and unavailable generated
50
+ speech switches to browser voice without repeated 404 responses.
51
+
52
+ - Removes the obsolete source-owned template copy and transcript probes from
53
+ the video-chat starter. The package, lower-level examples, and remaining
54
+ documentation now consistently present voice-and-video chat as the primary
55
+ path, with packed onboarding as its single clean-room browser gate.
56
+
57
+ - Makes the complete voice-and-video chat the primary README, getting-started,
58
+ agent, provider, concepts, and architecture path. The one-shot composition
59
+ APIs remain documented as an advanced non-chat integration.
60
+
61
+ - Adds `vanillasky init` and `vanillasky doctor`. Init creates the canonical
62
+ thin video-chat app, installs its app-owned provider packages, starts with
63
+ packaged templates and browser voice, and requests one server-only text key.
64
+ Doctor checks the generated shell and reports optional speech, media,
65
+ transcription, and generated-video capabilities without calling providers or
66
+ printing secrets. Template ownership commands now live under the breaking
67
+ pre-1.0 `vanillasky templates` namespace.
68
+
69
+ - Adds the complete default `VideoChat` interface and its explicitly imported,
70
+ fully scoped `video-chat.css`. Applications can now mount the same polished
71
+ voice-and-video chat in one component while retaining `useVideoChat` for
72
+ custom interfaces. The canonical starter no longer duplicates the UI,
73
+ interaction, accessibility, appearance, or voice-input code.
74
+
75
+ - Adds `useVideoChat` and `createVideoChatVoice` to the React entry. The SDK
76
+ now owns video-chat turns, conversation context, cancellation, retry,
77
+ opening speech, narration pacing, suggestions, replay, history selection,
78
+ and the browser-voice fallback behind one provider-neutral endpoint.
79
+
80
+ - Adds `VideoPlayer.onPlaybackEnd`, which fires when the visible playhead
81
+ actually reaches the end without changing the existing stream-composition
82
+ meaning of `onComplete`.
83
+
84
+ - Adds `createVideoChatHandler`, one provider-neutral server endpoint for the
85
+ general-purpose video-chat experience. The SDK now owns the response,
86
+ opening, narration, suggestion, speech, transcription, welcome, capability,
87
+ and server-held generated-video budget contracts; applications supply model
88
+ and media callbacks without exposing providers or keys to the browser.
89
+
90
+ - Reframes the canonical interactive demo as a general-purpose video chat
91
+ starter for explanations, stories, recommendations, and creative prompts.
92
+ It now runs with one text-model key, falls back to browser speech, and unlocks
93
+ generated speech, stock media, transcription, and generated video only when
94
+ their application-owned provider keys are configured.
95
+
96
+ - Adds `paused` to `VideoPlayer`: hold the playhead where it is, and release it
97
+ from the same frame. It completes `controls={false}`. Turning the player's own
98
+ controls off hands playback to the application, and until now there was no
99
+ lever to drive it with - a narrated answer could silence its voice but not
100
+ stop its picture, and the two came apart. Leaving the prop undefined keeps the
101
+ player's own behaviour untouched, and a video already at its end is not
102
+ resumed by it, since starting again is a replay rather than a continuation.
103
+
104
+ - Adds `maxResolvedMedia` to `createVideoHandler`: how many scenes in one
105
+ request may resolve media at all. Unbounded by default, which is right when
106
+ media is searched for and wrong when it is generated - the planner decides the
107
+ scene count and every scene is then a paid clip. Past the ceiling a scene keeps
108
+ its copy on the brand gradient, and a `media_budget_reached` warning reaches
109
+ `onWarning` once, since a spend policy is the application's business rather
110
+ than the browser's.
111
+
112
+ - Adds `useNarration` to the React entry: say a video's narration as it plays.
113
+ The line belongs to the scene, so it begins when that scene does, stops when
114
+ the picture moves on, and can be interrupted. The provider stays with the
115
+ application - it supplies anything that can `speak(text, { signal })`, whether
116
+ a realtime session, a speech model, or the browser's own synthesiser - so the
117
+ package gains no dependency and the choice of voice sits where the choice of
118
+ model already does.
119
+
120
+ - Adds `getSceneDuration`, `getSceneDurationBounds` and `getSpokenDuration`.
121
+ Templates already declare `minDuration`, `preferredDuration` and which fields
122
+ hold their content, and the runtime already computed a content-aware readable
123
+ duration from them - but internally, so applications reached for `minDuration`
124
+ instead. That is a compression bound, the least a template survives being
125
+ squeezed to when a video must fit a fixed length: 1 second for `media`, 1.5
126
+ for `bigNumber`. A narrated response built on it flashes past. `getSceneDuration`
127
+ answers the real question, taking a scene's `narration` into account, since
128
+ speech is slower than reading.
129
+
130
+ - Adds `generatedLook` to the video style and to `VideoInput.style`: the visual
131
+ language generated media is produced in. A style has two halves once media can
132
+ be generated - the brand decides how captions are drawn, this decides what the
133
+ footage behind them looks like - and they have to travel together, or a pale
134
+ illustrated ground ends up under dark documentary footage. It reaches
135
+ `resolveMedia` on the resolver context, so a provider prompt no longer has to
136
+ be threaded with it by hand, and it is stored with the video so a replay keeps
137
+ its look. Nothing is rendered from it.
138
+
139
+ - Adds `mediaConcurrency` to `createVideoHandler`: how many scenes may resolve
140
+ media at once, defaulting to one. Media resolution ran strictly in turn, which
141
+ is invisible for a stock search and costly for generated video - five clips at
142
+ a few seconds each is half a minute of nothing, and the only way around it was
143
+ to plan the shots and generate them outside the plan stream. Scenes are still
144
+ emitted in the order they were planned; only the waiting overlaps.
145
+
146
+ - Adds `createSceneTimeline`: compose a playable video from scenes the
147
+ application builds itself, appended through the player's `stream` prop.
148
+ `VideoInput.opening` holds a single line of copy, and replacing the `video`
149
+ prop restarts playback, so an application with its own opening scenes
150
+ previously had to emit protocol envelopes by hand - where the sequence, event
151
+ id, scene position and completion snapshot must all be exact and any mistake
152
+ rejects the whole stream silently. `awaitAudio` covers the related trap that
153
+ `audio.set` is only valid before the first scene, by holding openings until
154
+ the soundtrack is known.
155
+
156
+ - Adds optional `narration` to `VideoScene`: the line spoken aloud while that
157
+ scene is showing. A narrated video previously had to be carried as scenes plus
158
+ a separate script kept in step by index, which drifts; holding the line on the
159
+ scene keeps the two together through planning, playback, storage and replay,
160
+ so a stored video can be spoken again without the model that wrote it. The
161
+ renderer never draws it. Additive and optional, so existing videos parse
162
+ unchanged.
163
+
7
164
  ## 0.6.0
8
165
 
9
166
  - Adds opt-in native scene-video audio to `VideoPlayer`, mixed with the existing
package/PUBLIC-API.md CHANGED
@@ -92,11 +92,18 @@ provider-neutral text-delta escape hatch.
92
92
 
93
93
  ### Values
94
94
 
95
+ - `createVideoChatHandler(options)`
95
96
  - `createVideoHandler(options)`
96
97
  - `createServerTemplateRegistry(options)`
97
98
 
98
99
  ### Types
99
100
 
101
+ - `VideoChatCapabilities`
102
+ - `VideoChatConversationTurn`
103
+ - `VideoChatHandlerOptions`
104
+ - `VideoChatMode`
105
+ - `VideoChatWelcomeOptions`
106
+ - `VideoChatWelcomePrompt`
100
107
  - `VideoHandlerOptions`
101
108
  - `MediaResolver`
102
109
  - `MediaResolverContext`
@@ -111,6 +118,38 @@ provider-neutral text-delta escape hatch.
111
118
 
112
119
  ### Handler contract
113
120
 
121
+ `createVideoChatHandler` is the opinionated general-purpose video-chat route.
122
+ Mount it once and use its bounded `action` query parameter for capabilities,
123
+ responses, opening media, narration, suggestions, speech,
124
+ transcription, and the welcome screen. It owns the general response prompts,
125
+ fixed visual-mode spend limits, capability fallbacks, and auxiliary response
126
+ shapes. The application supplies provider-neutral `streamText`, `generateText`,
127
+ `generateSpeech`, `transcribe`, `searchMedia`, and `generateVideo` callbacks.
128
+ Only the two text callbacks are required. Missing optional callbacks remove
129
+ their capability; templates and browser speech remain available.
130
+
131
+ - A response accepts `prompt`, `mode`, `orientation`, optional bounded
132
+ `conversation`, `opening`, `brand`, and `style`. `opening` is an optional
133
+ prewritten hook from a selected suggestion. The response returns protocol
134
+ `0.5` SSE and negotiates `data.video-chat-opening`, which carries the bounded
135
+ 6-9 word hook and optional stock-search keyword before the first scene.
136
+ - The planner produces that opening as the first line of the same model stream
137
+ that produces the scenes. The separate opening-media action resolves its
138
+ keyword through the application-owned `searchMedia` callback, so stock lookup
139
+ never delays speech or planning. In `full` mode the first line also directs
140
+ the exact first generated scene; the handler consumes that private direction
141
+ and starts the clip while the model continues with scenes two through five.
142
+ - `templates` and `full` map to server-owned generated-media budgets of zero
143
+ and five. Without `generateVideo`, only `templates` is exposed
144
+ and forged generated-mode requests degrade to it.
145
+ - The response planner writes narration on each scene. The narration action is
146
+ retained as a compatibility fallback for missing lines, not used in the
147
+ normal path.
148
+ - Every action applies the same authorization, origin, request-size,
149
+ cancellation, safe-error, and server-only-provider boundaries.
150
+
151
+ `createVideoHandler` remains the lower-level video composition route:
152
+
114
153
  - `authorize` is required for HTTP handlers. Use `authorize: "none"` only for
115
154
  an intentionally non-public in-process/test handler.
116
155
  - `streamText` receives the generated system prompt, grounded user prompt, and
@@ -145,18 +184,84 @@ provider-neutral text-delta escape hatch.
145
184
 
146
185
  ### Values
147
186
 
187
+ - `VideoChat(props)`
188
+ - `useVideoChat(options?)`
189
+ - `createVideoChatVoice(options?)`
148
190
  - `useVideo(options?)`
149
191
  - `VideoPlayer`
150
192
  - `VideoError`
151
193
 
152
194
  ### Types
153
195
 
196
+ - `VideoChatProps`
197
+ - `UseVideoChatOptions`
198
+ - `UseVideoChatResult`
199
+ - `VideoChatAskOptions`
200
+ - `VideoChatFirstFrameMetric`
201
+ - `VideoChatTurn`
202
+ - `VideoChatStatus`
203
+ - `VideoChatMode`
204
+ - `VideoChatCapabilities`
205
+ - `VideoChatWelcome`
206
+ - `VideoChatSuggestion`
207
+ - `VideoChatMedia`
208
+ - `VideoChatVoice`
209
+ - `VideoChatPreparedSpeech`
210
+ - `CreateVideoChatVoiceOptions`
154
211
  - `UseVideoOptions`
155
212
  - `UseVideoResult`
156
213
  - `VideoPlaybackMode`
157
214
  - `VideoPlayerProps`
158
215
  - `VideoErrorOptions`
159
216
 
217
+ `VideoChat` is the complete default interface. Import its scoped stylesheet
218
+ explicitly so the host application keeps control over CSS loading:
219
+
220
+ ```tsx
221
+ import { VideoChat } from "@vanillaskyai/video/react";
222
+ import "@vanillaskyai/video/video-chat.css";
223
+
224
+ export function App() {
225
+ return <VideoChat />;
226
+ }
227
+ ```
228
+
229
+ Pass the same session configuration through `options`, for example
230
+ `<VideoChat options={{ endpoint, headers, templates, brand }} />`. `className`
231
+ and `welcomeTitle` are the only shell-level customizations. The stylesheet is
232
+ scoped under `.vanillasky-video-chat` and does not style the host document.
233
+
234
+ `useVideoChat` is the headless client for `createVideoChatHandler`. It owns
235
+ the conversation lifecycle and returns UI-neutral state plus a spread-ready
236
+ player binding:
237
+
238
+ ```tsx
239
+ const chat = useVideoChat();
240
+
241
+ await chat.ask("Tell me a tiny mystery set on a night train");
242
+
243
+ return chat.playerProps
244
+ ? <VideoPlayer key={chat.playerKey} {...chat.playerProps} />
245
+ : null;
246
+ ```
247
+
248
+ The hook loads capabilities and welcome suggestions, sends completed turns as
249
+ bounded context, cancels replaced prompts, retries once only before playback,
250
+ paces each scene to its prepared speech, and keeps pause, mute, replay, history,
251
+ captions, and actual playback completion synchronized. The default voice tries
252
+ the handler's generated-speech action and falls back to browser speech. Pass a
253
+ `VideoChatVoice` to replace it without rebuilding session orchestration.
254
+ `ask(prompt, { opening, openingMedia })` lets a custom interface start a
255
+ prewritten suggestion hook immediately and reuse its image or video without
256
+ another model or media lookup. The hook otherwise reads the opening from the
257
+ response stream, resolves its media keyword, holds that media while it is
258
+ spoken, and starts the planned timeline only when both speech and the first
259
+ scene are ready. `UseVideoChatOptions.onFirstFrame` receives a
260
+ `VideoChatFirstFrameMetric` once when a fresh response displays its first real
261
+ scene. Each `VideoChatTurn` exposes `openingMedia` and `completed`, so
262
+ custom interfaces can render the same handoff while partial or cancelled
263
+ responses remain visible without being mistaken for conversation context.
264
+
160
265
  `UseVideoResult` has this conceptual shape:
161
266
 
162
267
  ```ts
@@ -208,7 +313,9 @@ the internal reducer state are not public types.
208
313
  completed video from the beginning instead of showing the replay affordance;
209
314
  `onSceneChange(scene, index)` fires whenever the scene under the playhead
210
315
  changes, including when a loop wraps back to the first scene. Streaming
211
- playback is unaffected by either.
316
+ playback is unaffected by either. `onComplete(video)` reports that a streamed
317
+ response finished composing; `onPlaybackEnd(video)` reports that the visible
318
+ playhead actually reached the end for either a stream or saved replay.
212
319
 
213
320
  `resolveVideoBrand` fills a partial brand with the documented defaults and
214
321
  preset backgrounds, producing the fully resolved `VideoBrand` that `parseVideo`
@@ -285,14 +392,21 @@ invalid scenes, provider failure, content filtering, abort, and timeout.
285
392
 
286
393
  The `vanillasky` binary supports:
287
394
 
288
- - `list`
289
- - `describe`
290
- - `create`
291
- - `add`
292
- - `sync`
293
- - `check`
294
- - `add --dry-run`
295
- - `add --diff`
395
+ - `init`
396
+ - `doctor`
397
+ - `templates list`
398
+ - `templates describe`
399
+ - `templates create`
400
+ - `templates add`
401
+ - `templates sync`
402
+ - `templates check`
403
+ - `templates add --dry-run`
404
+ - `templates add --diff`
405
+
406
+ `init` writes the canonical app-owned provider and configuration shell while
407
+ using the SDK's packaged interface, session, handler, and built-in templates.
408
+ `doctor` is read-only and offline. The old top-level template commands are not
409
+ aliases; this is an intentional pre-1.0 namespace change.
296
410
 
297
411
  Generated customer files import only the public entry points in this document.
298
412
  Browser registries import `createTemplateRegistry` from `/templates`. Server
package/README.md CHANGED
@@ -1,164 +1,118 @@
1
- # Give your AI a video output
1
+ # Give your AI a voice and a face
2
2
 
3
- **VanillaSky is the open-source video response layer.** Turn text, structured
4
- data, and live application context into personalized video responses that start
5
- playing while your LLM composes them.
3
+ **VanillaSky is the open-source voice-and-video chat layer.** Add a polished,
4
+ general-purpose AI conversation that speaks and starts playing visual answers
5
+ while they are still being composed.
6
6
 
7
7
  > **Status: Beta.** VanillaSky is pre-1.0 and its public API may change as we
8
8
  > test it in real applications. Pin an exact version before production use.
9
9
 
10
- Your application owns the model, data, authentication, and UI. VanillaSky owns
11
- the planning prompt, trusted templates, validation, streaming, and player.
10
+ Your application owns the providers, keys, authentication, persistence,
11
+ branding, and product copy. VanillaSky owns the chat flow, planning prompts,
12
+ trusted templates, validation, streaming, voice timing, and player.
12
13
 
13
- ## See it first
14
+ ## Start on localhost
14
15
 
15
- Generate a video in the browser, with no install and no API key, in the
16
- [playground](https://vanillasky.ai/playground/). The
17
- [website](https://vanillasky.ai/) has the overview and full
18
- [documentation](https://vanillasky.ai/docs/).
19
-
20
- ## Start
16
+ ```bash
17
+ npx @vanillaskyai/video init
18
+ ```
21
19
 
22
- For humans:
20
+ Add `ANTHROPIC_API_KEY` to the generated, ignored `.env.local`, then run:
23
21
 
24
22
  ```bash
25
- npm install @vanillaskyai/video
23
+ npx vanillasky doctor
24
+ npm run dev
26
25
  ```
27
26
 
27
+ Open the reported localhost URL. One text key gives you the complete chat with
28
+ packaged templates and browser voice. No template setup is required.
29
+
30
+ Optional server-only keys progressively add capabilities without changing the
31
+ client:
32
+
33
+ | Key | Adds |
34
+ | --- | --- |
35
+ | `XAI_API_KEY` | Generated speech |
36
+ | `FAL_KEY` | Generated video and voice transcription |
37
+ | `PEXELS_API_KEY` | Stock media, including opening backgrounds |
38
+
39
+ `npx vanillasky doctor` reports readiness by key name and never prints values.
40
+ Provider SDKs remain dependencies of the generated application, not the core
41
+ package.
42
+
28
43
  For coding agents:
29
44
 
30
45
  ```bash
31
46
  npx skills add VanillaSkyAi/video@vanillasky
32
47
  ```
33
48
 
34
- Then prompt: `Use $vanillasky to turn this application's data into a personalized video response.`
35
-
36
- ## Connect your LLM
37
-
38
- VanillaSky never chooses a provider or model. Configure that separately on the
39
- server, then connect the model your application already owns. The route below
40
- assumes an AI SDK `LanguageModel` exported as `videoModel`; if you do not have
41
- one yet, choose a provider and current model in [Provider integration](docs/provider-integration.md).
42
-
43
- Create one authenticated server route:
44
-
45
- ```ts
46
- // app/api/video/route.ts
47
- import { streamText } from "ai";
48
- import { createVideoHandler } from "@vanillaskyai/video/server";
49
- import { videoModel } from "@/lib/video-model";
50
-
51
- const handle = createVideoHandler({
52
- // Local development only. Replace with your session check before deploying.
53
- authorize: (request) => {
54
- if (process.env.VANILLASKY_LOCAL_DEMO !== "1") return false;
55
- const hostname = new URL(request.url).hostname;
56
- return hostname === "localhost" || hostname === "127.0.0.1";
57
- },
58
- streamText: ({ systemPrompt, userPrompt, signal }) => streamText({
59
- model: videoModel,
60
- system: systemPrompt,
61
- prompt: userPrompt,
62
- abortSignal: signal,
63
- }),
64
- });
65
-
66
- export const POST = handle;
67
- export const OPTIONS = handle;
68
- ```
49
+ Then prompt: `Use $vanillasky to set up and verify a general-purpose video chat in this project.`
69
50
 
70
- The packaged development command supplies the local marker only to `next dev`,
71
- so production builds and `next start` deny every request. Replace it before
72
- deploying. The model can come from Anthropic, OpenAI, an AI SDK registry or
73
- gateway, or any compatible streaming adapter. The
74
- [Next.js quickstart](examples/nextjs-quickstart) shows the later, explicit
75
- provider-selection step with a quality-oriented Claude Sonnet model.
51
+ ## What init creates
76
52
 
77
- For planner-selected image and video backgrounds, configure the optional
78
- server-only `resolveMedia` callback described in
79
- [Media and soundtrack audio](docs/media-and-audio.md#media-providers).
53
+ The generated application is a thin, editable shell:
80
54
 
81
- ## Generate a video
55
+ - `src/main.tsx` mounts the complete SDK-owned `<VideoChat />` interface;
56
+ - `server.ts` connects app-owned text, speech, transcription, stock, and video
57
+ providers through one `createVideoChatHandler`;
58
+ - `vite.config.ts` serves the UI and the single `/api/video-chat` endpoint;
59
+ - `.env.local` holds server-only keys and is ignored by Git.
82
60
 
83
- Call the route from React and render the player:
61
+ The UI stays this small:
84
62
 
85
63
  ```tsx
86
- "use client";
87
-
88
- import { VideoPlayer, useVideo } from "@vanillaskyai/video/react";
89
-
90
- export function VideoResponse() {
91
- const video = useVideo();
64
+ import { VideoChat } from "@vanillaskyai/video/react";
65
+ import "@vanillaskyai/video/video-chat.css";
92
66
 
93
- return <>
94
- <button onClick={() => { void video.generate({
95
- input: "Activation increased from 41% to 58% after guided onboarding.",
96
- personalization: { firstName: "Maya" },
97
- }); }}>
98
- Generate video
99
- </button>
100
-
101
- {video.error && <p role="alert">Video generation failed.</p>}
102
- <VideoPlayer {...video.playerProps} />
103
- </>;
67
+ export function App() {
68
+ return <VideoChat />;
104
69
  }
105
70
  ```
106
71
 
107
- That is the complete path. Built-in templates require no setup. VanillaSky
108
- shows each complete, validated scene as soon as it is ready and returns a
109
- deterministic `Video` object when generation finishes.
72
+ Use `useVideoChat` when you want a custom interface while keeping the SDK-owned
73
+ conversation and playback lifecycle. Edit the generated server when you want a
74
+ different provider. The [provider guide](docs/provider-integration.md) explains
75
+ both boundaries.
110
76
 
111
- A copy-and-run app is in [`examples/nextjs-quickstart`](examples/nextjs-quickstart).
77
+ ## Templates are the built-in fallback
112
78
 
113
- ## Shape the response
79
+ The packaged visual templates are the fast, inexpensive default and require no
80
+ copied source tree. Add a video provider when you want generated footage; the
81
+ same conversation can mix both modes.
114
82
 
115
- Start with `input`. It is the complete factual boundary by default:
83
+ Copy template source only when you want to own and edit it:
116
84
 
117
- ```ts
118
- video.generate({
119
- input: JSON.stringify({
120
- period: "Q2",
121
- activation: { previous: 41, current: 58 },
122
- cause: "guided onboarding",
123
- }),
124
- instructions: "Lead with the improvement, then explain what changed.",
125
- personalization: { firstName: "Maya", plan: "Pro" },
126
- });
85
+ ```bash
86
+ npm install --save-dev tsx
87
+ npx vanillasky templates add bigNumber
127
88
  ```
128
89
 
129
- - Put claims, numbers, names, dates, and quotations in `input`.
130
- - Keep `knowledgeMode: "input-only"` (the default) for source-grounded video,
131
- or choose `knowledgeMode: "general"` when the model should answer a question
132
- or develop content with stable general knowledge.
133
- - Put presentation direction in `instructions`.
134
- - Put viewer or account context in `personalization`.
135
- - Add brand, approved media, soundtrack audio, or a smaller template set only
136
- when the experience needs them.
90
+ That compiler is needed only for source-owned templates. See
91
+ [Custom templates](docs/custom-templates.md).
137
92
 
138
- VanillaSky does not provide an LLM, hosted generation service, narration, TTS,
139
- or speech synchronization. MP4/WebM encoding and export are application-owned.
140
- Completed videos can be stored as JSON and replayed without calling the LLM.
93
+ ## Go deeper
141
94
 
142
- Custom templates are optional. Only source-owned templates need the local TSX
143
- compiler: `npm install --save-dev tsx`.
95
+ Completed chat responses are deterministic JSON and can be stored and replayed.
96
+ MP4/WebM export remains application-owned.
144
97
 
145
98
  ## Documentation
146
99
 
147
100
  | Goal | Guide |
148
101
  | --- | --- |
149
- | See a video without installing | [Playground](https://vanillasky.ai/playground/) |
150
- | Integrate with a coding agent | [Agent integration guide](docs/agent-integration.md) |
151
- | Build the first response | [Getting started](docs/getting-started.md) |
152
- | Copy the Next.js route and component | [Next.js integration](docs/integrate-nextjs.md) |
153
- | Connect another model | [Provider integration](docs/provider-integration.md) |
154
- | Understand grounding and prompts | [Prompt and input](docs/prompt-and-input.md) |
102
+ | Run the complete chat | [Getting started](docs/getting-started.md) |
103
+ | Set it up with a coding agent | [Agent integration guide](docs/agent-integration.md) |
104
+ | Change or add providers | [Provider integration](docs/provider-integration.md) |
105
+ | Customize the interface | [Customization](docs/customization.md) |
106
+ | Understand prompts and grounding | [Prompt and input](docs/prompt-and-input.md) |
155
107
  | Add brand or viewer context | [Branding and personalization](docs/branding-and-personalization.md) |
156
108
  | Add media or soundtrack audio | [Media and soundtrack audio](docs/media-and-audio.md) |
157
109
  | Persist and replay results | [Persistence and replay](docs/persistence.md) |
158
- | Run a looping, self-refreshing channel | [Live channels](docs/live-channels.md) |
159
- | Create source-owned templates | [Custom templates](docs/custom-templates.md) |
160
110
  | Test routes and streams | [Test integrations](docs/testing.md) |
161
111
  | Deploy securely | [Production](docs/production.md) · [Security](docs/security.md) |
162
112
  | Inspect the API contract | [Public API](PUBLIC-API.md) · [Protocol](docs/reference/protocol.md) |
163
113
 
114
+ Try a keyless template response in the
115
+ [playground](https://vanillasky.ai/playground/), or visit
116
+ [vanillasky.ai](https://vanillasky.ai/).
117
+
164
118
  Apache-2.0
@@ -4,7 +4,7 @@ import {
4
4
  import "./chunk-QAAO7OAY.js";
5
5
  import {
6
6
  TemplateText
7
- } from "./chunk-TCEHK2BW.js";
7
+ } from "./chunk-YJJC4N4D.js";
8
8
  import "./chunk-L5BTTCZV.js";
9
9
  import {
10
10
  BrandGradientOverlay
@@ -3,7 +3,7 @@ import {
3
3
  } from "./chunk-QAAO7OAY.js";
4
4
  import {
5
5
  TemplateText
6
- } from "./chunk-TCEHK2BW.js";
6
+ } from "./chunk-YJJC4N4D.js";
7
7
  import {
8
8
  Emoji
9
9
  } from "./chunk-L5BTTCZV.js";
@@ -5,10 +5,10 @@ import "./chunk-QAAO7OAY.js";
5
5
  import {
6
6
  SceneBackground,
7
7
  getMediaBackgroundProps
8
- } from "./chunk-7BGAU6C3.js";
8
+ } from "./chunk-MXZSDGZQ.js";
9
9
  import {
10
10
  TemplateText
11
- } from "./chunk-TCEHK2BW.js";
11
+ } from "./chunk-YJJC4N4D.js";
12
12
  import "./chunk-L5BTTCZV.js";
13
13
  import "./chunk-FRN6WKHA.js";
14
14
  import "./chunk-W3VDIBGR.js";
@@ -4,7 +4,7 @@ import {
4
4
  import {
5
5
  SceneBackground,
6
6
  getMediaBackgroundProps
7
- } from "./chunk-7BGAU6C3.js";
7
+ } from "./chunk-MXZSDGZQ.js";
8
8
  import {
9
9
  renderWithEmoji
10
10
  } from "./chunk-L5BTTCZV.js";
@@ -1,7 +1,7 @@
1
1
  import {
2
2
  BUILTIN_SERVER_TEMPLATE_KIT
3
- } from "./chunk-FVTMYS6U.js";
4
- import "./chunk-3RV4YKB3.js";
3
+ } from "./chunk-RGF452LL.js";
4
+ import "./chunk-JNN3EYVP.js";
5
5
  import "./chunk-2E6T633S.js";
6
6
  export {
7
7
  BUILTIN_SERVER_TEMPLATE_KIT
@@ -1,10 +1,10 @@
1
1
  import {
2
2
  SceneBackground,
3
3
  getMediaBackgroundProps
4
- } from "./chunk-7BGAU6C3.js";
4
+ } from "./chunk-MXZSDGZQ.js";
5
5
  import {
6
6
  TemplateText
7
- } from "./chunk-TCEHK2BW.js";
7
+ } from "./chunk-YJJC4N4D.js";
8
8
  import "./chunk-L5BTTCZV.js";
9
9
  import "./chunk-FRN6WKHA.js";
10
10
  import {
@@ -8,10 +8,10 @@ import "./chunk-QAAO7OAY.js";
8
8
  import {
9
9
  SceneBackground,
10
10
  getMediaBackgroundProps
11
- } from "./chunk-7BGAU6C3.js";
11
+ } from "./chunk-MXZSDGZQ.js";
12
12
  import {
13
13
  TemplateText
14
- } from "./chunk-TCEHK2BW.js";
14
+ } from "./chunk-YJJC4N4D.js";
15
15
  import "./chunk-L5BTTCZV.js";
16
16
  import "./chunk-FRN6WKHA.js";
17
17
  import {