@vanillaskyai/video 0.7.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. package/CHANGELOG.md +66 -0
  2. package/PUBLIC-API.md +83 -79
  3. package/README.md +16 -15
  4. package/dist/{builtin-server-KB7FKF6A.js → builtin-server-CUYPIDE7.js} +2 -2
  5. package/dist/check-runtime.js +1 -1
  6. package/dist/{chunk-2PYO6VAC.js → chunk-2DOLF4QN.js} +1 -43
  7. package/dist/{chunk-GAIOHGNR.js → chunk-2OKB27TG.js} +42 -45
  8. package/dist/{chunk-K746NUIP.js → chunk-2USVB3CY.js} +80 -29
  9. package/dist/{chunk-RE4IMWJR.js → chunk-3VRMKFM7.js} +87 -14
  10. package/dist/{chunk-JNN3EYVP.js → chunk-4UNKZEAT.js} +46 -46
  11. package/dist/{chunk-2XT4MZ76.js → chunk-APQKY6P3.js} +36 -3
  12. package/dist/{chunk-RXHW4EP4.js → chunk-L7QGGKJ2.js} +71 -40
  13. package/dist/{chunk-RGF452LL.js → chunk-OWBT45EN.js} +1 -1
  14. package/dist/{chunk-LKBZX7GV.js → chunk-PP3MCPN4.js} +4 -4
  15. package/dist/chunk-ZD2VTUYR.js +28 -0
  16. package/dist/cli.js +131 -24
  17. package/dist/{compose-video-PD6LKKRK.js → compose-video-2NFH4DZY.js} +3 -3
  18. package/dist/{events-B6qS1Lsb.d.ts → events-BQ6z264n.d.ts} +1 -1
  19. package/dist/index.d.ts +5 -67
  20. package/dist/index.js +2 -4
  21. package/dist/{kit-8g46H2RZ.d.ts → kit-tQnCPMOM.d.ts} +1 -1
  22. package/dist/react.d.ts +33 -53
  23. package/dist/react.js +642 -536
  24. package/dist/server.d.ts +11 -13
  25. package/dist/server.js +170 -81
  26. package/dist/template-catalog.js +1 -1
  27. package/dist/templates.d.ts +3 -3
  28. package/dist/test.d.ts +2 -2
  29. package/dist/test.js +3 -3
  30. package/dist/{text-stream-FW4BLBAL.js → text-stream-54LXABJW.js} +1 -1
  31. package/dist/{types-2wHBqtg8.d.ts → types-CEh9eUFu.d.ts} +1 -1
  32. package/dist/{types-DrABlRa7.d.ts → types-DWFO6qca.d.ts} +1 -1
  33. package/docs/agent-integration.md +9 -3
  34. package/docs/architecture.md +8 -10
  35. package/docs/concepts.md +16 -40
  36. package/docs/custom-templates.md +27 -23
  37. package/docs/customization.md +12 -49
  38. package/docs/errors.md +58 -68
  39. package/docs/getting-started.md +21 -14
  40. package/docs/media-and-audio.md +2 -2
  41. package/docs/performance.md +69 -0
  42. package/docs/persistence.md +12 -27
  43. package/docs/production.md +2 -2
  44. package/docs/provider-integration.md +11 -5
  45. package/docs/reference/provider-adapters.md +37 -89
  46. package/docs/security.md +1 -1
  47. package/docs/streaming-protocol.md +2 -2
  48. package/examples/custom-template/README.md +6 -6
  49. package/package.json +6 -7
  50. package/registry/items/barChart.json +2 -2
  51. package/registry/items/bigNumber.json +2 -2
  52. package/registry/items/brandMessage.json +2 -2
  53. package/registry/items/cardList.json +2 -2
  54. package/registry/items/codeEditor.json +2 -2
  55. package/registry/items/ctaLogo.json +2 -2
  56. package/registry/items/ctaMedia.json +2 -2
  57. package/registry/items/incomingCall.json +2 -2
  58. package/registry/items/media.json +2 -2
  59. package/registry/items/milestone.json +2 -2
  60. package/registry/items/notification.json +2 -2
  61. package/registry/items/phoneMockup.json +2 -2
  62. package/registry/items/problemSolution.json +2 -2
  63. package/registry/items/progressRing.json +2 -2
  64. package/registry/items/promptInput.json +2 -2
  65. package/registry/items/reaction.json +2 -2
  66. package/registry/items/reviewStack.json +2 -2
  67. package/registry/items/steps.json +2 -2
  68. package/registry/items/terminal.json +2 -2
  69. package/registry/items/testimonial.json +2 -2
  70. package/registry/items/tripleStats.json +2 -2
  71. package/registry/items/tweet.json +2 -2
  72. package/registry/items/webMockup.json +2 -2
  73. package/starters/video-chat/.env.example +4 -2
  74. package/starters/video-chat/README.md +17 -5
  75. package/starters/video-chat/package.json +2 -4
  76. package/starters/video-chat/providers/speech.ts +20 -0
  77. package/starters/video-chat/providers/video.ts +66 -0
  78. package/starters/video-chat/providers.ts +3 -0
  79. package/starters/video-chat/server.ts +4 -80
  80. package/starters/video-chat/stock.ts +114 -99
  81. package/dist/chunk-SKRGRKHY.js +0 -142
  82. package/dist/state-DZcKuS32.d.ts +0 -3
  83. package/docs/branding-and-personalization.md +0 -87
  84. package/examples/custom-template/supplied-media.tsx +0 -92
package/CHANGELOG.md CHANGED
@@ -4,6 +4,72 @@ VanillaSky follows semantic versioning. This changelog begins with the 0.1 beta.
4
4
 
5
5
  ## Unreleased
6
6
 
7
+ ## 0.8.0
8
+
9
+ - The default chat silently recovers from optional failures. Non-fatal diagnostics remain available to applications without appearing in the viewer interface; actionable errors remain visible.
10
+
11
+ - Adds `maxGeneratedVideos` (default five attempts per response), aligns AI-video planning with that budget, and keeps stock footage available beyond it. Failed searches can reuse matching completed footage once before falling back to a readable template.
12
+ - Improves opening stock footage with documented Pexels video-host support and a planner-supplied broader atmospheric query. Video and photo searches share one three-second deadline; available descriptions screen obvious mismatches without an extra AI call.
13
+
14
+ - Adds local, content-free playback observations for scene presentation, actual speech onset, and waits for the next scene. Measurements ignore stale callbacks and exclude deliberate pauses from stall duration.
15
+
16
+ - Runs setup checks automatically after init and supports retrying interrupted
17
+ installation. The baseline needs one text-provider key, packaged templates,
18
+ and browser voice; optional speech and video packages install only through
19
+ `vanillasky providers add speech` or `vanillasky providers add video`.
20
+
21
+ - Bounds optional video, stock, speech, and narration waits so stalled providers fall back without holding the whole answer. Completed responses no longer wait for follow-up suggestions.
22
+
23
+ - Preserves playable video-chat openings and completed scenes when planning or
24
+ an optional provider fails. Malformed scene lines no longer discard later
25
+ valid scenes; generated footage can fall back to stock and safe templates.
26
+ - Continues narration with scene text and browser voice when generated speech
27
+ fails, keeping non-fatal diagnostics available to applications without
28
+ showing recovery notices to viewers.
29
+ - Keeps template rendering failures local to a scene, and lets stock searches
30
+ recover from failed lookups or malformed candidates.
31
+
32
+ ### Breaking changes
33
+
34
+ The supported generation path is now video chat. Removes `useVideo`,
35
+ `createVideoHandler`, `useNarration`, `createSceneTimeline`, their public types,
36
+ and generic root input types. Chat handler options no longer expose standalone
37
+ soundtrack selection, snapshot-retention overrides, or durable-stream replay.
38
+ The old one-shot integration is no longer available:
39
+
40
+ ```tsx
41
+ const video = useVideo({ endpoint: "/api/video" });
42
+ await video.generate({ input: "Explain the Moon's orbit" });
43
+ ```
44
+
45
+ ### Adoption
46
+
47
+ Use `npx @vanillaskyai/video init` for the complete app. Existing custom hosts
48
+ mount `createVideoChatHandler` with `streamText` and `generateText`, then render
49
+ the packaged chat:
50
+
51
+ ```tsx
52
+ import { VideoChat } from "@vanillaskyai/video/react";
53
+ import "@vanillaskyai/video/video-chat.css";
54
+
55
+ export function App() {
56
+ return <VideoChat options={{ endpoint: "/api/video-chat" }} />;
57
+ }
58
+ ```
59
+
60
+ Use `useVideoChat` for a custom interface, `createVideoChatVoice` for a custom
61
+ voice, and `parseVideo` plus `VideoPlayer` for saved completed responses. Pass
62
+ custom registries to the chat handler and `VideoChat` options. Persist completed
63
+ `chat.turns` video values instead of one-shot `generate()` results. Saved `Video`
64
+ JSON, including existing soundtrack data, keeps its current storage contract.
65
+
66
+ ## 0.7.1
67
+
68
+ - Keeps Full AI video responses running when a text provider repeats narration
69
+ beside a scene that already owns the canonical line. The redundant copy is
70
+ discarded before strict plan validation instead of aborting after the first
71
+ generated clip.
72
+
7
73
  ## 0.7.0
8
74
 
9
75
  - Makes blank-folder onboarding one safe scoped command:
package/PUBLIC-API.md CHANGED
@@ -62,6 +62,9 @@ a request, renders React, imports a provider, or accesses browser globals.
62
62
  ### Values
63
63
 
64
64
  - `getVideoDuration(video: Video): number`
65
+ - `getSceneDuration(scene, metadata)`
66
+ - `getSceneDurationBounds(scene, metadata)`
67
+ - `getSpokenDuration(text)`
65
68
  - `parseVideo(value: unknown): Video`
66
69
  - `resolveVideoBrand(input?: VideoBrandInput): VideoBrand`
67
70
  - `VideoValidationError`
@@ -72,17 +75,17 @@ a request, renders React, imports a provider, or accesses browser globals.
72
75
  - `VideoAudio`
73
76
  - `VideoBackground`
74
77
  - `VideoBrand`
75
- - `VideoInput`
78
+ - `VideoBrandInput`
76
79
  - `VideoOrientation`
77
80
  - `VideoScene`
78
81
  - `VideoStyle`
79
82
  - `VideoStyleOptions`
80
- - `VideoSuppliedMedia`
81
83
  - `VideoStatus`
82
84
  - `VideoValidationErrorCode`
85
+ - `SceneDurationBounds`
83
86
 
84
87
  `VideoState` remains internal protocol reducer state. Browser consumers use the
85
- normalized fields returned by `useVideo` instead.
88
+ normalized fields returned by `useVideoChat` instead.
86
89
 
87
90
  ## Server
88
91
 
@@ -93,7 +96,6 @@ provider-neutral text-delta escape hatch.
93
96
  ### Values
94
97
 
95
98
  - `createVideoChatHandler(options)`
96
- - `createVideoHandler(options)`
97
99
  - `createServerTemplateRegistry(options)`
98
100
 
99
101
  ### Types
@@ -104,10 +106,6 @@ provider-neutral text-delta escape hatch.
104
106
  - `VideoChatMode`
105
107
  - `VideoChatWelcomeOptions`
106
108
  - `VideoChatWelcomePrompt`
107
- - `VideoHandlerOptions`
108
- - `MediaResolver`
109
- - `MediaResolverContext`
110
- - `ResolvedMedia`
111
109
  - `ServerTemplateRegistry`
112
110
  - `ServerTemplateMetadata`
113
111
  - `VideoFinishReason`
@@ -122,25 +120,36 @@ provider-neutral text-delta escape hatch.
122
120
  Mount it once and use its bounded `action` query parameter for capabilities,
123
121
  responses, opening media, narration, suggestions, speech,
124
122
  transcription, and the welcome screen. It owns the general response prompts,
125
- fixed visual-mode spend limits, capability fallbacks, and auxiliary response
123
+ application-configured visual-mode attempt limits, capability fallbacks, and auxiliary response
126
124
  shapes. The application supplies provider-neutral `streamText`, `generateText`,
127
125
  `generateSpeech`, `transcribe`, `searchMedia`, and `generateVideo` callbacks.
128
126
  Only the two text callbacks are required. Missing optional callbacks remove
129
- their capability; templates and browser speech remain available.
127
+ their capability; templates and browser speech remain available. During response
128
+ creation, failed generated footage falls back to stock when available, then to
129
+ matching completed footage and finally a safe template. The default handler skips invalid planner parts and preserves
130
+ playable scenes on an interrupted plan, emitting non-fatal warnings. Explicit
131
+ `invalidPartBehavior: "fail"` retains strict generation semantics.
130
132
 
131
133
  - A response accepts `prompt`, `mode`, `orientation`, optional bounded
132
134
  `conversation`, `opening`, `brand`, and `style`. `opening` is an optional
133
135
  prewritten hook from a selected suggestion. The response returns protocol
134
136
  `0.5` SSE and negotiates `data.video-chat-opening`, which carries the bounded
135
- 6-9 word hook and optional stock-search keyword before the first scene.
137
+ 6-9 word hook and optional stock-search `keyword` and `fallbackKeyword` before the first scene.
136
138
  - The planner produces that opening as the first line of the same model stream
137
139
  that produces the scenes. The separate opening-media action resolves its
138
- keyword through the application-owned `searchMedia` callback, so stock lookup
140
+ keyword through the application-owned `searchMedia` callback, with an optional
141
+ bounded `fallbackQuery` for broader opening atmosphere. Stock lookup
139
142
  never delays speech or planning. In `full` mode the first line also directs
140
143
  the exact first generated scene; the handler consumes that private direction
141
144
  and starts the clip while the model continues with scenes two through five.
142
- - `templates` and `full` map to server-owned generated-media budgets of zero
143
- and five. Without `generateVideo`, only `templates` is exposed
145
+ - `templates` never generates video. `full` uses an application-owned
146
+ `maxGeneratedVideos` limit (default five, nonnegative safe integer). The limit
147
+ counts attempts including failed attempts and the reserved first shot; it is
148
+ neither a currency nor duration cap. Stock lookup remains available after
149
+ the limit. With zero, full mode uses stock only. The planner receives this
150
+ budget, and the server enforces it independently. If providers fail, an
151
+ already completed video with the identical search query and visual look may
152
+ be reused once in the same response before a readable template fallback. Without `generateVideo`, only `templates` is exposed
144
153
  and forged generated-mode requests degrade to it.
145
154
  - The response planner writes narration on each scene. The narration action is
146
155
  retained as a compatibility fallback for missing lines, not used in the
@@ -148,37 +157,20 @@ their capability; templates and browser speech remain available.
148
157
  - Every action applies the same authorization, origin, request-size,
149
158
  cancellation, safe-error, and server-only-provider boundaries.
150
159
 
151
- `createVideoHandler` remains the lower-level video composition route:
152
-
153
- - `authorize` is required for HTTP handlers. Use `authorize: "none"` only for
154
- an intentionally non-public in-process/test handler.
155
- - `streamText` receives the generated system prompt, grounded user prompt, and
156
- the request `AbortSignal`.
157
- - `resolveMedia` is an optional application-owned resolver. It receives a
158
- bounded semantic query only when configured, and returns an approved image
159
- or video URL plus an optional poster before the scene is validated or sent
160
- to the browser. Provider clients, keys, metadata, and unresolved queries
161
- remain server-only.
162
- - `invalidPartBehavior` is `"drop" | "fail"`. A behavior selector is never
163
- named like a callback.
164
- - `requireCloser` defaults to `true` on `createVideoHandler`. The planner marks
165
- one `scene.add` with `placement: "closer"`; the runtime reserves it and emits
166
- it last. Specialized deterministic or test handlers may opt out explicitly.
167
- - `onWarning` receives safe typed warnings.
168
- - `onComplete` receives one server-only `VideoGenerationSummary` after an
169
- actual `response.complete`; errors and aborts do not invoke it.
170
- - `onError` receives the full internal server error. Client responses remain
171
- redacted and typed separately.
172
- - Provider usage, raw usage, provider metadata, and model identifiers remain
173
- server-side unless the host deliberately persists them.
174
- - Provider-native usage and metadata require the bounded
175
- `includeRawProviderData` opt-in.
176
- - `snapshotRetention` opts into individually bounded source, instructions, or
177
- supplied-media URL metadata; all are omitted by default.
178
- - Provider credentials, provider selection, authentication, rate limiting,
179
- provider retries, logging, and tracing remain host-owned.
180
- - Request cancellation always reaches the provider through `AbortSignal`.
181
- - Callback failures are isolated and never alter the video response.
160
+ The chat handler requires `authorize`; `authorize: "none"` is only for an
161
+ intentionally non-public in-process/test handler. All provider callbacks receive
162
+ cancellation signals. The application owns authentication, origins, request
163
+ limits, provider credentials, deadlines, and retry budgets.
164
+
165
+ `onWarning` receives safe typed diagnostics; `onComplete` receives a server-only
166
+ `VideoGenerationSummary` after an actual `response.complete`, including recovered
167
+ playable responses. `onError` receives private internal errors. Observer failures
168
+ do not change playback. Provider metadata remains server-side, with raw usage
169
+ and metadata requiring the bounded `includeRawProviderData` opt-in.
170
+
171
+ The handler exposes chat-relevant policy, template, and provider options only.
172
+ Standalone soundtrack selection, snapshot-retention overrides, and durable
173
+ stream replay are not public chat options.
182
174
 
183
175
  ## React
184
176
 
@@ -187,7 +179,6 @@ their capability; templates and browser speech remain available.
187
179
  - `VideoChat(props)`
188
180
  - `useVideoChat(options?)`
189
181
  - `createVideoChatVoice(options?)`
190
- - `useVideo(options?)`
191
182
  - `VideoPlayer`
192
183
  - `VideoError`
193
184
 
@@ -208,8 +199,6 @@ their capability; templates and browser speech remain available.
208
199
  - `VideoChatVoice`
209
200
  - `VideoChatPreparedSpeech`
210
201
  - `CreateVideoChatVoiceOptions`
211
- - `UseVideoOptions`
212
- - `UseVideoResult`
213
202
  - `VideoPlaybackMode`
214
203
  - `VideoPlayerProps`
215
204
  - `VideoErrorOptions`
@@ -251,50 +240,42 @@ paces each scene to its prepared speech, and keeps pause, mute, replay, history,
251
240
  captions, and actual playback completion synchronized. The default voice tries
252
241
  the handler's generated-speech action and falls back to browser speech. Pass a
253
242
  `VideoChatVoice` to replace it without rebuilding session orchestration.
243
+ `createVideoChatVoice({ onFallback })` optionally observes a generated-speech
244
+ failure that selects browser speech. Observer exceptions and rejected promises
245
+ are isolated from playback; no provider diagnostics are passed to this callback.
246
+
247
+ `chat.warnings` exposes concise notices for the displayed turn, also retained
248
+ as optional `VideoChatTurn.warnings`. The default interface recovers silently; these diagnostics are
249
+ for application developers and are not rendered to viewers. Optional scene, narration, speech, or stream failures preserve
250
+ playable output; an error is shown only when no playable response remains.
254
251
  `ask(prompt, { opening, openingMedia })` lets a custom interface start a
255
252
  prewritten suggestion hook immediately and reuse its image or video without
256
253
  another model or media lookup. The hook otherwise reads the opening from the
257
254
  response stream, resolves its media keyword, holds that media while it is
258
255
  spoken, and starts the planned timeline only when both speech and the first
259
256
  scene are ready. `UseVideoChatOptions.onFirstFrame` receives a
260
- `VideoChatFirstFrameMetric` once when a fresh response displays its first real
261
- scene. Each `VideoChatTurn` exposes `openingMedia` and `completed`, so
257
+ `VideoChatFirstFrameMetric` once after its first active scene commits and reaches
258
+ an animation-frame opportunity. This measures presentation readiness, not physical
259
+ screen paint or media decoding. Each `VideoChatTurn` exposes `openingMedia` and `completed`, so
262
260
  custom interfaces can render the same handoff while partial or cancelled
263
261
  responses remain visible without being mistaken for conversation context.
264
262
 
265
- `UseVideoResult` has this conceptual shape:
266
-
267
- ```ts
268
- interface UseVideoResult {
269
- generate(input: VideoInput): Promise<Video>;
270
- abort(reason?: string): void;
271
- video?: Video;
272
- status: VideoStatus;
273
- error?: VideoError;
274
- warnings: readonly VideoWarning[];
275
- playerProps: VideoPlayerProps;
276
- }
277
- ```
278
-
279
263
  `VideoPlayer` accepts either streaming player props or a completed saved video:
280
264
 
281
265
  ```tsx
282
- <VideoPlayer {...video.playerProps} />
266
+ <VideoPlayer {...chat.playerProps} />
283
267
  <VideoPlayer video={savedVideo} />
284
268
  ```
285
269
 
286
270
  Built-in renderers are always available, so a streaming `VideoPlayer` does not
287
271
  require a template registry. `playerProps` is a spread-ready player binding; it
288
- contains the customer registry supplied to `useVideo` when one exists, but it
272
+ contains the customer registry supplied to `useVideoChat` when one exists, but it
289
273
  does not expose the built-in planning catalog. Import `builtinTemplates` from
290
274
  `@vanillaskyai/video/templates/catalog` for labels, schemas, selection guidance,
291
275
  and other React-free metadata.
292
276
 
293
- Set `playbackMode="autoplay-after-interaction"` for chat feeds: the first
294
- soundtrack waits on a visible scene-one poster, and later streams on the same
295
- mounted player autoplay with sound after the first successful viewer start.
296
- `manual`, `muted-autoplay`, and `autoplay-with-sound` cover the other browser
297
- startup policies.
277
+ The chat owns playback startup, speech, and synchronization. Custom saved-video
278
+ players may choose a `VideoPlaybackMode` to match browser autoplay rules.
298
279
 
299
280
  Set `nativeMediaAudio={{ volume: 0.85 }}` when scene video files contain an
300
281
  embedded audio track. The active clip's audio becomes a second layer alongside
@@ -302,10 +283,6 @@ the video's continuous `audio` soundtrack. Both follow the player's master
302
283
  mute control; `nativeMediaAudio.volume` and serialized `audio.volume` set their
303
284
  independent mix levels. Incoming preroll videos remain muted until active.
304
285
 
305
- `VideoInput.opening` accepts custom copy, uses the deterministic fallback when
306
- omitted, and accepts `false` when the application owns transient loading UI and
307
- wants the completed video to begin with the first generated scene.
308
-
309
286
  Saved-video playback performs no generation request. `VideoPlayerBinding` and
310
287
  the internal reducer state are not public types.
311
288
 
@@ -430,9 +407,8 @@ from `/server` without crossing a React type boundary.
430
407
  fail before any renderer runs; they are never rendered partially.
431
408
  - Raw prompts, provider payloads, and credentials are never retained by
432
409
  default.
433
- - Raw source, instructions, and the supplied-media URL index are opt-in and
434
- bounded. Hosts own the database, storage, tenant policy, deletion, and media
435
- URL expiry.
410
+ - Chat snapshots omit raw source, instructions, and the supplied-media URL
411
+ index. Hosts own storage, tenant policy, deletion, and media URL expiry.
436
412
  - Replay through `<VideoPlayer video={savedVideo} />` never calls an LLM.
437
413
  - Completion checksums detect accidental drift only; they do not provide
438
414
  authenticity, authorization, or tenancy security.
@@ -444,5 +420,33 @@ from `/server` without crossing a React type boundary.
444
420
  - Hosted persistence.
445
421
  - OpenTelemetry integration.
446
422
  - Automatic factual verification or scene repair.
447
- - `useVideo(initialVideo)`.
423
+ - Standalone video generation hooks, handlers, narration, and timeline factories.
448
424
  - Undocumented API aliases.
425
+
426
+
427
+ ## Playback measurements
428
+
429
+ `UseVideoChatOptions.onPlaybackMetric` receives the exported
430
+ `VideoChatPlaybackMetric` union. Every event contains an opaque `turnId`, `mode`,
431
+ and nonnegative `elapsedMs` since the prompt was submitted:
432
+
433
+ - `first-frame`: first committed active scene at an animation-frame opportunity;
434
+ - `first-speech`: actual speech playback onset, with `source` equal to `browser`,
435
+ `generated`, or `custom`;
436
+ - `stall`: a finished wait for the next prepared scene, with `durationMs` and
437
+ `reason: "scene-generation"`.
438
+
439
+ Metrics contain no prompt, narration, scene, URL, or provider diagnostics. There
440
+ is no automatic network reporting. Use opaque identifiers when supplying
441
+ `createTurnId`. Observer throws and rejected promises cannot affect playback.
442
+ Replay and stale/cancelled callbacks do not create fresh response measurements.
443
+ Pauses end a stall interval; paused time is excluded from stall duration.
444
+
445
+ Custom `VideoChatVoice.speak` implementations can call their optional
446
+ `onStart()` argument when audio actually begins. Without that signal, speech
447
+ onset is unavailable, not zero. Built-in voices use browser utterance `start`
448
+ and audio `playing` events, not request completion or preparation estimates.
449
+
450
+ `VideoPlayerProps.onFramePresented` and `onStallChange` expose the underlying
451
+ presentation and stream-starvation signals for custom players. They do not
452
+ measure media-decoder buffering. See [Performance](docs/performance.md).
package/README.md CHANGED
@@ -17,7 +17,8 @@ trusted templates, validation, streaming, voice timing, and player.
17
17
  npx @vanillaskyai/video init
18
18
  ```
19
19
 
20
- Add `ANTHROPIC_API_KEY` to the generated, ignored `.env.local`, then run:
20
+ Init installs the baseline dependencies and runs doctor automatically. Add
21
+ `ANTHROPIC_API_KEY` to the generated, ignored `.env.local`, then run:
21
22
 
22
23
  ```bash
23
24
  npx vanillasky doctor
@@ -25,20 +26,21 @@ npm run dev
25
26
  ```
26
27
 
27
28
  Open the reported localhost URL. One text key gives you the complete chat with
28
- packaged templates and browser voice. No template setup is required.
29
+ packaged templates and browser voice. No template setup or optional provider
30
+ packages are required. If installation is interrupted, rerun the init command.
29
31
 
30
- Optional server-only keys progressively add capabilities without changing the
31
- client:
32
+ Optional upgrades keep the same client:
32
33
 
33
- | Key | Adds |
34
- | --- | --- |
35
- | `XAI_API_KEY` | Generated speech |
36
- | `FAL_KEY` | Generated video and voice transcription |
37
- | `PEXELS_API_KEY` | Stock media, including opening backgrounds |
34
+ | Add | Setup command | Server-only key |
35
+ | --- | --- | --- |
36
+ | Generated speech | `npx vanillasky providers add speech` | `XAI_API_KEY` |
37
+ | Generated video and voice transcription | `npx vanillasky providers add video` | `FAL_KEY` |
38
+ | Stock media, including opening backgrounds | No install needed | `PEXELS_API_KEY` |
38
39
 
39
- `npx vanillasky doctor` reports readiness by key name and never prints values.
40
- Provider SDKs remain dependencies of the generated application, not the core
41
- package.
40
+ Run the selected command, add its key to `.env.local`, and restart the server.
41
+ Only selected provider packages are installed. `npx vanillasky doctor` reports
42
+ readiness by key name and never prints values. Provider SDKs remain application
43
+ dependencies, outside the core package.
42
44
 
43
45
  For coding agents:
44
46
 
@@ -104,9 +106,8 @@ MP4/WebM export remains application-owned.
104
106
  | Change or add providers | [Provider integration](docs/provider-integration.md) |
105
107
  | Customize the interface | [Customization](docs/customization.md) |
106
108
  | Understand prompts and grounding | [Prompt and input](docs/prompt-and-input.md) |
107
- | Add brand or viewer context | [Branding and personalization](docs/branding-and-personalization.md) |
108
- | Add media or soundtrack audio | [Media and soundtrack audio](docs/media-and-audio.md) |
109
- | Persist and replay results | [Persistence and replay](docs/persistence.md) |
109
+ | Add media or voice | [Media and voice](docs/media-and-audio.md) |
110
+ | Persist and replay results | [Performance measurements](docs/performance.md) · [Persistence and replay](docs/persistence.md) |
110
111
  | Test routes and streams | [Test integrations](docs/testing.md) |
111
112
  | Deploy securely | [Production](docs/production.md) · [Security](docs/security.md) |
112
113
  | Inspect the API contract | [Public API](PUBLIC-API.md) · [Protocol](docs/reference/protocol.md) |
@@ -1,7 +1,7 @@
1
1
  import {
2
2
  BUILTIN_SERVER_TEMPLATE_KIT
3
- } from "./chunk-RGF452LL.js";
4
- import "./chunk-JNN3EYVP.js";
3
+ } from "./chunk-OWBT45EN.js";
4
+ import "./chunk-4UNKZEAT.js";
5
5
  import "./chunk-2E6T633S.js";
6
6
  export {
7
7
  BUILTIN_SERVER_TEMPLATE_KIT
@@ -1,6 +1,6 @@
1
1
  import {
2
2
  VideoFrame
3
- } from "./chunk-K746NUIP.js";
3
+ } from "./chunk-2USVB3CY.js";
4
4
  import "./chunk-224QNWRA.js";
5
5
  import "./chunk-5CDAM24P.js";
6
6
  import "./chunk-XWWLKRNU.js";
@@ -1,5 +1,4 @@
1
1
  import {
2
- VIDEO_PROTOCOL_VERSION,
3
2
  VIDEO_SCHEMA_VERSION
4
3
  } from "./chunk-AOWSXU2K.js";
5
4
  import {
@@ -501,52 +500,11 @@ function parseVideo(value) {
501
500
  }
502
501
  }
503
502
 
504
- // src/protocol/stable-json.ts
505
- function stableJson(value) {
506
- if (value == null || typeof value !== "object") return JSON.stringify(value);
507
- if (Array.isArray(value)) return `[${value.map(stableJson).join(",")}]`;
508
- const entries = Object.entries(value).filter(([, child]) => child !== void 0).sort(([left], [right]) => left < right ? -1 : left > right ? 1 : 0);
509
- return `{${entries.map(([key, child]) => `${JSON.stringify(key)}:${stableJson(child)}`).join(",")}}`;
510
- }
511
-
512
- // src/protocol/checksum.ts
513
- function checksumVideo(config) {
514
- const input = stableJson(config);
515
- let hash = 2166136261;
516
- for (let index = 0; index < input.length; index += 1) {
517
- hash ^= input.charCodeAt(index);
518
- hash = Math.imul(hash, 16777619) >>> 0;
519
- }
520
- return `fnv1a32:${hash.toString(16).padStart(8, "0")}`;
521
- }
522
-
523
- // src/protocol/events.ts
524
- function createVideoEventFactory(options) {
525
- let sequence = options.initialSequence ?? 0;
526
- return {
527
- create(type, data) {
528
- const currentSequence = sequence;
529
- sequence += 1;
530
- return {
531
- protocolVersion: VIDEO_PROTOCOL_VERSION,
532
- runId: options.runId,
533
- sequence: currentSequence,
534
- eventId: `${options.runId}:${currentSequence}`,
535
- type,
536
- data
537
- };
538
- }
539
- };
540
- }
541
-
542
503
  export {
543
504
  MAX_RETAINED_SOURCE_LENGTH,
544
505
  MAX_RETAINED_INSTRUCTIONS_LENGTH,
545
506
  MAX_RETAINED_MEDIA_URLS,
546
507
  MAX_RETAINED_MEDIA_URL_LENGTH,
547
508
  VideoValidationError,
548
- parseVideo,
549
- stableJson,
550
- checksumVideo,
551
- createVideoEventFactory
509
+ parseVideo
552
510
  };
@@ -2,13 +2,48 @@ import {
2
2
  parseVideoEvent
3
3
  } from "./chunk-MMUXVA47.js";
4
4
  import {
5
- checksumVideo,
6
- stableJson
7
- } from "./chunk-2PYO6VAC.js";
8
- import {
5
+ VIDEO_PROTOCOL_VERSION,
9
6
  VIDEO_SCHEMA_VERSION
10
7
  } from "./chunk-AOWSXU2K.js";
11
8
 
9
+ // src/protocol/stable-json.ts
10
+ function stableJson(value) {
11
+ if (value == null || typeof value !== "object") return JSON.stringify(value);
12
+ if (Array.isArray(value)) return `[${value.map(stableJson).join(",")}]`;
13
+ const entries = Object.entries(value).filter(([, child]) => child !== void 0).sort(([left], [right]) => left < right ? -1 : left > right ? 1 : 0);
14
+ return `{${entries.map(([key, child]) => `${JSON.stringify(key)}:${stableJson(child)}`).join(",")}}`;
15
+ }
16
+
17
+ // src/protocol/checksum.ts
18
+ function checksumVideo(config) {
19
+ const input = stableJson(config);
20
+ let hash = 2166136261;
21
+ for (let index = 0; index < input.length; index += 1) {
22
+ hash ^= input.charCodeAt(index);
23
+ hash = Math.imul(hash, 16777619) >>> 0;
24
+ }
25
+ return `fnv1a32:${hash.toString(16).padStart(8, "0")}`;
26
+ }
27
+
28
+ // src/protocol/events.ts
29
+ function createVideoEventFactory(options) {
30
+ let sequence = options.initialSequence ?? 0;
31
+ return {
32
+ create(type, data) {
33
+ const currentSequence = sequence;
34
+ sequence += 1;
35
+ return {
36
+ protocolVersion: VIDEO_PROTOCOL_VERSION,
37
+ runId: options.runId,
38
+ sequence: currentSequence,
39
+ eventId: `${options.runId}:${currentSequence}`,
40
+ type,
41
+ data
42
+ };
43
+ }
44
+ };
45
+ }
46
+
12
47
  // src/protocol/state.ts
13
48
  function createVideoState() {
14
49
  return {
@@ -133,47 +168,9 @@ function applyVideoEvent(state, untrustedEvent) {
133
168
  throw new Error(`Reducer does not yet support ${event.type}`);
134
169
  }
135
170
 
136
- // src/replayable-stream.ts
137
- function createReplayableStream(source, onError) {
138
- const buffered = [];
139
- const waiters = /* @__PURE__ */ new Set();
140
- let done = false;
141
- let streamError;
142
- const notify = () => {
143
- for (const waiter of waiters) waiter();
144
- waiters.clear();
145
- };
146
- void (async () => {
147
- try {
148
- for await (const item of source) {
149
- buffered.push(item);
150
- notify();
151
- }
152
- } catch (cause) {
153
- streamError = cause;
154
- onError(cause);
155
- } finally {
156
- done = true;
157
- notify();
158
- }
159
- })();
160
- return {
161
- async *[Symbol.asyncIterator]() {
162
- let index = 0;
163
- while (true) {
164
- while (index < buffered.length) yield buffered[index++];
165
- if (done) {
166
- if (streamError) throw streamError;
167
- return;
168
- }
169
- await new Promise((resolve) => waiters.add(resolve));
170
- }
171
- }
172
- };
173
- }
174
-
175
171
  export {
172
+ checksumVideo,
173
+ createVideoEventFactory,
176
174
  createVideoState,
177
- applyVideoEvent,
178
- createReplayableStream
175
+ applyVideoEvent
179
176
  };