@vanillaskyai/video 0.7.1 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +59 -0
- package/PUBLIC-API.md +83 -79
- package/README.md +16 -15
- package/dist/{builtin-server-KB7FKF6A.js → builtin-server-CUYPIDE7.js} +2 -2
- package/dist/check-runtime.js +1 -1
- package/dist/{chunk-2PYO6VAC.js → chunk-2DOLF4QN.js} +1 -43
- package/dist/{chunk-GAIOHGNR.js → chunk-2OKB27TG.js} +42 -45
- package/dist/{chunk-K746NUIP.js → chunk-2USVB3CY.js} +80 -29
- package/dist/{chunk-RE4IMWJR.js → chunk-3VRMKFM7.js} +87 -14
- package/dist/{chunk-JNN3EYVP.js → chunk-4UNKZEAT.js} +46 -46
- package/dist/{chunk-2XT4MZ76.js → chunk-APQKY6P3.js} +36 -3
- package/dist/{chunk-LLPMVDJZ.js → chunk-L7QGGKJ2.js} +69 -38
- package/dist/{chunk-RGF452LL.js → chunk-OWBT45EN.js} +1 -1
- package/dist/{chunk-LKBZX7GV.js → chunk-PP3MCPN4.js} +4 -4
- package/dist/chunk-ZD2VTUYR.js +28 -0
- package/dist/cli.js +131 -24
- package/dist/{compose-video-PD6LKKRK.js → compose-video-2NFH4DZY.js} +3 -3
- package/dist/{events-B6qS1Lsb.d.ts → events-BQ6z264n.d.ts} +1 -1
- package/dist/index.d.ts +5 -67
- package/dist/index.js +2 -4
- package/dist/{kit-8g46H2RZ.d.ts → kit-tQnCPMOM.d.ts} +1 -1
- package/dist/react.d.ts +33 -53
- package/dist/react.js +642 -536
- package/dist/server.d.ts +11 -13
- package/dist/server.js +170 -81
- package/dist/template-catalog.js +1 -1
- package/dist/templates.d.ts +3 -3
- package/dist/test.d.ts +2 -2
- package/dist/test.js +3 -3
- package/dist/{text-stream-ROTNGSD4.js → text-stream-54LXABJW.js} +1 -1
- package/dist/{types-2wHBqtg8.d.ts → types-CEh9eUFu.d.ts} +1 -1
- package/dist/{types-DrABlRa7.d.ts → types-DWFO6qca.d.ts} +1 -1
- package/docs/agent-integration.md +9 -3
- package/docs/architecture.md +8 -10
- package/docs/concepts.md +16 -40
- package/docs/custom-templates.md +27 -23
- package/docs/customization.md +12 -49
- package/docs/errors.md +58 -68
- package/docs/getting-started.md +21 -14
- package/docs/media-and-audio.md +2 -2
- package/docs/performance.md +69 -0
- package/docs/persistence.md +12 -27
- package/docs/production.md +2 -2
- package/docs/provider-integration.md +11 -5
- package/docs/reference/provider-adapters.md +37 -89
- package/docs/security.md +1 -1
- package/docs/streaming-protocol.md +2 -2
- package/examples/custom-template/README.md +6 -6
- package/package.json +6 -7
- package/registry/items/barChart.json +2 -2
- package/registry/items/bigNumber.json +2 -2
- package/registry/items/brandMessage.json +2 -2
- package/registry/items/cardList.json +2 -2
- package/registry/items/codeEditor.json +2 -2
- package/registry/items/ctaLogo.json +2 -2
- package/registry/items/ctaMedia.json +2 -2
- package/registry/items/incomingCall.json +2 -2
- package/registry/items/media.json +2 -2
- package/registry/items/milestone.json +2 -2
- package/registry/items/notification.json +2 -2
- package/registry/items/phoneMockup.json +2 -2
- package/registry/items/problemSolution.json +2 -2
- package/registry/items/progressRing.json +2 -2
- package/registry/items/promptInput.json +2 -2
- package/registry/items/reaction.json +2 -2
- package/registry/items/reviewStack.json +2 -2
- package/registry/items/steps.json +2 -2
- package/registry/items/terminal.json +2 -2
- package/registry/items/testimonial.json +2 -2
- package/registry/items/tripleStats.json +2 -2
- package/registry/items/tweet.json +2 -2
- package/registry/items/webMockup.json +2 -2
- package/starters/video-chat/.env.example +4 -2
- package/starters/video-chat/README.md +17 -5
- package/starters/video-chat/package.json +2 -4
- package/starters/video-chat/providers/speech.ts +20 -0
- package/starters/video-chat/providers/video.ts +66 -0
- package/starters/video-chat/providers.ts +3 -0
- package/starters/video-chat/server.ts +4 -80
- package/starters/video-chat/stock.ts +114 -99
- package/dist/chunk-SKRGRKHY.js +0 -142
- package/dist/state-DZcKuS32.d.ts +0 -3
- package/docs/branding-and-personalization.md +0 -87
- package/examples/custom-template/supplied-media.tsx +0 -92
package/CHANGELOG.md
CHANGED
|
@@ -4,6 +4,65 @@ VanillaSky follows semantic versioning. This changelog begins with the 0.1 beta.
|
|
|
4
4
|
|
|
5
5
|
## Unreleased
|
|
6
6
|
|
|
7
|
+
## 0.8.0
|
|
8
|
+
|
|
9
|
+
- The default chat silently recovers from optional failures. Non-fatal diagnostics remain available to applications without appearing in the viewer interface; actionable errors remain visible.
|
|
10
|
+
|
|
11
|
+
- Adds `maxGeneratedVideos` (default five attempts per response), aligns AI-video planning with that budget, and keeps stock footage available beyond it. Failed searches can reuse matching completed footage once before falling back to a readable template.
|
|
12
|
+
- Improves opening stock footage with documented Pexels video-host support and a planner-supplied broader atmospheric query. Video and photo searches share one three-second deadline; available descriptions screen obvious mismatches without an extra AI call.
|
|
13
|
+
|
|
14
|
+
- Adds local, content-free playback observations for scene presentation, actual speech onset, and waits for the next scene. Measurements ignore stale callbacks and exclude deliberate pauses from stall duration.
|
|
15
|
+
|
|
16
|
+
- Runs setup checks automatically after init and supports retrying interrupted
|
|
17
|
+
installation. The baseline needs one text-provider key, packaged templates,
|
|
18
|
+
and browser voice; optional speech and video packages install only through
|
|
19
|
+
`vanillasky providers add speech` or `vanillasky providers add video`.
|
|
20
|
+
|
|
21
|
+
- Bounds optional video, stock, speech, and narration waits so stalled providers fall back without holding the whole answer. Completed responses no longer wait for follow-up suggestions.
|
|
22
|
+
|
|
23
|
+
- Preserves playable video-chat openings and completed scenes when planning or
|
|
24
|
+
an optional provider fails. Malformed scene lines no longer discard later
|
|
25
|
+
valid scenes; generated footage can fall back to stock and safe templates.
|
|
26
|
+
- Continues narration with scene text and browser voice when generated speech
|
|
27
|
+
fails, keeping non-fatal diagnostics available to applications without
|
|
28
|
+
showing recovery notices to viewers.
|
|
29
|
+
- Keeps template rendering failures local to a scene, and lets stock searches
|
|
30
|
+
recover from failed lookups or malformed candidates.
|
|
31
|
+
|
|
32
|
+
### Breaking changes
|
|
33
|
+
|
|
34
|
+
The supported generation path is now video chat. Removes `useVideo`,
|
|
35
|
+
`createVideoHandler`, `useNarration`, `createSceneTimeline`, their public types,
|
|
36
|
+
and generic root input types. Chat handler options no longer expose standalone
|
|
37
|
+
soundtrack selection, snapshot-retention overrides, or durable-stream replay.
|
|
38
|
+
The old one-shot integration is no longer available:
|
|
39
|
+
|
|
40
|
+
```tsx
|
|
41
|
+
const video = useVideo({ endpoint: "/api/video" });
|
|
42
|
+
await video.generate({ input: "Explain the Moon's orbit" });
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
### Adoption
|
|
46
|
+
|
|
47
|
+
Use `npx @vanillaskyai/video init` for the complete app. Existing custom hosts
|
|
48
|
+
mount `createVideoChatHandler` with `streamText` and `generateText`, then render
|
|
49
|
+
the packaged chat:
|
|
50
|
+
|
|
51
|
+
```tsx
|
|
52
|
+
import { VideoChat } from "@vanillaskyai/video/react";
|
|
53
|
+
import "@vanillaskyai/video/video-chat.css";
|
|
54
|
+
|
|
55
|
+
export function App() {
|
|
56
|
+
return <VideoChat options={{ endpoint: "/api/video-chat" }} />;
|
|
57
|
+
}
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
Use `useVideoChat` for a custom interface, `createVideoChatVoice` for a custom
|
|
61
|
+
voice, and `parseVideo` plus `VideoPlayer` for saved completed responses. Pass
|
|
62
|
+
custom registries to the chat handler and `VideoChat` options. Persist completed
|
|
63
|
+
`chat.turns` video values instead of one-shot `generate()` results. Saved `Video`
|
|
64
|
+
JSON, including existing soundtrack data, keeps its current storage contract.
|
|
65
|
+
|
|
7
66
|
## 0.7.1
|
|
8
67
|
|
|
9
68
|
- Keeps Full AI video responses running when a text provider repeats narration
|
package/PUBLIC-API.md
CHANGED
|
@@ -62,6 +62,9 @@ a request, renders React, imports a provider, or accesses browser globals.
|
|
|
62
62
|
### Values
|
|
63
63
|
|
|
64
64
|
- `getVideoDuration(video: Video): number`
|
|
65
|
+
- `getSceneDuration(scene, metadata)`
|
|
66
|
+
- `getSceneDurationBounds(scene, metadata)`
|
|
67
|
+
- `getSpokenDuration(text)`
|
|
65
68
|
- `parseVideo(value: unknown): Video`
|
|
66
69
|
- `resolveVideoBrand(input?: VideoBrandInput): VideoBrand`
|
|
67
70
|
- `VideoValidationError`
|
|
@@ -72,17 +75,17 @@ a request, renders React, imports a provider, or accesses browser globals.
|
|
|
72
75
|
- `VideoAudio`
|
|
73
76
|
- `VideoBackground`
|
|
74
77
|
- `VideoBrand`
|
|
75
|
-
- `
|
|
78
|
+
- `VideoBrandInput`
|
|
76
79
|
- `VideoOrientation`
|
|
77
80
|
- `VideoScene`
|
|
78
81
|
- `VideoStyle`
|
|
79
82
|
- `VideoStyleOptions`
|
|
80
|
-
- `VideoSuppliedMedia`
|
|
81
83
|
- `VideoStatus`
|
|
82
84
|
- `VideoValidationErrorCode`
|
|
85
|
+
- `SceneDurationBounds`
|
|
83
86
|
|
|
84
87
|
`VideoState` remains internal protocol reducer state. Browser consumers use the
|
|
85
|
-
normalized fields returned by `
|
|
88
|
+
normalized fields returned by `useVideoChat` instead.
|
|
86
89
|
|
|
87
90
|
## Server
|
|
88
91
|
|
|
@@ -93,7 +96,6 @@ provider-neutral text-delta escape hatch.
|
|
|
93
96
|
### Values
|
|
94
97
|
|
|
95
98
|
- `createVideoChatHandler(options)`
|
|
96
|
-
- `createVideoHandler(options)`
|
|
97
99
|
- `createServerTemplateRegistry(options)`
|
|
98
100
|
|
|
99
101
|
### Types
|
|
@@ -104,10 +106,6 @@ provider-neutral text-delta escape hatch.
|
|
|
104
106
|
- `VideoChatMode`
|
|
105
107
|
- `VideoChatWelcomeOptions`
|
|
106
108
|
- `VideoChatWelcomePrompt`
|
|
107
|
-
- `VideoHandlerOptions`
|
|
108
|
-
- `MediaResolver`
|
|
109
|
-
- `MediaResolverContext`
|
|
110
|
-
- `ResolvedMedia`
|
|
111
109
|
- `ServerTemplateRegistry`
|
|
112
110
|
- `ServerTemplateMetadata`
|
|
113
111
|
- `VideoFinishReason`
|
|
@@ -122,25 +120,36 @@ provider-neutral text-delta escape hatch.
|
|
|
122
120
|
Mount it once and use its bounded `action` query parameter for capabilities,
|
|
123
121
|
responses, opening media, narration, suggestions, speech,
|
|
124
122
|
transcription, and the welcome screen. It owns the general response prompts,
|
|
125
|
-
|
|
123
|
+
application-configured visual-mode attempt limits, capability fallbacks, and auxiliary response
|
|
126
124
|
shapes. The application supplies provider-neutral `streamText`, `generateText`,
|
|
127
125
|
`generateSpeech`, `transcribe`, `searchMedia`, and `generateVideo` callbacks.
|
|
128
126
|
Only the two text callbacks are required. Missing optional callbacks remove
|
|
129
|
-
their capability; templates and browser speech remain available.
|
|
127
|
+
their capability; templates and browser speech remain available. During response
|
|
128
|
+
creation, failed generated footage falls back to stock when available, then to
|
|
129
|
+
matching completed footage and finally a safe template. The default handler skips invalid planner parts and preserves
|
|
130
|
+
playable scenes on an interrupted plan, emitting non-fatal warnings. Explicit
|
|
131
|
+
`invalidPartBehavior: "fail"` retains strict generation semantics.
|
|
130
132
|
|
|
131
133
|
- A response accepts `prompt`, `mode`, `orientation`, optional bounded
|
|
132
134
|
`conversation`, `opening`, `brand`, and `style`. `opening` is an optional
|
|
133
135
|
prewritten hook from a selected suggestion. The response returns protocol
|
|
134
136
|
`0.5` SSE and negotiates `data.video-chat-opening`, which carries the bounded
|
|
135
|
-
6-9 word hook and optional stock-search keyword before the first scene.
|
|
137
|
+
6-9 word hook and optional stock-search `keyword` and `fallbackKeyword` before the first scene.
|
|
136
138
|
- The planner produces that opening as the first line of the same model stream
|
|
137
139
|
that produces the scenes. The separate opening-media action resolves its
|
|
138
|
-
keyword through the application-owned `searchMedia` callback,
|
|
140
|
+
keyword through the application-owned `searchMedia` callback, with an optional
|
|
141
|
+
bounded `fallbackQuery` for broader opening atmosphere. Stock lookup
|
|
139
142
|
never delays speech or planning. In `full` mode the first line also directs
|
|
140
143
|
the exact first generated scene; the handler consumes that private direction
|
|
141
144
|
and starts the clip while the model continues with scenes two through five.
|
|
142
|
-
- `templates`
|
|
143
|
-
|
|
145
|
+
- `templates` never generates video. `full` uses an application-owned
|
|
146
|
+
`maxGeneratedVideos` limit (default five, nonnegative safe integer). The limit
|
|
147
|
+
counts attempts including failed attempts and the reserved first shot; it is
|
|
148
|
+
neither a currency nor duration cap. Stock lookup remains available after
|
|
149
|
+
the limit. With zero, full mode uses stock only. The planner receives this
|
|
150
|
+
budget, and the server enforces it independently. If providers fail, an
|
|
151
|
+
already completed video with the identical search query and visual look may
|
|
152
|
+
be reused once in the same response before a readable template fallback. Without `generateVideo`, only `templates` is exposed
|
|
144
153
|
and forged generated-mode requests degrade to it.
|
|
145
154
|
- The response planner writes narration on each scene. The narration action is
|
|
146
155
|
retained as a compatibility fallback for missing lines, not used in the
|
|
@@ -148,37 +157,20 @@ their capability; templates and browser speech remain available.
|
|
|
148
157
|
- Every action applies the same authorization, origin, request-size,
|
|
149
158
|
cancellation, safe-error, and server-only-provider boundaries.
|
|
150
159
|
|
|
151
|
-
`
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
one `scene.add` with `placement: "closer"`; the runtime reserves it and emits
|
|
166
|
-
it last. Specialized deterministic or test handlers may opt out explicitly.
|
|
167
|
-
- `onWarning` receives safe typed warnings.
|
|
168
|
-
- `onComplete` receives one server-only `VideoGenerationSummary` after an
|
|
169
|
-
actual `response.complete`; errors and aborts do not invoke it.
|
|
170
|
-
- `onError` receives the full internal server error. Client responses remain
|
|
171
|
-
redacted and typed separately.
|
|
172
|
-
- Provider usage, raw usage, provider metadata, and model identifiers remain
|
|
173
|
-
server-side unless the host deliberately persists them.
|
|
174
|
-
- Provider-native usage and metadata require the bounded
|
|
175
|
-
`includeRawProviderData` opt-in.
|
|
176
|
-
- `snapshotRetention` opts into individually bounded source, instructions, or
|
|
177
|
-
supplied-media URL metadata; all are omitted by default.
|
|
178
|
-
- Provider credentials, provider selection, authentication, rate limiting,
|
|
179
|
-
provider retries, logging, and tracing remain host-owned.
|
|
180
|
-
- Request cancellation always reaches the provider through `AbortSignal`.
|
|
181
|
-
- Callback failures are isolated and never alter the video response.
|
|
160
|
+
The chat handler requires `authorize`; `authorize: "none"` is only for an
|
|
161
|
+
intentionally non-public in-process/test handler. All provider callbacks receive
|
|
162
|
+
cancellation signals. The application owns authentication, origins, request
|
|
163
|
+
limits, provider credentials, deadlines, and retry budgets.
|
|
164
|
+
|
|
165
|
+
`onWarning` receives safe typed diagnostics; `onComplete` receives a server-only
|
|
166
|
+
`VideoGenerationSummary` after an actual `response.complete`, including recovered
|
|
167
|
+
playable responses. `onError` receives private internal errors. Observer failures
|
|
168
|
+
do not change playback. Provider metadata remains server-side, with raw usage
|
|
169
|
+
and metadata requiring the bounded `includeRawProviderData` opt-in.
|
|
170
|
+
|
|
171
|
+
The handler exposes chat-relevant policy, template, and provider options only.
|
|
172
|
+
Standalone soundtrack selection, snapshot-retention overrides, and durable
|
|
173
|
+
stream replay are not public chat options.
|
|
182
174
|
|
|
183
175
|
## React
|
|
184
176
|
|
|
@@ -187,7 +179,6 @@ their capability; templates and browser speech remain available.
|
|
|
187
179
|
- `VideoChat(props)`
|
|
188
180
|
- `useVideoChat(options?)`
|
|
189
181
|
- `createVideoChatVoice(options?)`
|
|
190
|
-
- `useVideo(options?)`
|
|
191
182
|
- `VideoPlayer`
|
|
192
183
|
- `VideoError`
|
|
193
184
|
|
|
@@ -208,8 +199,6 @@ their capability; templates and browser speech remain available.
|
|
|
208
199
|
- `VideoChatVoice`
|
|
209
200
|
- `VideoChatPreparedSpeech`
|
|
210
201
|
- `CreateVideoChatVoiceOptions`
|
|
211
|
-
- `UseVideoOptions`
|
|
212
|
-
- `UseVideoResult`
|
|
213
202
|
- `VideoPlaybackMode`
|
|
214
203
|
- `VideoPlayerProps`
|
|
215
204
|
- `VideoErrorOptions`
|
|
@@ -251,50 +240,42 @@ paces each scene to its prepared speech, and keeps pause, mute, replay, history,
|
|
|
251
240
|
captions, and actual playback completion synchronized. The default voice tries
|
|
252
241
|
the handler's generated-speech action and falls back to browser speech. Pass a
|
|
253
242
|
`VideoChatVoice` to replace it without rebuilding session orchestration.
|
|
243
|
+
`createVideoChatVoice({ onFallback })` optionally observes a generated-speech
|
|
244
|
+
failure that selects browser speech. Observer exceptions and rejected promises
|
|
245
|
+
are isolated from playback; no provider diagnostics are passed to this callback.
|
|
246
|
+
|
|
247
|
+
`chat.warnings` exposes concise notices for the displayed turn, also retained
|
|
248
|
+
as optional `VideoChatTurn.warnings`. The default interface recovers silently; these diagnostics are
|
|
249
|
+
for application developers and are not rendered to viewers. Optional scene, narration, speech, or stream failures preserve
|
|
250
|
+
playable output; an error is shown only when no playable response remains.
|
|
254
251
|
`ask(prompt, { opening, openingMedia })` lets a custom interface start a
|
|
255
252
|
prewritten suggestion hook immediately and reuse its image or video without
|
|
256
253
|
another model or media lookup. The hook otherwise reads the opening from the
|
|
257
254
|
response stream, resolves its media keyword, holds that media while it is
|
|
258
255
|
spoken, and starts the planned timeline only when both speech and the first
|
|
259
256
|
scene are ready. `UseVideoChatOptions.onFirstFrame` receives a
|
|
260
|
-
`VideoChatFirstFrameMetric` once
|
|
261
|
-
|
|
257
|
+
`VideoChatFirstFrameMetric` once after its first active scene commits and reaches
|
|
258
|
+
an animation-frame opportunity. This measures presentation readiness, not physical
|
|
259
|
+
screen paint or media decoding. Each `VideoChatTurn` exposes `openingMedia` and `completed`, so
|
|
262
260
|
custom interfaces can render the same handoff while partial or cancelled
|
|
263
261
|
responses remain visible without being mistaken for conversation context.
|
|
264
262
|
|
|
265
|
-
`UseVideoResult` has this conceptual shape:
|
|
266
|
-
|
|
267
|
-
```ts
|
|
268
|
-
interface UseVideoResult {
|
|
269
|
-
generate(input: VideoInput): Promise<Video>;
|
|
270
|
-
abort(reason?: string): void;
|
|
271
|
-
video?: Video;
|
|
272
|
-
status: VideoStatus;
|
|
273
|
-
error?: VideoError;
|
|
274
|
-
warnings: readonly VideoWarning[];
|
|
275
|
-
playerProps: VideoPlayerProps;
|
|
276
|
-
}
|
|
277
|
-
```
|
|
278
|
-
|
|
279
263
|
`VideoPlayer` accepts either streaming player props or a completed saved video:
|
|
280
264
|
|
|
281
265
|
```tsx
|
|
282
|
-
<VideoPlayer {...
|
|
266
|
+
<VideoPlayer {...chat.playerProps} />
|
|
283
267
|
<VideoPlayer video={savedVideo} />
|
|
284
268
|
```
|
|
285
269
|
|
|
286
270
|
Built-in renderers are always available, so a streaming `VideoPlayer` does not
|
|
287
271
|
require a template registry. `playerProps` is a spread-ready player binding; it
|
|
288
|
-
contains the customer registry supplied to `
|
|
272
|
+
contains the customer registry supplied to `useVideoChat` when one exists, but it
|
|
289
273
|
does not expose the built-in planning catalog. Import `builtinTemplates` from
|
|
290
274
|
`@vanillaskyai/video/templates/catalog` for labels, schemas, selection guidance,
|
|
291
275
|
and other React-free metadata.
|
|
292
276
|
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
mounted player autoplay with sound after the first successful viewer start.
|
|
296
|
-
`manual`, `muted-autoplay`, and `autoplay-with-sound` cover the other browser
|
|
297
|
-
startup policies.
|
|
277
|
+
The chat owns playback startup, speech, and synchronization. Custom saved-video
|
|
278
|
+
players may choose a `VideoPlaybackMode` to match browser autoplay rules.
|
|
298
279
|
|
|
299
280
|
Set `nativeMediaAudio={{ volume: 0.85 }}` when scene video files contain an
|
|
300
281
|
embedded audio track. The active clip's audio becomes a second layer alongside
|
|
@@ -302,10 +283,6 @@ the video's continuous `audio` soundtrack. Both follow the player's master
|
|
|
302
283
|
mute control; `nativeMediaAudio.volume` and serialized `audio.volume` set their
|
|
303
284
|
independent mix levels. Incoming preroll videos remain muted until active.
|
|
304
285
|
|
|
305
|
-
`VideoInput.opening` accepts custom copy, uses the deterministic fallback when
|
|
306
|
-
omitted, and accepts `false` when the application owns transient loading UI and
|
|
307
|
-
wants the completed video to begin with the first generated scene.
|
|
308
|
-
|
|
309
286
|
Saved-video playback performs no generation request. `VideoPlayerBinding` and
|
|
310
287
|
the internal reducer state are not public types.
|
|
311
288
|
|
|
@@ -430,9 +407,8 @@ from `/server` without crossing a React type boundary.
|
|
|
430
407
|
fail before any renderer runs; they are never rendered partially.
|
|
431
408
|
- Raw prompts, provider payloads, and credentials are never retained by
|
|
432
409
|
default.
|
|
433
|
-
-
|
|
434
|
-
|
|
435
|
-
URL expiry.
|
|
410
|
+
- Chat snapshots omit raw source, instructions, and the supplied-media URL
|
|
411
|
+
index. Hosts own storage, tenant policy, deletion, and media URL expiry.
|
|
436
412
|
- Replay through `<VideoPlayer video={savedVideo} />` never calls an LLM.
|
|
437
413
|
- Completion checksums detect accidental drift only; they do not provide
|
|
438
414
|
authenticity, authorization, or tenancy security.
|
|
@@ -444,5 +420,33 @@ from `/server` without crossing a React type boundary.
|
|
|
444
420
|
- Hosted persistence.
|
|
445
421
|
- OpenTelemetry integration.
|
|
446
422
|
- Automatic factual verification or scene repair.
|
|
447
|
-
-
|
|
423
|
+
- Standalone video generation hooks, handlers, narration, and timeline factories.
|
|
448
424
|
- Undocumented API aliases.
|
|
425
|
+
|
|
426
|
+
|
|
427
|
+
## Playback measurements
|
|
428
|
+
|
|
429
|
+
`UseVideoChatOptions.onPlaybackMetric` receives the exported
|
|
430
|
+
`VideoChatPlaybackMetric` union. Every event contains an opaque `turnId`, `mode`,
|
|
431
|
+
and nonnegative `elapsedMs` since the prompt was submitted:
|
|
432
|
+
|
|
433
|
+
- `first-frame`: first committed active scene at an animation-frame opportunity;
|
|
434
|
+
- `first-speech`: actual speech playback onset, with `source` equal to `browser`,
|
|
435
|
+
`generated`, or `custom`;
|
|
436
|
+
- `stall`: a finished wait for the next prepared scene, with `durationMs` and
|
|
437
|
+
`reason: "scene-generation"`.
|
|
438
|
+
|
|
439
|
+
Metrics contain no prompt, narration, scene, URL, or provider diagnostics. There
|
|
440
|
+
is no automatic network reporting. Use opaque identifiers when supplying
|
|
441
|
+
`createTurnId`. Observer throws and rejected promises cannot affect playback.
|
|
442
|
+
Replay and stale/cancelled callbacks do not create fresh response measurements.
|
|
443
|
+
Pauses end a stall interval; paused time is excluded from stall duration.
|
|
444
|
+
|
|
445
|
+
Custom `VideoChatVoice.speak` implementations can call their optional
|
|
446
|
+
`onStart()` argument when audio actually begins. Without that signal, speech
|
|
447
|
+
onset is unavailable, not zero. Built-in voices use browser utterance `start`
|
|
448
|
+
and audio `playing` events, not request completion or preparation estimates.
|
|
449
|
+
|
|
450
|
+
`VideoPlayerProps.onFramePresented` and `onStallChange` expose the underlying
|
|
451
|
+
presentation and stream-starvation signals for custom players. They do not
|
|
452
|
+
measure media-decoder buffering. See [Performance](docs/performance.md).
|
package/README.md
CHANGED
|
@@ -17,7 +17,8 @@ trusted templates, validation, streaming, voice timing, and player.
|
|
|
17
17
|
npx @vanillaskyai/video init
|
|
18
18
|
```
|
|
19
19
|
|
|
20
|
-
|
|
20
|
+
Init installs the baseline dependencies and runs doctor automatically. Add
|
|
21
|
+
`ANTHROPIC_API_KEY` to the generated, ignored `.env.local`, then run:
|
|
21
22
|
|
|
22
23
|
```bash
|
|
23
24
|
npx vanillasky doctor
|
|
@@ -25,20 +26,21 @@ npm run dev
|
|
|
25
26
|
```
|
|
26
27
|
|
|
27
28
|
Open the reported localhost URL. One text key gives you the complete chat with
|
|
28
|
-
packaged templates and browser voice. No template setup
|
|
29
|
+
packaged templates and browser voice. No template setup or optional provider
|
|
30
|
+
packages are required. If installation is interrupted, rerun the init command.
|
|
29
31
|
|
|
30
|
-
Optional
|
|
31
|
-
client:
|
|
32
|
+
Optional upgrades keep the same client:
|
|
32
33
|
|
|
33
|
-
|
|
|
34
|
-
| --- | --- |
|
|
35
|
-
|
|
|
36
|
-
|
|
|
37
|
-
|
|
|
34
|
+
| Add | Setup command | Server-only key |
|
|
35
|
+
| --- | --- | --- |
|
|
36
|
+
| Generated speech | `npx vanillasky providers add speech` | `XAI_API_KEY` |
|
|
37
|
+
| Generated video and voice transcription | `npx vanillasky providers add video` | `FAL_KEY` |
|
|
38
|
+
| Stock media, including opening backgrounds | No install needed | `PEXELS_API_KEY` |
|
|
38
39
|
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
40
|
+
Run the selected command, add its key to `.env.local`, and restart the server.
|
|
41
|
+
Only selected provider packages are installed. `npx vanillasky doctor` reports
|
|
42
|
+
readiness by key name and never prints values. Provider SDKs remain application
|
|
43
|
+
dependencies, outside the core package.
|
|
42
44
|
|
|
43
45
|
For coding agents:
|
|
44
46
|
|
|
@@ -104,9 +106,8 @@ MP4/WebM export remains application-owned.
|
|
|
104
106
|
| Change or add providers | [Provider integration](docs/provider-integration.md) |
|
|
105
107
|
| Customize the interface | [Customization](docs/customization.md) |
|
|
106
108
|
| Understand prompts and grounding | [Prompt and input](docs/prompt-and-input.md) |
|
|
107
|
-
| Add
|
|
108
|
-
|
|
|
109
|
-
| Persist and replay results | [Persistence and replay](docs/persistence.md) |
|
|
109
|
+
| Add media or voice | [Media and voice](docs/media-and-audio.md) |
|
|
110
|
+
| Persist and replay results | [Performance measurements](docs/performance.md) · [Persistence and replay](docs/persistence.md) |
|
|
110
111
|
| Test routes and streams | [Test integrations](docs/testing.md) |
|
|
111
112
|
| Deploy securely | [Production](docs/production.md) · [Security](docs/security.md) |
|
|
112
113
|
| Inspect the API contract | [Public API](PUBLIC-API.md) · [Protocol](docs/reference/protocol.md) |
|
package/dist/check-runtime.js
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import {
|
|
2
|
-
VIDEO_PROTOCOL_VERSION,
|
|
3
2
|
VIDEO_SCHEMA_VERSION
|
|
4
3
|
} from "./chunk-AOWSXU2K.js";
|
|
5
4
|
import {
|
|
@@ -501,52 +500,11 @@ function parseVideo(value) {
|
|
|
501
500
|
}
|
|
502
501
|
}
|
|
503
502
|
|
|
504
|
-
// src/protocol/stable-json.ts
|
|
505
|
-
function stableJson(value) {
|
|
506
|
-
if (value == null || typeof value !== "object") return JSON.stringify(value);
|
|
507
|
-
if (Array.isArray(value)) return `[${value.map(stableJson).join(",")}]`;
|
|
508
|
-
const entries = Object.entries(value).filter(([, child]) => child !== void 0).sort(([left], [right]) => left < right ? -1 : left > right ? 1 : 0);
|
|
509
|
-
return `{${entries.map(([key, child]) => `${JSON.stringify(key)}:${stableJson(child)}`).join(",")}}`;
|
|
510
|
-
}
|
|
511
|
-
|
|
512
|
-
// src/protocol/checksum.ts
|
|
513
|
-
function checksumVideo(config) {
|
|
514
|
-
const input = stableJson(config);
|
|
515
|
-
let hash = 2166136261;
|
|
516
|
-
for (let index = 0; index < input.length; index += 1) {
|
|
517
|
-
hash ^= input.charCodeAt(index);
|
|
518
|
-
hash = Math.imul(hash, 16777619) >>> 0;
|
|
519
|
-
}
|
|
520
|
-
return `fnv1a32:${hash.toString(16).padStart(8, "0")}`;
|
|
521
|
-
}
|
|
522
|
-
|
|
523
|
-
// src/protocol/events.ts
|
|
524
|
-
function createVideoEventFactory(options) {
|
|
525
|
-
let sequence = options.initialSequence ?? 0;
|
|
526
|
-
return {
|
|
527
|
-
create(type, data) {
|
|
528
|
-
const currentSequence = sequence;
|
|
529
|
-
sequence += 1;
|
|
530
|
-
return {
|
|
531
|
-
protocolVersion: VIDEO_PROTOCOL_VERSION,
|
|
532
|
-
runId: options.runId,
|
|
533
|
-
sequence: currentSequence,
|
|
534
|
-
eventId: `${options.runId}:${currentSequence}`,
|
|
535
|
-
type,
|
|
536
|
-
data
|
|
537
|
-
};
|
|
538
|
-
}
|
|
539
|
-
};
|
|
540
|
-
}
|
|
541
|
-
|
|
542
503
|
export {
|
|
543
504
|
MAX_RETAINED_SOURCE_LENGTH,
|
|
544
505
|
MAX_RETAINED_INSTRUCTIONS_LENGTH,
|
|
545
506
|
MAX_RETAINED_MEDIA_URLS,
|
|
546
507
|
MAX_RETAINED_MEDIA_URL_LENGTH,
|
|
547
508
|
VideoValidationError,
|
|
548
|
-
parseVideo
|
|
549
|
-
stableJson,
|
|
550
|
-
checksumVideo,
|
|
551
|
-
createVideoEventFactory
|
|
509
|
+
parseVideo
|
|
552
510
|
};
|
|
@@ -2,13 +2,48 @@ import {
|
|
|
2
2
|
parseVideoEvent
|
|
3
3
|
} from "./chunk-MMUXVA47.js";
|
|
4
4
|
import {
|
|
5
|
-
|
|
6
|
-
stableJson
|
|
7
|
-
} from "./chunk-2PYO6VAC.js";
|
|
8
|
-
import {
|
|
5
|
+
VIDEO_PROTOCOL_VERSION,
|
|
9
6
|
VIDEO_SCHEMA_VERSION
|
|
10
7
|
} from "./chunk-AOWSXU2K.js";
|
|
11
8
|
|
|
9
|
+
// src/protocol/stable-json.ts
|
|
10
|
+
function stableJson(value) {
|
|
11
|
+
if (value == null || typeof value !== "object") return JSON.stringify(value);
|
|
12
|
+
if (Array.isArray(value)) return `[${value.map(stableJson).join(",")}]`;
|
|
13
|
+
const entries = Object.entries(value).filter(([, child]) => child !== void 0).sort(([left], [right]) => left < right ? -1 : left > right ? 1 : 0);
|
|
14
|
+
return `{${entries.map(([key, child]) => `${JSON.stringify(key)}:${stableJson(child)}`).join(",")}}`;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
// src/protocol/checksum.ts
|
|
18
|
+
function checksumVideo(config) {
|
|
19
|
+
const input = stableJson(config);
|
|
20
|
+
let hash = 2166136261;
|
|
21
|
+
for (let index = 0; index < input.length; index += 1) {
|
|
22
|
+
hash ^= input.charCodeAt(index);
|
|
23
|
+
hash = Math.imul(hash, 16777619) >>> 0;
|
|
24
|
+
}
|
|
25
|
+
return `fnv1a32:${hash.toString(16).padStart(8, "0")}`;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
// src/protocol/events.ts
|
|
29
|
+
function createVideoEventFactory(options) {
|
|
30
|
+
let sequence = options.initialSequence ?? 0;
|
|
31
|
+
return {
|
|
32
|
+
create(type, data) {
|
|
33
|
+
const currentSequence = sequence;
|
|
34
|
+
sequence += 1;
|
|
35
|
+
return {
|
|
36
|
+
protocolVersion: VIDEO_PROTOCOL_VERSION,
|
|
37
|
+
runId: options.runId,
|
|
38
|
+
sequence: currentSequence,
|
|
39
|
+
eventId: `${options.runId}:${currentSequence}`,
|
|
40
|
+
type,
|
|
41
|
+
data
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
|
|
12
47
|
// src/protocol/state.ts
|
|
13
48
|
function createVideoState() {
|
|
14
49
|
return {
|
|
@@ -133,47 +168,9 @@ function applyVideoEvent(state, untrustedEvent) {
|
|
|
133
168
|
throw new Error(`Reducer does not yet support ${event.type}`);
|
|
134
169
|
}
|
|
135
170
|
|
|
136
|
-
// src/replayable-stream.ts
|
|
137
|
-
function createReplayableStream(source, onError) {
|
|
138
|
-
const buffered = [];
|
|
139
|
-
const waiters = /* @__PURE__ */ new Set();
|
|
140
|
-
let done = false;
|
|
141
|
-
let streamError;
|
|
142
|
-
const notify = () => {
|
|
143
|
-
for (const waiter of waiters) waiter();
|
|
144
|
-
waiters.clear();
|
|
145
|
-
};
|
|
146
|
-
void (async () => {
|
|
147
|
-
try {
|
|
148
|
-
for await (const item of source) {
|
|
149
|
-
buffered.push(item);
|
|
150
|
-
notify();
|
|
151
|
-
}
|
|
152
|
-
} catch (cause) {
|
|
153
|
-
streamError = cause;
|
|
154
|
-
onError(cause);
|
|
155
|
-
} finally {
|
|
156
|
-
done = true;
|
|
157
|
-
notify();
|
|
158
|
-
}
|
|
159
|
-
})();
|
|
160
|
-
return {
|
|
161
|
-
async *[Symbol.asyncIterator]() {
|
|
162
|
-
let index = 0;
|
|
163
|
-
while (true) {
|
|
164
|
-
while (index < buffered.length) yield buffered[index++];
|
|
165
|
-
if (done) {
|
|
166
|
-
if (streamError) throw streamError;
|
|
167
|
-
return;
|
|
168
|
-
}
|
|
169
|
-
await new Promise((resolve) => waiters.add(resolve));
|
|
170
|
-
}
|
|
171
|
-
}
|
|
172
|
-
};
|
|
173
|
-
}
|
|
174
|
-
|
|
175
171
|
export {
|
|
172
|
+
checksumVideo,
|
|
173
|
+
createVideoEventFactory,
|
|
176
174
|
createVideoState,
|
|
177
|
-
applyVideoEvent
|
|
178
|
-
createReplayableStream
|
|
175
|
+
applyVideoEvent
|
|
179
176
|
};
|