@vanillaskyai/video 0.10.2 → 0.10.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/CHANGELOG.md +17 -0
  2. package/PUBLIC-API.md +43 -31
  3. package/README.md +18 -9
  4. package/dist/check-runtime.js +2 -2
  5. package/dist/{chunk-KHLO5OHT.js → chunk-G5TYLTL5.js} +1 -1
  6. package/dist/{chunk-77VY4O7A.js → chunk-O6FBIB4L.js} +58 -10
  7. package/dist/{chunk-TVIU23OM.js → chunk-OOBT4X46.js} +13 -3
  8. package/dist/{chunk-5C6HZNHY.js → chunk-QZAZT44G.js} +3 -2
  9. package/dist/{chunk-6KZGF63O.js → chunk-WZFLEPLM.js} +58 -35
  10. package/dist/{chunk-VQH3JTQC.js → chunk-Z2ZI5G7O.js} +1 -1
  11. package/dist/{cinema-media-APDJS7SC.js → cinema-media-3SIGVVCY.js} +4 -4
  12. package/dist/{comparison-ZVE2DE7U.js → comparison-VDGRWPB5.js} +4 -4
  13. package/dist/{editorial-timeline-YIXXVJUV.js → editorial-timeline-QJU4JJWB.js} +4 -4
  14. package/dist/{key-figure-I2C37FWV.js → key-figure-YUAQW6OE.js} +4 -4
  15. package/dist/{mobile-message-JSGBU6EX.js → mobile-message-ZMMTUDUQ.js} +4 -4
  16. package/dist/{quote-XH54G3FS.js → quote-BB6UJT6L.js} +4 -4
  17. package/dist/react.d.ts +3 -1
  18. package/dist/react.js +94 -84
  19. package/dist/{scene-video-backdrop-PCTARAAJ.js → scene-video-backdrop-4EZVVQY7.js} +2 -2
  20. package/dist/server.d.ts +15 -3
  21. package/dist/server.js +416 -275
  22. package/dist/{types-CdBmNTGD.d.ts → types-BqB8zC9u.d.ts} +1 -1
  23. package/docs/agent-integration.md +5 -4
  24. package/docs/concepts.md +2 -2
  25. package/docs/development.md +23 -0
  26. package/docs/getting-started.md +7 -5
  27. package/docs/media-and-audio.md +37 -18
  28. package/docs/performance.md +21 -10
  29. package/docs/production.md +11 -10
  30. package/docs/prompt-and-input.md +24 -15
  31. package/docs/provider-integration.md +11 -13
  32. package/docs/reference/protocol.md +7 -1
  33. package/docs/reference/provider-adapters.md +7 -4
  34. package/docs/testing.md +7 -2
  35. package/package.json +5 -2
  36. package/registry/items/backgrounds.json +2 -2
  37. package/registry/items/cinemaMedia.json +1 -1
  38. package/registry/items/comparison.json +1 -1
  39. package/registry/items/editorialTimeline.json +1 -1
  40. package/registry/items/keyFigure.json +1 -1
  41. package/registry/items/mobileMessage.json +1 -1
  42. package/registry/items/quote.json +1 -1
  43. package/starters/video-chat/README.md +25 -26
  44. package/starters/video-chat/package.json +1 -1
  45. package/starters/video-chat/providers/video.ts +10 -6
  46. package/starters/video-chat/server.ts +1 -1
  47. package/starters/video-chat/stock.ts +59 -31
  48. package/styles/video-chat.css +6 -61
@@ -1,4 +1,4 @@
1
- type VideoChatMode = "cinematic";
1
+ type VideoChatMode = "cinematic" | "pexels";
2
2
  interface VideoChatConversationTurn {
3
3
  prompt: string;
4
4
  response?: string;
@@ -22,8 +22,9 @@ npm run dev
22
22
  ```
23
23
 
24
24
  Init generates the thin application-owned shell around the SDK's complete
25
- `VideoChat` experience and runs doctor automatically. The default uses packaged
26
- templates + browser voice and installs no optional provider packages.
25
+ `VideoChat` experience and runs doctor automatically. The baseline uses an immediate introduction and browser voice, and installs
26
+ no optional provider packages. Add the video adapter for moving video answers;
27
+ without a media provider, narration and subtitles remain available.
27
28
  `ANTHROPIC_API_KEY` is the only required key. Rerun init after an interrupted
28
29
  installation.
29
30
 
@@ -48,8 +49,8 @@ URL in a real browser. Browser automation should use normal motion
48
49
  its final frame, and returns to a usable composer.
49
50
  3. Ask one unrelated creative question in the same conversation and confirm it
50
51
  produces a distinct response.
51
- 4. If doctor reports generated video ready, choose that mode and verify one
52
- suitable prompt without editing client code.
52
+ 4. If doctor reports generated video ready, verify moving footage throughout
53
+ one complete answer, including its ending, without editing client code.
53
54
  5. Check failed network responses before reporting success.
54
55
 
55
56
  The handoff should name the localhost URL and ready capabilities. It should
package/docs/concepts.md CHANGED
@@ -23,7 +23,7 @@ trusted templates. It begins before the full plan is available and ends as an
23
23
  editable deterministic configuration.
24
24
 
25
25
  It is not an encoded video stream. The browser renders normal React components
26
- from validated scene instructions. The 0.1 SDK does not include MP4 or WebM
26
+ from validated scene instructions. The SDK does not include MP4 or WebM
27
27
  encoding; pass the completed deterministic JSON to an application-owned render
28
28
  or export pipeline when an encoded file is required.
29
29
 
@@ -31,7 +31,7 @@ or export pipeline when an encoded file is required.
31
31
 
32
32
  The viewer sends a prompt with bounded completed conversation turns. The server
33
33
  adds trusted application `instructions`, template capabilities, and the selected
34
- cinematic mode. `VideoChat` options control style, orientation, and custom
34
+ AI video or Pexels mode. `VideoChat` options control style, orientation, and custom
35
35
  templates. Exact facts belong in the authorized prompt or conversation; secrets
36
36
  and provider configuration stay on the server.
37
37
 
@@ -0,0 +1,23 @@
1
+ # Develop the chat experience
2
+
3
+ Run `npm ci --no-audit`, then `npm run dev:chat`. The localhost HMR surface renders the actual SDK `VideoChat`, templates and server handler from source. It never loads provider credentials. Select an intent and fault condition in the development toolbar; use the chat's normal media-mode controls. The prompt box remains the real product UI.
4
+
5
+ Offline answers are deterministic for explanation, story, comedy, imagination, practical steps and golf. They use local waterfall footage and a recorded timing cue, **not narration that matches the displayed script**. This harness is for loading, playback, controls and recovery. It cannot prove generated answer quality. Conditions cover ready, delayed footage, missing media, decode failure, speech failure, exhausted video allowance and request throttling. Browser speech may be used in the speech-failure condition.
6
+
7
+ The toolbar labels source and fixture identity. It separates the first body surface from the first decoded moving-footage frame; the former is a renderer paint opportunity and can precede decode. Its bounded safe phase log records browser request/stream arrival phases, speech response completion, first speech, footage and buffer pauses. No prompts, narration, scene IDs or provider bodies are retained. Media start/end/skip reasons come from the handler’s separate host-only `onDiagnostic` callback, shown in the local terminal for offline fixtures; live hosts own that callback themselves. Stream arrival is not the server’s exact authorship timestamp. Local fixtures never fetch external footage or invoke a paid model. Mode boundaries, provider deadlines and host admission remain covered by their dedicated server and host suites.
8
+
9
+ ## Optional live host
10
+
11
+ Set `VANILLASKY_CHAT_LIVE_ENDPOINT` to an application-owned video-chat endpoint before starting the harness. The toolbar then offers **Connect live endpoint (uses allowance)**. It stays offline until that explicit click. Live fetches include host cookies, subject to browser cookie policy. The host must allow credentialed requests from the localhost origin and provide its normal authorization; credentials and allowances stay with that application. This tool does not proxy secrets, reset limits, or retry generated answers. Capabilities and welcome requests may run as soon as you connect. Request only the bounded live examples needed to evaluate actual quality.
12
+
13
+ ## Fast checks
14
+
15
+ `npm run check:chat` runs the harness/unit recovery cases, checks harness TypeScript and runs one Chromium recording with short footage looping under actual audio. Install Chromium once with `npx playwright install chromium`. The harness smoke additionally records submit-to-visible-chapter paint opportunity with a 200 ms warm-UI target. This is a browser animation-frame opportunity, not a physical display measurement. The target for the complete command is under a minute on a warm machine; the command reports its measured duration. It does not replace release checks or claim browser-wide/live-provider coverage.
16
+
17
+ ## One candidate for release verification
18
+
19
+ `npm run verify:release` runs registry, lint, type, unit, acceptance, size, production dependency audit, API, packed-consumer, onboarding, provider and all existing browser checks. It builds and packs once into ignored `artifacts/chat-candidate/`, then sends the exact tarball and integrity to every consumer verifier. Each verifier still uses its independent clean install. Browser and provider compatibility remain independent gates; no paid API calls are enabled by this command.
20
+
21
+ `candidate.json` records version, source commit, dirty-tree status, SHA-256 and npm integrity. A local candidate may include uncommitted work; it is never called a published artifact. `VANILLASKY_CANDIDATE_DIR` selects a different output directory. The command verifies but never publishes, merges or deploys. CI additionally retains the existing Node and React version checks.
22
+
23
+ For one targeted consumer check, run `node scripts/chat-candidate.mjs`, then provide `VANILLASKY_PACKED_TARBALL`, `VANILLASKY_EXPECTED_INTEGRITY` and `VANILLASKY_EXPECTED_SHA256` from that manifest to an existing verifier. Do not repack between checks of the same candidate.
@@ -3,9 +3,10 @@
3
3
  # Getting started
4
4
 
5
5
  The fastest VanillaSky integration is the complete, general-purpose video chat.
6
- It starts with packaged templates and browser voice, then turns on optional
7
- speech and generated video when you install their adapters and add server keys.
8
- Stock media needs only its server key.
6
+ It starts with a template introduction and browser voice. Add the video adapter
7
+ and its server key for generated footage, or choose Pexels for stock footage.
8
+ Without a media provider, the answer retains narration and subtitles with an
9
+ chapter template. Generated speech is optional.
9
10
 
10
11
  ## Create the app
11
12
 
@@ -96,8 +97,9 @@ spoken hook from the response stream before the full plan is complete, holds its
96
97
  scene is ready, reaches its final frame, and leaves the composer ready for
97
98
  another turn. With stock media enabled, click a welcome or follow-up card and
98
99
  confirm its footage carries directly into that opening. With generated video
99
- enabled, confirm that relevant generated shots and editorial templates form one
100
- cinematic response, and the first shot continues the hook without repeating it.
100
+ enabled, confirm that moving shots develop a complete answer with narration
101
+ and subtitles, and that the first shot continues the hook without repeating it.
102
+ Check that no finished clip freezes while narration continues.
101
103
 
102
104
  The generated local authorization accepts localhost only. Replace it with your
103
105
  real session check, rate limits, and usage policy before deploying.
@@ -6,22 +6,23 @@ VanillaSky keeps provider choice in the application. The SDK defines small
6
6
  server callbacks, advertises only the capabilities you configure, and keeps
7
7
  all credentials out of React and the browser bundle.
8
8
 
9
- ## One cinematic mode
9
+ ## AI-first video answers
10
10
 
11
- The director combines relevant footage and seven editorial templates. Graphics
12
- use black and white; footage carries atmosphere and concrete action. Each media
13
- scene declares `mediaSource: "stock"` or `"generate"`. Provider availability and
14
- host budgets constrain those choices. There is no all-stock or all-AI switch.
11
+ The default chat displays the real chapter template immediately, then prepares
12
+ speech and footage concurrently. Choose `mode: "cinematic"` for AI video or
13
+ `mode: "pexels"` for stock. The UI labels these choices **AI video** and **Pexels**.
14
+ Each mode uses only its selected media provider. A failure, exhausted video
15
+ allowance, or missed deadline becomes an authored chapter with complete narration.
16
+ The chapter retains narration and subtitles for the whole beat.
15
17
 
16
- The canonical templates are `cinemaMedia`, `chapterTitle`,
17
- `editorialTimeline`, `mobileMessage`, `comparison`, `quote`, and `keyFigure`.
18
- All except `chapterTitle` accept optional media backgrounds. Editorial overlays use soft text shadows and a dark contrast scrim over relevant media, and fall back to black. Use at most one explanatory overlay per typical thirty-second answer. Full-bleed
19
- footage has no headline; narration and subtitles carry the explanation.
18
+ ## Pexels search
20
19
 
21
- ## Reviewed stock
20
+ The starter performs bounded full-catalog Pexels video search with subject
21
+ matching, orientation-aware renditions and a bounded cache. Add `PEXELS_API_KEY`
22
+ on the server and choose Pexels in Settings. The SDK UI links to Pexels; custom
23
+ interfaces must also display the attribution required by their media provider.
22
24
 
23
- Add `searchMedia` when template answers, the welcome screen, and follow-up
24
- cards should use approved photography or footage:
25
+ Applications can replace `searchMedia` with their own licensed catalog:
25
26
 
26
27
  ```ts
27
28
  createVideoChatHandler({
@@ -45,14 +46,14 @@ createVideoChatHandler({
45
46
  The planner emits a short semantic keyword, not a URL. The callback returns an
46
47
  application-approved image or video URL, and the SDK validates it before it
47
48
  reaches a scene. Return `null` when no licensed, safe, relevant asset exists;
48
- `cinemaMedia` becomes a chapter using its grounded `fallbackText`; a message can
49
- keep its content on black. Missing fallback copy produces an explicit error.
49
+ default chat displays an authored chapter while retaining the spoken answer. An explicit custom `templates` registry retains its structured planner,
50
+ validation, and fallback contracts.
50
51
 
51
52
  For Pexels, keep `PEXELS_API_KEY` on the server, enforce a deadline, filter for
52
53
  orientation, and return only validated Pexels asset domains. Licensing,
53
54
  attribution, caching, MIME checks, and byte limits remain application-owned.
54
55
 
55
- ## Planned generated shots
56
+ ## Generated shots
56
57
 
57
58
  Add `generateVideo` to enable generated shots within cinematic responses. It receives the planned visual
58
59
  subject plus the generated look so every clip can follow the same direction:
@@ -75,6 +76,8 @@ createVideoChatHandler({
75
76
  orientation,
76
77
  requestId,
77
78
  sceneId: scene?.id,
79
+ shotDirection: typeof scene?.variables.shotDirection === "string"
80
+ ? scene.variables.shotDirection : undefined,
78
81
  signal,
79
82
  maxRetries: 0,
80
83
  });
@@ -83,8 +86,9 @@ createVideoChatHandler({
83
86
  });
84
87
  ```
85
88
 
86
- The first streamed object reserves the first generated shot while the same
87
- model call continues planning later scenes. Use `requestId` and `scene.id` as
89
+ One model stream supplies an answer brief and shot descriptions. The runtime
90
+ prepares ordered shots while the model continues planning; the first and later
91
+ body shots follow the same path. Use `requestId` and `scene.id` as
88
92
  an idempotency key, because generated clips are billable. Keep `maxRetries: 0`
89
93
  inside provider calls, honour `signal`, and make retries an explicit product
90
94
  decision with a known budget.
@@ -92,6 +96,19 @@ decision with a known budget.
92
96
  VanillaSky does not depend on a video model or storage service. The application
93
97
  owns the provider key, model, spend, generated bytes, retention, and delivery.
94
98
 
99
+ ### Timing and recovery
100
+
101
+ Set `generatedClipDurationSec` to the duration your video adapter actually
102
+ requests (default 5, supported range 2–20 seconds). Keep it aligned with provider
103
+ settings and host spending limits. The planner fits natural spoken beats to that
104
+ budget; measured audio determines the final scene timing. Silent footage may
105
+ loop until the finite scene ends. Audible footage is not looped.
106
+
107
+ `generateVideoTimeoutMs` sets the first-shot preparation budget (default 15 seconds).
108
+ Later deadlines account for their position in the answer rather than restarting
109
+ an unlimited wait. Hosts must honor cancellation. A missed deadline selects the
110
+ authored chapter instead of a second paid generation or cross-mode stock search.
111
+
95
112
  ## Voice and transcription
96
113
 
97
114
  Without `generateSpeech`, `VideoChat` uses the browser voice. Add a speech
@@ -129,7 +146,9 @@ interaction before audible playback on many devices.
129
146
  - Keep every provider key in server-only environment variables.
130
147
  - Never let a planner return arbitrary final media URLs.
131
148
  - Bound query length, response size, duration, concurrency, and generated spend.
132
- - Preload the next asset and keep the current visual when media is late.
149
+ - Preload upcoming footage and pause narration honestly when usable media is late.
150
+ - Plan enough moving footage for the spoken beat; do not hold a finished frame
151
+ while narration continues.
133
152
  - Return a safe fallback instead of leaving the response waiting forever.
134
153
 
135
154
  [← Documentation home](../README.md) · [Previous: Customization](customization.md) · [Next: Custom templates →](custom-templates.md)
@@ -20,6 +20,13 @@ The SDK sends nothing to a telemetry service. Events contain only an opaque
20
20
  turn ID, mode, relative timing, and fixed event categories. Keep custom turn IDs
21
21
  opaque; do not put prompts or customer information into them.
22
22
 
23
+ The handler's optional `onDiagnostic(event)` observes accepted requests,
24
+ authored openings and shots, and media start/end/skip timings on the host.
25
+ Fixed reasons distinguish allowance, deadline, timeout, provider error, empty
26
+ results, cancellation and absent configuration. These records contain no
27
+ prompt, narration, query, media URL or provider response. They are never sent
28
+ to the browser automatically, and callback failures cannot stop an answer.
29
+
23
30
  ## What the measurements mean
24
31
 
25
32
  `first-frame` is the first committed active scene reaching an animation-frame
@@ -55,15 +62,19 @@ these against an explicitly authorized, bounded live run before treating them
55
62
  as tuned provider budgets. Compare first-frame/speech times, stalled duration,
56
63
  and visual/voice quality together; faster fallback alone does not prove quality.
57
64
 
58
- ## Opening stock footage
65
+ ## Opening and media preparation
66
+
67
+ The submitted prompt immediately appears in the chapter template; the streamed
68
+ opening replaces that topic with an authored spoken beat. The default UI makes
69
+ no opening stock request. Each body beat starts speech preparation while its
70
+ selected footage source prepares. At most two speech preparations run together.
59
71
 
60
- The opening's stock lookup runs alongside narration and body generation. Ready
61
- footage stays behind the hook until the body starts; late results are ignored.
62
- Playback never waits solely for intro footage.
72
+ `first-media-frame` reports the first decoded footage frame presented by the
73
+ mounted media surface. It is separate from `first-frame`, which also includes
74
+ chapter scenes. Neither callback measures the immediate opening template;
75
+ measure that surface separately when checking submit-to-template latency.
63
76
 
64
- The packaged stock adapter tries the specific and optional broader video query
65
- before photo recovery, within one three-second budget. The planner supplies the
66
- broader query only for opening atmosphere; body demonstrations retain their
67
- specific subject. Pexels resource-page slugs and photo descriptions can screen
68
- obvious mismatches, but missing metadata remains unknown and is accepted. This
69
- is not visual relevance verification. See the [Pexels API contract](https://www.pexels.com/api/documentation/).
77
+ AI and Pexels modes remain separate. Missing, late, or unplayable footage uses
78
+ the authored chapter and complete narration. Silent clips loop for the finite
79
+ narrated scene. See [the local chat harness](development.md) for fixture timing
80
+ and explicit live-provider checks.
@@ -22,10 +22,11 @@ player. Read the [security guide](security.md) for the complete controls.
22
22
 
23
23
  ## Cinematic direction and providers
24
24
 
25
- One cinematic mode combines footage and editorial graphics according to the
26
- story. Keep templates available as grounded fallbacks. Add `searchMedia` for
27
- reviewed stock and `generateVideo` for selected illustrative shots. The planner
28
- sets `mediaSource` per media scene; the host enforces the generation budget.
25
+ Default chat shows an immediate chapter while speech and selected footage prepare.
26
+ Configure `generateVideo` for AI mode and `searchMedia` for Pexels mode. Neither
27
+ mode calls the other footage source. Missing or late footage uses the authored
28
+ chapter with complete narration. Explicit custom template registries keep their
29
+ existing composition and fallback contracts.
29
30
  A stock candidate must match the subject, action and permitted crop. Return
30
31
  `null` for uncertainty rather than broadening an essential detail.
31
32
 
@@ -36,11 +37,11 @@ licensing before use.
36
37
 
37
38
  ## Fast first response
38
39
 
39
- The planner's first streamed object supplies the spoken hook and media keyword.
40
- Start speech immediately, resolve stock in parallel, and keep the opening
41
- playing until the contiguous preparation cushion and first frame are ready. Welcome cards
42
- should carry a prepared hook and preloaded footage so they can start without a
43
- model round trip.
40
+ The prompt appears immediately in the chapter template. The planner's first
41
+ streamed object supplies an authored spoken opening and reserves the ending.
42
+ Prepare each shot's speech alongside its footage, and keep the opening readable
43
+ until its narration and the contiguous preparation cushion are ready. Welcome
44
+ cards may carry a prepared opening so its speech starts without a model round trip.
44
45
 
45
46
  Do not wait for the complete plan before showing the first validated scene.
46
47
  Preload upcoming assets and keep the current visual if the next one is late.
@@ -103,7 +104,7 @@ npm test
103
104
  - [ ] Keys exist only in the server secret store.
104
105
  - [ ] Authentication, tenant policy, rate limits, and origin allowlist are live.
105
106
  - [ ] Cancellation, timeouts, fallbacks, and safe errors are tested.
106
- - [ ] Grounded graphic recovery works when every optional provider is unavailable.
107
+ - [ ] Authored chapter recovery works when every optional provider is unavailable.
107
108
  - [ ] Per-scene media choices obey provider availability and spending limits.
108
109
  - [ ] Both orientations render and narration stays synchronized.
109
110
  - [ ] A packed-artifact consumer and deterministic browser chat pass.
@@ -55,7 +55,7 @@ viewer prompt separate from these trusted server-side instructions.
55
55
 
56
56
  ## What reaches the model
57
57
 
58
- `createVideoChatHandler` builds the trusted template catalog, video rules,
58
+ `createVideoChatHandler` builds the shot-planning instructions, video rules,
59
59
  conversation context, and application guidance. Your provider adapter receives
60
60
  two complete strings:
61
61
 
@@ -68,9 +68,9 @@ streamText: ({ systemPrompt, userPrompt, signal }) => streamText({
68
68
  });
69
69
  ```
70
70
 
71
- Pass both strings unchanged. The system prompt describes the installed
72
- templates, schema limits, pacing, narration, opening contract, and safe media
73
- fields. The user prompt contains the current request, bounded prior turns,
71
+ Pass both strings unchanged. The system prompt describes the internal answer
72
+ brief and shot format, pacing, narration, opening contract, and safe media
73
+ directions. The user prompt contains the current request, bounded prior turns,
74
74
  orientation, visual mode, and whether an opening was already spoken.
75
75
 
76
76
  Provider credentials and raw media URLs never belong in either prompt. Media
@@ -79,19 +79,27 @@ output has been parsed.
79
79
 
80
80
  ## One stream, one answer
81
81
 
82
- The planner first emits a 6–9 word spoken hook and a media keyword, then keeps
83
- streaming complete scenes. It is not a separate hook call followed by a second
84
- planning call. This keeps the opening consistent with the scenes that follow
85
- and lets the first playable scene arrive without waiting for the complete plan.
82
+ The planner streams an answer brief containing the opening, creative direction,
83
+ and ending, followed by narrated shot descriptions. One model request serves
84
+ both suggested and typed prompts; a suggestion may already supply its opening.
85
+ There is no separate classification or first-shot planning request.
86
+
87
+ The runtime assigns scene IDs, uses the footage renderer, resolves media, and
88
+ finalizes the response when planning ends. The model does not choose body
89
+ layouts, media providers, or lifecycle events. Narration and visible action are
90
+ planned together: explanations show mechanisms, stories develop consequences,
91
+ comedy times its reveal, imaginative requests depict their invented world, and
92
+ practical answers demonstrate usable steps. These are directions, not fixed
93
+ scene counts or one universal story structure.
86
94
 
87
95
  Every `scene.add` is validated before the browser receives it. The model never
88
- returns React, HTML, CSS, or executable JavaScript. Accepted scenes are
89
- immutable; invalid scenes are reported through safe warnings and omitted.
96
+ returns React, HTML, CSS, or executable JavaScript. Invalid planning content
97
+ produces safe diagnostics. A media failure does not delete valid narration.
90
98
 
91
- In template mode, the planner can choose any trusted template and request stock
92
- media through `searchMedia`. In full mode, it plans a complete generated-video
93
- answer and `generateVideo` resolves every visual beat. There is no mixed
94
- "generate a few clips" mode.
99
+ AI mode generates footage within the host allowance. Pexels mode searches
100
+ stock without calling the video generator. Each authored beat includes a short
101
+ chapter title; missing footage becomes that chapter with its complete narration. Handlers configured with an explicit custom `templates` registry continue to
102
+ support the trusted catalog and its existing structured planner contract.
95
103
 
96
104
  ## Grounding
97
105
 
@@ -110,7 +118,8 @@ Check these boundaries in order:
110
118
  3. Does the provider pass `systemPrompt` and `userPrompt` unchanged?
111
119
  4. Is extended reasoning delaying the first streamed object?
112
120
  5. Do `onWarning` and `onComplete` show rejected scenes or a length limit?
113
- 6. Does the trusted registry contain a suitable template for the requested answer?
121
+ 6. Do the planned actions develop the answer, and does its ending resolve the request?
122
+ 7. Does resolved footage actually play for the spoken duration?
114
123
 
115
124
  Log request IDs, safe warning codes, provider finish reasons, model IDs, and
116
125
  token usage. Never log credentials or expose raw provider errors in the video.
@@ -12,7 +12,7 @@ npm run dev
12
12
  ```
13
13
 
14
14
  Init runs doctor automatically. The generated `server.ts` starts with one
15
- `ANTHROPIC_API_KEY`, packaged templates, and browser voice; it installs no
15
+ `ANTHROPIC_API_KEY`, a template introduction, and browser voice; it installs no
16
16
  optional speech or video packages.
17
17
 
18
18
  Use `npx vanillasky providers add speech` to install xAI speech, or
@@ -58,14 +58,12 @@ browser speech. Supplying `generateSpeech`, `transcribe`, `searchMedia`, or
58
58
  structural and provider-neutral; their SDKs and credentials remain application
59
59
  dependencies and never enter the browser bundle.
60
60
 
61
- The planner emits a 6-9 word hook first, then continues into the scenes in the
62
- same stream. The stock lookup is a separate cancellable request, so it cannot
63
- delay speech or planning. A welcome card can carry a prewritten `opening`, which
64
- starts immediately with its already-loaded media. When generated shots are available, the
65
- same first streamed object reserves the exact first body scene. Its clip starts
66
- generating while the planner continues the story. Playback begins after its
67
- contiguous preparation cushion, without requiring the entire plan, and normal narration is written in that same planner call rather than
68
- through a second model round trip.
61
+ The planner emits a short spoken opening, then continues the answer in the same
62
+ stream. The default UI shows a chapter immediately. A welcome card can carry a
63
+ prewritten `opening`, whose narration starts without waiting for the model.
64
+ Each authored body beat prepares speech and selected footage together. Playback
65
+ starts after its contiguous preparation cushion, without requiring the entire
66
+ plan or a second model call for narration.
69
67
 
70
68
  The matching complete React interface is one component and one scoped style
71
69
  import:
@@ -84,9 +82,9 @@ export function App() {
84
82
  For a custom interface, use `useVideoChat` and render its `turns`, `welcome`,
85
83
  `suggestions`, `caption`, and `status`; the hook owns their network and playback
86
84
  lifecycle. Pass a selected card through
87
- `chat.ask(card.prompt, { opening: card.opening, openingMedia: card.media })` to
88
- start its hook and reuse its footage immediately. For typed prompts, the hook
89
- and media keyword arrive through the response stream automatically.
85
+ `chat.ask(card.prompt, { opening: card.opening })` to start its hook immediately.
86
+ Custom interfaces can still pass and render `openingMedia`; the default UI uses
87
+ the chapter. Typed prompts receive their authored opening through the stream.
90
88
 
91
89
  Any AI SDK `LanguageModel` works in both `streamText` and `generateText`. Keep
92
90
  selection in one server-only module when an application supports several text
@@ -180,7 +178,7 @@ is intentionally substantial. It is stable for the same SDK version, template
180
178
  kit, media policy, and base prompt. Record input-token usage, keep the selected
181
179
  kit no broader than the product needs, and enable provider-side prompt caching
182
180
  where the chosen provider/model supports it. VanillaSky does not assume one
183
- provider's cache controls in its provider-neutral adapter. The cinematic catalog contains seven templates; use provider-reported token
181
+ provider's cache controls in its provider-neutral adapter. Default chat streams an answer brief and shot directions rather than the template catalog; use provider-reported token
184
182
  usage as the authoritative measurement rather than a character estimate.
185
183
 
186
184
  Provider finish reasons `error` and `tool-calls` are terminal failures.
@@ -70,7 +70,13 @@ change core video state.
70
70
 
71
71
  ## Planning boundary
72
72
 
73
- LLMs do not emit public protocol envelopes. A server-only planner emits
73
+ LLMs do not emit public protocol envelopes. Default `createVideoChatHandler`
74
+ planning uses an internal answer brief and shot descriptions. The runtime
75
+ translates them into footage scenes and completes the answer at stream end,
76
+ without asking the model for lifecycle commands.
77
+
78
+ Handlers with an explicit custom `templates` registry keep the existing
79
+ composition planner contract. A server-only planner emits
74
80
  validated `scene.add` or `plan.complete` parts. The runtime assigns sequences,
75
81
  IDs, terminal snapshots, and checksums. Generated HTML, React, JavaScript, CSS, component
76
82
  source, audio events, protocol envelopes, and unknown part types are rejected.
@@ -69,7 +69,9 @@ VanillaSky or its protocol. The same boundary also accepts a self-hosted model
69
69
  or a provider-native async text stream when it is not represented in the AI
70
70
  SDK. VanillaSky has no model allowlist.
71
71
 
72
- The provider must emit NDJSON text: one complete VanillaSky plan part per line.
72
+ The provider must emit NDJSON text matching the supplied prompt: one complete
73
+ object per line. Default chat planning uses an answer brief and shot descriptions;
74
+ the SDK translates them into validated scene events.
73
75
  The SDK buffers arbitrary text chunks until a newline, parses the completed
74
76
  object, validates it, and only then forwards it to the motion runtime. Do not
75
77
  replace that per-line validator with whole-response structured output: motion
@@ -93,9 +95,10 @@ stream-reconnect contract.
93
95
  ## Generated-video budget
94
96
 
95
97
  Set `maxGeneratedVideos` on `createVideoChatHandler` to a nonnegative safe integer
96
- (default `5`). This is a per-response attempt limit for full mode, including the
97
- reserved first shot and failures. Use `0` to keep the configured full mode on
98
- stock footage. Stock lookup is independent of this limit. Retries inside your
98
+ (default `5`). This is a per-response generation attempt limit, including
99
+ failures. AI mode uses authored chapter recovery after the allowance is reached;
100
+ `0` skips all generated footage. Pexels mode searches stock independently and
101
+ never consumes the generated-video allowance. Retries inside your
99
102
  provider callback can incur additional charges; bound those separately. Never
100
103
  copy an untrusted request value into this application-owned option.
101
104
 
package/docs/testing.md CHANGED
@@ -7,12 +7,15 @@ no network request, and needs no model key.
7
7
  ## Test the chat route with Vitest
8
8
 
9
9
  Pass `createMockVideoPlanner()` to the same `createVideoChatHandler` used by the
10
- application. A standard `Request` exercises parsing, validation, pacing, the
10
+ application, using an explicit template registry for the structured composition
11
+ fixtures. Default AI-first chat tests should instead return an authored answer
12
+ brief and shot descriptions matching the supplied planning prompt. A standard
13
+ `Request` exercises parsing, validation, pacing, the
11
14
  opening extension, and SSE without starting an HTTP server.
12
15
 
13
16
  ```ts
14
17
  import { describe, expect, it } from "vitest";
15
- import { createVideoChatHandler } from "@vanillaskyai/video/server";
18
+ import { createVideoChatHandler, createServerTemplateRegistry } from "@vanillaskyai/video/server";
16
19
  import { createMockVideoPlanner } from "@vanillaskyai/video/test";
17
20
 
18
21
  describe("POST /api/video-chat", () => {
@@ -20,6 +23,7 @@ describe("POST /api/video-chat", () => {
20
23
  const handle = createVideoChatHandler({
21
24
  authorize: "none", // Only acceptable because this handler stays in process.
22
25
  heartbeatMs: false,
26
+ templates: createServerTemplateRegistry({ templates: [] }),
23
27
  streamText: createMockVideoPlanner(),
24
28
  generateText: async ({ task }) => task === "suggestions"
25
29
  ? JSON.stringify({ suggestions: [] })
@@ -43,6 +47,7 @@ describe("POST /api/video-chat", () => {
43
47
 
44
48
  expect(response.status).toBe(200);
45
49
  expect(body).toContain('"type":"data.video-chat-opening"');
50
+ expect(body).toContain('"type":"scene.add"');
46
51
  expect(body).toContain('"type":"response.complete"');
47
52
  });
48
53
  });
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@vanillaskyai/video",
3
- "version": "0.10.2",
3
+ "version": "0.10.4",
4
4
  "description": "Open-source voice-and-video chat SDK for AI applications.",
5
5
  "keywords": [
6
6
  "video-chat",
@@ -116,7 +116,10 @@
116
116
  "verify:api": "npm run build && node scripts/verify-public-api-surface.mjs",
117
117
  "verify:package": "npm run build && node scripts/verify-packed-package.mjs",
118
118
  "release:build": "node scripts/release-build.mjs",
119
- "acceptance:chat": "tsx scripts/acceptance/run.ts"
119
+ "acceptance:chat": "tsx scripts/acceptance/run.ts",
120
+ "dev:chat": "vite --config dev/chat/vite.config.ts --open /dev/chat/",
121
+ "check:chat": "node scripts/chat-check.mjs",
122
+ "verify:release": "node scripts/chat-verify.mjs"
120
123
  },
121
124
  "peerDependencies": {
122
125
  "react": ">=18 <20",
@@ -28,7 +28,7 @@
28
28
  "path": "src/visual-system/scene-templates/external-video-backdrop.tsx",
29
29
  "type": "registry:lib",
30
30
  "target": "vanillasky/scene-templates/external-video-backdrop.tsx",
31
- "content": "import React from \"react\";\n\n// Source-owned templates may live in a consumer's tree while VideoFrame comes\n// from the package. Both copies must observe the same internal context or the\n// consumer template would mount a second video over the player-owned plane,\n// and would never inherit the player's native media audio state.\nexport type ExternalVideoBackdropMode = false | \"pending\" | \"ready\" | \"fallback\";\n\ninterface BackdropContextValue {\n mode: ExternalVideoBackdropMode;\n audioMuted: boolean;\n audioVolume: number;\n}\n\nconst DEFAULT: BackdropContextValue = { mode: false, audioMuted: true, audioVolume: 1 };\n\nconst sharedContext = globalThis as typeof globalThis & {\n __vanillaskyVideoBackdropContext?: React.Context<BackdropContextValue>;\n};\nconst BackdropContext = sharedContext.__vanillaskyVideoBackdropContext\n ??= React.createContext<BackdropContextValue>(DEFAULT);\n\nexport function ExternalVideoBackdropProvider({\n mode,\n audioMuted = true,\n audioVolume = 1,\n children,\n}: {\n mode: ExternalVideoBackdropMode;\n audioMuted?: boolean;\n audioVolume?: number;\n children: React.ReactNode;\n}) {\n const value = React.useMemo(\n () => ({ mode, audioMuted, audioVolume }),\n [mode, audioMuted, audioVolume],\n );\n return (\n <BackdropContext.Provider value={value}>\n {children}\n </BackdropContext.Provider>\n );\n}\n\nexport function useExternalVideoBackdrop(): ExternalVideoBackdropMode {\n return React.useContext(BackdropContext).mode;\n}\n\nexport function useMediaAudio(): { muted: boolean; volume: number } {\n const { audioMuted, audioVolume } = React.useContext(BackdropContext);\n return { muted: audioMuted, volume: audioVolume };\n}\n"
31
+ "content": "import React from \"react\";\n\n// Source-owned templates may live in a consumer's tree while VideoFrame comes\n// from the package. Both copies must observe the same internal context or the\n// consumer template would mount a second video over the player-owned plane,\n// and would never inherit the player's native media audio state.\nexport type ExternalVideoBackdropMode = false | \"pending\" | \"ready\" | \"fallback\";\n\ninterface BackdropContextValue {\n mode: ExternalVideoBackdropMode;\n audioMuted: boolean;\n audioVolume: number;\n preparingNarration?: boolean;\n onMediaError?: () => void;\n}\n\nconst DEFAULT: BackdropContextValue = { mode: false, audioMuted: true, audioVolume: 1 };\n\nconst sharedContext = globalThis as typeof globalThis & {\n __vanillaskyVideoBackdropContext?: React.Context<BackdropContextValue>;\n};\nconst BackdropContext = sharedContext.__vanillaskyVideoBackdropContext\n ??= React.createContext<BackdropContextValue>(DEFAULT);\n\nexport function ExternalVideoBackdropProvider({\n mode,\n audioMuted = true,\n audioVolume = 1,\n preparingNarration = false,\n onMediaError,\n children,\n}: {\n mode: ExternalVideoBackdropMode;\n audioMuted?: boolean;\n audioVolume?: number;\n preparingNarration?: boolean;\n onMediaError?: () => void;\n children: React.ReactNode;\n}) {\n const value = React.useMemo(\n () => ({ mode, audioMuted, audioVolume, preparingNarration, onMediaError }),\n [mode, audioMuted, audioVolume, preparingNarration, onMediaError],\n );\n return (\n <BackdropContext.Provider value={value}>\n {children}\n </BackdropContext.Provider>\n );\n}\n\nexport function useExternalVideoBackdrop(): ExternalVideoBackdropMode {\n return React.useContext(BackdropContext).mode;\n}\n\nexport function useMediaAudio(): { muted: boolean; volume: number } {\n const { audioMuted, audioVolume } = React.useContext(BackdropContext);\n return { muted: audioMuted, volume: audioVolume };\n}\n\n/** Internal first-frame priming state; an explicit viewer pause never sets it. */\nexport function useNarrationPreroll(): boolean {\n return React.useContext(BackdropContext).preparingNarration === true;\n}\n\n/** Routes local decoder/playback failures to the scene-owned recovery surface. */\nexport function useMediaFailure(): (() => void) | undefined {\n return React.useContext(BackdropContext).onMediaError;\n}\n"
32
32
  },
33
33
  {
34
34
  "path": "src/visual-system/scene-templates/media-position.ts",
@@ -40,7 +40,7 @@
40
40
  "path": "src/visual-system/scene-templates/scene-video-backdrop.tsx",
41
41
  "type": "registry:lib",
42
42
  "target": "vanillasky/scene-templates/scene-video-backdrop.tsx",
43
- "content": "import React, { useEffect, useRef, useState } from \"react\";\nimport { getBackgroundTransform } from \"../backgrounds\";\nimport { useMediaAudio } from \"./external-video-backdrop\";\nimport { resolveMediaPosition } from \"./media-position\";\n\nexport interface SceneVideoBackdropProps {\n mediaUrl: string;\n mediaPoster?: string;\n mediaPosition?: string;\n backgroundEffect?: string;\n progress: number;\n beatIntensity?: number;\n isPlaying: boolean;\n muted?: boolean;\n volume?: number;\n playbackId?: string;\n retainPoster?: boolean;\n persistent?: boolean;\n preparedPoster?: {\n presentationKey: string;\n mediaPoster: string;\n mediaPosition: string;\n backgroundEffect?: string;\n /** Existing global transition progress. On decoder-constrained Safari,\n * this fades the decoded incoming still above the outgoing video before\n * the single video element changes source. */\n opacity?: number;\n };\n onReady?: () => void;\n onError?: () => void;\n}\n\nexport const SceneVideoBackdrop: React.FC<SceneVideoBackdropProps> = ({\n mediaUrl,\n mediaPoster,\n mediaPosition = \"center\",\n backgroundEffect,\n progress,\n beatIntensity = 0,\n isPlaying,\n muted,\n volume,\n playbackId = mediaUrl,\n retainPoster = false,\n persistent = false,\n preparedPoster,\n onReady,\n onError,\n}) => {\n const inheritedAudio = useMediaAudio();\n const resolvedMuted = muted ?? inheritedAudio.muted;\n const resolvedVolume = volume ?? inheritedAudio.volume;\n const resolvedPosition = resolveMediaPosition(mediaPosition);\n const bgTransform = getBackgroundTransform(backgroundEffect, progress, beatIntensity);\n const [decodedVideoUrl, setDecodedVideoUrl] = useState<string>();\n const videoRef = useRef<HTMLVideoElement>(null);\n const startedVideoUrl = useRef<string | undefined>(undefined);\n const startedPlaybackId = useRef<string | undefined>(undefined);\n const videoPresentationKey = `${playbackId}\\0${mediaUrl}`;\n\n useEffect(() => {\n const video = videoRef.current;\n if (!video) return;\n // React Strict Mode rehearses setup → cleanup → setup in development.\n // The cleanup deliberately releases the decoder, so the repeated setup\n // must restore the declarative source before the playback effect runs.\n if (video.getAttribute(\"src\") !== mediaUrl) {\n video.setAttribute(\"src\", mediaUrl);\n video.load();\n }\n return () => {\n video.pause();\n video.removeAttribute(\"src\");\n video.load();\n startedVideoUrl.current = undefined;\n startedPlaybackId.current = undefined;\n };\n }, [mediaUrl]);\n\n useEffect(() => {\n const video = videoRef.current;\n if (video) video.volume = resolvedVolume;\n }, [resolvedVolume]);\n\n useEffect(() => {\n const video = videoRef.current;\n if (!video) return;\n if (!isPlaying) {\n video.pause();\n return;\n }\n if (startedPlaybackId.current === playbackId) {\n if (!video.ended) void video.play().catch(() => {});\n return;\n }\n const changingSource = startedVideoUrl.current !== undefined && startedVideoUrl.current !== mediaUrl;\n video.playbackRate = 1;\n if (!changingSource && video.currentTime > 0) video.currentTime = 0;\n video.play().catch(() => {});\n startedVideoUrl.current = mediaUrl;\n startedPlaybackId.current = playbackId;\n }, [isPlaying, mediaUrl, playbackId]);\n\n const mediaStyle: React.CSSProperties = {\n position: \"absolute\",\n inset: 0,\n width: \"100%\",\n height: \"100%\",\n objectFit: \"cover\",\n objectPosition: resolvedPosition,\n transform: bgTransform.transform,\n transformOrigin: bgTransform.transformOrigin,\n zIndex: persistent ? 1 : undefined,\n };\n const preparedPosition = preparedPoster\n ? resolveMediaPosition(preparedPoster.mediaPosition)\n : resolvedPosition;\n const preparedTransform = getBackgroundTransform(preparedPoster?.backgroundEffect, 0, 0);\n const posterPlanes = [\n ...(persistent && mediaPoster ? [{\n presentationKey: videoPresentationKey,\n mediaPoster,\n mediaPosition: resolvedPosition,\n transform: bgTransform.transform,\n transformOrigin: bgTransform.transformOrigin,\n opacity: 1,\n zIndex: 0,\n role: \"current\",\n }] : []),\n ...(preparedPoster && preparedPoster.presentationKey !== videoPresentationKey ? [{\n presentationKey: preparedPoster.presentationKey,\n mediaPoster: preparedPoster.mediaPoster,\n mediaPosition: preparedPosition,\n transform: preparedTransform.transform,\n transformOrigin: preparedTransform.transformOrigin,\n opacity: preparedPoster.opacity ?? 0,\n zIndex: 2,\n role: \"prepared\",\n }] : []),\n ];\n\n return (\n <>\n {posterPlanes.map((posterPlane) => (\n <img\n key={posterPlane.presentationKey}\n src={posterPlane.mediaPoster}\n alt=\"\"\n aria-hidden=\"true\"\n draggable={false}\n data-video-poster-plane={posterPlane.role}\n data-video-poster-visible={posterPlane.opacity > 0 ? \"true\" : \"false\"}\n style={{\n position: \"absolute\",\n inset: 0,\n width: \"100%\",\n height: \"100%\",\n objectFit: \"cover\",\n objectPosition: posterPlane.mediaPosition,\n transform: posterPlane.transform,\n transformOrigin: posterPlane.transformOrigin,\n zIndex: posterPlane.zIndex,\n opacity: posterPlane.opacity,\n pointerEvents: \"none\",\n }}\n />\n ))}\n <video\n ref={videoRef}\n src={mediaUrl}\n poster={retainPoster || decodedVideoUrl !== mediaUrl ? mediaPoster || undefined : undefined}\n muted={resolvedMuted}\n loop={false}\n playsInline\n preload=\"auto\"\n onLoadedData={(event) => {\n const video = event.currentTarget;\n const markPresented = () => {\n if (!video.isConnected) return;\n onReady?.();\n if (!retainPoster) setDecodedVideoUrl(mediaUrl);\n };\n if (video.requestVideoFrameCallback) {\n video.requestVideoFrameCallback(markPresented);\n return;\n }\n markPresented();\n }}\n onError={onError}\n data-media-position={mediaPosition}\n data-video-backdrop={persistent ? \"persistent\" : \"scene\"}\n style={mediaStyle}\n />\n </>\n );\n};\n"
43
+ "content": "import React, { useCallback, useEffect, useRef, useState } from \"react\";\nimport { getBackgroundTransform } from \"../backgrounds\";\nimport { useMediaAudio, useMediaFailure, useNarrationPreroll } from \"./external-video-backdrop\";\nimport { resolveMediaPosition } from \"./media-position\";\n\nexport interface SceneVideoBackdropProps {\n mediaUrl: string;\n mediaPoster?: string;\n mediaPosition?: string;\n backgroundEffect?: string;\n progress: number;\n /** Narration-led visible duration; muted or pitch-preserving footage may be gently retimed. */\n sceneDuration?: number;\n /** Internal player-owned decoder priming, distinct from viewer pause. */\n preparingNarration?: boolean;\n beatIntensity?: number;\n isPlaying: boolean;\n muted?: boolean;\n volume?: number;\n playbackId?: string;\n retainPoster?: boolean;\n persistent?: boolean;\n preparedPoster?: {\n presentationKey: string;\n mediaPoster: string;\n mediaPosition: string;\n backgroundEffect?: string;\n /** Existing global transition progress. On decoder-constrained Safari,\n * this fades the decoded incoming still above the outgoing video before\n * the single video element changes source. */\n opacity?: number;\n };\n onReady?: () => void;\n onError?: () => void;\n}\n\nexport const SceneVideoBackdrop: React.FC<SceneVideoBackdropProps> = ({\n mediaUrl,\n mediaPoster,\n mediaPosition = \"center\",\n backgroundEffect,\n progress,\n sceneDuration,\n preparingNarration = false,\n beatIntensity = 0,\n isPlaying,\n muted,\n volume,\n playbackId = mediaUrl,\n retainPoster = false,\n persistent = false,\n preparedPoster,\n onReady,\n onError,\n}) => {\n const inheritedAudio = useMediaAudio();\n const reportMediaFailure = useMediaFailure();\n const inheritedPreroll = useNarrationPreroll();\n const rewindPreroll = preparingNarration || inheritedPreroll;\n const resolvedMuted = muted ?? inheritedAudio.muted;\n const resolvedVolume = volume ?? inheritedAudio.volume;\n const resolvedPosition = resolveMediaPosition(mediaPosition);\n const bgTransform = getBackgroundTransform(backgroundEffect, progress, beatIntensity);\n const [decodedVideoUrl, setDecodedVideoUrl] = useState<string>();\n const [waitingKey, setWaitingKey] = useState<string>();\n const [exhaustedKey, setExhaustedKey] = useState<string>();\n\n const videoRef = useRef<HTMLVideoElement>(null);\n const startedVideoUrl = useRef<string | undefined>(undefined);\n const startedPlaybackId = useRef<string | undefined>(undefined);\n const videoPresentationKey = `${playbackId}\\0${mediaUrl}`;\n\n const presentationRef = useRef({ key: videoPresentationKey, playing: isPlaying });\n presentationRef.current = { key: videoPresentationKey, playing: isPlaying };\n const unavailable = () => {\n if (presentationRef.current.key === videoPresentationKey && presentationRef.current.playing) {\n setExhaustedKey(videoPresentationKey);\n onError?.();\n reportMediaFailure?.();\n }\n };\n const fitDuration = useCallback((video: HTMLVideoElement) => {\n // Allow a small decode-to-speech onset margin without changing narration.\n video.playbackRate = (resolvedMuted || video.preservesPitch === true) && sceneDuration && Number.isFinite(video.duration) && video.duration > 0\n ? Math.max(.75, Math.min(1, video.duration / (sceneDuration + .2))) : 1;\n }, [resolvedMuted, sceneDuration]);\n useEffect(() => {\n if (videoRef.current) fitDuration(videoRef.current);\n }, [fitDuration]);\n const continueMotion = (video: HTMLVideoElement) => {\n if (!isPlaying) return;\n // The finite scene clock bounds silent coverage. Speech may outlast a\n // short clip; repeat motion until the scene ends, never audible dialogue.\n if (!resolvedMuted) {\n unavailable();\n return;\n }\n video.currentTime = 0;\n void video.play().catch(unavailable);\n };\n\n useEffect(() => {\n const video = videoRef.current;\n if (!video) return;\n // React Strict Mode rehearses setup → cleanup → setup in development.\n // The cleanup deliberately releases the decoder, so the repeated setup\n // must restore the declarative source before the playback effect runs.\n if (video.getAttribute(\"src\") !== mediaUrl) {\n video.setAttribute(\"src\", mediaUrl);\n video.load();\n }\n return () => {\n video.pause();\n video.removeAttribute(\"src\");\n video.load();\n startedVideoUrl.current = undefined;\n startedPlaybackId.current = undefined;\n };\n }, [mediaUrl]);\n\n useEffect(() => {\n const video = videoRef.current;\n if (video) video.volume = resolvedVolume;\n }, [resolvedVolume]);\n\n useEffect(() => {\n const video = videoRef.current;\n if (!video) return;\n if (!isPlaying) {\n video.pause();\n if (rewindPreroll && video.currentTime > 0) video.currentTime = 0;\n return;\n }\n if (startedPlaybackId.current === playbackId) {\n if (video.ended) continueMotion(video);\n else void video.play().catch(unavailable);\n return;\n }\n const changingSource = startedVideoUrl.current !== undefined && startedVideoUrl.current !== mediaUrl;\n fitDuration(video);\n if (!changingSource && video.currentTime > 0) video.currentTime = 0;\n video.play().catch(unavailable);\n startedVideoUrl.current = mediaUrl;\n startedPlaybackId.current = playbackId;\n }, [isPlaying, mediaUrl, playbackId, rewindPreroll]);\n\n const mediaStyle: React.CSSProperties = {\n position: \"absolute\",\n inset: 0,\n width: \"100%\",\n height: \"100%\",\n objectFit: \"cover\",\n objectPosition: resolvedPosition,\n transform: bgTransform.transform,\n transformOrigin: bgTransform.transformOrigin,\n zIndex: persistent ? 1 : undefined,\n };\n const preparedPosition = preparedPoster\n ? resolveMediaPosition(preparedPoster.mediaPosition)\n : resolvedPosition;\n const preparedTransform = getBackgroundTransform(preparedPoster?.backgroundEffect, 0, 0);\n const posterPlanes = [\n ...(persistent && mediaPoster ? [{\n presentationKey: videoPresentationKey,\n mediaPoster,\n mediaPosition: resolvedPosition,\n transform: bgTransform.transform,\n transformOrigin: bgTransform.transformOrigin,\n opacity: 1,\n zIndex: 0,\n role: \"current\",\n }] : []),\n ...(preparedPoster && preparedPoster.presentationKey !== videoPresentationKey ? [{\n presentationKey: preparedPoster.presentationKey,\n mediaPoster: preparedPoster.mediaPoster,\n mediaPosition: preparedPosition,\n transform: preparedTransform.transform,\n transformOrigin: preparedTransform.transformOrigin,\n opacity: preparedPoster.opacity ?? 0,\n zIndex: 2,\n role: \"prepared\",\n }] : []),\n ];\n\n return (\n <>\n {posterPlanes.map((posterPlane) => (\n <img\n key={posterPlane.presentationKey}\n src={posterPlane.mediaPoster}\n alt=\"\"\n aria-hidden=\"true\"\n draggable={false}\n data-video-poster-plane={posterPlane.role}\n data-video-poster-visible={posterPlane.opacity > 0 ? \"true\" : \"false\"}\n style={{\n position: \"absolute\",\n inset: 0,\n width: \"100%\",\n height: \"100%\",\n objectFit: \"cover\",\n objectPosition: posterPlane.mediaPosition,\n transform: posterPlane.transform,\n transformOrigin: posterPlane.transformOrigin,\n zIndex: posterPlane.zIndex,\n opacity: posterPlane.opacity,\n pointerEvents: \"none\",\n }}\n />\n ))}\n {(exhaustedKey === videoPresentationKey || waitingKey === videoPresentationKey) && <div\n role=\"status\" data-media-continuity={exhaustedKey === videoPresentationKey ? \"exhausted\" : \"waiting\"}\n style={{ position: \"absolute\", inset: 0, zIndex: 3, background: \"#000\", color: \"#bbb\", display: \"grid\", placeContent: \"center\", font: \"14px system-ui\" }}\n >{exhaustedKey === videoPresentationKey ? \"Visual unavailable\" : \"Loading visual\"}</div>}\n <video\n ref={videoRef}\n src={mediaUrl}\n poster={retainPoster || decodedVideoUrl !== mediaUrl ? mediaPoster || undefined : undefined}\n muted={resolvedMuted}\n loop={false}\n playsInline\n preload=\"auto\"\n onLoadedMetadata={event => fitDuration(event.currentTarget)}\n onEnded={event => continueMotion(event.currentTarget)}\n onWaiting={() => { if (isPlaying) setWaitingKey(videoPresentationKey); }}\n onPlaying={() => setWaitingKey(undefined)}\n onLoadedData={(event) => {\n const video = event.currentTarget;\n const markPresented = () => {\n if (!video.isConnected) return;\n onReady?.();\n if (!retainPoster) setDecodedVideoUrl(mediaUrl);\n };\n if (video.requestVideoFrameCallback) {\n video.requestVideoFrameCallback(markPresented);\n return;\n }\n markPresented();\n }}\n onError={onError}\n data-media-position={mediaPosition}\n data-video-backdrop={persistent ? \"persistent\" : \"scene\"}\n style={{ ...mediaStyle, visibility: exhaustedKey === videoPresentationKey || waitingKey === videoPresentationKey ? \"hidden\" : undefined }}\n />\n </>\n );\n};\n"
44
44
  },
45
45
  {
46
46
  "path": "src/visual-system/scene-templates/color-utils.ts",