@vanillaskyai/video 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +45 -0
- package/LICENSE +201 -0
- package/PUBLIC-API.md +276 -0
- package/README.md +152 -0
- package/SECURITY.md +22 -0
- package/SUPPORT.md +28 -0
- package/bin/vanillasky.js +3 -0
- package/dist/bg-confetti-WNQXS7ZS.js +89 -0
- package/dist/bg-emoji-ETI47GLS.js +150 -0
- package/dist/bg-media-SVUZAMGX.js +99 -0
- package/dist/brand-message-F3RD4P3P.js +253 -0
- package/dist/builtin-server-F7YXRVCH.js +8 -0
- package/dist/catalog-types-BIhSpOWK.d.ts +74 -0
- package/dist/chart-bar-CNSC7BNK.js +304 -0
- package/dist/chart-counter-MCRHKN77.js +299 -0
- package/dist/chart-progress-ring-RMI4A7ZK.js +296 -0
- package/dist/check-runtime.d.ts +15 -0
- package/dist/check-runtime.js +102 -0
- package/dist/chunk-2E6T633S.js +27 -0
- package/dist/chunk-352ZZCET.js +83 -0
- package/dist/chunk-73NTSFFI.js +81 -0
- package/dist/chunk-A2X45FET.js +46 -0
- package/dist/chunk-ABPKM4GK.js +187 -0
- package/dist/chunk-ABZACD4G.js +216 -0
- package/dist/chunk-BCRUUJ6A.js +756 -0
- package/dist/chunk-CHLGW22F.js +3551 -0
- package/dist/chunk-CJ7F2CE6.js +104 -0
- package/dist/chunk-CR7FE7BW.js +109 -0
- package/dist/chunk-E7CL7UPB.js +109 -0
- package/dist/chunk-EE6PK6XC.js +18 -0
- package/dist/chunk-ERY4S6QV.js +385 -0
- package/dist/chunk-EVG2ZYIY.js +125 -0
- package/dist/chunk-FNAEQ7QO.js +155 -0
- package/dist/chunk-FQ47WXAJ.js +38 -0
- package/dist/chunk-FSG2PYWG.js +336 -0
- package/dist/chunk-GHJEAP4O.js +504 -0
- package/dist/chunk-GRMUV7QY.js +262 -0
- package/dist/chunk-IFOW65Z6.js +205 -0
- package/dist/chunk-IXCUY3B3.js +786 -0
- package/dist/chunk-L5BTTCZV.js +97 -0
- package/dist/chunk-LMG3GHLL.js +107 -0
- package/dist/chunk-OOPIETLW.js +101 -0
- package/dist/chunk-P22HHFDK.js +43 -0
- package/dist/chunk-QAAO7OAY.js +19 -0
- package/dist/chunk-S7HFLYNN.js +35 -0
- package/dist/chunk-SPVTJH3F.js +24 -0
- package/dist/chunk-W2XJKG6C.js +3177 -0
- package/dist/chunk-XB7LH4J2.js +301 -0
- package/dist/chunk-XGRA2MUV.js +242 -0
- package/dist/chunk-XTOXFMLM.js +105 -0
- package/dist/chunk-YC5MFXA5.js +132 -0
- package/dist/cli.d.ts +1 -0
- package/dist/cli.js +1681 -0
- package/dist/compose-video-CN3VSWST.js +13 -0
- package/dist/cta-logo-SVA5GP2A.js +209 -0
- package/dist/cta-media-E7VLLK42.js +215 -0
- package/dist/events-_wfjbmEp.d.ts +81 -0
- package/dist/incoming-call-62MFZL3Z.js +320 -0
- package/dist/index.d.ts +15 -0
- package/dist/index.js +14 -0
- package/dist/infographic-before-after-AXCDAWND.js +460 -0
- package/dist/infographic-feature-list-GW5GUE6O.js +266 -0
- package/dist/infographic-problem-solution-7NURJANA.js +340 -0
- package/dist/infographic-stat-row-J7IZR6ZN.js +241 -0
- package/dist/infographic-steps-VSZM6F5M.js +400 -0
- package/dist/kit-BMNcHqa9.d.ts +79 -0
- package/dist/prompt-input-LOUCRXCY.js +236 -0
- package/dist/react.d.ts +81 -0
- package/dist/react.js +835 -0
- package/dist/reaction-QG4CYZLQ.js +119 -0
- package/dist/server.d.ts +141 -0
- package/dist/server.js +498 -0
- package/dist/showcase-code-RUTFZR3H.js +351 -0
- package/dist/showcase-phone-6CP6XEU5.js +272 -0
- package/dist/showcase-terminal-MGAEQWPR.js +306 -0
- package/dist/showcase-web-Y4W2HVRJ.js +390 -0
- package/dist/social-conversation-75SD2637.js +695 -0
- package/dist/social-milestone-XI2QE5BO.js +292 -0
- package/dist/social-notification-2UOHQCUY.js +244 -0
- package/dist/social-review-stack-FDYZGZTJ.js +239 -0
- package/dist/social-testimonial-OQ7N52GJ.js +284 -0
- package/dist/social-tweet-5OLCNPPC.js +295 -0
- package/dist/state-DZcKuS32.d.ts +3 -0
- package/dist/system-prompt-A6UXNTPK.js +8 -0
- package/dist/template-catalog.d.ts +24 -0
- package/dist/template-catalog.js +6 -0
- package/dist/templates.d.ts +25 -0
- package/dist/templates.js +19 -0
- package/dist/test.d.ts +149 -0
- package/dist/test.js +272 -0
- package/dist/text-stream-6XHC7J47.js +10 -0
- package/dist/types-B-PZR6G_.d.ts +208 -0
- package/dist/validate-T7GBU2YF.js +8 -0
- package/docs/agent-integration.md +39 -0
- package/docs/architecture.md +84 -0
- package/docs/branding-and-personalization.md +87 -0
- package/docs/concepts.md +106 -0
- package/docs/custom-templates.md +329 -0
- package/docs/customization.md +141 -0
- package/docs/errors.md +70 -0
- package/docs/getting-started.md +111 -0
- package/docs/input-and-first-scene.md +41 -0
- package/docs/integrate-nextjs.md +79 -0
- package/docs/media-and-audio.md +69 -0
- package/docs/motion-and-effects.md +107 -0
- package/docs/persistence.md +95 -0
- package/docs/production.md +122 -0
- package/docs/prompt-and-input.md +185 -0
- package/docs/provider-integration.md +115 -0
- package/docs/reference/protocol.md +96 -0
- package/docs/reference/provider-adapters.md +158 -0
- package/docs/responsive-orientation.md +37 -0
- package/docs/security.md +38 -0
- package/docs/streaming-protocol.md +15 -0
- package/docs/testing.md +137 -0
- package/docs/use-cases.md +59 -0
- package/examples/custom-template/README.md +19 -0
- package/examples/custom-template/minimal-text.tsx +82 -0
- package/examples/custom-template/structured-data.tsx +104 -0
- package/examples/custom-template/supplied-media.tsx +92 -0
- package/examples/nextjs-quickstart/.env.example +2 -0
- package/examples/nextjs-quickstart/README.md +22 -0
- package/examples/nextjs-quickstart/next-env.d.ts +4 -0
- package/examples/nextjs-quickstart/package.json +24 -0
- package/examples/nextjs-quickstart/src/app/api/video/route.ts +22 -0
- package/examples/nextjs-quickstart/src/app/layout.tsx +5 -0
- package/examples/nextjs-quickstart/src/app/page.tsx +31 -0
- package/examples/nextjs-quickstart/tsconfig.json +26 -0
- package/package.json +167 -0
- package/registry/items/backgrounds.json +52 -0
- package/registry/items/barChart.json +237 -0
- package/registry/items/beforeAfter.json +180 -0
- package/registry/items/bigNumber.json +225 -0
- package/registry/items/brandMessage.json +158 -0
- package/registry/items/cardList.json +206 -0
- package/registry/items/chatMessenger.json +151 -0
- package/registry/items/chatWhatsapp.json +151 -0
- package/registry/items/codeEditor.json +184 -0
- package/registry/items/confetti.json +107 -0
- package/registry/items/ctaLogo.json +160 -0
- package/registry/items/ctaMedia.json +193 -0
- package/registry/items/emojiBurst.json +101 -0
- package/registry/items/incomingCall.json +170 -0
- package/registry/items/media.json +179 -0
- package/registry/items/milestone.json +205 -0
- package/registry/items/motion.json +45 -0
- package/registry/items/notification.json +182 -0
- package/registry/items/phoneMockup.json +254 -0
- package/registry/items/problemSolution.json +186 -0
- package/registry/items/progressRing.json +199 -0
- package/registry/items/promptInput.json +148 -0
- package/registry/items/reaction.json +170 -0
- package/registry/items/reviewStack.json +224 -0
- package/registry/items/steps.json +207 -0
- package/registry/items/template-context.json +31 -0
- package/registry/items/terminal.json +212 -0
- package/registry/items/testimonial.json +164 -0
- package/registry/items/theme.json +47 -0
- package/registry/items/tripleStats.json +228 -0
- package/registry/items/tweet.json +203 -0
- package/registry/items/typography.json +47 -0
- package/registry/items/webMockup.json +273 -0
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
[← Documentation home](../README.md) · [Previous: Errors and recovery](errors.md)
|
|
2
|
+
|
|
3
|
+
# Production guide
|
|
4
|
+
|
|
5
|
+
Use this checklist before serving a video response to customers.
|
|
6
|
+
|
|
7
|
+
## Server boundary
|
|
8
|
+
|
|
9
|
+
- Keep provider keys, the system prompt, and tools on the server.
|
|
10
|
+
- Authenticate the user and tenant before reading the request body.
|
|
11
|
+
- Set an explicit origin allowlist. CORS is not authentication.
|
|
12
|
+
- Apply per-user and per-tenant request, token, and concurrency limits.
|
|
13
|
+
- Set route, model, media, and export timeouts with cancellation propagation.
|
|
14
|
+
- Bound raw input size, media count, scene count, and maximum duration.
|
|
15
|
+
|
|
16
|
+
Use `createVideoHandler()` for validated SSE and read
|
|
17
|
+
[the security guide](security.md) for the mandatory controls.
|
|
18
|
+
The handler automatically configures scene validation: it rejects unknown
|
|
19
|
+
templates and fields, missing required variables, non-supplied media URLs, and
|
|
20
|
+
fabricated quote-template content before a scene reaches the player.
|
|
21
|
+
Applications with a custom stream adapter may authorize additional final URLs
|
|
22
|
+
with `allowMediaUrl`; the callback validates URLs and does not resolve them.
|
|
23
|
+
Invalid generated parts are dropped by default, after `onError` receives their
|
|
24
|
+
private reason. Accepted scenes continue streaming with a safe recoverable
|
|
25
|
+
diagnostic. Use `invalidPartBehavior: "fail"` only for deliberate fail-fast behavior.
|
|
26
|
+
|
|
27
|
+
## Data and privacy
|
|
28
|
+
|
|
29
|
+
- Send the provider only the source and personalization required for the story.
|
|
30
|
+
- Do not log raw prompts, personalization, authorization headers, signed URLs,
|
|
31
|
+
or provider deltas.
|
|
32
|
+
- Record request IDs, model ID, duration, event counts, safe error codes, token
|
|
33
|
+
usage, and acceptance metrics.
|
|
34
|
+
- Review your provider's retention settings and data-processing terms.
|
|
35
|
+
- Keep signed asset URLs valid for the expected viewing and replay window.
|
|
36
|
+
|
|
37
|
+
## Reliability
|
|
38
|
+
|
|
39
|
+
- Emit the supplied opening and selected audio before starting model work.
|
|
40
|
+
- Abort provider work when the client disconnects.
|
|
41
|
+
- Persist terminal snapshots for replay and export, then validate loaded values
|
|
42
|
+
with `parseVideo` before use.
|
|
43
|
+
- Persist event logs when resume is required; validate them with
|
|
44
|
+
validate stored event logs before replay.
|
|
45
|
+
- Treat the [persistence contract](persistence.md) as the storage boundary;
|
|
46
|
+
database, tenant policy, object storage, deletion, and URL expiry remain
|
|
47
|
+
host-owned.
|
|
48
|
+
- Use idempotency keys around durable generation requests.
|
|
49
|
+
- Retry only before visible output or from a validated resume point. Do not
|
|
50
|
+
silently restart a response after the viewer has begun watching.
|
|
51
|
+
|
|
52
|
+
## Failure experience
|
|
53
|
+
|
|
54
|
+
- Keep private provider details in `onError`; send only safe typed errors.
|
|
55
|
+
- Do not display stack traces or protocol diagnostics inside the video.
|
|
56
|
+
- Hold the current scene and continue audio through a recoverable generation
|
|
57
|
+
gap.
|
|
58
|
+
- End cleanly on a terminal error; never append blank media after completion.
|
|
59
|
+
- Provide a normal application retry control outside the player.
|
|
60
|
+
|
|
61
|
+
## Media and audio
|
|
62
|
+
|
|
63
|
+
- Start generated playback with an asset-free scene.
|
|
64
|
+
- Preload media for the next scene and commit it only when ready.
|
|
65
|
+
- Resolve media before generation and pass approved results through
|
|
66
|
+
`suppliedMedia`. Stock-keyword resolution is not part of the 0.1 handler.
|
|
67
|
+
- Use customer-approved media domains and enforce type/byte limits.
|
|
68
|
+
- Select soundtrack audio from an already-loaded catalog; declare a positive
|
|
69
|
+
fade-out. Narration and speech synchronization are application-owned.
|
|
70
|
+
- Respect browser autoplay rules and provide an explicit unmute control.
|
|
71
|
+
|
|
72
|
+
## Observability
|
|
73
|
+
|
|
74
|
+
Track at minimum:
|
|
75
|
+
|
|
76
|
+
- time to supplied opening;
|
|
77
|
+
- time to first generated scene;
|
|
78
|
+
- time to complete plan;
|
|
79
|
+
- seconds of future content buffered;
|
|
80
|
+
- planner and protocol error codes;
|
|
81
|
+
- media readiness failures;
|
|
82
|
+
- completion and abandonment rate;
|
|
83
|
+
- provider model and token usage.
|
|
84
|
+
|
|
85
|
+
Use the repository acceptance harness in smoke tests. The defaults
|
|
86
|
+
require an opening within 250 ms, a generated scene within 15 seconds,
|
|
87
|
+
completion within 30 seconds, resolved media, audio before the opening, three
|
|
88
|
+
body scenes, three distinct templates, and a human quality score of 80.
|
|
89
|
+
|
|
90
|
+
## Testing
|
|
91
|
+
|
|
92
|
+
Use the React-free deterministic helpers in [Test integrations](testing.md) for
|
|
93
|
+
Vitest, in-process streaming, and route-handler tests without a live provider.
|
|
94
|
+
|
|
95
|
+
Test the public path at its HTTP boundary. Pass a deterministic `streamText`
|
|
96
|
+
generator to `createVideoHandler`, call the route with grounded input, and
|
|
97
|
+
assert the validated terminal result through `useVideo`. Keep separate
|
|
98
|
+
component tests for each trusted template.
|
|
99
|
+
|
|
100
|
+
In CI, build and test the application. If the project owns copied templates,
|
|
101
|
+
also verify that its registry is current:
|
|
102
|
+
|
|
103
|
+
```bash
|
|
104
|
+
npx vanillasky sync --check
|
|
105
|
+
npm run build
|
|
106
|
+
npm test
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
Before release, build one clean consumer from the packed SDK tarball. This
|
|
110
|
+
catches missing exports, React/server boundary leaks, code-generation drift,
|
|
111
|
+
and dependency-resolution problems that workspace tests cannot detect.
|
|
112
|
+
|
|
113
|
+
## Deployment checklist
|
|
114
|
+
|
|
115
|
+
- [ ] Provider keys exist only in the server secret store.
|
|
116
|
+
- [ ] Authentication, tenant policy, rate limits, and origin allowlist are live.
|
|
117
|
+
- [ ] Cancellation, timeouts, and safe errors are tested.
|
|
118
|
+
- [ ] Every installed template renders in both orientations.
|
|
119
|
+
- [ ] Chromium, Firefox, WebKit, React, and Node compatibility checks pass.
|
|
120
|
+
- [ ] A real provider run passes latency and human quality review.
|
|
121
|
+
- [ ] Final snapshots replay exactly and export through the configured adapter.
|
|
122
|
+
- [ ] Package and application dependency audits meet your severity policy.
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
[← Documentation home](../README.md) · [Next: Generate your first video →](getting-started.md)
|
|
2
|
+
|
|
3
|
+
# Prompt and input
|
|
4
|
+
|
|
5
|
+
VanillaSky separates application truth from visual direction. You provide the
|
|
6
|
+
facts and your server-side model. VanillaSky supplies the planner contract that
|
|
7
|
+
turns those facts into a finite sequence of trusted scenes.
|
|
8
|
+
|
|
9
|
+
## The four layers
|
|
10
|
+
|
|
11
|
+
### 1. VanillaSky system prompt
|
|
12
|
+
|
|
13
|
+
`createVideoHandler` generates `systemPrompt` from the trusted template
|
|
14
|
+
registry. It tells the model:
|
|
15
|
+
|
|
16
|
+
- which templates and variables exist;
|
|
17
|
+
- how to emit the typed plan;
|
|
18
|
+
- how to order and time complete scenes;
|
|
19
|
+
- how to stay inside the factual and media boundaries;
|
|
20
|
+
- how to finish a finite response.
|
|
21
|
+
|
|
22
|
+
Normal applications should not build, copy, or expose this prompt in browser
|
|
23
|
+
code. Pass it unchanged to the provider adapter.
|
|
24
|
+
|
|
25
|
+
For maintainers, the base system rules live in
|
|
26
|
+
`src/server/prompts/system-prompt.ts`, request-context formatting lives in
|
|
27
|
+
`src/server/prompts/user-prompt.ts`, and template-specific catalog guidance
|
|
28
|
+
lives in `src/visual-system/catalog/prompt.ts`. See the
|
|
29
|
+
[architecture map](architecture.md) for the complete request flow.
|
|
30
|
+
|
|
31
|
+
### 2. Application instructions
|
|
32
|
+
|
|
33
|
+
Use `instructions` for optional presentation direction:
|
|
34
|
+
|
|
35
|
+
```ts
|
|
36
|
+
video.generate({
|
|
37
|
+
input: "Account alerts launched today. They refresh every 15 minutes.",
|
|
38
|
+
instructions: "Make the launch feel direct and energetic. End with adoption.",
|
|
39
|
+
});
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
Instructions can influence selection, emphasis, ordering, tone, and pacing.
|
|
43
|
+
They never expand the factual boundary, authorize a new media URL, or weaken
|
|
44
|
+
the event and validation contract.
|
|
45
|
+
|
|
46
|
+
For durable product-wide direction, use the server handler's `basePrompt`.
|
|
47
|
+
Keep per-request creative direction in `instructions`.
|
|
48
|
+
|
|
49
|
+
### 3. Grounded input
|
|
50
|
+
|
|
51
|
+
`input` is required and is the complete factual source for the video. It may be
|
|
52
|
+
plain text or a serialized structured object:
|
|
53
|
+
|
|
54
|
+
```ts
|
|
55
|
+
video.generate({
|
|
56
|
+
input: "Activation increased from 41% to 58% after guided onboarding.",
|
|
57
|
+
});
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
```ts
|
|
61
|
+
video.generate({
|
|
62
|
+
input: JSON.stringify({
|
|
63
|
+
period: "Q2",
|
|
64
|
+
activation: { previous: 41, current: 58 },
|
|
65
|
+
cause: "guided onboarding",
|
|
66
|
+
}),
|
|
67
|
+
});
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
Include exact numbers, quote wording, attribution, names, dates, and product
|
|
71
|
+
facts that may appear on screen. Do not place secrets, provider keys, or hidden
|
|
72
|
+
policy in input.
|
|
73
|
+
|
|
74
|
+
`personalization`, `brand`, and `suppliedMedia` are separate structured context.
|
|
75
|
+
They do not replace the source material.
|
|
76
|
+
|
|
77
|
+
### 4. Streamed plan
|
|
78
|
+
|
|
79
|
+
Your provider streams text deltas. VanillaSky decodes them into typed planner
|
|
80
|
+
parts, validates each complete scene, and emits deterministic protocol events.
|
|
81
|
+
The model never returns React, HTML, CSS, or executable JavaScript.
|
|
82
|
+
|
|
83
|
+
Generated `scene.add`, `scene.patch`, and `asset.patch` operations all pass
|
|
84
|
+
through the same merged-scene validation before the player sees them.
|
|
85
|
+
|
|
86
|
+
## What reaches the LLM
|
|
87
|
+
|
|
88
|
+
The provider adapter receives:
|
|
89
|
+
|
|
90
|
+
```ts
|
|
91
|
+
streamText: ({ systemPrompt, userPrompt, signal }) => streamText({
|
|
92
|
+
model,
|
|
93
|
+
system: systemPrompt,
|
|
94
|
+
prompt: userPrompt,
|
|
95
|
+
abortSignal: signal,
|
|
96
|
+
})
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
`userPrompt` is assembled by VanillaSky from:
|
|
100
|
+
|
|
101
|
+
- orientation and maximum duration;
|
|
102
|
+
- whether an opening scene already exists;
|
|
103
|
+
- raw `input`;
|
|
104
|
+
- creative `instructions`;
|
|
105
|
+
- personalization;
|
|
106
|
+
- descriptions and opaque references for approved supplied media;
|
|
107
|
+
- brand context.
|
|
108
|
+
|
|
109
|
+
Provider credentials and application authentication never belong in either
|
|
110
|
+
prompt. Original supplied-media URLs and data URIs also remain outside the
|
|
111
|
+
model prompt; the server restores an exact SDK-issued opaque reference only
|
|
112
|
+
after provider output has been parsed.
|
|
113
|
+
|
|
114
|
+
Generated template values are validated exactly. VanillaSky does not silently
|
|
115
|
+
truncate factual labels to satisfy a layout schema because truncation can drop
|
|
116
|
+
qualifiers or change meaning. A rejected generated scene contributes to
|
|
117
|
+
`rejectedSceneCount`; hosts can use the completion quality fields to retry with
|
|
118
|
+
a stronger model or revised source.
|
|
119
|
+
|
|
120
|
+
## Input examples
|
|
121
|
+
|
|
122
|
+
### Product update
|
|
123
|
+
|
|
124
|
+
```ts
|
|
125
|
+
video.generate({
|
|
126
|
+
input: "Account alerts launched on August 16. They refresh every 15 minutes, filter by segment, and are available to all plans.",
|
|
127
|
+
});
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
### Metrics recap
|
|
131
|
+
|
|
132
|
+
```ts
|
|
133
|
+
video.generate({
|
|
134
|
+
input: JSON.stringify({
|
|
135
|
+
period: "Q2",
|
|
136
|
+
customerConversations: 142,
|
|
137
|
+
escalationsResolved: "96%",
|
|
138
|
+
improvementsLaunched: 4,
|
|
139
|
+
}),
|
|
140
|
+
personalization: { firstName: "Joris" },
|
|
141
|
+
});
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
### Grounded review
|
|
145
|
+
|
|
146
|
+
```ts
|
|
147
|
+
video.generate({
|
|
148
|
+
input: 'Review by Maya Chen, VP Product: "Setup took minutes, not weeks." Rating: 5/5.',
|
|
149
|
+
});
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
Every visible quote must occur in the input exactly. Attribution and ratings
|
|
153
|
+
should be explicit.
|
|
154
|
+
|
|
155
|
+
### Article or long source
|
|
156
|
+
|
|
157
|
+
Pass the source text and let the planner select the most decision-relevant
|
|
158
|
+
takeaways that fit the duration. The planner summarizes; it should not attempt
|
|
159
|
+
to place every paragraph on screen.
|
|
160
|
+
|
|
161
|
+
## Grounding and media safety
|
|
162
|
+
|
|
163
|
+
- Numeric templates require real numbers present in the source.
|
|
164
|
+
- Every grounded quote value must occur in the source, not merely one quote in a scene.
|
|
165
|
+
- Screenshot fields require an exact supplied image.
|
|
166
|
+
- Media URLs must be supplied or approved by the server's `allowMediaUrl` policy.
|
|
167
|
+
- Patches are validated after merging with the existing scene.
|
|
168
|
+
|
|
169
|
+
These checks happen at runtime. Prompt instructions improve model behavior but
|
|
170
|
+
are not treated as a security boundary.
|
|
171
|
+
|
|
172
|
+
## Debugging
|
|
173
|
+
|
|
174
|
+
If the plan is rejected or the result is weak, inspect the boundary in this order:
|
|
175
|
+
|
|
176
|
+
1. **Input:** Does it contain the exact facts, numbers, quotes, and attribution?
|
|
177
|
+
2. **Instructions:** Are they presentation guidance rather than new claims?
|
|
178
|
+
3. **Provider adapter:** Does it pass both prompts unchanged and stream only text deltas?
|
|
179
|
+
4. **Finish reason:** Did the provider report `length`, a content filter, or an execution error?
|
|
180
|
+
5. **Template fit:** Does the trusted registry contain a suitable visual for the requested story?
|
|
181
|
+
|
|
182
|
+
Log request IDs, provider finish reasons, and terminal SDK status. Do not log
|
|
183
|
+
secrets or expose raw provider diagnostics inside the video.
|
|
184
|
+
|
|
185
|
+
[← Documentation home](../README.md) · [Next: Generate your first video →](getting-started.md)
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
[← Documentation home](../README.md) · [Previous: Next.js](integrate-nextjs.md) · [Next: Input and first scene →](input-and-first-scene.md)
|
|
2
|
+
|
|
3
|
+
# Provider integration
|
|
4
|
+
|
|
5
|
+
The model connection enters the SDK through `streamText` in
|
|
6
|
+
`src/server/create-video-handler.ts`. The SDK does not instantiate or choose a
|
|
7
|
+
model; the application passes the generated `systemPrompt` and `userPrompt` to
|
|
8
|
+
its provider here.
|
|
9
|
+
|
|
10
|
+
Use the AI SDK as the normal app-owned provider adapter. Any AI SDK
|
|
11
|
+
`LanguageModel` works. Keep selection in one server-only module so OpenAI and
|
|
12
|
+
Anthropic use the same route and React component:
|
|
13
|
+
|
|
14
|
+
```ts
|
|
15
|
+
import "server-only";
|
|
16
|
+
import { anthropic } from "@ai-sdk/anthropic";
|
|
17
|
+
import { openai } from "@ai-sdk/openai";
|
|
18
|
+
import { streamText } from "ai";
|
|
19
|
+
import { createVideoHandler } from "@vanillaskyai/video/server";
|
|
20
|
+
|
|
21
|
+
const provider = process.env.VIDEO_PROVIDER;
|
|
22
|
+
const modelId = process.env.VIDEO_MODEL;
|
|
23
|
+
if (!modelId) throw new Error("Set VIDEO_MODEL in the server environment");
|
|
24
|
+
const model = provider === "openai" ? openai(modelId)
|
|
25
|
+
: provider === "anthropic" ? anthropic(modelId)
|
|
26
|
+
: (() => { throw new Error("Set VIDEO_PROVIDER to openai or anthropic"); })();
|
|
27
|
+
|
|
28
|
+
export const POST = createVideoHandler({
|
|
29
|
+
authorize: verifySession,
|
|
30
|
+
streamText: ({ systemPrompt, userPrompt, signal }) => streamText({
|
|
31
|
+
model,
|
|
32
|
+
system: systemPrompt,
|
|
33
|
+
prompt: userPrompt,
|
|
34
|
+
abortSignal: signal,
|
|
35
|
+
}),
|
|
36
|
+
});
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
The AI SDK result can be returned directly: its `textStream`, `finishReason`,
|
|
40
|
+
`rawFinishReason`, usage, provider metadata, warnings, and response metadata
|
|
41
|
+
match VanillaSky's structural callback contract. Keep the
|
|
42
|
+
provider, model, and credentials on the server. The callback runs per request,
|
|
43
|
+
so an application can route cheap/fast and high-capability models through the
|
|
44
|
+
same handler without adding a VanillaSky abstraction. See
|
|
45
|
+
[provider adapter reference](reference/provider-adapters.md) for model alternatives and advanced native
|
|
46
|
+
provider loops.
|
|
47
|
+
|
|
48
|
+
## Completion and usage
|
|
49
|
+
|
|
50
|
+
Use `onComplete` for server-side cost and completion measurement:
|
|
51
|
+
|
|
52
|
+
```ts
|
|
53
|
+
createVideoHandler({
|
|
54
|
+
authorize: verifySession,
|
|
55
|
+
streamText: ({ systemPrompt, userPrompt, signal }) => streamText({
|
|
56
|
+
model,
|
|
57
|
+
system: systemPrompt,
|
|
58
|
+
prompt: userPrompt,
|
|
59
|
+
abortSignal: signal,
|
|
60
|
+
}),
|
|
61
|
+
onWarning: (warning) => logSafeWarning(warning.code, warning.category),
|
|
62
|
+
onComplete: (summary) => recordGeneration({
|
|
63
|
+
finishReason: summary.finishReason,
|
|
64
|
+
usage: summary.usage,
|
|
65
|
+
requestedModelId: summary.requestedModelId,
|
|
66
|
+
resolvedModelId: summary.resolvedModelId,
|
|
67
|
+
totalDurationMs: summary.totalDurationMs,
|
|
68
|
+
}),
|
|
69
|
+
onError: (error) => recordPrivateFailure(error),
|
|
70
|
+
});
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
`onComplete` fires once only after `response.complete`. It does not fire for a
|
|
74
|
+
terminal error, abort, disconnect, or timeout. Callback failures are isolated
|
|
75
|
+
from the event stream. Normalized token usage and model IDs remain server-only;
|
|
76
|
+
they never enter SSE or the persisted `Video`. Set `includeRawProviderData:
|
|
77
|
+
true` only when the host deliberately needs bounded provider-native usage and
|
|
78
|
+
metadata and has an appropriate retention policy.
|
|
79
|
+
|
|
80
|
+
`acceptedSceneCount`, `rejectedSceneCount`, and `timeToFirstSceneMs` describe
|
|
81
|
+
model-generated scene additions; a deterministic supplied opening is not
|
|
82
|
+
counted. Their sum is the proposed scene count. `videoDurationSec` is the
|
|
83
|
+
duration actually committed; compare it with the `maxDurationSec` supplied by
|
|
84
|
+
your application when applying a retry policy. These fields provide a
|
|
85
|
+
server-side quality signal without exposing model metadata in the browser.
|
|
86
|
+
Warnings include the same bounded typed warnings emitted to the client.
|
|
87
|
+
For non-interactive backfills, define an application threshold (for example,
|
|
88
|
+
no rejected scenes and a useful committed-duration ratio) and retry a bounded
|
|
89
|
+
number of times. Keep the best accepted result rather than treating
|
|
90
|
+
`finishReason: "stop"` alone as a quality score.
|
|
91
|
+
|
|
92
|
+
The generated system prompt includes the selected trusted-template catalog and
|
|
93
|
+
is intentionally substantial. It is stable for the same SDK version, template
|
|
94
|
+
kit, media policy, and base prompt. Record input-token usage, keep the selected
|
|
95
|
+
kit no broader than the product needs, and enable provider-side prompt caching
|
|
96
|
+
where the chosen provider/model supports it. VanillaSky does not assume one
|
|
97
|
+
provider's cache controls in its provider-neutral adapter. With the 28 built-in
|
|
98
|
+
templates, the current catalog prompt is roughly 28,000 characters (about
|
|
99
|
+
7,000 tokens before user input; tokenizer-dependent); provider-reported usage
|
|
100
|
+
is the authoritative measurement.
|
|
101
|
+
|
|
102
|
+
Provider finish reasons `error` and `tool-calls` are terminal failures.
|
|
103
|
+
`length` and `content-filter` may complete with already accepted scenes; a
|
|
104
|
+
truncation before the first generated scene fails instead of returning an empty
|
|
105
|
+
success. The request signal is forwarded to the provider. Configure route and
|
|
106
|
+
provider timeouts with that signal, and keep retries host-owned and within the
|
|
107
|
+
same explicit request budget.
|
|
108
|
+
|
|
109
|
+
## Product-level planner guidance
|
|
110
|
+
|
|
111
|
+
`createVideoHandler` constructs the planner prompt from the generated server
|
|
112
|
+
template registry. Normal integrations do not build prompts or capabilities.
|
|
113
|
+
Use the handler's `basePrompt` option only for durable product-level direction.
|
|
114
|
+
Grounded facts still come from `VideoInput.input`; presentation guidance must
|
|
115
|
+
never override that factual boundary.
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
# Video Response Protocol 0.4
|
|
2
|
+
|
|
3
|
+
## Transport
|
|
4
|
+
|
|
5
|
+
The public transport is UTF-8 Server-Sent Events returned from a `POST` request.
|
|
6
|
+
Responses use `Content-Type: text/event-stream`,
|
|
7
|
+
`x-vanillasky-video-stream: 0.4`, `Cache-Control: no-cache, no-transform`, and
|
|
8
|
+
`X-Accel-Buffering: no`. Each block has an SSE `id`, the event name
|
|
9
|
+
`video`, and one JSON envelope in `data`. A final `data: [DONE]`
|
|
10
|
+
closes the transport after a terminal protocol event. Comment heartbeats do not
|
|
11
|
+
change protocol state.
|
|
12
|
+
|
|
13
|
+
## Envelope
|
|
14
|
+
|
|
15
|
+
```ts
|
|
16
|
+
type VideoEvent<T extends string, D> = {
|
|
17
|
+
protocolVersion: "0.4";
|
|
18
|
+
runId: string;
|
|
19
|
+
sequence: number;
|
|
20
|
+
eventId: string; // exactly `${runId}:${sequence}`
|
|
21
|
+
type: T;
|
|
22
|
+
data: D;
|
|
23
|
+
};
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
Sequences start at zero and increase by exactly one. Unknown fields are
|
|
27
|
+
rejected. Replaying the same `eventId` with identical content is idempotent;
|
|
28
|
+
replaying it with different content is an error. A run ID cannot change
|
|
29
|
+
midstream and events cannot follow a terminal state.
|
|
30
|
+
|
|
31
|
+
## Lifecycle
|
|
32
|
+
|
|
33
|
+
1. `response.start` establishes request ID, orientation, style, and negotiated
|
|
34
|
+
templates/extensions.
|
|
35
|
+
2. Optional `audio.set` occurs at most once and before the first scene.
|
|
36
|
+
3. `scene.add` appends one trusted-template scene at revision `0`.
|
|
37
|
+
4. `scene.patch` or `asset.patch` advances that scene revision by exactly one.
|
|
38
|
+
A host must reject patches after the scene has played.
|
|
39
|
+
5. Exactly one terminal event ends the run:
|
|
40
|
+
`response.complete`, terminal `response.error`, or `response.abort`.
|
|
41
|
+
|
|
42
|
+
`response.complete` carries the complete replayable `Video`, a
|
|
43
|
+
finish reason, and a deterministic checksum. The reducer verifies that the
|
|
44
|
+
snapshot equals the state produced by all prior events. The checksum detects
|
|
45
|
+
accidental drift; it is not a cryptographic signature.
|
|
46
|
+
|
|
47
|
+
The terminal snapshot carries persisted `schemaVersion: "0.1"`. That storage
|
|
48
|
+
version is independent from this streaming protocol version. Load stored
|
|
49
|
+
snapshots through the universal `parseVideo` boundary described in the
|
|
50
|
+
[persistence guide](../persistence.md).
|
|
51
|
+
|
|
52
|
+
Recoverable `response.error` events may be followed by more events. Terminal
|
|
53
|
+
errors contain a safe public message. Provider details belong only in a private
|
|
54
|
+
server callback.
|
|
55
|
+
|
|
56
|
+
### Duration ceiling
|
|
57
|
+
|
|
58
|
+
`maxDurationSec` is enforced by the runtime for scene additions and timing
|
|
59
|
+
patches. A final scene may be shortened to the remaining duration. If a later
|
|
60
|
+
scene would start at the ceiling, the runtime preserves the partial response
|
|
61
|
+
and emits `response.complete` with `finishReason: "length"`; reaching the
|
|
62
|
+
declared maximum is normal completion, not a generation error. The value is a
|
|
63
|
+
ceiling, not a target: a grounded response may complete below that ceiling when
|
|
64
|
+
the source does not support another distinct readable scene.
|
|
65
|
+
|
|
66
|
+
## Extensions
|
|
67
|
+
|
|
68
|
+
Extension events use a namespaced `data.*` type, for example
|
|
69
|
+
`data.customer.status`. They are accepted only when the exact name was
|
|
70
|
+
negotiated in `response.start.capabilities.extensions`. Extensions cannot
|
|
71
|
+
change core video state.
|
|
72
|
+
|
|
73
|
+
## Planning boundary
|
|
74
|
+
|
|
75
|
+
LLMs do not emit public protocol envelopes. A server-only planner emits
|
|
76
|
+
validated `scene.add`, `scene.patch`, `asset.patch`, `plan.complete`, or
|
|
77
|
+
`plan.error` parts. The runtime assigns sequences, revisions, IDs, terminal
|
|
78
|
+
snapshots, and checksums. Generated HTML, React, JavaScript, CSS, component
|
|
79
|
+
source, audio events, protocol envelopes, and unknown part types are rejected.
|
|
80
|
+
|
|
81
|
+
## Resume
|
|
82
|
+
|
|
83
|
+
A resume request repeats the public input and includes:
|
|
84
|
+
|
|
85
|
+
```json
|
|
86
|
+
{"resume":{"runId":"run-123","afterSequence":7}}
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
It also sends `Last-Event-ID: run-123:7`. The server validates that both cursors
|
|
90
|
+
match, then calls customer-owned replay storage. Replay begins at sequence `8`
|
|
91
|
+
and remains subject to normal run, order, validation, and terminal rules. The
|
|
92
|
+
SDK does not prescribe or operate a persistence service.
|
|
93
|
+
|
|
94
|
+
Validate adapter output and persisted replay logs against this protocol before
|
|
95
|
+
accepting them. A replay log must preserve ordering, checksums, and a terminal
|
|
96
|
+
event.
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
# Secure LLM provider adapters
|
|
2
|
+
|
|
3
|
+
This page covers the planner boundary. For Pexels and other visual providers,
|
|
4
|
+
read [Media and audio](../media-and-audio.md#media-providers).
|
|
5
|
+
|
|
6
|
+
VanillaSky deliberately does not depend on a model provider or AI framework.
|
|
7
|
+
Your server owns the model and credentials; `createVideoHandler` accepts one
|
|
8
|
+
small `streamText` callback. The recommended adapter is the [AI SDK](https://ai-sdk.dev/docs/reference/ai-sdk-core/stream-text),
|
|
9
|
+
which gives the application one `LanguageModel` interface across official,
|
|
10
|
+
community, AI Gateway, OpenAI-compatible, and custom providers.
|
|
11
|
+
|
|
12
|
+
## Recommended: AI SDK
|
|
13
|
+
|
|
14
|
+
Install the AI SDK plus the provider package your application chooses:
|
|
15
|
+
|
|
16
|
+
```bash
|
|
17
|
+
npm install ai @ai-sdk/openai
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
```ts
|
|
21
|
+
import { openai } from "@ai-sdk/openai";
|
|
22
|
+
import { streamText } from "ai";
|
|
23
|
+
import { createVideoHandler } from "@vanillaskyai/video/server";
|
|
24
|
+
|
|
25
|
+
const modelId = process.env.OPENAI_MODEL;
|
|
26
|
+
if (!modelId) throw new Error("Set OPENAI_MODEL in the server environment");
|
|
27
|
+
|
|
28
|
+
export const POST = createVideoHandler({
|
|
29
|
+
authorize: verifySession,
|
|
30
|
+
streamText: ({ systemPrompt, userPrompt, signal }) => streamText({
|
|
31
|
+
model: openai(modelId),
|
|
32
|
+
system: systemPrompt,
|
|
33
|
+
prompt: userPrompt,
|
|
34
|
+
abortSignal: signal,
|
|
35
|
+
}),
|
|
36
|
+
});
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
Return the AI SDK `StreamTextResult` directly. It is structurally compatible
|
|
40
|
+
with VanillaSky's callback: VanillaSky reads `textStream`, finish metadata,
|
|
41
|
+
usage, safe provider warnings, provider metadata, and response/final-step model
|
|
42
|
+
metadata. Usage and model IDs are available only through the server-side
|
|
43
|
+
`onComplete` summary. Provider-native usage and metadata require the explicit
|
|
44
|
+
bounded `includeRawProviderData` opt-in and never enter SSE. The forwarded abort
|
|
45
|
+
signal cancels provider work when the request disconnects or the host timeout
|
|
46
|
+
fires.
|
|
47
|
+
|
|
48
|
+
Only the model expression changes:
|
|
49
|
+
|
|
50
|
+
- Use any [official AI SDK provider](https://ai-sdk.dev/providers/ai-sdk-providers).
|
|
51
|
+
- Use an [AI Gateway model ID](https://ai-sdk.dev/providers/ai-sdk-providers/ai-gateway).
|
|
52
|
+
- Use an [OpenAI-compatible provider](https://ai-sdk.dev/providers/openai-compatible-providers).
|
|
53
|
+
- Use a [community or custom provider](https://ai-sdk.dev/providers/community-providers)
|
|
54
|
+
implementing the Language Model Specification.
|
|
55
|
+
|
|
56
|
+
The callback runs for every video request, so the application may select a
|
|
57
|
+
different model each time. A product can route routine planning to a cheap,
|
|
58
|
+
fast model and reserve a stronger model for difficult inputs without changing
|
|
59
|
+
VanillaSky or its protocol. The same boundary also accepts a self-hosted model
|
|
60
|
+
or a provider-native async text stream when it is not represented in the AI
|
|
61
|
+
SDK. VanillaSky has no model allowlist.
|
|
62
|
+
|
|
63
|
+
The provider must emit NDJSON text: one complete VanillaSky plan part per line.
|
|
64
|
+
The SDK buffers arbitrary text chunks until a newline, parses the completed
|
|
65
|
+
object, validates it, and only then forwards it to the motion runtime. Do not
|
|
66
|
+
replace that per-line validator with whole-response structured output: motion
|
|
67
|
+
streaming intentionally renders the first scene before the full composition is
|
|
68
|
+
complete.
|
|
69
|
+
|
|
70
|
+
## Advanced: native provider loops
|
|
71
|
+
|
|
72
|
+
Use the lower-level callback directly when the AI SDK does not expose a
|
|
73
|
+
provider-specific feature you need. `streamText` may return either an
|
|
74
|
+
`AsyncIterable<string>` or an object with `textStream` plus optional
|
|
75
|
+
`finishReason`, `rawFinishReason`, `usage`, `providerMetadata`, and
|
|
76
|
+
response/final-step metadata. Requested model IDs are read from the final
|
|
77
|
+
step's `model.modelId`; resolved IDs are read from response metadata. Prefer the
|
|
78
|
+
richer form when the native provider exposes completion metadata.
|
|
79
|
+
|
|
80
|
+
Provider retries remain host-owned. Retry only within an explicit request and
|
|
81
|
+
time budget, and never silently restart after visible output unless the host
|
|
82
|
+
has validated durable resume storage.
|
|
83
|
+
|
|
84
|
+
### OpenAI Responses API
|
|
85
|
+
|
|
86
|
+
```ts
|
|
87
|
+
import OpenAI from "openai";
|
|
88
|
+
import { createVideoHandler } from "@vanillaskyai/video/server";
|
|
89
|
+
|
|
90
|
+
const apiKey = process.env.OPENAI_API_KEY;
|
|
91
|
+
const model = process.env.OPENAI_MODEL;
|
|
92
|
+
if (!apiKey || !model) throw new Error("Set OPENAI_API_KEY and OPENAI_MODEL");
|
|
93
|
+
|
|
94
|
+
const openai = new OpenAI({ apiKey });
|
|
95
|
+
|
|
96
|
+
export const POST = createVideoHandler({
|
|
97
|
+
authorize: verifySession,
|
|
98
|
+
streamText: async function* ({ systemPrompt, userPrompt, signal }) {
|
|
99
|
+
const stream = await openai.responses.create({
|
|
100
|
+
model,
|
|
101
|
+
input: [
|
|
102
|
+
{ role: "system", content: systemPrompt },
|
|
103
|
+
{ role: "user", content: userPrompt },
|
|
104
|
+
],
|
|
105
|
+
stream: true,
|
|
106
|
+
}, { signal });
|
|
107
|
+
|
|
108
|
+
for await (const event of stream) {
|
|
109
|
+
if (event.type === "response.output_text.delta") yield event.delta;
|
|
110
|
+
}
|
|
111
|
+
},
|
|
112
|
+
});
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
OpenAI documents the typed `response.output_text.delta` event in its
|
|
116
|
+
[Responses streaming guide](https://developers.openai.com/api/docs/guides/streaming-responses).
|
|
117
|
+
|
|
118
|
+
### Anthropic Messages API
|
|
119
|
+
|
|
120
|
+
```ts
|
|
121
|
+
import Anthropic from "@anthropic-ai/sdk";
|
|
122
|
+
import { createVideoHandler } from "@vanillaskyai/video/server";
|
|
123
|
+
|
|
124
|
+
const apiKey = process.env.ANTHROPIC_API_KEY;
|
|
125
|
+
const model = process.env.ANTHROPIC_MODEL;
|
|
126
|
+
if (!apiKey || !model) throw new Error("Set ANTHROPIC_API_KEY and ANTHROPIC_MODEL");
|
|
127
|
+
|
|
128
|
+
const anthropic = new Anthropic({ apiKey });
|
|
129
|
+
|
|
130
|
+
export const POST = createVideoHandler({
|
|
131
|
+
authorize: verifySession,
|
|
132
|
+
streamText: async function* ({ systemPrompt, userPrompt, signal }) {
|
|
133
|
+
const stream = anthropic.messages.stream({
|
|
134
|
+
model,
|
|
135
|
+
max_tokens: 4096,
|
|
136
|
+
system: systemPrompt,
|
|
137
|
+
messages: [{ role: "user", content: userPrompt }],
|
|
138
|
+
}, { signal });
|
|
139
|
+
|
|
140
|
+
for await (const event of stream) {
|
|
141
|
+
if (event.type === "content_block_delta" && event.delta.type === "text_delta") {
|
|
142
|
+
yield event.delta.text;
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
},
|
|
146
|
+
});
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
Anthropic documents `content_block_delta` and `text_delta` in its
|
|
150
|
+
[Messages streaming guide](https://platform.claude.com/docs/en/build-with-claude/streaming).
|
|
151
|
+
|
|
152
|
+
## Optional web fetch and other tools
|
|
153
|
+
|
|
154
|
+
Keep tools server-side and outside the SDK core. Fetch or search before motion
|
|
155
|
+
generation, append only the approved grounded result to `input`, and
|
|
156
|
+
record provenance separately. Apply domain allowlists, SSRF protection,
|
|
157
|
+
timeouts, redirect limits, and response-size limits. Do not give a model a
|
|
158
|
+
general-purpose fetch tool unless the application truly needs one.
|