@vanillaskyai/video 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/PUBLIC-API.md +20 -2
- package/README.md +7 -3
- package/dist/{builtin-server-F7YXRVCH.js → builtin-server-BRTZN4Q7.js} +2 -2
- package/dist/check-runtime.js +1 -1
- package/dist/{chunk-EVG2ZYIY.js → chunk-3O7OMMMF.js} +7 -4
- package/dist/{chunk-GRMUV7QY.js → chunk-BKF3A357.js} +1 -1
- package/dist/{chunk-A2X45FET.js → chunk-I5YWVQ5R.js} +1 -1
- package/dist/{chunk-ERY4S6QV.js → chunk-JW47XCRL.js} +6 -1
- package/dist/{chunk-FQ47WXAJ.js → chunk-K5SRF7D3.js} +4 -0
- package/dist/{chunk-YC5MFXA5.js → chunk-Q2DDQKEG.js} +34 -12
- package/dist/{chunk-XB7LH4J2.js → chunk-QTMHS7JD.js} +2 -1
- package/dist/{chunk-W2XJKG6C.js → chunk-RBQN3VSY.js} +2 -2
- package/dist/{chunk-OOPIETLW.js → chunk-S5ZU5WUU.js} +1 -1
- package/dist/{chunk-ABPKM4GK.js → chunk-STIFILQG.js} +1 -1
- package/dist/{chunk-IXCUY3B3.js → chunk-TGFO454Q.js} +169 -45
- package/dist/{chunk-CHLGW22F.js → chunk-XAMUOSX7.js} +3 -3
- package/dist/cli.js +5 -5
- package/dist/{compose-video-CN3VSWST.js → compose-video-34GRZIF2.js} +4 -4
- package/dist/{events-_wfjbmEp.d.ts → events-B-28kERX.d.ts} +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/{kit-BMNcHqa9.d.ts → kit-CBedbj1n.d.ts} +1 -1
- package/dist/react.d.ts +7 -4
- package/dist/react.js +226 -30
- package/dist/server.d.ts +21 -4
- package/dist/server.js +193 -14
- package/dist/{system-prompt-A6UXNTPK.js → system-prompt-GYTYGWGO.js} +1 -1
- package/dist/template-catalog.js +1 -1
- package/dist/templates.d.ts +3 -3
- package/dist/test.d.ts +3 -2
- package/dist/test.js +3 -3
- package/dist/{text-stream-6XHC7J47.js → text-stream-UPGUD2TD.js} +2 -2
- package/dist/{types-B-PZR6G_.d.ts → types-_t4bduwX.d.ts} +1 -0
- package/docs/concepts.md +7 -0
- package/docs/custom-templates.md +3 -3
- package/docs/errors.md +3 -1
- package/docs/getting-started.md +8 -6
- package/docs/input-and-first-scene.md +7 -4
- package/docs/integrate-nextjs.md +2 -2
- package/docs/media-and-audio.md +73 -14
- package/docs/prompt-and-input.md +7 -0
- package/docs/provider-integration.md +7 -0
- package/docs/reference/protocol.md +9 -0
- package/examples/nextjs-quickstart/package.json +1 -1
- package/package.json +1 -1
- package/registry/items/brandMessage.json +1 -2
- package/registry/items/media.json +2 -1
package/dist/server.js
CHANGED
|
@@ -1,27 +1,27 @@
|
|
|
1
1
|
import {
|
|
2
2
|
createTextDeltaVideoPlanner
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-BKF3A357.js";
|
|
4
4
|
import {
|
|
5
5
|
createVideo
|
|
6
|
-
} from "./chunk-
|
|
6
|
+
} from "./chunk-TGFO454Q.js";
|
|
7
7
|
import "./chunk-E7CL7UPB.js";
|
|
8
8
|
import {
|
|
9
9
|
BUILTIN_SERVER_TEMPLATE_KIT,
|
|
10
10
|
createServerTemplateRegistry,
|
|
11
11
|
overlayServerTemplateRegistry
|
|
12
|
-
} from "./chunk-
|
|
12
|
+
} from "./chunk-I5YWVQ5R.js";
|
|
13
13
|
import {
|
|
14
14
|
createTemplateSceneValidator
|
|
15
15
|
} from "./chunk-LMG3GHLL.js";
|
|
16
16
|
import {
|
|
17
17
|
encodeVideoSseEvent,
|
|
18
18
|
videoSseHeaders
|
|
19
|
-
} from "./chunk-
|
|
20
|
-
import "./chunk-
|
|
19
|
+
} from "./chunk-S5ZU5WUU.js";
|
|
20
|
+
import "./chunk-STIFILQG.js";
|
|
21
21
|
import {
|
|
22
22
|
parseVideoEvent,
|
|
23
23
|
safePublicDiagnostic
|
|
24
|
-
} from "./chunk-
|
|
24
|
+
} from "./chunk-JW47XCRL.js";
|
|
25
25
|
import {
|
|
26
26
|
MAX_RETAINED_MEDIA_URL_LENGTH
|
|
27
27
|
} from "./chunk-GHJEAP4O.js";
|
|
@@ -29,14 +29,15 @@ import {
|
|
|
29
29
|
VIDEO_PROTOCOL_VERSION
|
|
30
30
|
} from "./chunk-EE6PK6XC.js";
|
|
31
31
|
import {
|
|
32
|
-
createTemplateSystemPrompt
|
|
33
|
-
|
|
32
|
+
createTemplateSystemPrompt,
|
|
33
|
+
getStandardMediaResolverContract
|
|
34
|
+
} from "./chunk-Q2DDQKEG.js";
|
|
34
35
|
import {
|
|
35
36
|
DEFAULT_VIDEO_SYSTEM_PROMPT
|
|
36
|
-
} from "./chunk-
|
|
37
|
-
import "./chunk-
|
|
37
|
+
} from "./chunk-K5SRF7D3.js";
|
|
38
|
+
import "./chunk-3O7OMMMF.js";
|
|
38
39
|
import "./chunk-352ZZCET.js";
|
|
39
|
-
import "./chunk-
|
|
40
|
+
import "./chunk-XAMUOSX7.js";
|
|
40
41
|
import "./chunk-2E6T633S.js";
|
|
41
42
|
import {
|
|
42
43
|
validateBrandInput
|
|
@@ -397,6 +398,7 @@ function createVideoStreamHandler(options) {
|
|
|
397
398
|
validateScene: options.validateScene,
|
|
398
399
|
getTemplatePacing: options.getTemplatePacing,
|
|
399
400
|
invalidPartBehavior: options.invalidPartBehavior ?? "drop",
|
|
401
|
+
requireCloser: options.requireCloser,
|
|
400
402
|
onError: options.onError,
|
|
401
403
|
onWarning: options.onWarning,
|
|
402
404
|
onComplete: options.onComplete,
|
|
@@ -459,6 +461,155 @@ function createVideoStreamHandler(options) {
|
|
|
459
461
|
};
|
|
460
462
|
}
|
|
461
463
|
|
|
464
|
+
// src/server/media-resolver.ts
|
|
465
|
+
var MAX_MEDIA_QUERY_CHARACTERS = 80;
|
|
466
|
+
var MAX_MEDIA_QUERY_WORDS = 8;
|
|
467
|
+
function queryIsBounded(query) {
|
|
468
|
+
const length = [...query].length;
|
|
469
|
+
const words = query.match(/\S+/gu)?.length ?? 0;
|
|
470
|
+
return length >= 2 && length <= MAX_MEDIA_QUERY_CHARACTERS && words <= MAX_MEDIA_QUERY_WORDS;
|
|
471
|
+
}
|
|
472
|
+
function preferredType(value) {
|
|
473
|
+
if (value === "photo") return "image";
|
|
474
|
+
if (value === "video") return "video";
|
|
475
|
+
return "any";
|
|
476
|
+
}
|
|
477
|
+
function cleanUrl(value, field) {
|
|
478
|
+
if (value == null || value === "") return void 0;
|
|
479
|
+
if (typeof value !== "string" || !value.trim() || value.length > MAX_RETAINED_MEDIA_URL_LENGTH) {
|
|
480
|
+
throw new Error(`Resolved media ${field} is invalid`);
|
|
481
|
+
}
|
|
482
|
+
return value.trim();
|
|
483
|
+
}
|
|
484
|
+
function fallbackVariables(variables) {
|
|
485
|
+
const fallback = { ...variables };
|
|
486
|
+
delete fallback.mediaKeyword;
|
|
487
|
+
delete fallback.mediaUrl;
|
|
488
|
+
delete fallback.mediaPoster;
|
|
489
|
+
fallback.mediaType = "gradient";
|
|
490
|
+
return fallback;
|
|
491
|
+
}
|
|
492
|
+
function isAbortError(error) {
|
|
493
|
+
return error instanceof Error && error.name === "AbortError";
|
|
494
|
+
}
|
|
495
|
+
async function resolveVariables(options) {
|
|
496
|
+
if (!Object.hasOwn(options.variables, "mediaKeyword")) return options.variables;
|
|
497
|
+
const variables = { ...options.variables };
|
|
498
|
+
const rawQuery = typeof variables.mediaKeyword === "string" ? variables.mediaKeyword.trim() : "";
|
|
499
|
+
delete variables.mediaKeyword;
|
|
500
|
+
if (!options.contract) return variables;
|
|
501
|
+
if (!options.openingReady || !options.resolveMedia || !queryIsBounded(rawQuery)) {
|
|
502
|
+
return fallbackVariables(variables);
|
|
503
|
+
}
|
|
504
|
+
let resolved;
|
|
505
|
+
try {
|
|
506
|
+
resolved = await options.resolveMedia(rawQuery, {
|
|
507
|
+
input: options.input,
|
|
508
|
+
templateId: options.templateId,
|
|
509
|
+
preferredType: preferredType(variables.mediaType),
|
|
510
|
+
signal: options.signal
|
|
511
|
+
});
|
|
512
|
+
} catch (error) {
|
|
513
|
+
if (options.signal.aborted) throw options.signal.reason ?? error;
|
|
514
|
+
if (isAbortError(error)) throw error;
|
|
515
|
+
return fallbackVariables(variables);
|
|
516
|
+
}
|
|
517
|
+
if (options.signal.aborted) {
|
|
518
|
+
throw options.signal.reason ?? new DOMException("The media request was aborted", "AbortError");
|
|
519
|
+
}
|
|
520
|
+
if (!resolved) return fallbackVariables(variables);
|
|
521
|
+
if (resolved.type !== "image" && resolved.type !== "video") {
|
|
522
|
+
throw new Error("Resolved media type is invalid");
|
|
523
|
+
}
|
|
524
|
+
const url = cleanUrl(resolved.url, "URL");
|
|
525
|
+
if (!url) throw new Error("Resolved media URL is required");
|
|
526
|
+
const posterUrl = options.contract.acceptsPoster ? cleanUrl(resolved.posterUrl, "poster URL") : void 0;
|
|
527
|
+
options.approveUrl(options.input, url);
|
|
528
|
+
if (posterUrl) options.approveUrl(options.input, posterUrl);
|
|
529
|
+
variables.mediaUrl = url;
|
|
530
|
+
variables.mediaType = resolved.type === "image" ? "photo" : "video";
|
|
531
|
+
if (posterUrl) variables.mediaPoster = posterUrl;
|
|
532
|
+
else delete variables.mediaPoster;
|
|
533
|
+
return variables;
|
|
534
|
+
}
|
|
535
|
+
function templateContract(templates, templateId) {
|
|
536
|
+
const metadata = templates.getTemplateMetadata(templateId);
|
|
537
|
+
return metadata == null ? void 0 : getStandardMediaResolverContract(metadata.schema);
|
|
538
|
+
}
|
|
539
|
+
function withoutKeyword(variables) {
|
|
540
|
+
if (!Object.hasOwn(variables, "mediaKeyword")) return variables;
|
|
541
|
+
const clean = { ...variables };
|
|
542
|
+
delete clean.mediaKeyword;
|
|
543
|
+
return clean;
|
|
544
|
+
}
|
|
545
|
+
function sanitizeUnknownTemplatePart(part) {
|
|
546
|
+
if (part.type === "scene.add") {
|
|
547
|
+
return { ...part, scene: { ...part.scene, variables: withoutKeyword(part.scene.variables) } };
|
|
548
|
+
}
|
|
549
|
+
if (part.type === "scene.patch" && part.patch.variables) {
|
|
550
|
+
return { ...part, patch: { ...part.patch, variables: withoutKeyword(part.patch.variables) } };
|
|
551
|
+
}
|
|
552
|
+
if (part.type === "asset.patch") {
|
|
553
|
+
return { ...part, variables: withoutKeyword(part.variables) };
|
|
554
|
+
}
|
|
555
|
+
return part;
|
|
556
|
+
}
|
|
557
|
+
async function resolvePartVariables(options) {
|
|
558
|
+
if (!options.templateId) return sanitizeUnknownTemplatePart(options.part);
|
|
559
|
+
const shared = {
|
|
560
|
+
contract: templateContract(options.templates, options.templateId),
|
|
561
|
+
input: options.input,
|
|
562
|
+
templateId: options.templateId,
|
|
563
|
+
signal: options.signal,
|
|
564
|
+
resolveMedia: options.resolveMedia,
|
|
565
|
+
approveUrl: options.approveUrl,
|
|
566
|
+
openingReady: options.openingReady
|
|
567
|
+
};
|
|
568
|
+
if (options.part.type === "scene.add") {
|
|
569
|
+
const variables = await resolveVariables({
|
|
570
|
+
...shared,
|
|
571
|
+
variables: options.part.scene.variables
|
|
572
|
+
});
|
|
573
|
+
return { ...options.part, scene: { ...options.part.scene, variables } };
|
|
574
|
+
}
|
|
575
|
+
if (options.part.type === "scene.patch" && options.part.patch.variables) {
|
|
576
|
+
const variables = await resolveVariables({
|
|
577
|
+
...shared,
|
|
578
|
+
variables: options.part.patch.variables
|
|
579
|
+
});
|
|
580
|
+
return { ...options.part, patch: { ...options.part.patch, variables } };
|
|
581
|
+
}
|
|
582
|
+
if (options.part.type === "asset.patch") {
|
|
583
|
+
const variables = await resolveVariables({
|
|
584
|
+
...shared,
|
|
585
|
+
variables: options.part.variables
|
|
586
|
+
});
|
|
587
|
+
return { ...options.part, variables };
|
|
588
|
+
}
|
|
589
|
+
return options.part;
|
|
590
|
+
}
|
|
591
|
+
function createMediaResolvingPlanner(options) {
|
|
592
|
+
return async function* resolveMediaPlan(context) {
|
|
593
|
+
const sceneTemplates = /* @__PURE__ */ new Map();
|
|
594
|
+
for await (const part of options.planner(context)) {
|
|
595
|
+
const templateId = part.type === "scene.add" ? part.scene.templateId : part.type === "scene.patch" || part.type === "asset.patch" ? sceneTemplates.get(part.sceneId) : void 0;
|
|
596
|
+
if (part.type === "scene.add") {
|
|
597
|
+
sceneTemplates.set(part.scene.id, part.scene.templateId);
|
|
598
|
+
}
|
|
599
|
+
yield await resolvePartVariables({
|
|
600
|
+
part,
|
|
601
|
+
templateId,
|
|
602
|
+
input: context.request.input,
|
|
603
|
+
signal: context.signal,
|
|
604
|
+
templates: options.templates,
|
|
605
|
+
resolveMedia: options.resolveMedia,
|
|
606
|
+
approveUrl: options.approveUrl,
|
|
607
|
+
openingReady: options.isOpeningReady(context.request.input)
|
|
608
|
+
});
|
|
609
|
+
}
|
|
610
|
+
};
|
|
611
|
+
}
|
|
612
|
+
|
|
462
613
|
// src/server/create-video-handler.ts
|
|
463
614
|
function createVideoHandler(options) {
|
|
464
615
|
if (options?.authorize !== "none" && typeof options?.authorize !== "function") {
|
|
@@ -473,22 +624,50 @@ function createVideoHandler(options) {
|
|
|
473
624
|
includeRawProviderData,
|
|
474
625
|
basePrompt,
|
|
475
626
|
allowMediaUrl,
|
|
627
|
+
resolveMedia,
|
|
628
|
+
requireCloser = true,
|
|
476
629
|
...handlerOptions
|
|
477
630
|
} = options;
|
|
478
631
|
const templates = configuredTemplates ? overlayServerTemplateRegistry(BUILTIN_SERVER_TEMPLATE_KIT, configuredTemplates) : BUILTIN_SERVER_TEMPLATE_KIT;
|
|
632
|
+
const approvedMediaUrls = /* @__PURE__ */ new WeakMap();
|
|
633
|
+
const openingReadyInputs = /* @__PURE__ */ new WeakSet();
|
|
634
|
+
const approveUrl = (input, url) => {
|
|
635
|
+
const approved = approvedMediaUrls.get(input) ?? /* @__PURE__ */ new Set();
|
|
636
|
+
approved.add(url);
|
|
637
|
+
approvedMediaUrls.set(input, approved);
|
|
638
|
+
};
|
|
639
|
+
const planner = createTextDeltaVideoPlanner({ streamText, includeRawProviderData });
|
|
640
|
+
const validateTemplateScene = createTemplateSceneValidator({
|
|
641
|
+
kit: templates,
|
|
642
|
+
allowMediaUrl: (url, context) => approvedMediaUrls.get(context.input)?.has(url) === true || allowMediaUrl?.(url, context) === true
|
|
643
|
+
});
|
|
479
644
|
return createVideoStreamHandler({
|
|
480
645
|
...handlerOptions,
|
|
481
|
-
|
|
646
|
+
requireCloser,
|
|
647
|
+
generate: createMediaResolvingPlanner({
|
|
648
|
+
planner,
|
|
649
|
+
templates,
|
|
650
|
+
resolveMedia,
|
|
651
|
+
approveUrl,
|
|
652
|
+
isOpeningReady: (input) => openingReadyInputs.has(input)
|
|
653
|
+
}),
|
|
482
654
|
systemPrompt: ({ capabilities: capabilities2 }) => {
|
|
483
655
|
const selectedIds = capabilities2?.templates == null ? void 0 : new Set(capabilities2.templates);
|
|
484
656
|
const selectedTemplates = selectedIds == null ? templates.listTemplateMetadata() : templates.listTemplateMetadata().filter(({ id }) => selectedIds.has(id));
|
|
485
657
|
return createTemplateSystemPrompt({
|
|
486
658
|
kit: { listTemplateMetadata: () => selectedTemplates },
|
|
487
|
-
basePrompt
|
|
659
|
+
basePrompt,
|
|
660
|
+
mediaResolverAvailable: resolveMedia != null
|
|
488
661
|
});
|
|
489
662
|
},
|
|
490
663
|
supportedCapabilities: templates.capabilities,
|
|
491
|
-
validateScene:
|
|
664
|
+
validateScene: (scene, context) => {
|
|
665
|
+
validateTemplateScene(scene, context);
|
|
666
|
+
const isAsk = templates.getTemplateMetadata(scene.templateId)?.jobs?.includes("ask") === true;
|
|
667
|
+
if (!isAsk) {
|
|
668
|
+
openingReadyInputs.add(context.input);
|
|
669
|
+
}
|
|
670
|
+
},
|
|
492
671
|
getTemplatePacing: (templateId) => templates.getTemplateMetadata(templateId)
|
|
493
672
|
});
|
|
494
673
|
}
|
package/dist/template-catalog.js
CHANGED
package/dist/templates.d.ts
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { I as InferTemplateJsonSchema, S as SceneTemplateProps, a as SceneTemplate } from './kit-
|
|
2
|
-
export { T as TemplateRegistry, c as createTemplateRegistry } from './kit-
|
|
1
|
+
import { I as InferTemplateJsonSchema, S as SceneTemplateProps, a as SceneTemplate } from './kit-CBedbj1n.js';
|
|
2
|
+
export { T as TemplateRegistry, c as createTemplateRegistry } from './kit-CBedbj1n.js';
|
|
3
3
|
import { ComponentType } from 'react';
|
|
4
4
|
import { d as TemplateJsonSchema, S as SceneTemplateMetadata } from './catalog-types-BIhSpOWK.js';
|
|
5
5
|
export { T as TemplateFamily, c as TemplateTimingMetadata, e as TemplateTransitionTiming } from './catalog-types-BIhSpOWK.js';
|
|
6
|
-
import './types-
|
|
6
|
+
import './types-_t4bduwX.js';
|
|
7
7
|
|
|
8
8
|
interface TemplateExample<Variables extends Record<string, unknown> = Record<string, unknown>> {
|
|
9
9
|
name: string;
|
package/dist/test.d.ts
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
import { f as VideoScene, e as VideoOrientation, g as VideoStyle, V as Video, a as VideoAudio, d as VideoInput } from './types-
|
|
2
|
-
import { a as VideoWarning, b as VideoFinishReason } from './events-
|
|
1
|
+
import { f as VideoScene, e as VideoOrientation, g as VideoStyle, V as Video, a as VideoAudio, d as VideoInput } from './types-_t4bduwX.js';
|
|
2
|
+
import { a as VideoWarning, b as VideoFinishReason } from './events-B-28kERX.js';
|
|
3
3
|
|
|
4
4
|
type MockVideoStreamPart = {
|
|
5
5
|
type: "scene.add";
|
|
6
|
+
placement?: "closer";
|
|
6
7
|
scene: VideoScene;
|
|
7
8
|
} | {
|
|
8
9
|
type: "scene.patch";
|
package/dist/test.js
CHANGED
|
@@ -233,9 +233,9 @@ async function* simulateVideoStream(parts, options = {}) {
|
|
|
233
233
|
if (timeoutMs != null && (!Number.isFinite(timeoutMs) || timeoutMs < 0)) {
|
|
234
234
|
throw new Error("Simulation timeoutMs must be a non-negative finite number");
|
|
235
235
|
}
|
|
236
|
-
const { createVideo } = await import("./compose-video-
|
|
237
|
-
const { createTextDeltaVideoPlanner } = await import("./text-stream-
|
|
238
|
-
const { BUILTIN_SERVER_TEMPLATE_KIT } = await import("./builtin-server-
|
|
236
|
+
const { createVideo } = await import("./compose-video-34GRZIF2.js");
|
|
237
|
+
const { createTextDeltaVideoPlanner } = await import("./text-stream-UPGUD2TD.js");
|
|
238
|
+
const { BUILTIN_SERVER_TEMPLATE_KIT } = await import("./builtin-server-BRTZN4Q7.js");
|
|
239
239
|
const { createTemplateSceneValidator } = await import("./validate-T7GBU2YF.js");
|
|
240
240
|
const controller = new AbortController();
|
|
241
241
|
const forwardAbort = () => controller.abort(options.signal?.reason);
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import {
|
|
2
2
|
createTextDeltaVideoPlanner
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-BKF3A357.js";
|
|
4
4
|
import "./chunk-E7CL7UPB.js";
|
|
5
|
-
import "./chunk-
|
|
5
|
+
import "./chunk-JW47XCRL.js";
|
|
6
6
|
import "./chunk-EE6PK6XC.js";
|
|
7
7
|
import "./chunk-XGRA2MUV.js";
|
|
8
8
|
export {
|
package/docs/concepts.md
CHANGED
|
@@ -54,6 +54,12 @@ another streaming text model.
|
|
|
54
54
|
The planner does not create public event IDs, checksums, revisions, or final
|
|
55
55
|
snapshots. The runtime owns those guarantees.
|
|
56
56
|
|
|
57
|
+
`createVideoHandler` asks the planner to emit one early
|
|
58
|
+
`scene.add` with `placement: "closer"` after the first playable body scene.
|
|
59
|
+
That placement is planner-only: the runtime holds the validated closer,
|
|
60
|
+
reserves its readable duration while later body scenes stream, and appends it
|
|
61
|
+
last without persisting `placement` into the completed `Video`.
|
|
62
|
+
|
|
57
63
|
| Boundary | Type | Owner |
|
|
58
64
|
| --- | --- | --- |
|
|
59
65
|
| Planner parts | Internal validated plan data | Server/provider adapter |
|
|
@@ -73,6 +79,7 @@ audio.set optional, before any scene
|
|
|
73
79
|
scene.add supplied opening
|
|
74
80
|
scene.add generated body
|
|
75
81
|
scene.add generated body
|
|
82
|
+
scene.add reserved closer
|
|
76
83
|
response.complete exact terminal snapshot
|
|
77
84
|
```
|
|
78
85
|
|
package/docs/custom-templates.md
CHANGED
|
@@ -311,9 +311,9 @@ Templates whose core proof needs a real statistic can add
|
|
|
311
311
|
`"x-vanillasky": { "requiresStat": true }`. See the structured-data and
|
|
312
312
|
supplied-media references for complete examples.
|
|
313
313
|
|
|
314
|
-
The automatic `opening` uses the built-in `
|
|
315
|
-
|
|
316
|
-
keep that variable contract.
|
|
314
|
+
The automatic `opening` uses the built-in `media` variables (`texts` and
|
|
315
|
+
`mediaType: "gradient"`). If you replace `media` while using automatic
|
|
316
|
+
openings, keep that variable contract.
|
|
317
317
|
|
|
318
318
|
## What belongs where
|
|
319
319
|
|
package/docs/errors.md
CHANGED
|
@@ -35,7 +35,9 @@ try {
|
|
|
35
35
|
`response.error` or `response.abort` rejects it. The hook still retains the
|
|
36
36
|
latest validated `video.video`, and its player stream still receives terminal
|
|
37
37
|
events, so an application may keep already accepted scenes visible while it
|
|
38
|
-
offers a retry.
|
|
38
|
+
offers a retry. If the planner already supplied a validated reserved closer,
|
|
39
|
+
the runtime appends it to that playable terminal snapshot before reporting a
|
|
40
|
+
late provider failure; it never invents or rewrites closer copy during recovery.
|
|
39
41
|
|
|
40
42
|
Both a server `response.abort` and an explicit `video.abort(reason)` keep
|
|
41
43
|
`video.status` at `aborted`; `video.error` contains the same safe typed abort
|
package/docs/getting-started.md
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
Install VanillaSky:
|
|
6
6
|
|
|
7
7
|
```bash
|
|
8
|
-
npm install @vanillaskyai/video@0.
|
|
8
|
+
npm install @vanillaskyai/video@0.2.0 ai @ai-sdk/openai
|
|
9
9
|
```
|
|
10
10
|
|
|
11
11
|
Set your provider key in `.env.local` (never commit it):
|
|
@@ -49,7 +49,7 @@ The local bypass is intentionally fail-closed: it accepts only localhost while
|
|
|
49
49
|
Next.js is in development and denies every production request. Replace it with
|
|
50
50
|
your real session validation before deploying. For literal files and commands,
|
|
51
51
|
use the tested
|
|
52
|
-
[`examples/nextjs-quickstart` directory](https://github.com/VanillaSkyAi/video/tree/v0.
|
|
52
|
+
[`examples/nextjs-quickstart` directory](https://github.com/VanillaSkyAi/video/tree/v0.2.0/examples/nextjs-quickstart).
|
|
53
53
|
|
|
54
54
|
`model` can come from any AI SDK provider, registry, gateway, compatible API,
|
|
55
55
|
or custom implementation. The application can choose a cheaper or faster model
|
|
@@ -90,10 +90,12 @@ reactive UI.
|
|
|
90
90
|
|
|
91
91
|
`video.status` is `idle`, `streaming`, `complete`, `error`, or `aborted`.
|
|
92
92
|
`video.video` is the latest deterministic video, and `video.warnings` contains
|
|
93
|
-
bounded typed diagnostics safe to show or branch on.
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
93
|
+
bounded typed diagnostics safe to show or branch on. A playable response that
|
|
94
|
+
stops at a planner length limit includes a `plan_incomplete` warning because
|
|
95
|
+
requested scenes or the ending may be missing. Provider finish reasons and
|
|
96
|
+
content-filter details remain available to the server through the `onComplete`
|
|
97
|
+
summary; surface that server-owned state separately when completeness matters
|
|
98
|
+
to your product.
|
|
97
99
|
|
|
98
100
|
Persist a completed `Video` as JSON and play it later without another model
|
|
99
101
|
request:
|
|
@@ -28,14 +28,17 @@ Bound request bytes and reject secret-shaped fields on the server. Do not pass p
|
|
|
28
28
|
|
|
29
29
|
## Opening
|
|
30
30
|
|
|
31
|
-
When supplied, `opening` becomes a deterministic
|
|
32
|
-
|
|
33
|
-
|
|
31
|
+
When supplied, `opening` becomes a deterministic `media` scene emitted before
|
|
32
|
+
provider work. It contains only the supplied sentence and explicitly uses the
|
|
33
|
+
brand gradient without stock media. The SDK requests a three-second opening,
|
|
34
|
+
then applies its normal readability and overall-duration budget. It owns the
|
|
35
|
+
scene ID, template, variables, and timing so callers only provide the copy. It
|
|
36
|
+
should:
|
|
34
37
|
|
|
35
38
|
- be personal or situational enough to feel intentional;
|
|
36
39
|
- require no network media lookup;
|
|
37
40
|
- remain true if generation later fails;
|
|
38
|
-
-
|
|
41
|
+
- be one concise sentence that fits comfortably in both supported orientations;
|
|
39
42
|
- be part of the final story, not a spinner disguised as a scene.
|
|
40
43
|
|
|
41
44
|
Omit `opening` when the generated story should begin with its first planned scene.
|
package/docs/integrate-nextjs.md
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
Install VanillaSky and one AI SDK provider:
|
|
6
6
|
|
|
7
7
|
```bash
|
|
8
|
-
npm install @vanillaskyai/video@0.
|
|
8
|
+
npm install @vanillaskyai/video@0.2.0 ai @ai-sdk/openai
|
|
9
9
|
```
|
|
10
10
|
|
|
11
11
|
Create an ignored `.env.local`:
|
|
@@ -71,7 +71,7 @@ requests; replace it with your application's session validation before
|
|
|
71
71
|
deploying.
|
|
72
72
|
|
|
73
73
|
The copy-and-run app is in the
|
|
74
|
-
[`examples/nextjs-quickstart` directory](https://github.com/VanillaSkyAi/video/tree/v0.
|
|
74
|
+
[`examples/nextjs-quickstart` directory](https://github.com/VanillaSkyAi/video/tree/v0.2.0/examples/nextjs-quickstart).
|
|
75
75
|
|
|
76
76
|
For another LLM, replace `openai(...)` with the matching AI SDK model. The route
|
|
77
77
|
shape and React code stay the same. See [Provider integration](provider-integration.md)
|
package/docs/media-and-audio.md
CHANGED
|
@@ -10,9 +10,10 @@ The 0.1 SDK does not provide narration, TTS, or speech synchronization.
|
|
|
10
10
|
If a product needs spoken audio, the application must create and synchronize
|
|
11
11
|
that experience outside this contract.
|
|
12
12
|
|
|
13
|
-
Only send source URLs you trust.
|
|
14
|
-
|
|
15
|
-
|
|
13
|
+
Only send source URLs you trust. Pass known approved assets through
|
|
14
|
+
`suppliedMedia`, or configure the server-only `resolveMedia` callback for
|
|
15
|
+
planner-selected backgrounds. Preload the next scene's asset before it becomes
|
|
16
|
+
active. Keep provider credentials on the server.
|
|
16
17
|
|
|
17
18
|
Supplied URLs and data URIs are not copied into the LLM prompt. The model sees
|
|
18
19
|
an optional pool of opaque HTTPS-shaped references plus safe descriptive
|
|
@@ -47,23 +48,81 @@ Keep the catalog and files in your application so you control caching,
|
|
|
47
48
|
licensing, and deployment. The SDK continues to handle playback, timing,
|
|
48
49
|
serialization, replay, and export from the supplied URL.
|
|
49
50
|
|
|
51
|
+
## Start with sound
|
|
52
|
+
|
|
53
|
+
Browsers block audible autoplay unless the viewer has already interacted with
|
|
54
|
+
the page. For a sound-first experience, keep the branded generation intro
|
|
55
|
+
visible and let the player's start control provide that interaction:
|
|
56
|
+
|
|
57
|
+
```tsx
|
|
58
|
+
<VideoPlayer
|
|
59
|
+
{...video.playerProps}
|
|
60
|
+
playbackMode="autoplay-after-interaction"
|
|
61
|
+
/>
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
The generation cover immediately includes a centered **Play with sound**
|
|
65
|
+
button. A click starts the soundtrack and holds that branded cover as a
|
|
66
|
+
three-second generation intro while planning continues. Once both the intro
|
|
67
|
+
and the first validated scene are ready, the generated timeline begins from
|
|
68
|
+
time zero. The generation intro remains visible indefinitely if the viewer has
|
|
69
|
+
not clicked yet, even when the complete generated video is already ready. The
|
|
70
|
+
server keeps the generated first scene on screen for at least three more
|
|
71
|
+
seconds when the duration budget permits. After that first successful sound start,
|
|
72
|
+
replacement streams on the same mounted player autoplay the same generation
|
|
73
|
+
intro with sound and fall back to a start control if the browser blocks them.
|
|
74
|
+
|
|
75
|
+
When `VideoInput.opening` is supplied, its asset-free gradient `media` scene
|
|
76
|
+
replaces the generic generation cover as soon as it is available. It remains
|
|
77
|
+
as the static start poster until the viewer clicks, then begins the actual
|
|
78
|
+
timeline with sound; there is no additional generic pre-roll before it.
|
|
79
|
+
|
|
80
|
+
Use `playbackMode="manual"` to require the button on every run,
|
|
81
|
+
`playbackMode="muted-autoplay"` for browser-safe muted autoplay, or
|
|
82
|
+
`playbackMode="autoplay-with-sound"` to try audible autoplay immediately. The
|
|
83
|
+
lower-level `autoPlay` and `startMuted` props remain available when no playback
|
|
84
|
+
mode is set.
|
|
85
|
+
|
|
50
86
|
## Media providers
|
|
51
87
|
|
|
52
|
-
VanillaSky is provider-independent.
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
88
|
+
VanillaSky is provider-independent. When `resolveMedia` is configured, the
|
|
89
|
+
built-in planner may emit a bounded semantic query for later media-capable
|
|
90
|
+
scenes. The SDK calls the application-owned resolver on the server, replaces
|
|
91
|
+
the query with the approved URL, type, and optional poster, then validates the
|
|
92
|
+
scene before emitting it. Without the callback, media intent stays hidden from
|
|
93
|
+
the planner. The first accepted generated scene remains asset-free.
|
|
94
|
+
|
|
95
|
+
```ts
|
|
96
|
+
import { createVideoHandler } from "@vanillaskyai/video/server";
|
|
97
|
+
|
|
98
|
+
createVideoHandler({
|
|
99
|
+
authorize: checkSession,
|
|
100
|
+
streamText: planWithYourModel,
|
|
101
|
+
resolveMedia: async (query, { preferredType, signal }) => {
|
|
102
|
+
const asset = await searchYourApprovedCatalog({ query, preferredType, signal });
|
|
103
|
+
return asset
|
|
104
|
+
? { url: asset.url, type: asset.type, posterUrl: asset.posterUrl }
|
|
105
|
+
: null;
|
|
106
|
+
},
|
|
107
|
+
});
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
The resolver query is 2–80 characters and at most eight words. Return `null`
|
|
111
|
+
when no licensed, safe, relevant asset exists; media-capable templates fall
|
|
112
|
+
back to the brand gradient when their schema permits it. The browser never
|
|
113
|
+
receives `mediaKeyword`, provider keys, or raw provider metadata.
|
|
56
114
|
|
|
57
115
|
`allowMediaUrl` is an authorization hook for applications with their own custom
|
|
58
116
|
stream adapter. It validates a final URL; it does not search for, fetch, or
|
|
59
117
|
resolve media. The default 0.1 path needs no callback because every planner URL
|
|
60
118
|
must already be present in `suppliedMedia`.
|
|
61
119
|
|
|
62
|
-
Do not expose provider keys to React
|
|
63
|
-
|
|
64
|
-
|
|
120
|
+
Do not expose provider keys to React or allow arbitrary planner URLs. Templates
|
|
121
|
+
describe visual building blocks; the application owns media retrieval,
|
|
122
|
+
caching, licensing, and delivery.
|
|
65
123
|
|
|
66
|
-
For Pexels, keep `PEXELS_API_KEY` on the server
|
|
67
|
-
|
|
68
|
-
`
|
|
69
|
-
orientation filtering, MIME checks, timeouts, caching,
|
|
124
|
+
For Pexels, keep `PEXELS_API_KEY` on the server and implement `resolveMedia`
|
|
125
|
+
with the Pexels API. Return only validated `images.pexels.com` or
|
|
126
|
+
`videos.pexels.com` results. The application remains responsible for
|
|
127
|
+
attribution, search, orientation filtering, MIME checks, timeouts, caching,
|
|
128
|
+
and fallback behavior.
|
package/docs/prompt-and-input.md
CHANGED
|
@@ -83,6 +83,13 @@ The model never returns React, HTML, CSS, or executable JavaScript.
|
|
|
83
83
|
Generated `scene.add`, `scene.patch`, and `asset.patch` operations all pass
|
|
84
84
|
through the same merged-scene validation before the player sees them.
|
|
85
85
|
|
|
86
|
+
The standard planner contract requires one `scene.add` with
|
|
87
|
+
`placement: "closer"` immediately after the first playable body scene. The
|
|
88
|
+
model writes short grounded conclusion copy: a supplied action when one
|
|
89
|
+
exists, otherwise a declarative payoff that answers the story's “so what.” The
|
|
90
|
+
runtime holds the closer and emits it last, so a long body plan cannot displace
|
|
91
|
+
an ending that was already generated.
|
|
92
|
+
|
|
86
93
|
## What reaches the LLM
|
|
87
94
|
|
|
88
95
|
The provider adapter receives:
|
|
@@ -84,6 +84,13 @@ duration actually committed; compare it with the `maxDurationSec` supplied by
|
|
|
84
84
|
your application when applying a retry policy. These fields provide a
|
|
85
85
|
server-side quality signal without exposing model metadata in the browser.
|
|
86
86
|
Warnings include the same bounded typed warnings emitted to the client.
|
|
87
|
+
`plan_incomplete` identifies a playable partial response whose planner reported
|
|
88
|
+
a length limit; applications should show that result as incomplete and may
|
|
89
|
+
offer a bounded retry with a larger output or duration budget.
|
|
90
|
+
`plan_missing_closer` identifies a playable plan that ended without the
|
|
91
|
+
explicit final scene required by the standard handler. `createVideoHandler`
|
|
92
|
+
sets `requireCloser: true` by default. Specialized deterministic integrations
|
|
93
|
+
may set `requireCloser: false`; ordinary AI planners should keep the default.
|
|
87
94
|
For non-interactive backfills, define an application threshold (for example,
|
|
88
95
|
no rejected scenes and a useful committed-duration ratio) and retry a bounded
|
|
89
96
|
number of times. Keep the best accepted result rather than treating
|
|
@@ -78,6 +78,15 @@ validated `scene.add`, `scene.patch`, `asset.patch`, `plan.complete`, or
|
|
|
78
78
|
snapshots, and checksums. Generated HTML, React, JavaScript, CSS, component
|
|
79
79
|
source, audio events, protocol envelopes, and unknown part types are rejected.
|
|
80
80
|
|
|
81
|
+
A planner may add `placement: "closer"` to exactly one `scene.add`. The
|
|
82
|
+
standard handler requires that closer by default, holds it outside the public
|
|
83
|
+
event stream while body scenes continue, and commits it as the final scene.
|
|
84
|
+
Only templates advertised for `jobs:[ask]` or `jobs:[payoff]` qualify. The
|
|
85
|
+
placement marker is not part of `VideoScene` and never enters a replay
|
|
86
|
+
snapshot. If the planner completes without a valid closer, the handler emits
|
|
87
|
+
`plan_missing_closer` and uses `finishReason: "other"`; provider `length` and
|
|
88
|
+
`content-filter` reasons remain unchanged.
|
|
89
|
+
|
|
81
90
|
## Resume
|
|
82
91
|
|
|
83
92
|
A resume request repeats the public input and includes:
|