@hypit/hypit 0.2.7 → 0.2.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/examples/provider-package/README.md +2 -2
- package/package.json +1 -1
- package/packages/media-execution/src/execute.ts +1 -1
- package/packages/pixverse/README.md +22 -13
- package/packages/pixverse/src/activation.ts +3 -1
- package/packages/pixverse/src/index.ts +181 -67
- package/packages/pixverse/src/surface.ts +181 -72
- package/packages/pixverse/src/validation.ts +13 -0
- package/packages/provider-hiapi/README.md +1 -1
- package/packages/provider-hypihub/README.md +12 -5
- package/packages/provider-hypihub/src/mapping.ts +23 -0
- package/packages/provider-monid/README.md +1 -1
- package/packages/provider-tokendance/README.md +1 -1
- package/packages/seedance/README.md +11 -18
- package/packages/seedance/src/index.ts +7 -7
- package/packages/seedance/src/surface.ts +9 -8
- package/packages/seedance-kits/README.md +4 -3
- package/packages/video-cli/src/snapshot.ts +18 -4
|
@@ -68,7 +68,7 @@ differ from the image example, and each is the reason the video example exists.
|
|
|
68
68
|
**The reference vocabulary is wider, and each role maps to its own field.** The Model declares
|
|
69
69
|
`referenceImage`, `referenceVideo` and `referenceAudio` as separate ports precisely so each can map
|
|
70
70
|
to one wire field — a service that mixes them into one array cannot tell the roles apart. An image
|
|
71
|
-
reference also
|
|
71
|
+
reference also requires a boolean `personReference` classification, which is why those ports use
|
|
72
72
|
`itemObject` rather than `urlArray`: the item field travels with the URL it belongs to.
|
|
73
73
|
|
|
74
74
|
```json
|
|
@@ -76,7 +76,7 @@ reference also carries an optional `personReference` classification, which is wh
|
|
|
76
76
|
```
|
|
77
77
|
|
|
78
78
|
`firstFrame` and `lastFrame` accept at most one item each, so their body fields are URLs.
|
|
79
|
-
This illustrative service accepts their
|
|
79
|
+
This illustrative service accepts their required visual-reference person classification on `POST /uploads` through
|
|
80
80
|
`x-person-reference: true` or `false`. Their mappings declare `resourceFields: ["personReference"]`,
|
|
81
81
|
and the URL resolver sends that header when supplied. Replace this with the real service’s
|
|
82
82
|
documented transport; a field that the service cannot carry must be refused, never discarded.
|
package/package.json
CHANGED
|
@@ -802,6 +802,7 @@ export async function executeRenderStillVideo(
|
|
|
802
802
|
"select=eq(n\\,0)",
|
|
803
803
|
"loop=loop=-1:size=1:start=0",
|
|
804
804
|
`trim=start_frame=0:end_frame=${frames}`,
|
|
805
|
+
`settb=expr=${denominator}/${numerator}`,
|
|
805
806
|
`setpts=N*${denominator}/(${numerator}*TB)`,
|
|
806
807
|
].join(",");
|
|
807
808
|
let argv: string[];
|
|
@@ -1278,7 +1279,6 @@ export async function executeMuxProgramMedia(
|
|
|
1278
1279
|
"-y", "-i", visualPath, "-i", audioPath,
|
|
1279
1280
|
"-map", `0:${visual.index}`, "-map", "1:0",
|
|
1280
1281
|
"-c:v", "copy", "-c:a", "aac", "-ar", "48000", "-ac", "2",
|
|
1281
|
-
"-frames:v", String(need.visual.frameCount),
|
|
1282
1282
|
"-movflags", "+faststart", output,
|
|
1283
1283
|
],
|
|
1284
1284
|
timeoutMs: env.processTimeoutMs,
|
|
@@ -1,16 +1,18 @@
|
|
|
1
1
|
# `@hypit/pixverse`
|
|
2
2
|
|
|
3
|
-
Exact author/compute contracts and
|
|
3
|
+
Exact author/compute contracts and package-owned author Surfaces for PixVerse. The package carries
|
|
4
|
+
two exact models, `pixverse-v6` and `pixverse-c1`, and selects no Provider, API key or network
|
|
5
|
+
execution. The selected Provider implements the exact capability.
|
|
4
6
|
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
Connect prompt and frames as ordinary graph edges:
|
|
7
|
+
Both models render 1 to 15 seconds at `360p`, `540p`, `720p` or `1080p` from a prompt of up to
|
|
8
|
+
5,000 characters. `<pix:Video>` generates from the prompt, from a first frame, or from a first and
|
|
9
|
+
last frame; `<pix:ReferenceVideo>` generates from the image and video subjects its `Reference`
|
|
10
|
+
children carry.
|
|
10
11
|
|
|
11
12
|
```xml
|
|
12
13
|
<pix:Video
|
|
13
14
|
id="opening"
|
|
15
|
+
model="v6"
|
|
14
16
|
prompt={line}
|
|
15
17
|
duration="5"
|
|
16
18
|
quality="720p"
|
|
@@ -18,17 +20,24 @@ Connect prompt and frames as ordinary graph edges:
|
|
|
18
20
|
generate-audio="true"
|
|
19
21
|
/>
|
|
20
22
|
|
|
21
|
-
<pix:Video id="bridge" prompt={motion} duration="5" quality="720p"
|
|
23
|
+
<pix:Video id="bridge" model="c1" prompt={motion} duration="5" quality="720p"
|
|
22
24
|
first-frame={hero.image} last-frame={product.image}/>
|
|
25
|
+
|
|
26
|
+
<pix:ReferenceVideo id="fusion" model="v6" prompt={outfit} duration="5" quality="720p" aspect-ratio="16:9">
|
|
27
|
+
<pix:Reference image={character.image}/>
|
|
28
|
+
<pix:Reference image={clothes.image}/>
|
|
29
|
+
</pix:ReferenceVideo>
|
|
23
30
|
```
|
|
24
31
|
|
|
25
|
-
The
|
|
26
|
-
|
|
32
|
+
The prompt addresses the references in the order they appear, as `@ref_1`, `@ref_2` and so on. V6
|
|
33
|
+
takes up to ten image references and C1 up to seven. V6 also takes up to two video references
|
|
34
|
+
totalling 15 seconds; those carry the length of the run, so that element states no `duration`, and
|
|
35
|
+
`aspect-ratio="auto"` takes their shape.
|
|
27
36
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
`multi-clip
|
|
37
|
+
A prompt-only run states its `aspect-ratio`; a run that starts from a `first-frame` takes that
|
|
38
|
+
frame's shape instead. A `last-frame` bridges from the first frame into one continuous shot, as do
|
|
39
|
+
subject references, so neither is combined with V6's `multi-clip`, which renders the prompt as
|
|
40
|
+
several cuts. `seed` and `multi-clip` are V6's own switches.
|
|
32
41
|
|
|
33
42
|
`generate-audio` renders an audio track alongside the picture, including speech the prompt asks a
|
|
34
43
|
character to say. The model exposes no separate voice, language or dialogue field, so a spoken line
|
|
@@ -2,10 +2,12 @@ import { createMarkupSurfaceHostFacet } from "@hypit/markup";
|
|
|
2
2
|
import {
|
|
3
3
|
pixverseComponent, pixverseDefinition, pixverseManifest, pixverseModuleRef, pixverseMarkupSurfaces,
|
|
4
4
|
} from "./index.js";
|
|
5
|
-
import { decodePixverseVideoSurface } from "./surface.js";
|
|
5
|
+
import { decodePixverseReferenceVideoSurface, decodePixverseVideoSurface } from "./surface.js";
|
|
6
6
|
export const hypitPackage = { format: "hypit.node-package@1" as const, modules: [{ manifest: pixverseManifest }], components: [pixverseComponent], hostFacets: [
|
|
7
7
|
pixverseDefinition.hostFacet,
|
|
8
8
|
createMarkupSurfaceHostFacet({ module: pixverseModuleRef,
|
|
9
9
|
declaration: pixverseMarkupSurfaces.find((item) => item.name === "video")!, handler: decodePixverseVideoSurface }),
|
|
10
|
+
createMarkupSurfaceHostFacet({ module: pixverseModuleRef,
|
|
11
|
+
declaration: pixverseMarkupSurfaces.find((item) => item.name === "reference-video")!, handler: decodePixverseReferenceVideoSurface }),
|
|
10
12
|
] };
|
|
11
13
|
export default hypitPackage;
|
|
@@ -1,117 +1,231 @@
|
|
|
1
1
|
import { artifactTypes } from "@hypit/artifact";
|
|
2
2
|
import { sealGenerationPortRequest, sealGenerationPortTable } from "@hypit/generation";
|
|
3
|
-
import type {
|
|
4
|
-
|
|
3
|
+
import type {
|
|
4
|
+
GenerationPort,
|
|
5
|
+
GenerationPortRequirement,
|
|
6
|
+
GenerationPortTable,
|
|
7
|
+
GenerationPortValue,
|
|
8
|
+
GenerationRequest,
|
|
9
|
+
} from "@hypit/generation";
|
|
10
|
+
import type { SurfaceAttributeVocabulary, SurfacePortVocabulary } from "@hypit/markup";
|
|
5
11
|
import { defineExactModelModule } from "@hypit/model-kit";
|
|
6
12
|
import { textTypes } from "@hypit/text";
|
|
7
13
|
|
|
14
|
+
import { pixverseRequestValidator } from "./validation.js";
|
|
15
|
+
|
|
8
16
|
export const pixverseModuleRef = { name: "@hypit/pixverse", version: "1" } as const;
|
|
9
|
-
export const pixverseModels = ["pixverse-v6"] as const;
|
|
17
|
+
export const pixverseModels = ["pixverse-v6", "pixverse-c1"] as const;
|
|
10
18
|
export type PixverseModel = typeof pixverseModels[number];
|
|
11
19
|
|
|
12
20
|
const PIXVERSE_QUALITIES = ["360p", "540p", "720p", "1080p"] as const;
|
|
13
21
|
const PIXVERSE_ASPECT_RATIOS = ["16:9", "4:3", "1:1", "3:4", "9:16", "2:3", "3:2", "21:9"] as const;
|
|
22
|
+
/** V6 reads `auto` as the shape of the reference videos it generates from. */
|
|
23
|
+
const PIXVERSE_V6_ASPECT_RATIOS = [...PIXVERSE_ASPECT_RATIOS, "auto"] as const;
|
|
14
24
|
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
{ name: "
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
25
|
+
/**
|
|
26
|
+
* Exact PixVerse model inputs. V6 takes up to ten reference images and up to two reference videos,
|
|
27
|
+
* and carries the sampling seed and multi-clip switch its endpoints declare. C1 takes up to seven
|
|
28
|
+
* reference images and generates from a prompt, a frame or a pair of frames.
|
|
29
|
+
*/
|
|
30
|
+
function pixversePortTable(model: PixverseModel): GenerationPortTable {
|
|
31
|
+
const v6 = model === "pixverse-v6";
|
|
32
|
+
const referenceVideo: readonly GenerationPort[] = v6
|
|
33
|
+
? [{ name: "referenceVideo", value: { kind: "media", accepts: ["video"] }, minItems: 0, maxItems: 2 }]
|
|
34
|
+
: [];
|
|
35
|
+
const v6Switches: readonly GenerationPort[] = v6
|
|
36
|
+
? [
|
|
37
|
+
{ name: "multiClip", value: { kind: "boolean" }, minItems: 0, maxItems: 1 },
|
|
38
|
+
{ name: "seed", value: { kind: "number", integer: true, minimum: 0, maximum: 2_147_483_647 }, minItems: 0, maxItems: 1 },
|
|
39
|
+
]
|
|
40
|
+
: [];
|
|
41
|
+
const v6Requires: readonly GenerationPortRequirement[] = v6
|
|
42
|
+
? [
|
|
43
|
+
// A run that bridges two frames, and one that carries subject references, is a single
|
|
44
|
+
// continuous shot rather than several cuts.
|
|
45
|
+
{ kind: "atMostOneOf", ports: ["multiClip", "lastFrame"] },
|
|
46
|
+
{ kind: "atMostOneOf", ports: ["multiClip", "referenceImage"] },
|
|
47
|
+
// The reference videos carry the length of the generated run.
|
|
48
|
+
{ kind: "atMostOneOf", ports: ["duration", "referenceVideo"] },
|
|
49
|
+
]
|
|
50
|
+
: [];
|
|
51
|
+
return sealGenerationPortTable({
|
|
52
|
+
model,
|
|
53
|
+
result: "video",
|
|
54
|
+
ports: [
|
|
55
|
+
{ name: "prompt", value: { kind: "text", maxChars: 5_000 }, minItems: 1, maxItems: 1 },
|
|
56
|
+
{ name: "firstFrame", value: { kind: "media", accepts: ["image"] }, minItems: 0, maxItems: 1 },
|
|
57
|
+
{ name: "lastFrame", value: { kind: "media", accepts: ["image"] }, minItems: 0, maxItems: 1 },
|
|
58
|
+
{ name: "referenceImage", value: { kind: "media", accepts: ["image"] }, minItems: 0, maxItems: v6 ? 10 : 7 },
|
|
59
|
+
...referenceVideo,
|
|
60
|
+
// Required except in the reference-video mode, which pixverseRequestValidator states.
|
|
61
|
+
{ name: "duration", value: { kind: "number", integer: true, minimum: 1, maximum: 15 }, minItems: 0, maxItems: 1 },
|
|
62
|
+
{ name: "quality", value: { kind: "enum", values: [...PIXVERSE_QUALITIES] }, minItems: 1, maxItems: 1 },
|
|
63
|
+
{
|
|
64
|
+
name: "aspectRatio",
|
|
65
|
+
value: { kind: "enum", values: v6 ? [...PIXVERSE_V6_ASPECT_RATIOS] : [...PIXVERSE_ASPECT_RATIOS] },
|
|
66
|
+
minItems: 0,
|
|
67
|
+
maxItems: 1,
|
|
68
|
+
},
|
|
69
|
+
{ name: "generateAudio", value: { kind: "boolean" }, minItems: 0, maxItems: 1 },
|
|
70
|
+
...v6Switches,
|
|
71
|
+
],
|
|
72
|
+
requires: [
|
|
73
|
+
// A last frame states where a run that already has a first frame ends.
|
|
74
|
+
{ kind: "requiresPresent", port: "lastFrame", needs: ["firstFrame"] },
|
|
75
|
+
// A run that starts from a frame inherits that frame's shape.
|
|
76
|
+
{ kind: "atMostOneOf", ports: ["aspectRatio", "firstFrame"] },
|
|
77
|
+
// Frames and subject references are separate ways of placing an image in the run.
|
|
78
|
+
{ kind: "atMostOneOf", ports: ["referenceImage", "firstFrame"] },
|
|
79
|
+
...v6Requires,
|
|
80
|
+
],
|
|
81
|
+
});
|
|
82
|
+
}
|
|
38
83
|
|
|
39
84
|
export const pixversePorts: Readonly<Record<PixverseModel, GenerationPortTable>> = {
|
|
40
|
-
"pixverse-v6":
|
|
85
|
+
"pixverse-v6": pixversePortTable("pixverse-v6"),
|
|
86
|
+
"pixverse-c1": pixversePortTable("pixverse-c1"),
|
|
41
87
|
};
|
|
42
88
|
|
|
43
89
|
export function sealPixverseRequest(
|
|
90
|
+
model: PixverseModel,
|
|
44
91
|
ports: Readonly<Record<string, readonly GenerationPortValue[]>>,
|
|
45
92
|
): GenerationRequest {
|
|
46
|
-
return sealGenerationPortRequest(
|
|
93
|
+
return sealGenerationPortRequest(pixversePorts[model], ports);
|
|
47
94
|
}
|
|
48
95
|
|
|
49
96
|
const pixverseBaseDefinition = defineExactModelModule({
|
|
50
97
|
module: pixverseModuleRef,
|
|
51
|
-
endpoints: [{
|
|
52
|
-
key
|
|
53
|
-
requestTypeName: "PixverseV6Request",
|
|
54
|
-
producerName:
|
|
55
|
-
ports:
|
|
56
|
-
|
|
98
|
+
endpoints: ([["v6", "pixverse-v6"], ["c1", "pixverse-c1"]] as const).map(([key, model]) => ({
|
|
99
|
+
key,
|
|
100
|
+
requestTypeName: model === "pixverse-v6" ? "PixverseV6Request" : "PixverseC1Request",
|
|
101
|
+
producerName: `request-${model}`,
|
|
102
|
+
ports: pixversePorts[model],
|
|
103
|
+
validateRequest: pixverseRequestValidator(model),
|
|
104
|
+
})),
|
|
57
105
|
});
|
|
58
106
|
|
|
59
107
|
export const pixverseEndpoints = pixverseBaseDefinition.endpoints;
|
|
108
|
+
export const pixverseEndpointsByModel = {
|
|
109
|
+
"pixverse-v6": pixverseEndpoints.v6!,
|
|
110
|
+
"pixverse-c1": pixverseEndpoints.c1!,
|
|
111
|
+
} as const;
|
|
60
112
|
export const pixverseComponent = pixverseBaseDefinition.component;
|
|
61
|
-
const endpoint = pixverseEndpoints.video!;
|
|
62
113
|
|
|
63
|
-
const
|
|
114
|
+
const surfaceOutputs = Object.values(pixverseEndpoints).flatMap((endpoint) => [
|
|
115
|
+
endpoint.draftType,
|
|
116
|
+
...Object.values(endpoint.mediaBindings).map((binding) => binding.type),
|
|
117
|
+
]);
|
|
118
|
+
|
|
119
|
+
const pixverseCommonAttributes: readonly SurfaceAttributeVocabulary[] = [
|
|
64
120
|
{ name: "id", kind: "identifier", required: true,
|
|
65
121
|
summary: "Names this generation so its video Artifact can be referenced elsewhere in the Source." },
|
|
122
|
+
{ name: "model", kind: "literal", required: true, values: ["v6", "pixverse-v6", "c1", "pixverse-c1"],
|
|
123
|
+
summary: "Chooses the exact PixVerse model that renders the video." },
|
|
66
124
|
{ name: "prompt", kind: "reference", required: true, accepts: [textTypes.text],
|
|
67
125
|
summary: "The Text edge describing the shot, including any spoken line the model should voice." },
|
|
68
|
-
{ name: "first-frame", kind: "reference", required: false, accepts: [artifactTypes.blob],
|
|
69
|
-
summary: "Starts the video from one image Artifact." },
|
|
70
|
-
{ name: "last-frame", kind: "reference", required: false, accepts: [artifactTypes.blob],
|
|
71
|
-
summary: "Ends the video on one image Artifact, bridging from the first frame." },
|
|
72
|
-
{ name: "duration", kind: "literal", required: true,
|
|
73
|
-
summary: "How many seconds of video to render, from 1 to 15." },
|
|
74
126
|
{ name: "quality", kind: "literal", required: true, values: [...PIXVERSE_QUALITIES],
|
|
75
127
|
summary: "The size band the model renders at." },
|
|
76
|
-
{ name: "aspect-ratio", kind: "literal", required: false, values: [...PIXVERSE_ASPECT_RATIOS],
|
|
77
|
-
summary: "The Frame shape of the generated video." },
|
|
78
128
|
{ name: "generate-audio", kind: "literal", required: false, values: ["true", "false"],
|
|
79
129
|
summary: "Renders an audio track alongside the picture." },
|
|
80
|
-
{ name: "multi-clip", kind: "literal", required: false, values: ["true", "false"],
|
|
81
|
-
summary: "Renders the prompt as several cuts instead of one continuous shot." },
|
|
82
130
|
{ name: "seed", kind: "literal", required: false,
|
|
83
|
-
summary: "Seeds
|
|
131
|
+
summary: "Seeds V6's sampling so a rerun stays close to this one." },
|
|
84
132
|
];
|
|
85
133
|
|
|
86
|
-
|
|
134
|
+
const pixverseAspectRatio: SurfaceAttributeVocabulary = {
|
|
135
|
+
name: "aspect-ratio", kind: "literal", required: false, values: [...PIXVERSE_V6_ASPECT_RATIOS],
|
|
136
|
+
summary: "The Frame shape of the generated video.",
|
|
137
|
+
};
|
|
138
|
+
|
|
139
|
+
const pixverseVideoPort: readonly SurfacePortVocabulary[] = [{
|
|
87
140
|
name: "video",
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
141
|
+
type: artifactTypes.blob,
|
|
142
|
+
summary: "The generated video, addressed as `<id>.video`.",
|
|
143
|
+
}];
|
|
144
|
+
|
|
145
|
+
const pixverseQualityNote = "`quality` is `360p`, `540p`, `720p` or `1080p`, and `duration` is 1 to 15 seconds.";
|
|
146
|
+
const pixversePromptNote = "A spoken line belongs in the prompt; the model exposes no separate voice, language or dialogue field.";
|
|
147
|
+
const pixverseModelNote = "V6 accepts `seed` and up to ten references; C1 accepts up to seven references.";
|
|
148
|
+
|
|
149
|
+
export const pixverseMarkupSurfaces = [
|
|
150
|
+
{
|
|
151
|
+
name: "video",
|
|
152
|
+
tag: "Video",
|
|
153
|
+
mode: "structured" as const,
|
|
154
|
+
outputs: surfaceOutputs,
|
|
155
|
+
vocabulary: {
|
|
156
|
+
summary: "Generates one video with an exact PixVerse model from a Text prompt, optionally starting from a frame or bridging two.",
|
|
157
|
+
attributes: [
|
|
158
|
+
...pixverseCommonAttributes,
|
|
159
|
+
{ name: "duration", kind: "literal" as const, required: true,
|
|
160
|
+
summary: "How many seconds of video to render, from 1 to 15." },
|
|
161
|
+
{ name: "first-frame", kind: "reference" as const, required: false, accepts: [artifactTypes.blob],
|
|
162
|
+
summary: "Starts the video from one image Artifact." },
|
|
163
|
+
{ name: "last-frame", kind: "reference" as const, required: false, accepts: [artifactTypes.blob],
|
|
164
|
+
summary: "Ends the video on one image Artifact, bridging from the first frame." },
|
|
165
|
+
pixverseAspectRatio,
|
|
166
|
+
{ name: "multi-clip", kind: "literal" as const, required: false, values: ["true", "false"],
|
|
167
|
+
summary: "Renders the prompt as several cuts instead of one continuous shot, on V6." },
|
|
168
|
+
],
|
|
169
|
+
ports: pixverseVideoPort,
|
|
170
|
+
example: `<pix:Video
|
|
100
171
|
id="opening"
|
|
172
|
+
model="v6"
|
|
101
173
|
prompt={line}
|
|
102
174
|
duration="5"
|
|
103
175
|
quality="720p"
|
|
104
176
|
aspect-ratio="9:16"
|
|
105
177
|
generate-audio="true"
|
|
106
178
|
/>`,
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
179
|
+
notes: [
|
|
180
|
+
pixverseQualityNote,
|
|
181
|
+
pixverseModelNote,
|
|
182
|
+
"A run that starts from a frame takes its shape from that frame, so `aspect-ratio` states the shape only for a prompt-only run.",
|
|
183
|
+
"A `last-frame` bridges from the `first-frame` into one continuous shot, so it is not combined with `multi-clip`.",
|
|
184
|
+
pixversePromptNote,
|
|
185
|
+
"The element accepts no children and no text content.",
|
|
186
|
+
],
|
|
187
|
+
},
|
|
188
|
+
},
|
|
189
|
+
{
|
|
190
|
+
name: "reference-video",
|
|
191
|
+
tag: "ReferenceVideo",
|
|
192
|
+
mode: "structured" as const,
|
|
193
|
+
outputs: surfaceOutputs,
|
|
194
|
+
vocabulary: {
|
|
195
|
+
summary: "Generates one video with an exact PixVerse model from a Text prompt and the image or video subjects it carries.",
|
|
196
|
+
attributes: [
|
|
197
|
+
...pixverseCommonAttributes,
|
|
198
|
+
{ name: "duration", kind: "literal" as const, required: false,
|
|
199
|
+
summary: "How many seconds of video to render, from 1 to 15; video references carry their own length." },
|
|
200
|
+
pixverseAspectRatio,
|
|
201
|
+
],
|
|
202
|
+
children: [{
|
|
203
|
+
tag: "Reference",
|
|
204
|
+
cardinality: "many" as const,
|
|
205
|
+
summary: "Attaches one subject Artifact the model generates from, chosen by an `image` or `video` reference.",
|
|
206
|
+
attributes: [
|
|
207
|
+
{ name: "image", kind: "reference" as const, required: false, accepts: [artifactTypes.blob],
|
|
208
|
+
summary: "Selects the image Artifact whose subject the generated video carries." },
|
|
209
|
+
{ name: "video", kind: "reference" as const, required: false, accepts: [artifactTypes.blob],
|
|
210
|
+
summary: "Selects the video Artifact whose motion and subject the generated video carries." },
|
|
211
|
+
],
|
|
212
|
+
}],
|
|
213
|
+
ports: pixverseVideoPort,
|
|
214
|
+
example: `<pix:ReferenceVideo id="fusion" model="v6" prompt={outfit} duration="5" quality="720p" aspect-ratio="16:9">
|
|
215
|
+
<pix:Reference image={character.image}/>
|
|
216
|
+
<pix:Reference image={clothes.image}/>
|
|
217
|
+
</pix:ReferenceVideo>`,
|
|
218
|
+
notes: [
|
|
219
|
+
pixverseQualityNote,
|
|
220
|
+
pixverseModelNote,
|
|
221
|
+
"The prompt addresses the references in order as `@ref_1`, `@ref_2` and so on.",
|
|
222
|
+
"`Reference` carries exactly one of `image` or `video`, and is empty.",
|
|
223
|
+
"V6 accepts up to two video references totalling 15 seconds, which carry the length of the run in place of `duration`; `aspect-ratio=\"auto\"` takes their shape.",
|
|
224
|
+
pixversePromptNote,
|
|
225
|
+
],
|
|
226
|
+
},
|
|
113
227
|
},
|
|
114
|
-
|
|
228
|
+
] as const;
|
|
115
229
|
|
|
116
230
|
export const pixverseManifest = {
|
|
117
231
|
...pixverseBaseDefinition.manifest,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { artifactTypes } from "@hypit/artifact";
|
|
2
2
|
import { generationPort, sealGenerationMediaBinding, sealGenerationRequestDraft } from "@hypit/generation";
|
|
3
|
-
import type { GenerationMediaPort, GenerationPortValue } from "@hypit/generation";
|
|
3
|
+
import type { GenerationMediaPort, GenerationMediaRole, GenerationPortValue } from "@hypit/generation";
|
|
4
4
|
import {
|
|
5
5
|
createExactModelPrimaryGenerationFragment,
|
|
6
6
|
exactModelMediaInputNames,
|
|
@@ -16,23 +16,31 @@ import type {
|
|
|
16
16
|
import type { CanonicalValue, TypeRef } from "@hypit/protocol";
|
|
17
17
|
import { textTypes, verifyText } from "@hypit/text";
|
|
18
18
|
|
|
19
|
-
import {
|
|
19
|
+
import { pixverseEndpointsByModel } from "./index.js";
|
|
20
|
+
import type { PixverseModel } from "./index.js";
|
|
21
|
+
|
|
22
|
+
type MediaInput = {
|
|
23
|
+
readonly port: "firstFrame" | "lastFrame" | "referenceImage" | "referenceVideo";
|
|
24
|
+
readonly role: GenerationMediaRole;
|
|
25
|
+
readonly source: SurfaceResolvedReference;
|
|
26
|
+
};
|
|
20
27
|
|
|
21
28
|
function assert(condition: unknown, message: string): asserts condition {
|
|
22
29
|
if (!condition) throw new Error(message);
|
|
23
30
|
}
|
|
24
31
|
|
|
32
|
+
function localName(name: string): string {
|
|
33
|
+
return name.includes(":") ? name.slice(name.lastIndexOf(":") + 1) : name;
|
|
34
|
+
}
|
|
35
|
+
|
|
25
36
|
function sameType(left: TypeRef, right: TypeRef): boolean {
|
|
26
37
|
return left.name === right.name && left.module.name === right.module.name && left.module.version === right.module.version;
|
|
27
38
|
}
|
|
28
39
|
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
function exact(element: StructuredElement): void {
|
|
33
|
-
const unknown = Object.keys(element.attributes).filter((name) => !attributes.includes(name as typeof attributes[number]));
|
|
40
|
+
function exact(element: StructuredElement, allowed: readonly string[], required: readonly string[]): void {
|
|
41
|
+
const unknown = Object.keys(element.attributes).filter((name) => !allowed.includes(name));
|
|
34
42
|
assert(unknown.length === 0, `${element.name} does not accept ${unknown[0]}`);
|
|
35
|
-
const missing =
|
|
43
|
+
const missing = required.filter((name) => element.attributes[name] === undefined);
|
|
36
44
|
assert(missing.length === 0, `${element.name} requires ${missing.join(", ")}`);
|
|
37
45
|
}
|
|
38
46
|
|
|
@@ -60,6 +68,10 @@ function integer(element: StructuredElement, name: string): readonly GenerationP
|
|
|
60
68
|
return [Number(value)];
|
|
61
69
|
}
|
|
62
70
|
|
|
71
|
+
function optionalInteger(element: StructuredElement, name: string): readonly GenerationPortValue[] | undefined {
|
|
72
|
+
return element.attributes[name] === undefined ? undefined : integer(element, name);
|
|
73
|
+
}
|
|
74
|
+
|
|
63
75
|
function ref(
|
|
64
76
|
element: StructuredElement,
|
|
65
77
|
name: string,
|
|
@@ -73,74 +85,171 @@ function ref(
|
|
|
73
85
|
return result;
|
|
74
86
|
}
|
|
75
87
|
|
|
76
|
-
function
|
|
88
|
+
function media(
|
|
77
89
|
element: StructuredElement,
|
|
78
90
|
name: string,
|
|
91
|
+
role: GenerationMediaRole,
|
|
79
92
|
resolve: (path: string) => SurfaceResolvedReference | undefined,
|
|
80
|
-
): SurfaceResolvedReference
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
`${element.name}.${name} must reference image media`);
|
|
93
|
+
): SurfaceResolvedReference {
|
|
94
|
+
const artifact = ref(element, name, artifactTypes.blob, resolve);
|
|
95
|
+
if (artifact.record !== undefined) {
|
|
96
|
+
assert(artifact.record.value.kind === "blob" && artifact.record.value.mediaType.startsWith(`${role}/`),
|
|
97
|
+
`${element.name}.${name} must reference ${role} media`);
|
|
86
98
|
}
|
|
87
|
-
return
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
function
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
)
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
99
|
+
return artifact;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
function empty(element: StructuredElement): void {
|
|
103
|
+
assert(!element.children.some((item) => item.kind === "element" || item.value.trim()),
|
|
104
|
+
`${element.name} accepts no children`);
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
function selectModel(element: StructuredElement): { endpoint: ExactModelEndpoint; model: PixverseModel } {
|
|
108
|
+
const requested = text(element, "model");
|
|
109
|
+
if (requested === "v6" || requested === "pixverse-v6") {
|
|
110
|
+
return { endpoint: pixverseEndpointsByModel["pixverse-v6"], model: "pixverse-v6" };
|
|
111
|
+
}
|
|
112
|
+
if (requested === "c1" || requested === "pixverse-c1") {
|
|
113
|
+
return { endpoint: pixverseEndpointsByModel["pixverse-c1"], model: "pixverse-c1" };
|
|
114
|
+
}
|
|
115
|
+
throw new Error(`${element.name}.model must be v6 or c1`);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/** A switch one model carries and the other does not is refused where it was authored. */
|
|
119
|
+
function modelScoped(
|
|
120
|
+
element: StructuredElement,
|
|
121
|
+
endpoint: ExactModelEndpoint,
|
|
122
|
+
attribute: string,
|
|
123
|
+
port: string,
|
|
124
|
+
read: (element: StructuredElement, name: string) => readonly GenerationPortValue[] | undefined,
|
|
125
|
+
): readonly GenerationPortValue[] | undefined {
|
|
126
|
+
if (element.attributes[attribute] === undefined) return undefined;
|
|
127
|
+
assert(endpoint.ports.ports.some((item) => item.name === port),
|
|
128
|
+
`${element.name}.${attribute} is not accepted by ${endpoint.ports.model}`);
|
|
129
|
+
return read(element, attribute);
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
function capacity(element: StructuredElement, endpoint: ExactModelEndpoint, port: string, used: number): void {
|
|
133
|
+
if (used === 0) return;
|
|
134
|
+
const declared = endpoint.ports.ports.find((item) => item.name === port);
|
|
135
|
+
assert(declared !== undefined, `${element.name} references are not accepted by ${endpoint.ports.model}`);
|
|
136
|
+
assert(used <= declared.maxItems,
|
|
137
|
+
`${element.name} accepts at most ${declared.maxItems} ${port === "referenceVideo" ? "video" : "image"} references on ${endpoint.ports.model}`);
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
function assemble(
|
|
141
|
+
element: StructuredElement,
|
|
142
|
+
endpoint: ExactModelEndpoint,
|
|
143
|
+
prompt: SurfaceResolvedReference,
|
|
144
|
+
inputs: readonly MediaInput[],
|
|
145
|
+
stated: Readonly<Record<string, readonly GenerationPortValue[] | undefined>>,
|
|
146
|
+
) {
|
|
147
|
+
const id = text(element, "id");
|
|
148
|
+
const draft = sealGenerationRequestDraft(endpoint.ports, Object.fromEntries(
|
|
149
|
+
Object.entries(stated).filter((entry): entry is [string, readonly GenerationPortValue[]] => entry[1] !== undefined),
|
|
150
|
+
));
|
|
151
|
+
const records: Array<{ id: string; type: TypeRef; value: { kind: "inline"; value: CanonicalValue }; range: StructuredElement["range"] }> = [{
|
|
152
|
+
id: `${id}.draft`, type: endpoint.draftType,
|
|
153
|
+
value: { kind: "inline", value: draft as unknown as CanonicalValue }, range: element.range,
|
|
154
|
+
}];
|
|
155
|
+
const componentInputs: Record<string, SurfaceResolvedReference["ref"] | { kind: "record"; id: string }> = {
|
|
156
|
+
draft: { kind: "record", id: `${id}.draft` }, [exactModelTextInputName("prompt")]: prompt.ref,
|
|
143
157
|
};
|
|
158
|
+
const attached = inputs.map(({ port, role, source }, index) => {
|
|
159
|
+
const name = `media-${String(index + 1).padStart(4, "0")}`;
|
|
160
|
+
const bindingId = `${id}.${name}.binding`;
|
|
161
|
+
const mediaPort = generationPort(endpoint.ports, port);
|
|
162
|
+
assert(mediaPort.value.kind === "media", `PixVerse port ${port} is not media`);
|
|
163
|
+
records.push({
|
|
164
|
+
id: bindingId, type: endpoint.mediaBindings[port]!.type,
|
|
165
|
+
value: { kind: "inline", value: sealGenerationMediaBinding(mediaPort as GenerationMediaPort, { role }) as unknown as CanonicalValue },
|
|
166
|
+
range: element.range,
|
|
167
|
+
});
|
|
168
|
+
const names = exactModelMediaInputNames(name);
|
|
169
|
+
componentInputs[names.binding] = { kind: "record", id: bindingId };
|
|
170
|
+
componentInputs[names.artifact] = source.ref;
|
|
171
|
+
return { name, port } as const;
|
|
172
|
+
});
|
|
173
|
+
const fragment = createExactModelPrimaryGenerationFragment(endpoint, attached, [{ name: "prompt", port: "prompt" }]);
|
|
174
|
+
return { records, fragments: [fragment], components: [{
|
|
175
|
+
id, fragment: fragment.id, inputs: componentInputs, outputs: { video: `${id}.video` }, range: element.range,
|
|
176
|
+
}] };
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
function promptReference(
|
|
180
|
+
element: StructuredElement,
|
|
181
|
+
resolveReference: (path: string) => SurfaceResolvedReference | undefined,
|
|
182
|
+
): SurfaceResolvedReference {
|
|
183
|
+
const prompt = ref(element, "prompt", textTypes.text, resolveReference);
|
|
184
|
+
if (prompt.record !== undefined) {
|
|
185
|
+
assert(prompt.record.value.kind === "inline", `${element.name}.prompt must reference Text`);
|
|
186
|
+
verifyText(prompt.record.value.value);
|
|
187
|
+
}
|
|
188
|
+
return prompt;
|
|
144
189
|
}
|
|
145
190
|
|
|
146
|
-
|
|
191
|
+
const VIDEO_ATTRIBUTES = ["id", "model", "prompt", "first-frame", "last-frame", "duration", "quality",
|
|
192
|
+
"aspect-ratio", "generate-audio", "multi-clip", "seed"] as const;
|
|
193
|
+
|
|
194
|
+
export const decodePixverseVideoSurface: StructuredSurfaceHandler = ({ element, resolveReference }) => {
|
|
195
|
+
exact(element, VIDEO_ATTRIBUTES, ["id", "model", "prompt", "duration", "quality"]);
|
|
196
|
+
empty(element);
|
|
197
|
+
const { endpoint } = selectModel(element);
|
|
198
|
+
const frames = ([["firstFrame", "first-frame"], ["lastFrame", "last-frame"]] as const)
|
|
199
|
+
.filter(([, attribute]) => element.attributes[attribute] !== undefined)
|
|
200
|
+
.map(([port, attribute]) => ({
|
|
201
|
+
port, role: "image" as const, source: media(element, attribute, "image", resolveReference),
|
|
202
|
+
}));
|
|
203
|
+
return assemble(element, endpoint, promptReference(element, resolveReference), frames, {
|
|
204
|
+
duration: integer(element, "duration"),
|
|
205
|
+
quality: [text(element, "quality")],
|
|
206
|
+
aspectRatio: optionalText(element, "aspect-ratio"),
|
|
207
|
+
generateAudio: optionalFlag(element, "generate-audio"),
|
|
208
|
+
multiClip: modelScoped(element, endpoint, "multi-clip", "multiClip", optionalFlag),
|
|
209
|
+
seed: modelScoped(element, endpoint, "seed", "seed", optionalInteger),
|
|
210
|
+
});
|
|
211
|
+
};
|
|
212
|
+
|
|
213
|
+
const REFERENCE_ATTRIBUTES = ["id", "model", "prompt", "duration", "quality",
|
|
214
|
+
"aspect-ratio", "generate-audio", "seed"] as const;
|
|
215
|
+
|
|
216
|
+
export const decodePixverseReferenceVideoSurface: StructuredSurfaceHandler = ({ element, resolveReference }) => {
|
|
217
|
+
exact(element, REFERENCE_ATTRIBUTES, ["id", "model", "prompt", "quality"]);
|
|
218
|
+
const { endpoint } = selectModel(element);
|
|
219
|
+
const references: MediaInput[] = [];
|
|
220
|
+
for (const child of element.children) {
|
|
221
|
+
if (child.kind === "text") {
|
|
222
|
+
assert(child.value.trim().length === 0, `${element.name} accepts only Reference children`);
|
|
223
|
+
continue;
|
|
224
|
+
}
|
|
225
|
+
assert(localName(child.name) === "Reference", `${element.name} accepts only Reference children`);
|
|
226
|
+
empty(child);
|
|
227
|
+
const roles = (["image", "video"] as const).filter((role) => child.attributes[role] !== undefined);
|
|
228
|
+
exact(child, ["image", "video"], []);
|
|
229
|
+
assert(roles.length === 1, `${child.name} requires exactly one of image, video`);
|
|
230
|
+
const role = roles[0]!;
|
|
231
|
+
references.push({
|
|
232
|
+
port: role === "image" ? "referenceImage" : "referenceVideo",
|
|
233
|
+
role,
|
|
234
|
+
source: media(child, role, role, resolveReference),
|
|
235
|
+
});
|
|
236
|
+
}
|
|
237
|
+
assert(references.length > 0, `${element.name} requires at least one Reference`);
|
|
238
|
+
for (const port of ["referenceImage", "referenceVideo"] as const) {
|
|
239
|
+
capacity(element, endpoint, port, references.filter((item) => item.port === port).length);
|
|
240
|
+
}
|
|
241
|
+
const videos = references.some((item) => item.port === "referenceVideo");
|
|
242
|
+
if (videos) {
|
|
243
|
+
assert(element.attributes["duration"] === undefined,
|
|
244
|
+
`${element.name} takes no duration; its video references carry the length of the run`);
|
|
245
|
+
} else {
|
|
246
|
+
assert(element.attributes["duration"] !== undefined, `${element.name} requires duration`);
|
|
247
|
+
}
|
|
248
|
+
return assemble(element, endpoint, promptReference(element, resolveReference), references, {
|
|
249
|
+
duration: optionalInteger(element, "duration"),
|
|
250
|
+
quality: [text(element, "quality")],
|
|
251
|
+
aspectRatio: optionalText(element, "aspect-ratio"),
|
|
252
|
+
generateAudio: optionalFlag(element, "generate-audio"),
|
|
253
|
+
seed: modelScoped(element, endpoint, "seed", "seed", optionalInteger),
|
|
254
|
+
});
|
|
255
|
+
};
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { GenerationRequest } from "@hypit/generation";
|
|
2
|
+
import type { PixverseModel } from "./index.js";
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Reference videos carry the length of the run, so that mode states no duration.
|
|
6
|
+
* Every other mode states its own.
|
|
7
|
+
*/
|
|
8
|
+
export function pixverseRequestValidator(model: PixverseModel): (request: GenerationRequest) => void {
|
|
9
|
+
return (request) => {
|
|
10
|
+
if (request.ports.referenceVideo !== undefined || request.ports.duration !== undefined) return;
|
|
11
|
+
throw new Error(`${model} requires duration`);
|
|
12
|
+
};
|
|
13
|
+
}
|
|
@@ -37,7 +37,7 @@ Service limits this Provider reports as unsupported before submitting:
|
|
|
37
37
|
`1:3`, `9:21`; 4K excludes `1:1`, `3:1`, `1:3`, `9:21`; image-to-image takes up to six references.
|
|
38
38
|
- Grok Imagine renders 480p or 720p; `grok-imagine-1.5/image-to-video` animates exactly one image.
|
|
39
39
|
|
|
40
|
-
Seedance visual references
|
|
40
|
+
Seedance visual references require `person-reference`; the Provider accepts the declaration and
|
|
41
41
|
transmits nothing for it, since HiAPI has no field for it. Seedance rejects reference images and
|
|
42
42
|
videos that contain a real human face; HiAPI offers no way to register authorized portrait material,
|
|
43
43
|
so such a request fails with the service's moderation error.
|
|
@@ -20,6 +20,13 @@ Seedance 2.5 (`@hypit/seedance` model `2.5`) maps to `seedance-2.5` and supports
|
|
|
20
20
|
`480p`, `720p` and `1080p`. The Provider passes the authored `resolution` to `POST /v1/videos`;
|
|
21
21
|
omitting it in the Seedance Surface defaults to `720p`.
|
|
22
22
|
|
|
23
|
+
`@hypit/pixverse` models `pixverse-v6` and `pixverse-c1` map to `pixverse/v6` and `pixverse/c1` on
|
|
24
|
+
`POST /v1/videos`. The model's own `quality` band travels as `resolution` and its duration as
|
|
25
|
+
`seconds`; frames use `first_frame` and `last_frame`, image references use `reference_image_urls`,
|
|
26
|
+
and V6's video references use `reference_videos`. A reference-video request carries no `seconds`.
|
|
27
|
+
This body has no field for V6's `seed` or `multi-clip`, so a request that states either is refused
|
|
28
|
+
by name before any reference is uploaded.
|
|
29
|
+
|
|
23
30
|
The current HypiHub GPT Image 2 route has these service-specific limits:
|
|
24
31
|
|
|
25
32
|
| Resolution | Ratios unavailable at this Endpoint | `background` |
|
|
@@ -32,7 +39,8 @@ HypiHub owns this support check independently: it leaves the GPT Image model pac
|
|
|
32
39
|
the model or another Provider.
|
|
33
40
|
|
|
34
41
|
Model identity and input mode are separate. The mapping uses HypiHub's canonical model names:
|
|
35
|
-
`gpt-image-2`, `seedream-5-lite`, `minimax-h3`, `grok-imagine-video`
|
|
42
|
+
`gpt-image-2`, `seedream-5-lite`, `minimax-h3`, `grok-imagine-video`, `pixverse/v6`,
|
|
43
|
+
`pixverse/c1` and the individual Seedance names.
|
|
36
44
|
An image request without references uses `/images/generations`; image edits use `/images/edits`
|
|
37
45
|
with the same model name. Video requests use `/videos`, preserving reference images, reference
|
|
38
46
|
videos and first/last frames in their distinct fields. Old operation-specific names are not needed
|
|
@@ -123,12 +131,11 @@ size and part concurrency, retries a failed part with a fresh signed URL, comple
|
|
|
123
131
|
one upload, and then passes the returned HTTPS URL to generation or transcription. Signing requests
|
|
124
132
|
contain at most the service's 128-part limit; all batches belong to the same upload. One
|
|
125
133
|
Resource identity with the same declared person-reference classification is uploaded once within one Runtime operation. Hypit keeps no upload catalog or
|
|
126
|
-
cross-Build cache. Seedance visual references
|
|
134
|
+
cross-Build cache. Seedance visual references require boolean `personReference` in their media fields;
|
|
127
135
|
the mapping declares it as a resource-transport field and the upload session receives
|
|
128
|
-
`is_person_reference`, preserving true
|
|
136
|
+
`is_person_reference`, preserving true and false. It stays out of the generation body.
|
|
129
137
|
This covers reference images, reference videos, and first/last frames for every declared Seedance
|
|
130
|
-
variant.
|
|
131
|
-
so omission does not enable detection or person-reference preparation.
|
|
138
|
+
variant. Seedance rejects omission before upload; no automatic face detection is requested.
|
|
132
139
|
HypiHub stores the authored classification and prepares the applicable upstream person reference;
|
|
133
140
|
this Provider does not detect faces or select an upstream private-avatar group.
|
|
134
141
|
|
|
@@ -8,6 +8,7 @@ const NANO_BANANA: ModuleRef = { name: "@hypit/nano-banana", version: "1" };
|
|
|
8
8
|
const SEEDREAM: ModuleRef = { name: "@hypit/seedream", version: "1" };
|
|
9
9
|
const MINIMAX: ModuleRef = { name: "@hypit/minimax-h3", version: "1" };
|
|
10
10
|
const GROK: ModuleRef = { name: "@hypit/grok-imagine", version: "1" };
|
|
11
|
+
const PIXVERSE: ModuleRef = { name: "@hypit/pixverse", version: "1" };
|
|
11
12
|
const MIMO_SPEECH: ModuleRef = { name: "@hypit/mimo-speech", version: "1" };
|
|
12
13
|
const FISHAUDIO_SPEECH: ModuleRef = { name: "@hypit/fishaudio-speech", version: "1" };
|
|
13
14
|
const ELEVENLABS_SPEECH: ModuleRef = { name: "@hypit/elevenlabs-speech", version: "1" };
|
|
@@ -29,6 +30,26 @@ const seedance = (name: string): GenerationWireMapping => ({
|
|
|
29
30
|
},
|
|
30
31
|
});
|
|
31
32
|
|
|
33
|
+
/**
|
|
34
|
+
* PixVerse V6 and C1 on `POST /v1/videos`. Both take the same body; V6 additionally accepts
|
|
35
|
+
* reference videos, which carry the length of the run in place of `seconds`. The model's own
|
|
36
|
+
* `quality` band is HypiHub's `resolution`.
|
|
37
|
+
*/
|
|
38
|
+
const pixverse = (name: string, model: string): GenerationWireMapping => ({
|
|
39
|
+
capability: { module: PIXVERSE, name }, result: "video", routes: [{ model }],
|
|
40
|
+
fields: {
|
|
41
|
+
prompt: { as: "value", field: "prompt" },
|
|
42
|
+
firstFrame: { as: "url", field: "first_frame" },
|
|
43
|
+
lastFrame: { as: "url", field: "last_frame" },
|
|
44
|
+
referenceImage: { as: "urlArray", field: "reference_image_urls" },
|
|
45
|
+
...(name === "pixverse-v6" ? { referenceVideo: { as: "urlArray" as const, field: "reference_videos" } } : {}),
|
|
46
|
+
duration: { as: "value", field: "seconds" },
|
|
47
|
+
quality: { as: "value", field: "resolution" },
|
|
48
|
+
aspectRatio: { as: "value", field: "aspect_ratio" },
|
|
49
|
+
generateAudio: { as: "value", field: "generate_audio" },
|
|
50
|
+
},
|
|
51
|
+
});
|
|
52
|
+
|
|
32
53
|
export const hypiHubMappings: readonly GenerationWireMapping[] = [
|
|
33
54
|
{
|
|
34
55
|
capability: { module: { name: "@hypit/volcengine-matting", version: "1" }, name: "matte-portrait-video" },
|
|
@@ -42,6 +63,8 @@ export const hypiHubMappings: readonly GenerationWireMapping[] = [
|
|
|
42
63
|
seedance("seedance-2-fast"),
|
|
43
64
|
seedance("seedance-2-mini"),
|
|
44
65
|
seedance("seedance-2.5"),
|
|
66
|
+
pixverse("pixverse-v6", "pixverse/v6"),
|
|
67
|
+
pixverse("pixverse-c1", "pixverse/c1"),
|
|
45
68
|
{
|
|
46
69
|
capability: { module: GPT_IMAGE, name: "gpt-image-2" }, result: "image", routes: [{ model: "gpt-image-2" }],
|
|
47
70
|
fields: {
|
|
@@ -25,7 +25,7 @@ each media input as a typed item with its `role` (`first_frame`, `last_frame`, `
|
|
|
25
25
|
|
|
26
26
|
The Seedance endpoints add `generate_audio`. Monid documents no web search field for them, so
|
|
27
27
|
`web-search="true"` is unsupported, and Seedance 2.5 frame mode (`first-frame` present) requires
|
|
28
|
-
`aspect-ratio="adaptive"`. Seedance visual references
|
|
28
|
+
`aspect-ratio="adaptive"`. Seedance visual references require `person-reference`; the Provider
|
|
29
29
|
accepts the declaration and transmits nothing for it, since the endpoint has no field for it.
|
|
30
30
|
Seedance rejects reference images and videos that contain a real human face; Monid offers no way to
|
|
31
31
|
register authorized portrait material, so such a request fails with the upstream moderation error.
|
|
@@ -19,7 +19,7 @@ the models the Distribution already describes.
|
|
|
19
19
|
Video requests write the prompt and each media input as one item of the protocol's `content`
|
|
20
20
|
array with its `role` (`first_frame`, `last_frame`, `reference_image`, `reference_video`,
|
|
21
21
|
`reference_audio`), then `resolution`, `ratio`, `duration` and, for Seedance, `generate_audio`;
|
|
22
|
-
`web-search="true"` adds `tools: [{ "type": "web_search" }]`. Seedance visual references
|
|
22
|
+
`web-search="true"` adds `tools: [{ "type": "web_search" }]`. Seedance visual references require
|
|
23
23
|
`person-reference`; the Provider accepts the declaration and transmits nothing for it, since the Ark
|
|
24
24
|
protocol has no such field. Seedance 2.0 and 2.5 reject reference images and videos that contain a
|
|
25
25
|
real human face; TokenDance offers no way to register authorized portrait material, so such a request
|
|
@@ -34,22 +34,20 @@ responsible for any additional service-specific input limits.
|
|
|
34
34
|
|
|
35
35
|
## Visual reference metadata
|
|
36
36
|
|
|
37
|
-
|
|
38
|
-
|
|
37
|
+
Every supplied image or video must explicitly declare `person-reference`: `true` if it contains
|
|
38
|
+
a person, `false` otherwise. Classify the supplied material, not the requested result.
|
|
39
39
|
|
|
40
40
|
```xml
|
|
41
|
-
<seedance:ReferenceVideo id="
|
|
41
|
+
<seedance:ReferenceVideo id="take" model="mini" prompt={direction} duration="8">
|
|
42
42
|
<seedance:Reference image={presenter.image} person-reference="true"/>
|
|
43
|
-
<seedance:Reference video={
|
|
43
|
+
<seedance:Reference video={presenter.video} person-reference="true"/>
|
|
44
44
|
<seedance:Reference image={room.image} person-reference="false"/>
|
|
45
45
|
</seedance:ReferenceVideo>
|
|
46
46
|
```
|
|
47
47
|
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
`first-frame-person-reference` and `last-frame-person-reference` beside their respective frame
|
|
52
|
-
inputs. A last-frame classification requires a last-frame input.
|
|
48
|
+
Missing or non-boolean declarations are rejected; there is no default or automatic face detection.
|
|
49
|
+
Audio must omit this field. `FrameVideo` requires `first-frame-person-reference` and, when a last
|
|
50
|
+
frame is supplied, `last-frame-person-reference`. A last-frame classification requires a last frame.
|
|
53
51
|
|
|
54
52
|
| Supplied visual input | Authored attribute | Request port |
|
|
55
53
|
| --- | --- | --- |
|
|
@@ -66,23 +64,18 @@ These forms apply to `standard`, `fast`, `mini` and `2.5`. For example:
|
|
|
66
64
|
last-frame={empty-room.image} last-frame-person-reference="false"/>
|
|
67
65
|
```
|
|
68
66
|
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
An empty room stays `false` when the prompt asks to add a person. Inspect video across the selected
|
|
72
|
-
excerpt, not only its opening frame. This flag neither detects faces nor locks or names an identity.
|
|
73
|
-
Identity and action direction remain in the prompt and references.
|
|
67
|
+
Inspect the selected video excerpt, not only its opening frame. An empty room stays `false` when
|
|
68
|
+
the prompt asks to add a person. The flag does not lock identity; direction and references own that.
|
|
74
69
|
|
|
75
70
|
The SVML author declares this parameter on each reference input. Admitted files, generated
|
|
76
71
|
images/videos and reused Results use the same attributes. For a future output, declare the intended
|
|
77
72
|
reference classification explicitly; if its contents are uncertain, generate and inspect that
|
|
78
73
|
material before using it downstream.
|
|
79
74
|
|
|
80
|
-
|
|
81
|
-
service's media handling; it is not a prompt sentence or a Core-level identity. HypiHub sends it as
|
|
75
|
+
Direct requests require the same boolean in `fields.personReference`. Providers interpret it through
|
|
76
|
+
their service's media handling; it is not a prompt sentence or a Core-level identity. HypiHub sends it as
|
|
82
77
|
`is_person_reference` when uploading the file, then uses the returned URL in the ordinary video
|
|
83
78
|
request. A project Provider maps it according to its own API.
|
|
84
|
-
Omission does not request automatic face detection. HypiHub currently treats omitted upload flags
|
|
85
|
-
as unmarked (`false`); declare `true` explicitly for a person reference that needs its preparation.
|
|
86
79
|
|
|
87
80
|
Video references can carry motion or camera behavior while image references carry the target
|
|
88
81
|
appearance. Request duration and reference-clip duration are different limits. Check the selected
|
|
@@ -17,7 +17,7 @@ export type SeedanceModel = typeof seedanceModels[number];
|
|
|
17
17
|
|
|
18
18
|
const ASPECT_RATIOS = ["1:1", "4:3", "3:4", "16:9", "9:16", "21:9", "adaptive"] as const;
|
|
19
19
|
|
|
20
|
-
const PERSON_REFERENCE_FIELDS = [{ name: "personReference", value: { kind: "boolean" }
|
|
20
|
+
const PERSON_REFERENCE_FIELDS = [{ name: "personReference", value: { kind: "boolean" } }] as const;
|
|
21
21
|
|
|
22
22
|
const SEEDANCE_25_DURATIONS = [-1, ...Array.from({ length: 27 }, (_item, index) => index + 4)] as const;
|
|
23
23
|
|
|
@@ -175,9 +175,9 @@ const seedanceCommonAttributes: readonly SurfaceAttributeVocabulary[] = [
|
|
|
175
175
|
},
|
|
176
176
|
];
|
|
177
177
|
|
|
178
|
-
const personReferenceAttribute = (name: string): SurfaceAttributeVocabulary => ({
|
|
179
|
-
name, kind: "literal", required
|
|
180
|
-
summary: "
|
|
178
|
+
const personReferenceAttribute = (name: string, required = false): SurfaceAttributeVocabulary => ({
|
|
179
|
+
name, kind: "literal", required, values: ["true", "false"],
|
|
180
|
+
summary: "Required for each supplied image/video, including first/last frames: true if it contains a person, false otherwise. Audio must omit it.",
|
|
181
181
|
});
|
|
182
182
|
|
|
183
183
|
const seedanceVideoPort: readonly SurfacePortVocabulary[] = [{
|
|
@@ -235,7 +235,7 @@ export const seedanceMarkupSurfaces = [
|
|
|
235
235
|
summary: "Generates one video with an exact Seedance model from a Text prompt and the images the video opens and closes on.",
|
|
236
236
|
attributes: [
|
|
237
237
|
...seedanceCommonAttributes,
|
|
238
|
-
personReferenceAttribute("first-frame-person-reference"),
|
|
238
|
+
personReferenceAttribute("first-frame-person-reference", true),
|
|
239
239
|
personReferenceAttribute("last-frame-person-reference"),
|
|
240
240
|
{
|
|
241
241
|
name: "first-frame",
|
|
@@ -253,7 +253,7 @@ export const seedanceMarkupSurfaces = [
|
|
|
253
253
|
},
|
|
254
254
|
],
|
|
255
255
|
ports: seedanceVideoPort,
|
|
256
|
-
example: '<seedance:FrameVideo id="bridge" model="fast" prompt={direction} duration="5" first-frame={first.image} last-frame={last.image}/>',
|
|
256
|
+
example: '<seedance:FrameVideo id="bridge" model="fast" prompt={direction} duration="5" first-frame={first.image} first-frame-person-reference="true" last-frame={last.image} last-frame-person-reference="false"/>',
|
|
257
257
|
notes: [
|
|
258
258
|
...seedanceSettingNotes,
|
|
259
259
|
"Both frames are ordinary image Artifact edges; the Surface copies no runtime media into request metadata.",
|
|
@@ -316,7 +316,7 @@ export const seedanceMarkupSurfaces = [
|
|
|
316
316
|
notes: [
|
|
317
317
|
...seedanceSettingNotes,
|
|
318
318
|
"The element requires at least one `Reference` child, and the model's port limits cap how many of each role it accepts.",
|
|
319
|
-
"`person-reference
|
|
319
|
+
"Every image/video Reference requires `person-reference=\"true|false\"`. Classify the supplied material; audio must omit the field. The Provider transports it according to its API.",
|
|
320
320
|
"A `Reference` carries exactly one of `image`, `video` or `audio`, and is empty.",
|
|
321
321
|
],
|
|
322
322
|
},
|
|
@@ -146,8 +146,10 @@ function booleanAttribute(element: StructuredElement, name: string, fallback: bo
|
|
|
146
146
|
}
|
|
147
147
|
|
|
148
148
|
function personReferenceFields(element: StructuredElement, name: string) {
|
|
149
|
-
|
|
150
|
-
|
|
149
|
+
if (element.attributes[name] === undefined) {
|
|
150
|
+
throw new Error(`${element.name}.${name} is required for this visual reference; set true if it contains a person, false otherwise`);
|
|
151
|
+
}
|
|
152
|
+
return { personReference: booleanAttribute(element, name, false) };
|
|
151
153
|
}
|
|
152
154
|
|
|
153
155
|
function enumeratedPort(table: GenerationPortTable, name: string): readonly (string | number)[] {
|
|
@@ -217,8 +219,8 @@ function referenceInputs(
|
|
|
217
219
|
const kinds = accepted.filter((kind) => child.attributes[kind] !== undefined);
|
|
218
220
|
if (kinds.length !== 1) throw new Error(`${child.name} requires exactly one of ${accepted.join(", ")}`);
|
|
219
221
|
const role = kinds[0]!;
|
|
220
|
-
|
|
221
|
-
|
|
222
|
+
if (role === "audio" && child.attributes["person-reference"] !== undefined) throw new Error(`${child.name}.person-reference applies to image or video, not audio`);
|
|
223
|
+
const fields = role === "audio" ? undefined : personReferenceFields(child, "person-reference");
|
|
222
224
|
result.push({
|
|
223
225
|
...(fields === undefined ? {} : { fields }),
|
|
224
226
|
role,
|
|
@@ -242,13 +244,12 @@ function frameInputs(
|
|
|
242
244
|
): MediaInput[] {
|
|
243
245
|
const first = mediaReference(resolved(element, "first-frame", resolveReference), "image", `${element.name}.first-frame`);
|
|
244
246
|
const firstFields = personReferenceFields(element, "first-frame-person-reference");
|
|
245
|
-
const result: MediaInput[] = [{ port: "firstFrame", role: "image", source: first,
|
|
246
|
-
|
|
247
|
-
if (lastFields !== undefined && element.attributes["last-frame"] === undefined) throw new Error(`${element.name}.last-frame-person-reference requires last-frame`);
|
|
247
|
+
const result: MediaInput[] = [{ port: "firstFrame", role: "image", source: first, fields: firstFields }];
|
|
248
|
+
if (element.attributes["last-frame-person-reference"] !== undefined && element.attributes["last-frame"] === undefined) throw new Error(`${element.name}.last-frame-person-reference requires last-frame`);
|
|
248
249
|
if (element.attributes["last-frame"] !== undefined) {
|
|
249
250
|
result.push({
|
|
250
251
|
port: "lastFrame",
|
|
251
|
-
|
|
252
|
+
fields: personReferenceFields(element, "last-frame-person-reference"),
|
|
252
253
|
role: "image",
|
|
253
254
|
source: mediaReference(resolved(element, "last-frame", resolveReference), "image", `${element.name}.last-frame`),
|
|
254
255
|
});
|
|
@@ -23,8 +23,8 @@ ordinary explicit graph edges:
|
|
|
23
23
|
|
|
24
24
|
<seedance:ReferenceVideo id="broll" model="mini" prompt={broll-prompt}
|
|
25
25
|
duration="5" resolution="720p" aspect-ratio="9:16">
|
|
26
|
-
<seedance:Reference image={scene}/>
|
|
27
|
-
<seedance:Reference image={product}/>
|
|
26
|
+
<seedance:Reference image={scene} person-reference="false"/>
|
|
27
|
+
<seedance:Reference image={product} person-reference="false"/>
|
|
28
28
|
</seedance:ReferenceVideo>
|
|
29
29
|
```
|
|
30
30
|
|
|
@@ -154,7 +154,8 @@ For a multi-scene B-roll montage, `story` can name the references in scene order
|
|
|
154
154
|
action. Choose an edit language compatible with cuts. “Continuous within each scene” and “one
|
|
155
155
|
continuous shot for the whole montage” ask for different results.
|
|
156
156
|
|
|
157
|
-
|
|
157
|
+
Every Seedance image/video Reference requires `person-reference="true|false"`: true if the supplied
|
|
158
|
+
material contains a person, false otherwise. Audio must omit it. The
|
|
158
159
|
[Seedance author package](../seedance/README.md#visual-reference-metadata) explains `person-reference`
|
|
159
160
|
and its frame variants. The Kit supplies direction; the actual reference edges supply media and
|
|
160
161
|
metadata. For movement-led work, a reference video can carry the motion while the text explains what
|
|
@@ -23,7 +23,7 @@ const MEDIA_TYPES: Readonly<Record<string, string>> = {
|
|
|
23
23
|
|
|
24
24
|
async function readProject(source: string, html: string, resources: FileResourceStore): Promise<HyperframesHtmlProject> {
|
|
25
25
|
const assets = [];
|
|
26
|
-
const base =
|
|
26
|
+
const base = isSnapshotHtmlUrl(source) ? new URL(source) : pathToFileURL(source);
|
|
27
27
|
for (const url of hyperframesHtmlAssetUrls(html)) {
|
|
28
28
|
const address = new URL(url, base);
|
|
29
29
|
let mediaType: string | undefined;
|
|
@@ -52,6 +52,20 @@ async function readProject(source: string, html: string, resources: FileResource
|
|
|
52
52
|
|
|
53
53
|
const OPTIONS = ["--studio", "--to", "--at-frame", "--start-frame", "--end-frame-exclusive", "--step-frames", "--grid", "--cell", "--runtime", "--workspace"];
|
|
54
54
|
|
|
55
|
+
/** Capture already treats HTTP(S) case-insensitively; snapshot must not turn HTTPS:// into a local path. */
|
|
56
|
+
export function isSnapshotHtmlUrl(value: string): boolean {
|
|
57
|
+
return /^https?:\/\//iu.test(value);
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** `--studio` is a base URL, not a host:port token. `new URL` otherwise throws TypeError. */
|
|
61
|
+
export function studioDocumentUrl(studio: string): string {
|
|
62
|
+
try {
|
|
63
|
+
return new URL("/__studio/document", studio).href;
|
|
64
|
+
} catch {
|
|
65
|
+
throw new Error(`--studio needs an http(s) Studio URL, got ${studio}`);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
|
|
55
69
|
export function writeSnapshotHelp(io: CliIo): void {
|
|
56
70
|
io.write(`hypit snapshot\nCapture exact frames from an existing compiled HyperFrames HTML programme through the selected Runtime Profile.\n\n`
|
|
57
71
|
+ ` hypit snapshot --studio <studio-url> --at-frame <n[,n,…]> --to <directory>\n`
|
|
@@ -87,8 +101,8 @@ export async function runSnapshotCli(argv: readonly string[], io: CliIo, environ
|
|
|
87
101
|
return Number(raw);
|
|
88
102
|
};
|
|
89
103
|
const source = studio === undefined
|
|
90
|
-
?
|
|
91
|
-
:
|
|
104
|
+
? isSnapshotHtmlUrl(positionals[0]!) ? positionals[0]! : resolve(environment.cwd, positionals[0]!)
|
|
105
|
+
: studioDocumentUrl(studio);
|
|
92
106
|
let document: HyperframesDocument | undefined;
|
|
93
107
|
let html: string;
|
|
94
108
|
if (studio !== undefined) {
|
|
@@ -97,7 +111,7 @@ export async function runSnapshotCli(argv: readonly string[], io: CliIo, environ
|
|
|
97
111
|
document = await response.json() as HyperframesDocument;
|
|
98
112
|
assertHyperframesDocument(document);
|
|
99
113
|
html = document.html;
|
|
100
|
-
} else if (
|
|
114
|
+
} else if (isSnapshotHtmlUrl(source)) {
|
|
101
115
|
const response = await fetch(source);
|
|
102
116
|
if (!response.ok) throw new Error(`Snapshot HTML: HTTP ${response.status} ${await response.text()}`);
|
|
103
117
|
html = await response.text();
|