nixamp 0.62.3 → 0.63.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ai-label.d.ts +37 -5
- package/dist/ai-label.js +73 -57
- package/dist/veo-studio.d.ts +1 -1
- package/dist/veo-studio.js +3 -2
- package/package.json +1 -1
- package/src/ai-label.ts +74 -54
- package/src/veo-studio.ts +3 -2
- package/web/dist/assets/{ar-CH_edyao.js → ar-QDWDg5H0.js} +1 -1
- package/web/dist/assets/{cs-D8TiybS7.js → cs-rmoNHybA.js} +1 -1
- package/web/dist/assets/{da-BPa31neD.js → da-BG4jH0u0.js} +1 -1
- package/web/dist/assets/{de-C7b6JGxy.js → de-TGv7SJZy.js} +1 -1
- package/web/dist/assets/{es-C9DK47Ke.js → es-CBEdzJZS.js} +1 -1
- package/web/dist/assets/{fi-D8AyeyfK.js → fi-BG_Ki393.js} +1 -1
- package/web/dist/assets/{fr-CYhZv62i.js → fr-D4BPB6z3.js} +1 -1
- package/web/dist/assets/{hi-I-620NOZ.js → hi-prjYhCEG.js} +1 -1
- package/web/dist/assets/{hu-CMW42K9k.js → hu-DAEq51w9.js} +1 -1
- package/web/dist/assets/{id-C9nfqrLw.js → id-EmAT3-7o.js} +1 -1
- package/web/dist/assets/{it-iWLQrB4w.js → it-DWFflIKf.js} +1 -1
- package/web/dist/assets/{main-1IVDmgoi.js → main-BJlO7hc-.js} +2 -2
- package/web/dist/assets/music-LviR4dJP.js +1 -0
- package/web/dist/assets/{nl-D0Fd86cP.js → nl-Bm0WfZ8G.js} +1 -1
- package/web/dist/assets/{ru-DzWEvW3W.js → ru-DkrNKYXC.js} +1 -1
- package/web/dist/assets/{sv-BeZH_PU4.js → sv-CSyHIcIi.js} +1 -1
- package/web/dist/assets/{uk-Bs5VLBTF.js → uk-CXxVncRG.js} +1 -1
- package/web/dist/assets/{vi-BH8V1lRr.js → vi-VBIcSVs7.js} +1 -1
- package/web/dist/assets/{zh-iSN405_E.js → zh-CR6df955.js} +1 -1
- package/web/dist/index.html +1 -1
- package/web/dist/music.html +1 -1
- package/web/dist/sw.js +20 -20
- package/web/dist/assets/music-YhoIqwcn.js +0 -1
package/dist/ai-label.d.ts
CHANGED
|
@@ -86,11 +86,11 @@ export interface LabelSettings {
|
|
|
86
86
|
seedAlbums: number;
|
|
87
87
|
songsPerAlbum: number;
|
|
88
88
|
songSeconds: number;
|
|
89
|
-
/**
|
|
89
|
+
/** Music videos, and a TV station of them. Only ever filmed ones (Veo footage end to end); no song gets a slideshow. */
|
|
90
90
|
videos: boolean;
|
|
91
|
-
/** Each album's single (song 1)
|
|
91
|
+
/** Each album's single (song 1) is filmed with Veo, while the month's Veo budget lasts. */
|
|
92
92
|
veo: boolean;
|
|
93
|
-
/** How many 8 s Veo shots a single
|
|
93
|
+
/** How many 8 s Veo shots a single is cut from. */
|
|
94
94
|
veoShots: number;
|
|
95
95
|
/** Public, listed and owned, so the genre stations are on the guide. */
|
|
96
96
|
publish: boolean;
|
|
@@ -191,6 +191,25 @@ export declare function renderStoryVideo(ffmpeg: string[], input: {
|
|
|
191
191
|
seconds: number;
|
|
192
192
|
}>;
|
|
193
193
|
}): Promise<void>;
|
|
194
|
+
/** The version of a video cut from Veo footage only; anything older is a slideshow and is not shown. */
|
|
195
|
+
export declare const FILMED = 4;
|
|
196
|
+
/**
|
|
197
|
+
* A single's music video, cut from its filmed shots alone: 4 s cuts, each
|
|
198
|
+
* shot's first half then its second, round and round the shots until the song
|
|
199
|
+
* ends, so no picture holds still. The artist and title come up at the start;
|
|
200
|
+
* the MP3's audio and tags are copied in. 1080p H.264.
|
|
201
|
+
*/
|
|
202
|
+
export declare function renderFilmedVideo(ffmpeg: string[], input: {
|
|
203
|
+
audio: string;
|
|
204
|
+
clips: Array<{
|
|
205
|
+
path: string;
|
|
206
|
+
seconds: number;
|
|
207
|
+
}>;
|
|
208
|
+
seconds: number;
|
|
209
|
+
artist: string;
|
|
210
|
+
title: string;
|
|
211
|
+
out: string;
|
|
212
|
+
}): Promise<void>;
|
|
194
213
|
/**
|
|
195
214
|
* A song's music video without a storyboard: its album cover over a blurred
|
|
196
215
|
* copy of itself, the song's waveform along the bottom, 720p H.264 with the
|
|
@@ -233,6 +252,18 @@ export interface AiLabelDeps {
|
|
|
233
252
|
seconds: number;
|
|
234
253
|
references: Buffer[];
|
|
235
254
|
}) => Promise<Buffer>) | null;
|
|
255
|
+
/** Cuts a single's filmed shots to its song; ffmpeg by default (renderFilmedVideo). */
|
|
256
|
+
filmedVideo?: (input: {
|
|
257
|
+
audio: string;
|
|
258
|
+
clips: Array<{
|
|
259
|
+
path: string;
|
|
260
|
+
seconds: number;
|
|
261
|
+
}>;
|
|
262
|
+
seconds: number;
|
|
263
|
+
artist: string;
|
|
264
|
+
title: string;
|
|
265
|
+
out: string;
|
|
266
|
+
}) => Promise<void>;
|
|
236
267
|
/** Renders a storyboard music video; ffmpeg by default (renderStoryVideo). */
|
|
237
268
|
storyVideo?: (input: {
|
|
238
269
|
audio: string;
|
|
@@ -333,8 +364,9 @@ export declare class AiLabel {
|
|
|
333
364
|
private sync;
|
|
334
365
|
private filmPass;
|
|
335
366
|
/**
|
|
336
|
-
*
|
|
337
|
-
*
|
|
367
|
+
* A single's video: Veo shots of the band, cut to the whole song (FILMED).
|
|
368
|
+
* Fewer than half the shots filmed (a refusal, the budget spent) makes no
|
|
369
|
+
* video at all: footage stretched that thin would be a loop, not a video.
|
|
338
370
|
*/
|
|
339
371
|
private filmSong;
|
|
340
372
|
/** Whether this song gets filmed Veo shots: an album's single, Veo on, and the month's budget not spent. */
|
package/dist/ai-label.js
CHANGED
|
@@ -273,7 +273,7 @@ export const DEFAULTS = {
|
|
|
273
273
|
publish: true,
|
|
274
274
|
videos: true,
|
|
275
275
|
veo: false,
|
|
276
|
-
veoShots:
|
|
276
|
+
veoShots: 6,
|
|
277
277
|
parallel: 1,
|
|
278
278
|
genreOrder: ["Punk", "Blues", "Country", "Jazz", "Classic Rock", "Metal", "Hip-Hop", "Pop", "R&B"],
|
|
279
279
|
};
|
|
@@ -605,6 +605,41 @@ export function renderStoryVideo(ffmpeg, input) {
|
|
|
605
605
|
"-movflags", "+faststart+use_metadata_tags", input.out];
|
|
606
606
|
return runLow(ffmpeg, [...ffmpeg.slice(1), ...args], 60 * 60_000, "the video failed");
|
|
607
607
|
}
|
|
608
|
+
/** The version of a video cut from Veo footage only; anything older is a slideshow and is not shown. */
|
|
609
|
+
export const FILMED = 4;
|
|
610
|
+
/**
|
|
611
|
+
* A single's music video, cut from its filmed shots alone: 4 s cuts, each
|
|
612
|
+
* shot's first half then its second, round and round the shots until the song
|
|
613
|
+
* ends, so no picture holds still. The artist and title come up at the start;
|
|
614
|
+
* the MP3's audio and tags are copied in. 1080p H.264.
|
|
615
|
+
*/
|
|
616
|
+
export function renderFilmedVideo(ffmpeg, input) {
|
|
617
|
+
if (input.clips.length === 0)
|
|
618
|
+
return Promise.reject(new Error("no footage to cut"));
|
|
619
|
+
const cut = 4;
|
|
620
|
+
const cuts = [];
|
|
621
|
+
for (let at = 0, i = 0; at < input.seconds; i++) {
|
|
622
|
+
const clip = input.clips[i % input.clips.length];
|
|
623
|
+
const half = Math.floor(i / input.clips.length) % 2;
|
|
624
|
+
const start = Math.min(half * cut, Math.max(0, clip.seconds - cut));
|
|
625
|
+
const length = Math.min(cut, clip.seconds - start, input.seconds - at + 0.5);
|
|
626
|
+
cuts.push({ path: clip.path, start, length });
|
|
627
|
+
at += length;
|
|
628
|
+
}
|
|
629
|
+
const parts = cuts.map((_, i) => `[${i}:v]scale=1920:1080:force_original_aspect_ratio=increase,crop=1920:1080,fps=25,setsar=1,setpts=PTS-STARTPTS[c${i}]`);
|
|
630
|
+
parts.push(`${cuts.map((_, i) => `[c${i}]`).join("")}concat=n=${cuts.length}:v=1:a=0[cut]`);
|
|
631
|
+
const font = titleFont();
|
|
632
|
+
const card = `${input.artist} — ${input.title}`.replace(/[\\':%]/g, " ").slice(0, 80);
|
|
633
|
+
parts.push(font
|
|
634
|
+
? `[cut]drawtext=fontfile=${font}:text='${card}':fontcolor=white:fontsize=60:x=80:y=h-150:enable='between(t,0.5,6)':alpha='if(lt(t,1.5),t-0.5,if(gt(t,5),6-t,1))':shadowcolor=black@0.7:shadowx=3:shadowy=3,format=yuv420p[v]`
|
|
635
|
+
: "[cut]format=yuv420p[v]");
|
|
636
|
+
const args = ["-hide_banner", "-loglevel", "error", "-y", "-threads", "2",
|
|
637
|
+
...cuts.flatMap((one) => ["-ss", one.start.toFixed(2), "-t", one.length.toFixed(2), "-i", one.path]),
|
|
638
|
+
"-i", input.audio, "-filter_complex", parts.join(";"),
|
|
639
|
+
"-map", "[v]", "-map", `${cuts.length}:a`, "-map_metadata", String(cuts.length), "-c:v", "libx264", "-preset", "medium", "-crf", "20", "-r", "25", "-c:a", "copy", "-shortest",
|
|
640
|
+
"-movflags", "+faststart+use_metadata_tags", input.out];
|
|
641
|
+
return runLow(ffmpeg, [...ffmpeg.slice(1), ...args], 60 * 60_000, "the video failed");
|
|
642
|
+
}
|
|
608
643
|
/**
|
|
609
644
|
* A song's music video without a storyboard: its album cover over a blurred
|
|
610
645
|
* copy of itself, the song's waveform along the bottom, 720p H.264 with the
|
|
@@ -988,6 +1023,8 @@ export class AiLabel {
|
|
|
988
1023
|
// Songs that land while videos are being made are picked up by the next pass.
|
|
989
1024
|
for (let made = await this.filmPass(); made > 0; made = await this.filmPass())
|
|
990
1025
|
total += made;
|
|
1026
|
+
// Also when nothing was made: the station keeps up with what may be shown.
|
|
1027
|
+
await this.placeVideos().catch(() => undefined);
|
|
991
1028
|
return total;
|
|
992
1029
|
}
|
|
993
1030
|
/**
|
|
@@ -1035,9 +1072,10 @@ export class AiLabel {
|
|
|
1035
1072
|
const songs = this.list().flatMap((artist) => artist.albums
|
|
1036
1073
|
.filter((album) => album.cover || album.coverTried)
|
|
1037
1074
|
.flatMap((album) => album.songs.filter((song) => song.file && !this.videoFailed.has(song.file)).map((song) => ({ artist, album, song }))));
|
|
1038
|
-
|
|
1039
|
-
const
|
|
1040
|
-
const
|
|
1075
|
+
// Only a single is filmed, and only with Veo footage: a song with an older (slideshow) video is filmed when its turn comes.
|
|
1076
|
+
const missing = songs.filter(({ song }) => this.wantsVeo(song) && (song.videoVersion ?? 1) < FILMED && !this.upgradeTried.has(song.file));
|
|
1077
|
+
const untagged = songs.filter(({ song }) => song.video && !song.videoTagged && (song.videoVersion ?? 1) >= FILMED);
|
|
1078
|
+
const old = [];
|
|
1041
1079
|
let made = 0;
|
|
1042
1080
|
for (const { album, song } of untagged) {
|
|
1043
1081
|
if (!this.settings.videos)
|
|
@@ -1062,10 +1100,11 @@ export class AiLabel {
|
|
|
1062
1100
|
for (const { artist, album, song } of [...missing, ...old]) {
|
|
1063
1101
|
if (!this.settings.videos)
|
|
1064
1102
|
break;
|
|
1065
|
-
//
|
|
1103
|
+
// Tried once a run: a shot Veo refuses today it may make tomorrow.
|
|
1066
1104
|
const upgrade = Boolean(song.video);
|
|
1067
|
-
|
|
1068
|
-
|
|
1105
|
+
this.upgradeTried.add(song.file);
|
|
1106
|
+
if (!this.wantsVeo(song))
|
|
1107
|
+
break;
|
|
1069
1108
|
const out = join(album.folder, `${songBase(song)}.mp4`);
|
|
1070
1109
|
const partial = join(album.folder, `.${randomBytes(4).toString("hex")}.video.mp4`);
|
|
1071
1110
|
const scenes = join(album.folder, `.scenes-${song.n}-${randomBytes(3).toString("hex")}`);
|
|
@@ -1082,7 +1121,7 @@ export class AiLabel {
|
|
|
1082
1121
|
catch (error) {
|
|
1083
1122
|
if (!upgrade)
|
|
1084
1123
|
this.videoFailed.add(song.file);
|
|
1085
|
-
this.log(`no
|
|
1124
|
+
this.log(`no filmed video for "${song.title}": ${error.message}`);
|
|
1086
1125
|
}
|
|
1087
1126
|
finally {
|
|
1088
1127
|
rmSync(partial, { force: true });
|
|
@@ -1092,10 +1131,11 @@ export class AiLabel {
|
|
|
1092
1131
|
return made;
|
|
1093
1132
|
}
|
|
1094
1133
|
/**
|
|
1095
|
-
*
|
|
1096
|
-
*
|
|
1134
|
+
* A single's video: Veo shots of the band, cut to the whole song (FILMED).
|
|
1135
|
+
* Fewer than half the shots filmed (a refusal, the budget spent) makes no
|
|
1136
|
+
* video at all: footage stretched that thin would be a loop, not a video.
|
|
1097
1137
|
*/
|
|
1098
|
-
async filmSong(artist,
|
|
1138
|
+
async filmSong(artist, _album, song, scenes, out, _upgrade = false) {
|
|
1099
1139
|
let seconds = song.seconds;
|
|
1100
1140
|
try {
|
|
1101
1141
|
const probed = await this.deps.probe(song.file);
|
|
@@ -1103,41 +1143,11 @@ export class AiLabel {
|
|
|
1103
1143
|
seconds = probed.duration;
|
|
1104
1144
|
}
|
|
1105
1145
|
catch { /* the asked-for length */ }
|
|
1106
|
-
const
|
|
1107
|
-
if (this.
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
mkdirSync(scenes, { recursive: true });
|
|
1112
|
-
const draw = this.deps.image ?? ((prompt) => workersAiCover(prompt));
|
|
1113
|
-
// A few at a time: the image model is quick, the machine is shared.
|
|
1114
|
-
for (let i = 0; i < prompts.length; i += 3) {
|
|
1115
|
-
const drawn = await Promise.all(prompts.slice(i, i + 3).map((prompt) => draw(`${prompt} Cinematic 16:9 music video still, no text.`).catch(() => null)));
|
|
1116
|
-
drawn.forEach((picture, j) => {
|
|
1117
|
-
if (!picture)
|
|
1118
|
-
return;
|
|
1119
|
-
const file = join(scenes, `${String(i + j).padStart(2, "0")}.jpg`);
|
|
1120
|
-
writeFileSync(file, picture);
|
|
1121
|
-
images.push(file);
|
|
1122
|
-
});
|
|
1123
|
-
}
|
|
1124
|
-
}
|
|
1125
|
-
catch (error) {
|
|
1126
|
-
this.log(`no storyboard for "${song.title}" (${error.message}); the cover video instead`);
|
|
1127
|
-
}
|
|
1128
|
-
}
|
|
1129
|
-
// An album's single gets filmed shots of the band too, while the month's Veo budget lasts.
|
|
1130
|
-
const clips = this.wantsVeo(song) ? await this.filmShots(artist, song, scenes) : [];
|
|
1131
|
-
if (images.length >= 2 || clips.length > 0) {
|
|
1132
|
-
if (storyOnly && (song.videoVersion ?? 1) >= 2 && clips.length === 0)
|
|
1133
|
-
throw new Error("no filmed shots this time; the video it has stays");
|
|
1134
|
-
await (this.deps.storyVideo ?? ((input) => renderStoryVideo(this.deps.ffmpeg, input)))({ audio: song.file, images, seconds, artist: artist.name, title: song.title, out, ...(clips.length ? { clips } : {}) });
|
|
1135
|
-
return clips.length ? 3 : 2;
|
|
1136
|
-
}
|
|
1137
|
-
if (storyOnly)
|
|
1138
|
-
throw new Error("no storyboard could be drawn; the video it has stays");
|
|
1139
|
-
await (this.deps.video ?? ((audio, cover, to) => renderVideo(this.deps.ffmpeg, audio, cover, to)))(song.file, album.cover, out);
|
|
1140
|
-
return 1;
|
|
1146
|
+
const clips = await this.filmShots(artist, song, scenes);
|
|
1147
|
+
if (clips.length < Math.max(2, Math.ceil(this.settings.veoShots / 2)))
|
|
1148
|
+
throw new Error(`only ${clips.length} of ${this.settings.veoShots} shots were filmed`);
|
|
1149
|
+
await (this.deps.filmedVideo ?? ((input) => renderFilmedVideo(this.deps.ffmpeg, input)))({ audio: song.file, clips, seconds, artist: artist.name, title: song.title, out });
|
|
1150
|
+
return FILMED;
|
|
1141
1151
|
}
|
|
1142
1152
|
/** Whether this song gets filmed Veo shots: an album's single, Veo on, and the month's budget not spent. */
|
|
1143
1153
|
wantsVeo(song) {
|
|
@@ -1178,7 +1188,7 @@ export class AiLabel {
|
|
|
1178
1188
|
catch (error) {
|
|
1179
1189
|
if (error instanceof VeoUnavailable) {
|
|
1180
1190
|
this.veoSpentIn = new Date(this.now()).toISOString().slice(0, 7);
|
|
1181
|
-
this.log(`Veo: ${error.message};
|
|
1191
|
+
this.log(`Veo: ${error.message}; no more videos until next month`);
|
|
1182
1192
|
break;
|
|
1183
1193
|
}
|
|
1184
1194
|
this.log(`Veo shot ${i + 1} for "${song.title}": ${error.message}`);
|
|
@@ -1193,18 +1203,24 @@ export class AiLabel {
|
|
|
1193
1203
|
const next = this.placing.then(async () => {
|
|
1194
1204
|
await this.deps.index.scan(this.source(), this.deps.probe).done;
|
|
1195
1205
|
const files = this.list()
|
|
1196
|
-
.flatMap((artist) => artist.albums.flatMap((album) => album.songs.map((song) => song.video ?? "")))
|
|
1206
|
+
.flatMap((artist) => artist.albums.flatMap((album) => album.songs.map((song) => ((song.videoVersion ?? 1) >= FILMED ? song.video ?? "" : ""))))
|
|
1197
1207
|
.filter((file) => file && this.deps.index.itemAt(file)?.state === "ready");
|
|
1198
|
-
if (files.length === 0)
|
|
1199
|
-
return;
|
|
1200
1208
|
const existing = this.deps.store.byAuto("ai:label:videos");
|
|
1201
|
-
if (
|
|
1202
|
-
|
|
1209
|
+
if (files.length === 0) {
|
|
1210
|
+
// Nothing filmed yet (older slideshow videos are not shown): off the guide until there is.
|
|
1211
|
+
if (existing && existing.listed)
|
|
1212
|
+
this.deps.store.setAccess(existing.id, { visibility: "private", listed: false }, this.deps.owner || "ai-label");
|
|
1203
1213
|
return;
|
|
1204
1214
|
}
|
|
1205
1215
|
const label = this.settings.holder.replace(/,?\s+(inc|llc|ltd|gmbh|co)\.?$/i, "");
|
|
1206
|
-
|
|
1207
|
-
|
|
1216
|
+
if (existing) {
|
|
1217
|
+
this.deps.store.update(existing.id, {}, files);
|
|
1218
|
+
if (!this.settings.publish || existing.listed)
|
|
1219
|
+
return;
|
|
1220
|
+
}
|
|
1221
|
+
const made = existing ?? this.deps.store.create({ name: `${label} AI · Music Videos`, description: `Music videos by ${label}'s AI artists, every genre.`, shuffle: true, repeat: true, video: true }, files, this.deps.owner);
|
|
1222
|
+
if (!existing)
|
|
1223
|
+
this.deps.store.markAuto(made.id, "ai:label:videos");
|
|
1208
1224
|
if (this.settings.publish) {
|
|
1209
1225
|
this.deps.store.setAccess(made.id, {
|
|
1210
1226
|
visibility: "public",
|
|
@@ -1549,7 +1565,7 @@ export class AiLabel {
|
|
|
1549
1565
|
.filter((album) => album.songs.some((song) => song.file))
|
|
1550
1566
|
.map((album) => ({
|
|
1551
1567
|
key: album.key, title: album.title, year: album.year, concept: album.concept, cover: Boolean(album.cover), complete: album.doneAt > 0,
|
|
1552
|
-
songs: album.songs.filter((song) => song.file).map((song) => ({ n: song.n, title: song.title, seconds: song.seconds, lyrics: song.lyrics, ...(song.video ? { video: true } : {}) })),
|
|
1568
|
+
songs: album.songs.filter((song) => song.file).map((song) => ({ n: song.n, title: song.title, seconds: song.seconds, lyrics: song.lyrics, ...(song.video && (song.videoVersion ?? 1) >= FILMED ? { video: true } : {}) })),
|
|
1553
1569
|
}))
|
|
1554
1570
|
.reverse(),
|
|
1555
1571
|
}))
|
|
@@ -1569,7 +1585,7 @@ export class AiLabel {
|
|
|
1569
1585
|
if (!song)
|
|
1570
1586
|
return null;
|
|
1571
1587
|
if (match[2] === "mp4")
|
|
1572
|
-
return song.video ? { path: song.video, type: "video/mp4" } : null;
|
|
1588
|
+
return song.video && (song.videoVersion ?? 1) >= FILMED ? { path: song.video, type: "video/mp4" } : null;
|
|
1573
1589
|
return match[2] === "mp3" ? { path: song.file, type: "audio/mpeg" } : { path: song.lyricsFile, type: "text/plain; charset=utf-8" };
|
|
1574
1590
|
}
|
|
1575
1591
|
/** The label song at this path, for the now-playing card's Download buttons. */
|
|
@@ -1684,7 +1700,7 @@ export class AiLabel {
|
|
|
1684
1700
|
entries.push({ name: `${folder}/${songBase(song)}.lyrics.txt`, bytes: Buffer.from(`${song.lyrics}\n`) });
|
|
1685
1701
|
if (song.lrc)
|
|
1686
1702
|
entries.push({ name: `${folder}/${songBase(song)}.lrc`, path: song.lrc });
|
|
1687
|
-
if (song.video)
|
|
1703
|
+
if (song.video && (song.videoVersion ?? 1) >= FILMED)
|
|
1688
1704
|
entries.push({ name: `${folder}/${songBase(song)}.mp4`, path: song.video });
|
|
1689
1705
|
}
|
|
1690
1706
|
if (album.cover)
|
package/dist/veo-studio.d.ts
CHANGED
|
@@ -42,7 +42,7 @@ export declare class VeoStudio {
|
|
|
42
42
|
usdPerSecond: number;
|
|
43
43
|
secondsLeft: number;
|
|
44
44
|
}>;
|
|
45
|
-
/** One shot: 4, 6 or 8 seconds, 16:9, with up to three reference images (JPEG). */
|
|
45
|
+
/** One shot: 4, 6 or 8 seconds, 16:9 1080p, with up to three reference images (JPEG). */
|
|
46
46
|
clip(input: {
|
|
47
47
|
prompt: string;
|
|
48
48
|
seconds: number;
|
package/dist/veo-studio.js
CHANGED
|
@@ -92,7 +92,7 @@ export class VeoStudio {
|
|
|
92
92
|
const left = Math.max(0, this.cap - spent);
|
|
93
93
|
return { month: this.month(), spentUsd: Math.round(spent * 100) / 100, capUsd: this.cap, leftUsd: Math.round(left * 100) / 100, usdPerSecond: this.rate, secondsLeft: Math.floor(left / this.rate) };
|
|
94
94
|
}
|
|
95
|
-
/** One shot: 4, 6 or 8 seconds, 16:9, with up to three reference images (JPEG). */
|
|
95
|
+
/** One shot: 4, 6 or 8 seconds, 16:9 1080p, with up to three reference images (JPEG). */
|
|
96
96
|
async clip(input) {
|
|
97
97
|
const seconds = [4, 6, 8].includes(input.seconds) ? input.seconds : 8;
|
|
98
98
|
const cost = seconds * this.rate;
|
|
@@ -121,7 +121,8 @@ export class VeoStudio {
|
|
|
121
121
|
prompt: prompt.slice(0, 1800),
|
|
122
122
|
...(references.length ? { referenceImages: references.map((bytes) => ({ image: { bytesBase64Encoded: bytes.toString("base64"), mimeType: "image/jpeg" }, referenceType: "asset" })) } : {}),
|
|
123
123
|
}],
|
|
124
|
-
|
|
124
|
+
// 1080p, counted at the same rate; the cut is 1080p too (renderFilmedVideo).
|
|
125
|
+
parameters: { aspectRatio: "16:9", durationSeconds: seconds, resolution: "1080p" },
|
|
125
126
|
}),
|
|
126
127
|
});
|
|
127
128
|
const operation = await started.json().catch(() => ({}));
|
package/package.json
CHANGED
package/src/ai-label.ts
CHANGED
|
@@ -123,11 +123,11 @@ export interface LabelSettings {
|
|
|
123
123
|
seedAlbums: number;
|
|
124
124
|
songsPerAlbum: number;
|
|
125
125
|
songSeconds: number;
|
|
126
|
-
/**
|
|
126
|
+
/** Music videos, and a TV station of them. Only ever filmed ones (Veo footage end to end); no song gets a slideshow. */
|
|
127
127
|
videos: boolean;
|
|
128
|
-
/** Each album's single (song 1)
|
|
128
|
+
/** Each album's single (song 1) is filmed with Veo, while the month's Veo budget lasts. */
|
|
129
129
|
veo: boolean;
|
|
130
|
-
/** How many 8 s Veo shots a single
|
|
130
|
+
/** How many 8 s Veo shots a single is cut from. */
|
|
131
131
|
veoShots: number;
|
|
132
132
|
/** Public, listed and owned, so the genre stations are on the guide. */
|
|
133
133
|
publish: boolean;
|
|
@@ -383,7 +383,7 @@ export const DEFAULTS: Omit<LabelSettings, "pausedReason"> = {
|
|
|
383
383
|
publish: true,
|
|
384
384
|
videos: true,
|
|
385
385
|
veo: false,
|
|
386
|
-
veoShots:
|
|
386
|
+
veoShots: 6,
|
|
387
387
|
parallel: 1,
|
|
388
388
|
genreOrder: ["Punk", "Blues", "Country", "Jazz", "Classic Rock", "Metal", "Hip-Hop", "Pop", "R&B"],
|
|
389
389
|
};
|
|
@@ -732,6 +732,42 @@ export function renderStoryVideo(ffmpeg: string[], input: { audio: string; image
|
|
|
732
732
|
return runLow(ffmpeg, [...ffmpeg.slice(1), ...args], 60 * 60_000, "the video failed");
|
|
733
733
|
}
|
|
734
734
|
|
|
735
|
+
/** The version of a video cut from Veo footage only; anything older is a slideshow and is not shown. */
|
|
736
|
+
export const FILMED = 4;
|
|
737
|
+
|
|
738
|
+
/**
|
|
739
|
+
* A single's music video, cut from its filmed shots alone: 4 s cuts, each
|
|
740
|
+
* shot's first half then its second, round and round the shots until the song
|
|
741
|
+
* ends, so no picture holds still. The artist and title come up at the start;
|
|
742
|
+
* the MP3's audio and tags are copied in. 1080p H.264.
|
|
743
|
+
*/
|
|
744
|
+
export function renderFilmedVideo(ffmpeg: string[], input: { audio: string; clips: Array<{ path: string; seconds: number }>; seconds: number; artist: string; title: string; out: string }): Promise<void> {
|
|
745
|
+
if (input.clips.length === 0) return Promise.reject(new Error("no footage to cut"));
|
|
746
|
+
const cut = 4;
|
|
747
|
+
const cuts: Array<{ path: string; start: number; length: number }> = [];
|
|
748
|
+
for (let at = 0, i = 0; at < input.seconds; i++) {
|
|
749
|
+
const clip = input.clips[i % input.clips.length]!;
|
|
750
|
+
const half = Math.floor(i / input.clips.length) % 2;
|
|
751
|
+
const start = Math.min(half * cut, Math.max(0, clip.seconds - cut));
|
|
752
|
+
const length = Math.min(cut, clip.seconds - start, input.seconds - at + 0.5);
|
|
753
|
+
cuts.push({ path: clip.path, start, length });
|
|
754
|
+
at += length;
|
|
755
|
+
}
|
|
756
|
+
const parts = cuts.map((_, i) => `[${i}:v]scale=1920:1080:force_original_aspect_ratio=increase,crop=1920:1080,fps=25,setsar=1,setpts=PTS-STARTPTS[c${i}]`);
|
|
757
|
+
parts.push(`${cuts.map((_, i) => `[c${i}]`).join("")}concat=n=${cuts.length}:v=1:a=0[cut]`);
|
|
758
|
+
const font = titleFont();
|
|
759
|
+
const card = `${input.artist} — ${input.title}`.replace(/[\\':%]/g, " ").slice(0, 80);
|
|
760
|
+
parts.push(font
|
|
761
|
+
? `[cut]drawtext=fontfile=${font}:text='${card}':fontcolor=white:fontsize=60:x=80:y=h-150:enable='between(t,0.5,6)':alpha='if(lt(t,1.5),t-0.5,if(gt(t,5),6-t,1))':shadowcolor=black@0.7:shadowx=3:shadowy=3,format=yuv420p[v]`
|
|
762
|
+
: "[cut]format=yuv420p[v]");
|
|
763
|
+
const args = ["-hide_banner", "-loglevel", "error", "-y", "-threads", "2",
|
|
764
|
+
...cuts.flatMap((one) => ["-ss", one.start.toFixed(2), "-t", one.length.toFixed(2), "-i", one.path]),
|
|
765
|
+
"-i", input.audio, "-filter_complex", parts.join(";"),
|
|
766
|
+
"-map", "[v]", "-map", `${cuts.length}:a`, "-map_metadata", String(cuts.length), "-c:v", "libx264", "-preset", "medium", "-crf", "20", "-r", "25", "-c:a", "copy", "-shortest",
|
|
767
|
+
"-movflags", "+faststart+use_metadata_tags", input.out];
|
|
768
|
+
return runLow(ffmpeg, [...ffmpeg.slice(1), ...args], 60 * 60_000, "the video failed");
|
|
769
|
+
}
|
|
770
|
+
|
|
735
771
|
/**
|
|
736
772
|
* A song's music video without a storyboard: its album cover over a blurred
|
|
737
773
|
* copy of itself, the song's waveform along the bottom, 720p H.264 with the
|
|
@@ -819,6 +855,8 @@ export interface AiLabelDeps {
|
|
|
819
855
|
align?: ((audio: string, text: string) => Promise<TimedWord[]>) | null;
|
|
820
856
|
/** A filmed shot (Veo, through nixamp.com): the MP4, or a VeoUnavailable when the month's budget is spent. */
|
|
821
857
|
veo?: ((input: { prompt: string; seconds: number; references: Buffer[] }) => Promise<Buffer>) | null;
|
|
858
|
+
/** Cuts a single's filmed shots to its song; ffmpeg by default (renderFilmedVideo). */
|
|
859
|
+
filmedVideo?: (input: { audio: string; clips: Array<{ path: string; seconds: number }>; seconds: number; artist: string; title: string; out: string }) => Promise<void>;
|
|
822
860
|
/** Renders a storyboard music video; ffmpeg by default (renderStoryVideo). */
|
|
823
861
|
storyVideo?: (input: { audio: string; images: string[]; seconds: number; artist: string; title: string; out: string; clips?: Array<{ path: string; seconds: number }> }) => Promise<void>;
|
|
824
862
|
/** A scene's picture for this prompt, or null when nothing can draw one; Workers AI by default. */
|
|
@@ -1133,6 +1171,8 @@ export class AiLabel {
|
|
|
1133
1171
|
let total = 0;
|
|
1134
1172
|
// Songs that land while videos are being made are picked up by the next pass.
|
|
1135
1173
|
for (let made = await this.filmPass(); made > 0; made = await this.filmPass()) total += made;
|
|
1174
|
+
// Also when nothing was made: the station keeps up with what may be shown.
|
|
1175
|
+
await this.placeVideos().catch(() => undefined);
|
|
1136
1176
|
return total;
|
|
1137
1177
|
}
|
|
1138
1178
|
|
|
@@ -1178,9 +1218,10 @@ export class AiLabel {
|
|
|
1178
1218
|
const songs = this.list().flatMap((artist) => artist.albums
|
|
1179
1219
|
.filter((album) => album.cover || album.coverTried)
|
|
1180
1220
|
.flatMap((album) => album.songs.filter((song) => song.file && !this.videoFailed.has(song.file)).map((song) => ({ artist, album, song }))));
|
|
1181
|
-
|
|
1182
|
-
const
|
|
1183
|
-
const
|
|
1221
|
+
// Only a single is filmed, and only with Veo footage: a song with an older (slideshow) video is filmed when its turn comes.
|
|
1222
|
+
const missing = songs.filter(({ song }) => this.wantsVeo(song) && (song.videoVersion ?? 1) < FILMED && !this.upgradeTried.has(song.file));
|
|
1223
|
+
const untagged = songs.filter(({ song }) => song.video && !song.videoTagged && (song.videoVersion ?? 1) >= FILMED);
|
|
1224
|
+
const old: typeof songs = [];
|
|
1184
1225
|
let made = 0;
|
|
1185
1226
|
for (const { album, song } of untagged) {
|
|
1186
1227
|
if (!this.settings.videos) break;
|
|
@@ -1201,9 +1242,10 @@ export class AiLabel {
|
|
|
1201
1242
|
}
|
|
1202
1243
|
for (const { artist, album, song } of [...missing, ...old]) {
|
|
1203
1244
|
if (!this.settings.videos) break;
|
|
1204
|
-
//
|
|
1245
|
+
// Tried once a run: a shot Veo refuses today it may make tomorrow.
|
|
1205
1246
|
const upgrade = Boolean(song.video);
|
|
1206
|
-
|
|
1247
|
+
this.upgradeTried.add(song.file);
|
|
1248
|
+
if (!this.wantsVeo(song)) break;
|
|
1207
1249
|
const out = join(album.folder, `${songBase(song)}.mp4`);
|
|
1208
1250
|
const partial = join(album.folder, `.${randomBytes(4).toString("hex")}.video.mp4`);
|
|
1209
1251
|
const scenes = join(album.folder, `.scenes-${song.n}-${randomBytes(3).toString("hex")}`);
|
|
@@ -1218,7 +1260,7 @@ export class AiLabel {
|
|
|
1218
1260
|
await this.placeVideos().catch(() => undefined);
|
|
1219
1261
|
} catch (error) {
|
|
1220
1262
|
if (!upgrade) this.videoFailed.add(song.file);
|
|
1221
|
-
this.log(`no
|
|
1263
|
+
this.log(`no filmed video for "${song.title}": ${(error as Error).message}`);
|
|
1222
1264
|
} finally {
|
|
1223
1265
|
rmSync(partial, { force: true });
|
|
1224
1266
|
rmSync(scenes, { recursive: true, force: true });
|
|
@@ -1228,46 +1270,20 @@ export class AiLabel {
|
|
|
1228
1270
|
}
|
|
1229
1271
|
|
|
1230
1272
|
/**
|
|
1231
|
-
*
|
|
1232
|
-
*
|
|
1273
|
+
* A single's video: Veo shots of the band, cut to the whole song (FILMED).
|
|
1274
|
+
* Fewer than half the shots filmed (a refusal, the budget spent) makes no
|
|
1275
|
+
* video at all: footage stretched that thin would be a loop, not a video.
|
|
1233
1276
|
*/
|
|
1234
|
-
private async filmSong(artist: LabelArtist,
|
|
1277
|
+
private async filmSong(artist: LabelArtist, _album: LabelAlbum, song: LabelSong, scenes: string, out: string, _upgrade = false): Promise<number> {
|
|
1235
1278
|
let seconds = song.seconds;
|
|
1236
1279
|
try {
|
|
1237
1280
|
const probed = await this.deps.probe(song.file);
|
|
1238
1281
|
if (probed.duration > 5) seconds = probed.duration;
|
|
1239
1282
|
} catch { /* the asked-for length */ }
|
|
1240
|
-
const
|
|
1241
|
-
if (this.
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
const prompts = await writeStoryboard({ ...artist, cast: this.castOf(artist) }, album, song, count, this.deps.ask ?? askJson);
|
|
1245
|
-
mkdirSync(scenes, { recursive: true });
|
|
1246
|
-
const draw = this.deps.image ?? ((prompt: string) => workersAiCover(prompt));
|
|
1247
|
-
// A few at a time: the image model is quick, the machine is shared.
|
|
1248
|
-
for (let i = 0; i < prompts.length; i += 3) {
|
|
1249
|
-
const drawn = await Promise.all(prompts.slice(i, i + 3).map((prompt) => draw(`${prompt} Cinematic 16:9 music video still, no text.`).catch(() => null)));
|
|
1250
|
-
drawn.forEach((picture, j) => {
|
|
1251
|
-
if (!picture) return;
|
|
1252
|
-
const file = join(scenes, `${String(i + j).padStart(2, "0")}.jpg`);
|
|
1253
|
-
writeFileSync(file, picture);
|
|
1254
|
-
images.push(file);
|
|
1255
|
-
});
|
|
1256
|
-
}
|
|
1257
|
-
} catch (error) {
|
|
1258
|
-
this.log(`no storyboard for "${song.title}" (${(error as Error).message}); the cover video instead`);
|
|
1259
|
-
}
|
|
1260
|
-
}
|
|
1261
|
-
// An album's single gets filmed shots of the band too, while the month's Veo budget lasts.
|
|
1262
|
-
const clips = this.wantsVeo(song) ? await this.filmShots(artist, song, scenes) : [];
|
|
1263
|
-
if (images.length >= 2 || clips.length > 0) {
|
|
1264
|
-
if (storyOnly && (song.videoVersion ?? 1) >= 2 && clips.length === 0) throw new Error("no filmed shots this time; the video it has stays");
|
|
1265
|
-
await (this.deps.storyVideo ?? ((input) => renderStoryVideo(this.deps.ffmpeg, input)))({ audio: song.file, images, seconds, artist: artist.name, title: song.title, out, ...(clips.length ? { clips } : {}) });
|
|
1266
|
-
return clips.length ? 3 : 2;
|
|
1267
|
-
}
|
|
1268
|
-
if (storyOnly) throw new Error("no storyboard could be drawn; the video it has stays");
|
|
1269
|
-
await (this.deps.video ?? ((audio, cover, to) => renderVideo(this.deps.ffmpeg, audio, cover, to)))(song.file, album.cover, out);
|
|
1270
|
-
return 1;
|
|
1283
|
+
const clips = await this.filmShots(artist, song, scenes);
|
|
1284
|
+
if (clips.length < Math.max(2, Math.ceil(this.settings.veoShots / 2))) throw new Error(`only ${clips.length} of ${this.settings.veoShots} shots were filmed`);
|
|
1285
|
+
await (this.deps.filmedVideo ?? ((input) => renderFilmedVideo(this.deps.ffmpeg, input)))({ audio: song.file, clips, seconds, artist: artist.name, title: song.title, out });
|
|
1286
|
+
return FILMED;
|
|
1271
1287
|
}
|
|
1272
1288
|
|
|
1273
1289
|
/** Whether this song gets filmed Veo shots: an album's single, Veo on, and the month's budget not spent. */
|
|
@@ -1307,7 +1323,7 @@ export class AiLabel {
|
|
|
1307
1323
|
} catch (error) {
|
|
1308
1324
|
if (error instanceof VeoUnavailable) {
|
|
1309
1325
|
this.veoSpentIn = new Date(this.now()).toISOString().slice(0, 7);
|
|
1310
|
-
this.log(`Veo: ${(error as Error).message};
|
|
1326
|
+
this.log(`Veo: ${(error as Error).message}; no more videos until next month`);
|
|
1311
1327
|
break;
|
|
1312
1328
|
}
|
|
1313
1329
|
this.log(`Veo shot ${i + 1} for "${song.title}": ${(error as Error).message}`);
|
|
@@ -1322,17 +1338,21 @@ export class AiLabel {
|
|
|
1322
1338
|
const next = this.placing.then(async () => {
|
|
1323
1339
|
await this.deps.index.scan(this.source(), this.deps.probe).done;
|
|
1324
1340
|
const files = this.list()
|
|
1325
|
-
.flatMap((artist) => artist.albums.flatMap((album) => album.songs.map((song) => song.video ?? "")))
|
|
1341
|
+
.flatMap((artist) => artist.albums.flatMap((album) => album.songs.map((song) => ((song.videoVersion ?? 1) >= FILMED ? song.video ?? "" : ""))))
|
|
1326
1342
|
.filter((file) => file && this.deps.index.itemAt(file)?.state === "ready");
|
|
1327
|
-
if (files.length === 0) return;
|
|
1328
1343
|
const existing = this.deps.store.byAuto("ai:label:videos");
|
|
1329
|
-
if (
|
|
1330
|
-
|
|
1344
|
+
if (files.length === 0) {
|
|
1345
|
+
// Nothing filmed yet (older slideshow videos are not shown): off the guide until there is.
|
|
1346
|
+
if (existing && existing.listed) this.deps.store.setAccess(existing.id, { visibility: "private", listed: false }, this.deps.owner || "ai-label");
|
|
1331
1347
|
return;
|
|
1332
1348
|
}
|
|
1333
1349
|
const label = this.settings.holder.replace(/,?\s+(inc|llc|ltd|gmbh|co)\.?$/i, "");
|
|
1334
|
-
|
|
1335
|
-
|
|
1350
|
+
if (existing) {
|
|
1351
|
+
this.deps.store.update(existing.id, {}, files);
|
|
1352
|
+
if (!this.settings.publish || existing.listed) return;
|
|
1353
|
+
}
|
|
1354
|
+
const made = existing ?? this.deps.store.create({ name: `${label} AI · Music Videos`, description: `Music videos by ${label}'s AI artists, every genre.`, shuffle: true, repeat: true, video: true }, files, this.deps.owner);
|
|
1355
|
+
if (!existing) this.deps.store.markAuto(made.id, "ai:label:videos");
|
|
1336
1356
|
if (this.settings.publish) {
|
|
1337
1357
|
this.deps.store.setAccess(made.id, {
|
|
1338
1358
|
visibility: "public",
|
|
@@ -1662,7 +1682,7 @@ export class AiLabel {
|
|
|
1662
1682
|
.filter((album) => album.songs.some((song) => song.file))
|
|
1663
1683
|
.map((album) => ({
|
|
1664
1684
|
key: album.key, title: album.title, year: album.year, concept: album.concept, cover: Boolean(album.cover), complete: album.doneAt > 0,
|
|
1665
|
-
songs: album.songs.filter((song) => song.file).map((song) => ({ n: song.n, title: song.title, seconds: song.seconds, lyrics: song.lyrics, ...(song.video ? { video: true } : {}) })),
|
|
1685
|
+
songs: album.songs.filter((song) => song.file).map((song) => ({ n: song.n, title: song.title, seconds: song.seconds, lyrics: song.lyrics, ...(song.video && (song.videoVersion ?? 1) >= FILMED ? { video: true } : {}) })),
|
|
1666
1686
|
}))
|
|
1667
1687
|
.reverse(),
|
|
1668
1688
|
}))
|
|
@@ -1679,7 +1699,7 @@ export class AiLabel {
|
|
|
1679
1699
|
const match = /^(\d{1,3})\.(mp3|txt|mp4)$/.exec(what);
|
|
1680
1700
|
const song = match ? album.songs.find((one) => one.n === Number(match[1]) && one.file) : undefined;
|
|
1681
1701
|
if (!song) return null;
|
|
1682
|
-
if (match![2] === "mp4") return song.video ? { path: song.video, type: "video/mp4" } : null;
|
|
1702
|
+
if (match![2] === "mp4") return song.video && (song.videoVersion ?? 1) >= FILMED ? { path: song.video, type: "video/mp4" } : null;
|
|
1683
1703
|
return match![2] === "mp3" ? { path: song.file, type: "audio/mpeg" } : { path: song.lyricsFile, type: "text/plain; charset=utf-8" };
|
|
1684
1704
|
}
|
|
1685
1705
|
|
|
@@ -1791,7 +1811,7 @@ export class AiLabel {
|
|
|
1791
1811
|
entries.push({ name: `${folder}/${songBase(song)}.mp3`, path: song.file });
|
|
1792
1812
|
entries.push({ name: `${folder}/${songBase(song)}.lyrics.txt`, bytes: Buffer.from(`${song.lyrics}\n`) });
|
|
1793
1813
|
if (song.lrc) entries.push({ name: `${folder}/${songBase(song)}.lrc`, path: song.lrc });
|
|
1794
|
-
if (song.video) entries.push({ name: `${folder}/${songBase(song)}.mp4`, path: song.video });
|
|
1814
|
+
if (song.video && (song.videoVersion ?? 1) >= FILMED) entries.push({ name: `${folder}/${songBase(song)}.mp4`, path: song.video });
|
|
1795
1815
|
}
|
|
1796
1816
|
if (album.cover) entries.push({ name: `${folder}/cover.jpg`, path: album.cover });
|
|
1797
1817
|
entries.push({ name: `${folder}/album.json`, bytes: Buffer.from(`${JSON.stringify({
|
package/src/veo-studio.ts
CHANGED
|
@@ -111,7 +111,7 @@ export class VeoStudio {
|
|
|
111
111
|
return { month: this.month(), spentUsd: Math.round(spent * 100) / 100, capUsd: this.cap, leftUsd: Math.round(left * 100) / 100, usdPerSecond: this.rate, secondsLeft: Math.floor(left / this.rate) };
|
|
112
112
|
}
|
|
113
113
|
|
|
114
|
-
/** One shot: 4, 6 or 8 seconds, 16:9, with up to three reference images (JPEG). */
|
|
114
|
+
/** One shot: 4, 6 or 8 seconds, 16:9 1080p, with up to three reference images (JPEG). */
|
|
115
115
|
async clip(input: { prompt: string; seconds: number; references: Buffer[] }): Promise<Buffer> {
|
|
116
116
|
const seconds = [4, 6, 8].includes(input.seconds) ? input.seconds : 8;
|
|
117
117
|
const cost = seconds * this.rate;
|
|
@@ -139,7 +139,8 @@ export class VeoStudio {
|
|
|
139
139
|
prompt: prompt.slice(0, 1800),
|
|
140
140
|
...(references.length ? { referenceImages: references.map((bytes) => ({ image: { bytesBase64Encoded: bytes.toString("base64"), mimeType: "image/jpeg" }, referenceType: "asset" })) } : {}),
|
|
141
141
|
}],
|
|
142
|
-
|
|
142
|
+
// 1080p, counted at the same rate; the cut is 1080p too (renderFilmedVideo).
|
|
143
|
+
parameters: { aspectRatio: "16:9", durationSeconds: seconds, resolution: "1080p" },
|
|
143
144
|
}),
|
|
144
145
|
});
|
|
145
146
|
const operation = await started.json().catch(() => ({})) as { name?: string; error?: { message?: string } };
|