@neta-art/cohub-cli 10.18.0 → 10.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/board-export.d.ts +8 -2
- package/dist/board-export.js +75 -42
- package/dist/board-gpu/page.js +339 -133
- package/dist/board-gpu/protocol.d.ts +1 -0
- package/dist/board-kernel.d.ts +1 -1
- package/dist/board-kernel.js +1113 -188
- package/dist/board-media-export.d.ts +16 -0
- package/dist/board-media-export.js +147 -0
- package/dist/board-media.d.ts +2 -1
- package/dist/board-media.js +23 -0
- package/dist/commands/boards/authoring.js +45 -3
- package/dist/commands/space-displays.d.ts +1 -1
- package/dist/commands/space-displays.js +70 -18
- package/dist/output.d.ts +1 -0
- package/dist/output.js +3 -0
- package/dist/safe-remote-image.d.ts +4 -0
- package/dist/safe-remote-image.js +8 -0
- package/package.json +2 -2
package/dist/board-export.d.ts
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
import type { BoardDocument, BoardExportRegion, BoardSceneItem, BoardThemeId } from "@neta-art/cohub/board";
|
|
1
|
+
import type { BoardDocument, BoardExportRegion, BoardMediaState, BoardSceneItem, BoardThemeId } from "@neta-art/cohub/board";
|
|
2
2
|
import type { BoardNodeExportFormat, BoardNodeFont, BoardNodeTexture } from "@neta-art/cohub/board/export/node";
|
|
3
3
|
import type { BoardScriptHost } from "@neta-art/cohub/board/render";
|
|
4
4
|
import { type BoardGpuResources } from "./board-gpu/index.js";
|
|
5
5
|
import { createClient } from "./client.js";
|
|
6
|
+
import { type BoardMediaFiles } from "./board-media-export.js";
|
|
6
7
|
import { type BoardExportRenderer, type BoardExportDecodeImage } from "./board-export-resources.js";
|
|
7
8
|
export declare function resolveBundledFonts(): BoardNodeFont[];
|
|
8
9
|
export type BoardExportSource = {
|
|
@@ -69,6 +70,7 @@ export type BoardExportRunOptions = {
|
|
|
69
70
|
}) => void;
|
|
70
71
|
onFrame?: (frame: BoardExportFrame, index: number) => Promise<void>;
|
|
71
72
|
onProgress?: (completed: number, total: number) => void;
|
|
73
|
+
mediaFiles?: BoardMediaFiles;
|
|
72
74
|
};
|
|
73
75
|
export type BoardExportFrame = {
|
|
74
76
|
time: number | null;
|
|
@@ -82,6 +84,10 @@ export type BoardExportFrame = {
|
|
|
82
84
|
export type BoardExportRunResult = {
|
|
83
85
|
frames: BoardExportFrame[];
|
|
84
86
|
warnings: string[];
|
|
87
|
+
media: Array<{
|
|
88
|
+
at: number;
|
|
89
|
+
media: BoardMediaState[];
|
|
90
|
+
}>;
|
|
85
91
|
};
|
|
86
92
|
export type BoardVideoExportFormat = "mp4" | "webm";
|
|
87
93
|
export type BoardVideoExportOptions = Omit<BoardExportRunOptions, "format"> & {
|
|
@@ -97,5 +103,5 @@ export declare function runBoardVideoExport(options: BoardVideoExportOptions): P
|
|
|
97
103
|
warnings: string[];
|
|
98
104
|
}>;
|
|
99
105
|
export declare function runBoardExport(options: BoardExportRunOptions): Promise<BoardExportRunResult | null>;
|
|
100
|
-
export declare function gpuResources(spaceId: string, document: BoardDocument): BoardGpuResources;
|
|
106
|
+
export declare function gpuResources(spaceId: string, document: BoardDocument, framePicture?: (key: string) => Uint8Array | undefined): BoardGpuResources;
|
|
101
107
|
export {};
|
package/dist/board-export.js
CHANGED
|
@@ -11,12 +11,13 @@ import { tmpdir } from "node:os";
|
|
|
11
11
|
const execFile = promisify(execFileCallback);
|
|
12
12
|
import { boardMediaNaturalSize, parseBoardMediaSource } from "@neta-art/cohub/board";
|
|
13
13
|
import { imageVariantUrl, MEDIA_VARIANT_MAX, mediaVariantSize, videoFrameUrl } from "@neta-art/cohub/media";
|
|
14
|
-
import { hasBoardScripts, boardAssetKey, boardAssetKeySource,
|
|
14
|
+
import { hasBoardScripts, boardAssetKey, boardAssetKeySource, boardMediaRate, boardMediaSpans, boardRenderTheme, boardTimedMedia, boardTextureEdge, boardTexturePixels, createBoardRenderSampler, buildBoardScene, createNodeBoardRenderer, createNodeBoardScriptHost, describeBoardExportWarning, exportBoardImageBytes, parseBoardDocument, planBoardExport, prepareBoardExport, selectBoardExportAssets, } from "./board-kernel.js";
|
|
15
15
|
import { settleBoardDocument } from "./board-media.js";
|
|
16
16
|
import { findBoardGpuBrowser, launchBoardGpuExporter } from "./board-gpu/index.js";
|
|
17
17
|
import { BOARD_EXPORT_FONTS } from "./board-fonts.js";
|
|
18
18
|
import { createClient } from "./client.js";
|
|
19
|
-
import {
|
|
19
|
+
import { downloadPublicImage } from "./safe-remote-image.js";
|
|
20
|
+
import { boardAudioMix, createBoardMediaFiles, createBoardVideoFrames } from "./board-media-export.js";
|
|
20
21
|
import { BOARD_EXPORT_IMAGE_MEMORY_MIB, BoardExportResourceError, createBoardExportImageBudget, } from "./board-export-resources.js";
|
|
21
22
|
const debugExport = debuglog("cohub:board-export");
|
|
22
23
|
export function resolveBundledFonts() {
|
|
@@ -177,12 +178,8 @@ export async function runBoardVideoExport(options) {
|
|
|
177
178
|
try {
|
|
178
179
|
let count = 0;
|
|
179
180
|
let dimensions = { width: 0, height: 0 };
|
|
180
|
-
const
|
|
181
|
-
|
|
182
|
-
throw new Error(`MP4 needs even dimensions, but the export is ${plan.width}x${plan.height}. Use --camera --size with even dimensions, or adjust the export region.`);
|
|
183
|
-
}
|
|
184
|
-
options.onPlan?.(plan);
|
|
185
|
-
}, onFrame: async (frame, index) => {
|
|
181
|
+
const mediaFiles = createBoardMediaFiles(options.spaceId, tempDir, options.signal);
|
|
182
|
+
const result = await runBoardExport({ ...options, document, format: "png", mediaFiles, onFrame: async (frame, index) => {
|
|
186
183
|
if (count && (frame.width !== dimensions.width || frame.height !== dimensions.height))
|
|
187
184
|
throw new Error("Video frame dimensions changed. Use a fixed export region or a smaller camera viewport.");
|
|
188
185
|
await writeFile(join(tempDir, `${String(index + 1).padStart(6, "0")}.png`), frame.bytes, { signal: options.signal });
|
|
@@ -191,36 +188,18 @@ export async function runBoardVideoExport(options) {
|
|
|
191
188
|
} });
|
|
192
189
|
if (!result || !count)
|
|
193
190
|
throw new Error("Nothing to export: the region contains no items.");
|
|
194
|
-
const
|
|
195
|
-
const
|
|
196
|
-
const
|
|
197
|
-
|
|
198
|
-
const source = parseBoardMediaSource(item.props.src);
|
|
199
|
-
if (!source)
|
|
200
|
-
throw new Error("Invalid board audio source");
|
|
201
|
-
const { bytes } = source.kind === "url"
|
|
202
|
-
? await downloadPublicAudio(source.value)
|
|
203
|
-
: await readSpaceFileBytes(createClient(), options.spaceId, source.value);
|
|
204
|
-
const path = join(tempDir, `audio-${index}`);
|
|
205
|
-
await writeFile(path, bytes);
|
|
206
|
-
audioInputs.push({ path, offset: Math.max(0, item.props.time ?? 0) / 1000 });
|
|
207
|
-
}
|
|
191
|
+
const start = times[0];
|
|
192
|
+
const sampled = boardMediaSpans(result.media.map((sample) => ({ ...sample, at: sample.at - start })));
|
|
193
|
+
const spans = sampled.map((span) => sampled.some((next) => next.id === span.id && next.start === span.end) ? span : { ...span, end: span.end + step });
|
|
194
|
+
const mix = await boardAudioMix(spans, mediaFiles, 1);
|
|
208
195
|
const duration = Math.max(0.001, count / options.fps);
|
|
209
|
-
const args = ["-y", "-f", "image2", "-framerate", String(options.fps), "-i", join(tempDir, "%06d.png")];
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
args.push("-i", input.path);
|
|
214
|
-
}
|
|
215
|
-
if (audioInputs.length > 1) {
|
|
216
|
-
const inputs = audioInputs.map((_, index) => `[${index + 1}:a]`).join("");
|
|
217
|
-
args.push("-filter_complex", `${inputs}amix=inputs=${audioInputs.length}:duration=longest:dropout_transition=0[aout]`, "-map", "0:v:0", "-map", "[aout]");
|
|
218
|
-
}
|
|
219
|
-
else {
|
|
196
|
+
const args = ["-y", "-f", "image2", "-framerate", String(options.fps), "-i", join(tempDir, "%06d.png"), ...(mix?.inputs ?? [])];
|
|
197
|
+
if (mix)
|
|
198
|
+
args.push("-filter_complex", mix.filter, "-map", "0:v:0", "-map", "[aout]");
|
|
199
|
+
else
|
|
220
200
|
args.push("-map", "0:v:0");
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
}
|
|
201
|
+
if (options.format === "mp4" && (dimensions.width % 2 || dimensions.height % 2))
|
|
202
|
+
args.push("-vf", "pad=ceil(iw/2)*2:ceil(ih/2)*2");
|
|
224
203
|
if (options.format === "mp4")
|
|
225
204
|
args.push("-c:v", "libx264", "-pix_fmt", "yuv420p", "-c:a", "aac", "-movflags", "+faststart", "-f", "mp4");
|
|
226
205
|
else
|
|
@@ -282,6 +261,8 @@ export async function runBoardExport(options) {
|
|
|
282
261
|
let scriptHost = null;
|
|
283
262
|
let gpu = null;
|
|
284
263
|
const textureCache = { textures: new Map(), edges: new Map(), missed: new Map() };
|
|
264
|
+
const frameTextures = new Map();
|
|
265
|
+
let mediaDir = null;
|
|
285
266
|
let backgroundTexture;
|
|
286
267
|
try {
|
|
287
268
|
const theme = boardRenderTheme(options.theme);
|
|
@@ -336,6 +317,18 @@ export async function runBoardExport(options) {
|
|
|
336
317
|
return key && omittedKeys.has(key) ? null : key;
|
|
337
318
|
};
|
|
338
319
|
const frames = [];
|
|
320
|
+
const media = [];
|
|
321
|
+
const fps = times.length > 1 && times[0] !== null && times[1] !== null ? 1000 / (times[1] - times[0]) : 30;
|
|
322
|
+
const lastMedia = new Map();
|
|
323
|
+
const framePictures = new Map();
|
|
324
|
+
let videoFrames = null;
|
|
325
|
+
const frameOf = async (state, rate, edge) => {
|
|
326
|
+
if (!videoFrames) {
|
|
327
|
+
mediaDir = await mkdtemp(join(tmpdir(), "cohub-board-media-"));
|
|
328
|
+
videoFrames = createBoardVideoFrames(options.mediaFiles ?? createBoardMediaFiles(options.spaceId, mediaDir, options.signal), mediaDir, { fps, signal: options.signal });
|
|
329
|
+
}
|
|
330
|
+
return videoFrames.frame(state, rate, edge);
|
|
331
|
+
};
|
|
339
332
|
const reported = new Set();
|
|
340
333
|
const report = (text) => {
|
|
341
334
|
if (reported.has(text))
|
|
@@ -354,7 +347,7 @@ export async function runBoardExport(options) {
|
|
|
354
347
|
options.onProgress?.(completed, times.length);
|
|
355
348
|
};
|
|
356
349
|
const launchGpu = async (browser) => {
|
|
357
|
-
const exporter = await launchBoardGpuExporter(gpuResources(options.spaceId, document), { browser, signal: options.signal });
|
|
350
|
+
const exporter = await launchBoardGpuExporter(gpuResources(options.spaceId, document, (key) => framePictures.get(key)), { browser, signal: options.signal });
|
|
358
351
|
try {
|
|
359
352
|
await exporter.init({ document, animation: options.animation ?? null, theme: options.theme });
|
|
360
353
|
}
|
|
@@ -412,6 +405,38 @@ export async function runBoardExport(options) {
|
|
|
412
405
|
prepared = scripted;
|
|
413
406
|
}
|
|
414
407
|
}
|
|
408
|
+
const timed = boardTimedMedia(prepared.scene);
|
|
409
|
+
media.push({ at: time ?? 0, media: timed });
|
|
410
|
+
const frameKeys = {};
|
|
411
|
+
for (const texture of frameTextures.values())
|
|
412
|
+
texture.destroy(true);
|
|
413
|
+
frameTextures.clear();
|
|
414
|
+
framePictures.clear();
|
|
415
|
+
if (options.withImages) {
|
|
416
|
+
const planned = new Set(prepared.plan.items.map((item) => item.id));
|
|
417
|
+
for (const state of timed) {
|
|
418
|
+
const item = prepared.scene.get(state.id);
|
|
419
|
+
const base = item && state.kind === "video" && state.shown && planned.has(state.id) ? boardExportAssetKey(item) : null;
|
|
420
|
+
if (!item || !base)
|
|
421
|
+
continue;
|
|
422
|
+
const before = lastMedia.get(state.id);
|
|
423
|
+
if (time !== null)
|
|
424
|
+
lastMedia.set(state.id, { at: time, time: state.time });
|
|
425
|
+
const rate = times.length === 1 ? 0 : before && time !== null ? boardMediaRate(before, { at: time, time: state.time }) : 1;
|
|
426
|
+
const edge = Math.min(4096, textureEdge(item, prepared.plan.scale));
|
|
427
|
+
const bytes = await frameOf(state, rate, edge).catch((error) => {
|
|
428
|
+
report(`Frames of ${state.src} could not be decoded; its cover was drawn instead. ${error instanceof Error ? error.message : ""}`.trim());
|
|
429
|
+
return null;
|
|
430
|
+
});
|
|
431
|
+
if (!bytes?.length)
|
|
432
|
+
continue;
|
|
433
|
+
const key = `${base}@${Math.round(state.time)}`;
|
|
434
|
+
frameKeys[state.id] = key;
|
|
435
|
+
framePictures.set(key, bytes);
|
|
436
|
+
if (!gpu)
|
|
437
|
+
frameTextures.set(key, await nodeRenderer.decodeImage(bytes, "image/jpeg"));
|
|
438
|
+
}
|
|
439
|
+
}
|
|
415
440
|
if (gpu) {
|
|
416
441
|
trace("gpu", completed, prepared.plan);
|
|
417
442
|
const result = await gpu.render({
|
|
@@ -425,6 +450,7 @@ export async function runBoardExport(options) {
|
|
|
425
450
|
...(options.quality === undefined ? {} : { quality: options.quality }),
|
|
426
451
|
withImages: options.withImages,
|
|
427
452
|
imageMemory: imageMemoryBytes,
|
|
453
|
+
assetKeys: frameKeys,
|
|
428
454
|
});
|
|
429
455
|
for (const warning of result.warnings)
|
|
430
456
|
report(options.withImages && warning.kind === "images-missing" ? describeBoardExportWarning(warning) : describeWarning(warning));
|
|
@@ -433,7 +459,7 @@ export async function runBoardExport(options) {
|
|
|
433
459
|
}
|
|
434
460
|
trace("images", completed, prepared.plan);
|
|
435
461
|
if (options.withImages) {
|
|
436
|
-
const visible = prepared.plan.items.filter((item) => prepared.scene.opacity(item.id) > 0);
|
|
462
|
+
const visible = prepared.plan.items.filter((item) => prepared.scene.opacity(item.id) > 0 && !frameKeys[item.id]);
|
|
437
463
|
const loaded = await loadBoardTextures(images, options.spaceId, visible, { scale: prepared.plan.scale, maxBytes: imageMemoryBytes, cache: textureCache });
|
|
438
464
|
omittedKeys = new Set(loaded.omitted);
|
|
439
465
|
const sources = (keys) => keys.slice(0, 3).map((key) => boardAssetKeySource(key)?.value ?? key).join(", ");
|
|
@@ -454,10 +480,10 @@ export async function runBoardExport(options) {
|
|
|
454
480
|
...frameOptions,
|
|
455
481
|
...theme,
|
|
456
482
|
background: options.background,
|
|
457
|
-
textures,
|
|
483
|
+
textures: frameTextures.size ? new Map([...textures, ...frameTextures]) : textures,
|
|
458
484
|
scripts: options.scripts ?? scriptHost ?? undefined,
|
|
459
485
|
backgroundTexture,
|
|
460
|
-
assetKey,
|
|
486
|
+
assetKey: (item) => frameKeys[item.id] ?? assetKey(item),
|
|
461
487
|
format: options.format,
|
|
462
488
|
quality: options.quality,
|
|
463
489
|
});
|
|
@@ -481,7 +507,7 @@ export async function runBoardExport(options) {
|
|
|
481
507
|
});
|
|
482
508
|
trace("rendered", completed, result.plan);
|
|
483
509
|
}
|
|
484
|
-
return completed ? { frames, warnings } : null;
|
|
510
|
+
return completed ? { frames, warnings, media } : null;
|
|
485
511
|
}
|
|
486
512
|
finally {
|
|
487
513
|
await gpu?.close();
|
|
@@ -489,10 +515,14 @@ export async function runBoardExport(options) {
|
|
|
489
515
|
for (const texture of textureCache.textures.values())
|
|
490
516
|
texture.destroy(true);
|
|
491
517
|
backgroundTexture?.destroy(true);
|
|
518
|
+
for (const texture of frameTextures.values())
|
|
519
|
+
texture.destroy(true);
|
|
520
|
+
if (mediaDir)
|
|
521
|
+
await rm(mediaDir, { recursive: true, force: true });
|
|
492
522
|
renderer.destroy();
|
|
493
523
|
}
|
|
494
524
|
}
|
|
495
|
-
export function gpuResources(spaceId, document) {
|
|
525
|
+
export function gpuResources(spaceId, document, framePicture = () => undefined) {
|
|
496
526
|
const client = createClient();
|
|
497
527
|
const scene = buildBoardScene(document);
|
|
498
528
|
return {
|
|
@@ -505,6 +535,9 @@ export function gpuResources(spaceId, document) {
|
|
|
505
535
|
return null;
|
|
506
536
|
return source.kind === "url" ? downloadPublicImage(source.value) : readSpaceFileBytes(client, spaceId, source.value);
|
|
507
537
|
}
|
|
538
|
+
const frame = framePicture(request.key);
|
|
539
|
+
if (frame)
|
|
540
|
+
return { bytes: frame, mimeType: "image/jpeg" };
|
|
508
541
|
const source = boardAssetKeySource(request.key);
|
|
509
542
|
const item = scene.items.find((entry) => boardAssetKey(entry) === request.key);
|
|
510
543
|
return source && item ? fetchBoardPicture(client, spaceId, source, item, request.edge) : null;
|