@neta-art/cohub-cli 10.18.0 → 10.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,8 +1,9 @@
1
- import type { BoardDocument, BoardExportRegion, BoardSceneItem, BoardThemeId } from "@neta-art/cohub/board";
1
+ import type { BoardDocument, BoardExportRegion, BoardMediaState, BoardSceneItem, BoardThemeId } from "@neta-art/cohub/board";
2
2
  import type { BoardNodeExportFormat, BoardNodeFont, BoardNodeTexture } from "@neta-art/cohub/board/export/node";
3
3
  import type { BoardScriptHost } from "@neta-art/cohub/board/render";
4
4
  import { type BoardGpuResources } from "./board-gpu/index.js";
5
5
  import { createClient } from "./client.js";
6
+ import { type BoardMediaFiles } from "./board-media-export.js";
6
7
  import { type BoardExportRenderer, type BoardExportDecodeImage } from "./board-export-resources.js";
7
8
  export declare function resolveBundledFonts(): BoardNodeFont[];
8
9
  export type BoardExportSource = {
@@ -69,6 +70,7 @@ export type BoardExportRunOptions = {
69
70
  }) => void;
70
71
  onFrame?: (frame: BoardExportFrame, index: number) => Promise<void>;
71
72
  onProgress?: (completed: number, total: number) => void;
73
+ mediaFiles?: BoardMediaFiles;
72
74
  };
73
75
  export type BoardExportFrame = {
74
76
  time: number | null;
@@ -82,6 +84,10 @@ export type BoardExportFrame = {
82
84
  export type BoardExportRunResult = {
83
85
  frames: BoardExportFrame[];
84
86
  warnings: string[];
87
+ media: Array<{
88
+ at: number;
89
+ media: BoardMediaState[];
90
+ }>;
85
91
  };
86
92
  export type BoardVideoExportFormat = "mp4" | "webm";
87
93
  export type BoardVideoExportOptions = Omit<BoardExportRunOptions, "format"> & {
@@ -97,5 +103,5 @@ export declare function runBoardVideoExport(options: BoardVideoExportOptions): P
97
103
  warnings: string[];
98
104
  }>;
99
105
  export declare function runBoardExport(options: BoardExportRunOptions): Promise<BoardExportRunResult | null>;
100
- export declare function gpuResources(spaceId: string, document: BoardDocument): BoardGpuResources;
106
+ export declare function gpuResources(spaceId: string, document: BoardDocument, framePicture?: (key: string) => Uint8Array | undefined): BoardGpuResources;
101
107
  export {};
@@ -11,12 +11,13 @@ import { tmpdir } from "node:os";
11
11
  const execFile = promisify(execFileCallback);
12
12
  import { boardMediaNaturalSize, parseBoardMediaSource } from "@neta-art/cohub/board";
13
13
  import { imageVariantUrl, MEDIA_VARIANT_MAX, mediaVariantSize, videoFrameUrl } from "@neta-art/cohub/media";
14
- import { hasBoardScripts, boardAssetKey, boardAssetKeySource, boardDocumentAt, boardRenderTheme, boardTextureEdge, boardTexturePixels, createBoardRenderSampler, buildBoardScene, createNodeBoardRenderer, createNodeBoardScriptHost, describeBoardExportWarning, exportBoardImageBytes, parseBoardDocument, planBoardExport, prepareBoardExport, selectBoardExportAssets, } from "./board-kernel.js";
14
+ import { hasBoardScripts, boardAssetKey, boardAssetKeySource, boardMediaRate, boardMediaSpans, boardRenderTheme, boardTimedMedia, boardTextureEdge, boardTexturePixels, createBoardRenderSampler, buildBoardScene, createNodeBoardRenderer, createNodeBoardScriptHost, describeBoardExportWarning, exportBoardImageBytes, parseBoardDocument, planBoardExport, prepareBoardExport, selectBoardExportAssets, } from "./board-kernel.js";
15
15
  import { settleBoardDocument } from "./board-media.js";
16
16
  import { findBoardGpuBrowser, launchBoardGpuExporter } from "./board-gpu/index.js";
17
17
  import { BOARD_EXPORT_FONTS } from "./board-fonts.js";
18
18
  import { createClient } from "./client.js";
19
- import { downloadPublicAudio, downloadPublicImage } from "./safe-remote-image.js";
19
+ import { downloadPublicImage } from "./safe-remote-image.js";
20
+ import { boardAudioMix, createBoardMediaFiles, createBoardVideoFrames } from "./board-media-export.js";
20
21
  import { BOARD_EXPORT_IMAGE_MEMORY_MIB, BoardExportResourceError, createBoardExportImageBudget, } from "./board-export-resources.js";
21
22
  const debugExport = debuglog("cohub:board-export");
22
23
  export function resolveBundledFonts() {
@@ -177,12 +178,8 @@ export async function runBoardVideoExport(options) {
177
178
  try {
178
179
  let count = 0;
179
180
  let dimensions = { width: 0, height: 0 };
180
- const result = await runBoardExport({ ...options, document, format: "png", onPlan: (plan) => {
181
- if (options.format === "mp4" && (plan.width % 2 || plan.height % 2)) {
182
- throw new Error(`MP4 needs even dimensions, but the export is ${plan.width}x${plan.height}. Use --camera --size with even dimensions, or adjust the export region.`);
183
- }
184
- options.onPlan?.(plan);
185
- }, onFrame: async (frame, index) => {
181
+ const mediaFiles = createBoardMediaFiles(options.spaceId, tempDir, options.signal);
182
+ const result = await runBoardExport({ ...options, document, format: "png", mediaFiles, onFrame: async (frame, index) => {
186
183
  if (count && (frame.width !== dimensions.width || frame.height !== dimensions.height))
187
184
  throw new Error("Video frame dimensions changed. Use a fixed export region or a smaller camera viewport.");
188
185
  await writeFile(join(tempDir, `${String(index + 1).padStart(6, "0")}.png`), frame.bytes, { signal: options.signal });
@@ -191,36 +188,18 @@ export async function runBoardVideoExport(options) {
191
188
  } });
192
189
  if (!result || !count)
193
190
  throw new Error("Nothing to export: the region contains no items.");
194
- const audioDocument = boardDocumentAt(document, options.animation, times[0]).document;
195
- const audio = Object.values(audioDocument.items).filter((item) => item.type === "audio");
196
- const audioInputs = [];
197
- for (const [index, item] of audio.entries()) {
198
- const source = parseBoardMediaSource(item.props.src);
199
- if (!source)
200
- throw new Error("Invalid board audio source");
201
- const { bytes } = source.kind === "url"
202
- ? await downloadPublicAudio(source.value)
203
- : await readSpaceFileBytes(createClient(), options.spaceId, source.value);
204
- const path = join(tempDir, `audio-${index}`);
205
- await writeFile(path, bytes);
206
- audioInputs.push({ path, offset: Math.max(0, item.props.time ?? 0) / 1000 });
207
- }
191
+ const start = times[0];
192
+ const sampled = boardMediaSpans(result.media.map((sample) => ({ ...sample, at: sample.at - start })));
193
+ const spans = sampled.map((span) => sampled.some((next) => next.id === span.id && next.start === span.end) ? span : { ...span, end: span.end + step });
194
+ const mix = await boardAudioMix(spans, mediaFiles, 1);
208
195
  const duration = Math.max(0.001, count / options.fps);
209
- const args = ["-y", "-f", "image2", "-framerate", String(options.fps), "-i", join(tempDir, "%06d.png")];
210
- for (const input of audioInputs) {
211
- if (input.offset > 0)
212
- args.push("-ss", input.offset.toFixed(3));
213
- args.push("-i", input.path);
214
- }
215
- if (audioInputs.length > 1) {
216
- const inputs = audioInputs.map((_, index) => `[${index + 1}:a]`).join("");
217
- args.push("-filter_complex", `${inputs}amix=inputs=${audioInputs.length}:duration=longest:dropout_transition=0[aout]`, "-map", "0:v:0", "-map", "[aout]");
218
- }
219
- else {
196
+ const args = ["-y", "-f", "image2", "-framerate", String(options.fps), "-i", join(tempDir, "%06d.png"), ...(mix?.inputs ?? [])];
197
+ if (mix)
198
+ args.push("-filter_complex", mix.filter, "-map", "0:v:0", "-map", "[aout]");
199
+ else
220
200
  args.push("-map", "0:v:0");
221
- if (audioInputs.length === 1)
222
- args.push("-map", "1:a:0");
223
- }
201
+ if (options.format === "mp4" && (dimensions.width % 2 || dimensions.height % 2))
202
+ args.push("-vf", "pad=ceil(iw/2)*2:ceil(ih/2)*2");
224
203
  if (options.format === "mp4")
225
204
  args.push("-c:v", "libx264", "-pix_fmt", "yuv420p", "-c:a", "aac", "-movflags", "+faststart", "-f", "mp4");
226
205
  else
@@ -282,6 +261,8 @@ export async function runBoardExport(options) {
282
261
  let scriptHost = null;
283
262
  let gpu = null;
284
263
  const textureCache = { textures: new Map(), edges: new Map(), missed: new Map() };
264
+ const frameTextures = new Map();
265
+ let mediaDir = null;
285
266
  let backgroundTexture;
286
267
  try {
287
268
  const theme = boardRenderTheme(options.theme);
@@ -336,6 +317,18 @@ export async function runBoardExport(options) {
336
317
  return key && omittedKeys.has(key) ? null : key;
337
318
  };
338
319
  const frames = [];
320
+ const media = [];
321
+ const fps = times.length > 1 && times[0] !== null && times[1] !== null ? 1000 / (times[1] - times[0]) : 30;
322
+ const lastMedia = new Map();
323
+ const framePictures = new Map();
324
+ let videoFrames = null;
325
+ const frameOf = async (state, rate, edge) => {
326
+ if (!videoFrames) {
327
+ mediaDir = await mkdtemp(join(tmpdir(), "cohub-board-media-"));
328
+ videoFrames = createBoardVideoFrames(options.mediaFiles ?? createBoardMediaFiles(options.spaceId, mediaDir, options.signal), mediaDir, { fps, signal: options.signal });
329
+ }
330
+ return videoFrames.frame(state, rate, edge);
331
+ };
339
332
  const reported = new Set();
340
333
  const report = (text) => {
341
334
  if (reported.has(text))
@@ -354,7 +347,7 @@ export async function runBoardExport(options) {
354
347
  options.onProgress?.(completed, times.length);
355
348
  };
356
349
  const launchGpu = async (browser) => {
357
- const exporter = await launchBoardGpuExporter(gpuResources(options.spaceId, document), { browser, signal: options.signal });
350
+ const exporter = await launchBoardGpuExporter(gpuResources(options.spaceId, document, (key) => framePictures.get(key)), { browser, signal: options.signal });
358
351
  try {
359
352
  await exporter.init({ document, animation: options.animation ?? null, theme: options.theme });
360
353
  }
@@ -412,6 +405,38 @@ export async function runBoardExport(options) {
412
405
  prepared = scripted;
413
406
  }
414
407
  }
408
+ const timed = boardTimedMedia(prepared.scene);
409
+ media.push({ at: time ?? 0, media: timed });
410
+ const frameKeys = {};
411
+ for (const texture of frameTextures.values())
412
+ texture.destroy(true);
413
+ frameTextures.clear();
414
+ framePictures.clear();
415
+ if (options.withImages) {
416
+ const planned = new Set(prepared.plan.items.map((item) => item.id));
417
+ for (const state of timed) {
418
+ const item = prepared.scene.get(state.id);
419
+ const base = item && state.kind === "video" && state.shown && planned.has(state.id) ? boardExportAssetKey(item) : null;
420
+ if (!item || !base)
421
+ continue;
422
+ const before = lastMedia.get(state.id);
423
+ if (time !== null)
424
+ lastMedia.set(state.id, { at: time, time: state.time });
425
+ const rate = times.length === 1 ? 0 : before && time !== null ? boardMediaRate(before, { at: time, time: state.time }) : 1;
426
+ const edge = Math.min(4096, textureEdge(item, prepared.plan.scale));
427
+ const bytes = await frameOf(state, rate, edge).catch((error) => {
428
+ report(`Frames of ${state.src} could not be decoded; its cover was drawn instead. ${error instanceof Error ? error.message : ""}`.trim());
429
+ return null;
430
+ });
431
+ if (!bytes?.length)
432
+ continue;
433
+ const key = `${base}@${Math.round(state.time)}`;
434
+ frameKeys[state.id] = key;
435
+ framePictures.set(key, bytes);
436
+ if (!gpu)
437
+ frameTextures.set(key, await nodeRenderer.decodeImage(bytes, "image/jpeg"));
438
+ }
439
+ }
415
440
  if (gpu) {
416
441
  trace("gpu", completed, prepared.plan);
417
442
  const result = await gpu.render({
@@ -425,6 +450,7 @@ export async function runBoardExport(options) {
425
450
  ...(options.quality === undefined ? {} : { quality: options.quality }),
426
451
  withImages: options.withImages,
427
452
  imageMemory: imageMemoryBytes,
453
+ assetKeys: frameKeys,
428
454
  });
429
455
  for (const warning of result.warnings)
430
456
  report(options.withImages && warning.kind === "images-missing" ? describeBoardExportWarning(warning) : describeWarning(warning));
@@ -433,7 +459,7 @@ export async function runBoardExport(options) {
433
459
  }
434
460
  trace("images", completed, prepared.plan);
435
461
  if (options.withImages) {
436
- const visible = prepared.plan.items.filter((item) => prepared.scene.opacity(item.id) > 0);
462
+ const visible = prepared.plan.items.filter((item) => prepared.scene.opacity(item.id) > 0 && !frameKeys[item.id]);
437
463
  const loaded = await loadBoardTextures(images, options.spaceId, visible, { scale: prepared.plan.scale, maxBytes: imageMemoryBytes, cache: textureCache });
438
464
  omittedKeys = new Set(loaded.omitted);
439
465
  const sources = (keys) => keys.slice(0, 3).map((key) => boardAssetKeySource(key)?.value ?? key).join(", ");
@@ -454,10 +480,10 @@ export async function runBoardExport(options) {
454
480
  ...frameOptions,
455
481
  ...theme,
456
482
  background: options.background,
457
- textures,
483
+ textures: frameTextures.size ? new Map([...textures, ...frameTextures]) : textures,
458
484
  scripts: options.scripts ?? scriptHost ?? undefined,
459
485
  backgroundTexture,
460
- assetKey,
486
+ assetKey: (item) => frameKeys[item.id] ?? assetKey(item),
461
487
  format: options.format,
462
488
  quality: options.quality,
463
489
  });
@@ -481,7 +507,7 @@ export async function runBoardExport(options) {
481
507
  });
482
508
  trace("rendered", completed, result.plan);
483
509
  }
484
- return completed ? { frames, warnings } : null;
510
+ return completed ? { frames, warnings, media } : null;
485
511
  }
486
512
  finally {
487
513
  await gpu?.close();
@@ -489,10 +515,14 @@ export async function runBoardExport(options) {
489
515
  for (const texture of textureCache.textures.values())
490
516
  texture.destroy(true);
491
517
  backgroundTexture?.destroy(true);
518
+ for (const texture of frameTextures.values())
519
+ texture.destroy(true);
520
+ if (mediaDir)
521
+ await rm(mediaDir, { recursive: true, force: true });
492
522
  renderer.destroy();
493
523
  }
494
524
  }
495
- export function gpuResources(spaceId, document) {
525
+ export function gpuResources(spaceId, document, framePicture = () => undefined) {
496
526
  const client = createClient();
497
527
  const scene = buildBoardScene(document);
498
528
  return {
@@ -505,6 +535,9 @@ export function gpuResources(spaceId, document) {
505
535
  return null;
506
536
  return source.kind === "url" ? downloadPublicImage(source.value) : readSpaceFileBytes(client, spaceId, source.value);
507
537
  }
538
+ const frame = framePicture(request.key);
539
+ if (frame)
540
+ return { bytes: frame, mimeType: "image/jpeg" };
508
541
  const source = boardAssetKeySource(request.key);
509
542
  const item = scene.items.find((entry) => boardAssetKey(entry) === request.key);
510
543
  return source && item ? fetchBoardPicture(client, spaceId, source, item, request.edge) : null;