@neta-art/cohub-cli 10.17.1 → 10.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -0
- package/dist/board-export-resources.d.ts +2 -0
- package/dist/board-export-resources.js +1 -0
- package/dist/board-export.d.ts +11 -2
- package/dist/board-export.js +181 -62
- package/dist/board-gpu/index.d.ts +32 -0
- package/dist/board-gpu/index.js +188 -0
- package/dist/board-gpu/page.js +70384 -0
- package/dist/board-gpu/protocol.d.ts +38 -0
- package/dist/board-gpu/protocol.js +1 -0
- package/dist/board-kernel/{getPo2TextureFromSource.js → FilterPipe.js} +251 -1
- package/dist/board-kernel/browserAll.js +1 -1
- package/dist/board-kernel/init.js +1 -251
- package/dist/board-kernel/webworkerAll.js +1 -1
- package/dist/board-kernel.d.ts +3 -3
- package/dist/board-kernel.js +6501 -414
- package/dist/board-media-export.d.ts +16 -0
- package/dist/board-media-export.js +147 -0
- package/dist/board-media.d.ts +2 -1
- package/dist/board-media.js +23 -0
- package/dist/board-simulate.d.ts +4 -0
- package/dist/board-simulate.js +34 -4
- package/dist/client.d.ts +1 -0
- package/dist/client.js +4 -1
- package/dist/commands/apps.js +25 -3
- package/dist/commands/boards/authoring.js +45 -3
- package/dist/commands/boards/examples.js +21 -5
- package/dist/commands/boards.js +10 -2
- package/dist/commands/space-displays.d.ts +1 -1
- package/dist/commands/space-displays.js +356 -85
- package/dist/commands/spaces.js +11 -2
- package/dist/host/runtime.d.ts +12 -2
- package/dist/host/runtime.js +155 -10
- package/dist/host/serve.js +49 -11
- package/dist/host/workspaces.d.ts +2 -0
- package/dist/host/workspaces.js +18 -0
- package/dist/index.js +1 -0
- package/dist/output.d.ts +1 -0
- package/dist/output.js +3 -0
- package/dist/runtime/cli-path.d.ts +5 -0
- package/dist/runtime/cli-path.js +27 -0
- package/dist/runtime/connection.d.ts +22 -4
- package/dist/runtime/connection.js +13 -3
- package/dist/runtime/harnesses/claude-approval.d.ts +31 -0
- package/dist/runtime/harnesses/claude-approval.js +119 -0
- package/dist/runtime/harnesses/claude.js +42 -3
- package/dist/runtime/harnesses/codex-approval.d.ts +25 -0
- package/dist/runtime/harnesses/codex-approval.js +69 -0
- package/dist/runtime/harnesses/codex.js +36 -5
- package/dist/runtime/harnesses/index.js +12 -3
- package/dist/runtime/harnesses/stream-json.d.ts +16 -2
- package/dist/runtime/harnesses/stream-json.js +26 -6
- package/dist/runtime/harnesses/types.d.ts +3 -1
- package/dist/runtime/instance.d.ts +1 -0
- package/dist/runtime/instance.js +1 -0
- package/dist/runtime/launch.js +2 -4
- package/dist/runtime/local-workspace.d.ts +15 -0
- package/dist/runtime/local-workspace.js +52 -0
- package/dist/runtime/model-catalog.js +7 -4
- package/dist/runtime/native/approvals.d.ts +18 -0
- package/dist/runtime/native/approvals.js +54 -0
- package/dist/runtime/native/codex-link.d.ts +7 -0
- package/dist/runtime/native/codex-link.js +40 -5
- package/dist/runtime/native/daemon.d.ts +5 -1
- package/dist/runtime/native/daemon.js +6 -1
- package/dist/runtime/native/execution.d.ts +2 -0
- package/dist/runtime/preferences.d.ts +7 -0
- package/dist/runtime/preferences.js +22 -0
- package/dist/runtime/presentation.d.ts +3 -0
- package/dist/runtime/presentation.js +2 -1
- package/dist/runtime/supervisor.js +92 -9
- package/dist/safe-remote-image.d.ts +4 -0
- package/dist/safe-remote-image.js +8 -0
- package/dist/space-files/transfer.js +3 -2
- package/dist/space-files/upload.d.ts +5 -2
- package/dist/space-files/upload.js +41 -1
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -73,6 +73,12 @@ cohub boards export lesson.board --animation lecture --at 0:92s:40ms --camera --
|
|
|
73
73
|
NODE_DEBUG=cohub:board-export cohub boards export lesson.board --animation lecture --at 0:92s:40ms -o lecture.mp4
|
|
74
74
|
```
|
|
75
75
|
|
|
76
|
+
Boards whose scripts draw with shaders render in a local Chrome, Chromium or Edge (headless;
|
|
77
|
+
software WebGL without a GPU) so frames match the web app; the rest render in Node.
|
|
78
|
+
`--renderer canvas` keeps Node and warns about the shaders it could not run, `--renderer gpu`
|
|
79
|
+
always uses the browser, and `COHUB_CHROME` picks the browser executable. `boards simulate`
|
|
80
|
+
compiles the same shaders with that browser and reports their errors.
|
|
81
|
+
|
|
76
82
|
Debug output reports render stages, RSS, heap, external buffers, reserved image bytes,
|
|
77
83
|
and available/constrained process memory. Video encoding is reported separately.
|
|
78
84
|
MP4 exports require even output dimensions and fail before rendering if the initial
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import type { BoardNodeRenderer } from "@neta-art/cohub/board/export/node";
|
|
2
2
|
export declare const BOARD_EXPORT_IMAGE_MEMORY_MIB = 256;
|
|
3
|
+
export declare const BOARD_EXPORT_RENDERERS: readonly ["auto", "canvas", "gpu"];
|
|
4
|
+
export type BoardExportRenderer = (typeof BOARD_EXPORT_RENDERERS)[number];
|
|
3
5
|
export declare class BoardExportResourceError extends Error {
|
|
4
6
|
}
|
|
5
7
|
export type BoardExportDecodeImage = (bytes: Uint8Array, mimeType?: string, maxEdge?: number) => ReturnType<BoardNodeRenderer["decodeImage"]>;
|
package/dist/board-export.d.ts
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
|
-
import type { BoardDocument, BoardExportRegion, BoardSceneItem, BoardThemeId } from "@neta-art/cohub/board";
|
|
1
|
+
import type { BoardDocument, BoardExportRegion, BoardMediaState, BoardSceneItem, BoardThemeId } from "@neta-art/cohub/board";
|
|
2
2
|
import type { BoardNodeExportFormat, BoardNodeFont, BoardNodeTexture } from "@neta-art/cohub/board/export/node";
|
|
3
3
|
import type { BoardScriptHost } from "@neta-art/cohub/board/render";
|
|
4
|
+
import { type BoardGpuResources } from "./board-gpu/index.js";
|
|
4
5
|
import { createClient } from "./client.js";
|
|
5
|
-
import { type
|
|
6
|
+
import { type BoardMediaFiles } from "./board-media-export.js";
|
|
7
|
+
import { type BoardExportRenderer, type BoardExportDecodeImage } from "./board-export-resources.js";
|
|
6
8
|
export declare function resolveBundledFonts(): BoardNodeFont[];
|
|
7
9
|
export type BoardExportSource = {
|
|
8
10
|
document: BoardDocument;
|
|
@@ -58,6 +60,7 @@ export type BoardExportRunOptions = {
|
|
|
58
60
|
};
|
|
59
61
|
document?: BoardDocument;
|
|
60
62
|
scripts?: BoardScriptHost;
|
|
63
|
+
renderer?: BoardExportRenderer;
|
|
61
64
|
signal?: AbortSignal;
|
|
62
65
|
imageMemoryMiB?: number;
|
|
63
66
|
onPlan?: (plan: {
|
|
@@ -67,6 +70,7 @@ export type BoardExportRunOptions = {
|
|
|
67
70
|
}) => void;
|
|
68
71
|
onFrame?: (frame: BoardExportFrame, index: number) => Promise<void>;
|
|
69
72
|
onProgress?: (completed: number, total: number) => void;
|
|
73
|
+
mediaFiles?: BoardMediaFiles;
|
|
70
74
|
};
|
|
71
75
|
export type BoardExportFrame = {
|
|
72
76
|
time: number | null;
|
|
@@ -80,6 +84,10 @@ export type BoardExportFrame = {
|
|
|
80
84
|
export type BoardExportRunResult = {
|
|
81
85
|
frames: BoardExportFrame[];
|
|
82
86
|
warnings: string[];
|
|
87
|
+
media: Array<{
|
|
88
|
+
at: number;
|
|
89
|
+
media: BoardMediaState[];
|
|
90
|
+
}>;
|
|
83
91
|
};
|
|
84
92
|
export type BoardVideoExportFormat = "mp4" | "webm";
|
|
85
93
|
export type BoardVideoExportOptions = Omit<BoardExportRunOptions, "format"> & {
|
|
@@ -95,4 +103,5 @@ export declare function runBoardVideoExport(options: BoardVideoExportOptions): P
|
|
|
95
103
|
warnings: string[];
|
|
96
104
|
}>;
|
|
97
105
|
export declare function runBoardExport(options: BoardExportRunOptions): Promise<BoardExportRunResult | null>;
|
|
106
|
+
export declare function gpuResources(spaceId: string, document: BoardDocument, framePicture?: (key: string) => Uint8Array | undefined): BoardGpuResources;
|
|
98
107
|
export {};
|
package/dist/board-export.js
CHANGED
|
@@ -11,11 +11,13 @@ import { tmpdir } from "node:os";
|
|
|
11
11
|
const execFile = promisify(execFileCallback);
|
|
12
12
|
import { boardMediaNaturalSize, parseBoardMediaSource } from "@neta-art/cohub/board";
|
|
13
13
|
import { imageVariantUrl, MEDIA_VARIANT_MAX, mediaVariantSize, videoFrameUrl } from "@neta-art/cohub/media";
|
|
14
|
-
import { hasBoardScripts, boardAssetKey, boardAssetKeySource,
|
|
14
|
+
import { hasBoardScripts, boardAssetKey, boardAssetKeySource, boardMediaRate, boardMediaSpans, boardRenderTheme, boardTimedMedia, boardTextureEdge, boardTexturePixels, createBoardRenderSampler, buildBoardScene, createNodeBoardRenderer, createNodeBoardScriptHost, describeBoardExportWarning, exportBoardImageBytes, parseBoardDocument, planBoardExport, prepareBoardExport, selectBoardExportAssets, } from "./board-kernel.js";
|
|
15
15
|
import { settleBoardDocument } from "./board-media.js";
|
|
16
|
+
import { findBoardGpuBrowser, launchBoardGpuExporter } from "./board-gpu/index.js";
|
|
16
17
|
import { BOARD_EXPORT_FONTS } from "./board-fonts.js";
|
|
17
18
|
import { createClient } from "./client.js";
|
|
18
|
-
import {
|
|
19
|
+
import { downloadPublicImage } from "./safe-remote-image.js";
|
|
20
|
+
import { boardAudioMix, createBoardMediaFiles, createBoardVideoFrames } from "./board-media-export.js";
|
|
19
21
|
import { BOARD_EXPORT_IMAGE_MEMORY_MIB, BoardExportResourceError, createBoardExportImageBudget, } from "./board-export-resources.js";
|
|
20
22
|
const debugExport = debuglog("cohub:board-export");
|
|
21
23
|
export function resolveBundledFonts() {
|
|
@@ -49,7 +51,7 @@ export async function loadBoardDocument(spaceId, target) {
|
|
|
49
51
|
throw new Error("Board changed while reading. Retry the export.");
|
|
50
52
|
Object.assign(items, result.items);
|
|
51
53
|
}
|
|
52
|
-
const parsed = parseBoardDocument({ board: snapshot.board, items, animations: snapshot.animations ?? {} });
|
|
54
|
+
const parsed = parseBoardDocument({ board: snapshot.board, items, animations: snapshot.animations ?? {} }, { keepUnknown: true });
|
|
53
55
|
if (!parsed.ok)
|
|
54
56
|
throw new Error(`Board ${result.id} could not be read: ${parsed.diagnostics[0]?.path}: ${parsed.diagnostics[0]?.message}`);
|
|
55
57
|
return { document: parsed.document, boardId: result.id, title: result.title };
|
|
@@ -176,12 +178,8 @@ export async function runBoardVideoExport(options) {
|
|
|
176
178
|
try {
|
|
177
179
|
let count = 0;
|
|
178
180
|
let dimensions = { width: 0, height: 0 };
|
|
179
|
-
const
|
|
180
|
-
|
|
181
|
-
throw new Error(`MP4 needs even dimensions, but the export is ${plan.width}x${plan.height}. Use --camera --size with even dimensions, or adjust the export region.`);
|
|
182
|
-
}
|
|
183
|
-
options.onPlan?.(plan);
|
|
184
|
-
}, onFrame: async (frame, index) => {
|
|
181
|
+
const mediaFiles = createBoardMediaFiles(options.spaceId, tempDir, options.signal);
|
|
182
|
+
const result = await runBoardExport({ ...options, document, format: "png", mediaFiles, onFrame: async (frame, index) => {
|
|
185
183
|
if (count && (frame.width !== dimensions.width || frame.height !== dimensions.height))
|
|
186
184
|
throw new Error("Video frame dimensions changed. Use a fixed export region or a smaller camera viewport.");
|
|
187
185
|
await writeFile(join(tempDir, `${String(index + 1).padStart(6, "0")}.png`), frame.bytes, { signal: options.signal });
|
|
@@ -190,36 +188,18 @@ export async function runBoardVideoExport(options) {
|
|
|
190
188
|
} });
|
|
191
189
|
if (!result || !count)
|
|
192
190
|
throw new Error("Nothing to export: the region contains no items.");
|
|
193
|
-
const
|
|
194
|
-
const
|
|
195
|
-
const
|
|
196
|
-
|
|
197
|
-
const source = parseBoardMediaSource(item.props.src);
|
|
198
|
-
if (!source)
|
|
199
|
-
throw new Error("Invalid board audio source");
|
|
200
|
-
const { bytes } = source.kind === "url"
|
|
201
|
-
? await downloadPublicAudio(source.value)
|
|
202
|
-
: await readSpaceFileBytes(createClient(), options.spaceId, source.value);
|
|
203
|
-
const path = join(tempDir, `audio-${index}`);
|
|
204
|
-
await writeFile(path, bytes);
|
|
205
|
-
audioInputs.push({ path, offset: Math.max(0, item.props.time ?? 0) / 1000 });
|
|
206
|
-
}
|
|
191
|
+
const start = times[0];
|
|
192
|
+
const sampled = boardMediaSpans(result.media.map((sample) => ({ ...sample, at: sample.at - start })));
|
|
193
|
+
const spans = sampled.map((span) => sampled.some((next) => next.id === span.id && next.start === span.end) ? span : { ...span, end: span.end + step });
|
|
194
|
+
const mix = await boardAudioMix(spans, mediaFiles, 1);
|
|
207
195
|
const duration = Math.max(0.001, count / options.fps);
|
|
208
|
-
const args = ["-y", "-f", "image2", "-framerate", String(options.fps), "-i", join(tempDir, "%06d.png")];
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
args.push("-i", input.path);
|
|
213
|
-
}
|
|
214
|
-
if (audioInputs.length > 1) {
|
|
215
|
-
const inputs = audioInputs.map((_, index) => `[${index + 1}:a]`).join("");
|
|
216
|
-
args.push("-filter_complex", `${inputs}amix=inputs=${audioInputs.length}:duration=longest:dropout_transition=0[aout]`, "-map", "0:v:0", "-map", "[aout]");
|
|
217
|
-
}
|
|
218
|
-
else {
|
|
196
|
+
const args = ["-y", "-f", "image2", "-framerate", String(options.fps), "-i", join(tempDir, "%06d.png"), ...(mix?.inputs ?? [])];
|
|
197
|
+
if (mix)
|
|
198
|
+
args.push("-filter_complex", mix.filter, "-map", "0:v:0", "-map", "[aout]");
|
|
199
|
+
else
|
|
219
200
|
args.push("-map", "0:v:0");
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
}
|
|
201
|
+
if (options.format === "mp4" && (dimensions.width % 2 || dimensions.height % 2))
|
|
202
|
+
args.push("-vf", "pad=ceil(iw/2)*2:ceil(ih/2)*2");
|
|
223
203
|
if (options.format === "mp4")
|
|
224
204
|
args.push("-c:v", "libx264", "-pix_fmt", "yuv420p", "-c:a", "aac", "-movflags", "+faststart", "-f", "mp4");
|
|
225
205
|
else
|
|
@@ -279,7 +259,10 @@ export async function runBoardExport(options) {
|
|
|
279
259
|
});
|
|
280
260
|
};
|
|
281
261
|
let scriptHost = null;
|
|
262
|
+
let gpu = null;
|
|
282
263
|
const textureCache = { textures: new Map(), edges: new Map(), missed: new Map() };
|
|
264
|
+
const frameTextures = new Map();
|
|
265
|
+
let mediaDir = null;
|
|
283
266
|
let backgroundTexture;
|
|
284
267
|
try {
|
|
285
268
|
const theme = boardRenderTheme(options.theme);
|
|
@@ -321,7 +304,7 @@ export async function runBoardExport(options) {
|
|
|
321
304
|
warnings.push("The board background image could not be loaded; the fallback color was exported.");
|
|
322
305
|
}
|
|
323
306
|
}
|
|
324
|
-
if (!options.scripts && hasBoardScripts(document))
|
|
307
|
+
if (!options.scripts && options.renderer !== "gpu" && hasBoardScripts(document))
|
|
325
308
|
scriptHost = createNodeBoardScriptHost(renderer, {
|
|
326
309
|
concurrency: 2,
|
|
327
310
|
readModule: async (src) => {
|
|
@@ -329,16 +312,62 @@ export async function runBoardExport(options) {
|
|
|
329
312
|
return new TextDecoder().decode(bytes);
|
|
330
313
|
},
|
|
331
314
|
});
|
|
332
|
-
const host = scriptHost;
|
|
333
315
|
const assetKey = (item) => {
|
|
334
316
|
const key = boardExportAssetKey(item);
|
|
335
317
|
return key && omittedKeys.has(key) ? null : key;
|
|
336
318
|
};
|
|
337
319
|
const frames = [];
|
|
320
|
+
const media = [];
|
|
321
|
+
const fps = times.length > 1 && times[0] !== null && times[1] !== null ? 1000 / (times[1] - times[0]) : 30;
|
|
322
|
+
const lastMedia = new Map();
|
|
323
|
+
const framePictures = new Map();
|
|
324
|
+
let videoFrames = null;
|
|
325
|
+
const frameOf = async (state, rate, edge) => {
|
|
326
|
+
if (!videoFrames) {
|
|
327
|
+
mediaDir = await mkdtemp(join(tmpdir(), "cohub-board-media-"));
|
|
328
|
+
videoFrames = createBoardVideoFrames(options.mediaFiles ?? createBoardMediaFiles(options.spaceId, mediaDir, options.signal), mediaDir, { fps, signal: options.signal });
|
|
329
|
+
}
|
|
330
|
+
return videoFrames.frame(state, rate, edge);
|
|
331
|
+
};
|
|
338
332
|
const reported = new Set();
|
|
333
|
+
const report = (text) => {
|
|
334
|
+
if (reported.has(text))
|
|
335
|
+
return;
|
|
336
|
+
reported.add(text);
|
|
337
|
+
warnings.push(text);
|
|
338
|
+
};
|
|
339
339
|
let completed = 0;
|
|
340
|
+
let decided = options.renderer === "canvas";
|
|
341
|
+
const emit = async (frame) => {
|
|
342
|
+
if (options.onFrame)
|
|
343
|
+
await options.onFrame(frame, completed);
|
|
344
|
+
else
|
|
345
|
+
frames.push(frame);
|
|
346
|
+
completed++;
|
|
347
|
+
options.onProgress?.(completed, times.length);
|
|
348
|
+
};
|
|
349
|
+
const launchGpu = async (browser) => {
|
|
350
|
+
const exporter = await launchBoardGpuExporter(gpuResources(options.spaceId, document, (key) => framePictures.get(key)), { browser, signal: options.signal });
|
|
351
|
+
try {
|
|
352
|
+
await exporter.init({ document, animation: options.animation ?? null, theme: options.theme });
|
|
353
|
+
}
|
|
354
|
+
catch (error) {
|
|
355
|
+
await exporter.close();
|
|
356
|
+
throw error;
|
|
357
|
+
}
|
|
358
|
+
scriptHost?.destroy();
|
|
359
|
+
scriptHost = null;
|
|
360
|
+
return exporter;
|
|
361
|
+
};
|
|
362
|
+
if (options.renderer === "gpu") {
|
|
363
|
+
const browser = findBoardGpuBrowser();
|
|
364
|
+
if (!browser)
|
|
365
|
+
throw new Error("GPU export needs Chrome, Chromium or Edge. Install one, or set COHUB_CHROME to its path.");
|
|
366
|
+
gpu = await launchGpu(browser);
|
|
367
|
+
}
|
|
340
368
|
for (const time of times) {
|
|
341
369
|
options.signal?.throwIfAborted();
|
|
370
|
+
const host = scriptHost;
|
|
342
371
|
host?.clearFrames();
|
|
343
372
|
const frameOptions = { sampler, camera: options.camera, ...(time === null ? {} : { at: { animation: options.animation, time } }), region, scale: options.scale, padding: times.length > 1 ? 0 : options.padding };
|
|
344
373
|
const initial = prepareBoardExport(document, frameOptions);
|
|
@@ -346,21 +375,91 @@ export async function runBoardExport(options) {
|
|
|
346
375
|
continue;
|
|
347
376
|
let prepared = initial;
|
|
348
377
|
trace("prepare", completed, initial.plan);
|
|
349
|
-
|
|
350
|
-
await
|
|
351
|
-
for (const
|
|
352
|
-
const error =
|
|
378
|
+
const prepareScripts = async (scripts, shaders) => {
|
|
379
|
+
await scripts.prepare(initial.scene.items, [time ?? 0], initial.plan.scale, initial.document.board, { shaders });
|
|
380
|
+
for (const id of [...Object.keys(initial.document.board.scripts ?? {}), ...initial.scene.items.map((item) => item.id)]) {
|
|
381
|
+
const error = scripts.error(id);
|
|
353
382
|
if (error)
|
|
354
|
-
throw new Error(`${
|
|
383
|
+
throw new Error(`${initial.document.board.scripts?.[id]?.src ?? initial.scene.get(id)?.script?.src ?? id}: ${error}`);
|
|
355
384
|
}
|
|
356
|
-
|
|
385
|
+
return prepareBoardExport(document, { ...frameOptions, scripts });
|
|
386
|
+
};
|
|
387
|
+
if (host && !gpu) {
|
|
388
|
+
const scripted = await prepareScripts(host, !decided);
|
|
357
389
|
if (!scripted)
|
|
358
390
|
continue;
|
|
359
391
|
prepared = scripted;
|
|
360
392
|
}
|
|
393
|
+
if (!decided) {
|
|
394
|
+
decided = true;
|
|
395
|
+
const shaded = (host?.shaders?.(prepared.document, time ?? 0) ?? []).map((owner) => owner.id);
|
|
396
|
+
const browser = shaded.length ? findBoardGpuBrowser() : null;
|
|
397
|
+
if (browser)
|
|
398
|
+
gpu = await launchGpu(browser);
|
|
399
|
+
else if (host && shaded.length) {
|
|
400
|
+
report(`Shaders in ${shaded.join(", ")} need Chrome, Chromium or Edge; install one or set COHUB_CHROME to draw them.`);
|
|
401
|
+
host.clearFrames();
|
|
402
|
+
const scripted = await prepareScripts(host, false);
|
|
403
|
+
if (!scripted)
|
|
404
|
+
continue;
|
|
405
|
+
prepared = scripted;
|
|
406
|
+
}
|
|
407
|
+
}
|
|
408
|
+
const timed = boardTimedMedia(prepared.scene);
|
|
409
|
+
media.push({ at: time ?? 0, media: timed });
|
|
410
|
+
const frameKeys = {};
|
|
411
|
+
for (const texture of frameTextures.values())
|
|
412
|
+
texture.destroy(true);
|
|
413
|
+
frameTextures.clear();
|
|
414
|
+
framePictures.clear();
|
|
415
|
+
if (options.withImages) {
|
|
416
|
+
const planned = new Set(prepared.plan.items.map((item) => item.id));
|
|
417
|
+
for (const state of timed) {
|
|
418
|
+
const item = prepared.scene.get(state.id);
|
|
419
|
+
const base = item && state.kind === "video" && state.shown && planned.has(state.id) ? boardExportAssetKey(item) : null;
|
|
420
|
+
if (!item || !base)
|
|
421
|
+
continue;
|
|
422
|
+
const before = lastMedia.get(state.id);
|
|
423
|
+
if (time !== null)
|
|
424
|
+
lastMedia.set(state.id, { at: time, time: state.time });
|
|
425
|
+
const rate = times.length === 1 ? 0 : before && time !== null ? boardMediaRate(before, { at: time, time: state.time }) : 1;
|
|
426
|
+
const edge = Math.min(4096, textureEdge(item, prepared.plan.scale));
|
|
427
|
+
const bytes = await frameOf(state, rate, edge).catch((error) => {
|
|
428
|
+
report(`Frames of ${state.src} could not be decoded; its cover was drawn instead. ${error instanceof Error ? error.message : ""}`.trim());
|
|
429
|
+
return null;
|
|
430
|
+
});
|
|
431
|
+
if (!bytes?.length)
|
|
432
|
+
continue;
|
|
433
|
+
const key = `${base}@${Math.round(state.time)}`;
|
|
434
|
+
frameKeys[state.id] = key;
|
|
435
|
+
framePictures.set(key, bytes);
|
|
436
|
+
if (!gpu)
|
|
437
|
+
frameTextures.set(key, await nodeRenderer.decodeImage(bytes, "image/jpeg"));
|
|
438
|
+
}
|
|
439
|
+
}
|
|
440
|
+
if (gpu) {
|
|
441
|
+
trace("gpu", completed, prepared.plan);
|
|
442
|
+
const result = await gpu.render({
|
|
443
|
+
time,
|
|
444
|
+
region,
|
|
445
|
+
...(options.camera ? { camera: options.camera } : {}),
|
|
446
|
+
scale: options.scale,
|
|
447
|
+
...(times.length > 1 || options.padding === undefined ? {} : { padding: options.padding }),
|
|
448
|
+
background: options.background,
|
|
449
|
+
format: options.format,
|
|
450
|
+
...(options.quality === undefined ? {} : { quality: options.quality }),
|
|
451
|
+
withImages: options.withImages,
|
|
452
|
+
imageMemory: imageMemoryBytes,
|
|
453
|
+
assetKeys: frameKeys,
|
|
454
|
+
});
|
|
455
|
+
for (const warning of result.warnings)
|
|
456
|
+
report(options.withImages && warning.kind === "images-missing" ? describeBoardExportWarning(warning) : describeWarning(warning));
|
|
457
|
+
await emit({ time, bytes: result.bytes, width: result.width, height: result.height, scale: result.scale, itemCount: result.itemCount, format: options.format });
|
|
458
|
+
continue;
|
|
459
|
+
}
|
|
361
460
|
trace("images", completed, prepared.plan);
|
|
362
461
|
if (options.withImages) {
|
|
363
|
-
const visible = prepared.plan.items.filter((item) => prepared.scene.opacity(item.id) > 0);
|
|
462
|
+
const visible = prepared.plan.items.filter((item) => prepared.scene.opacity(item.id) > 0 && !frameKeys[item.id]);
|
|
364
463
|
const loaded = await loadBoardTextures(images, options.spaceId, visible, { scale: prepared.plan.scale, maxBytes: imageMemoryBytes, cache: textureCache });
|
|
365
464
|
omittedKeys = new Set(loaded.omitted);
|
|
366
465
|
const sources = (keys) => keys.slice(0, 3).map((key) => boardAssetKeySource(key)?.value ?? key).join(", ");
|
|
@@ -381,10 +480,10 @@ export async function runBoardExport(options) {
|
|
|
381
480
|
...frameOptions,
|
|
382
481
|
...theme,
|
|
383
482
|
background: options.background,
|
|
384
|
-
textures,
|
|
483
|
+
textures: frameTextures.size ? new Map([...textures, ...frameTextures]) : textures,
|
|
385
484
|
scripts: options.scripts ?? scriptHost ?? undefined,
|
|
386
485
|
backgroundTexture,
|
|
387
|
-
assetKey,
|
|
486
|
+
assetKey: (item) => frameKeys[item.id] ?? assetKey(item),
|
|
388
487
|
format: options.format,
|
|
389
488
|
quality: options.quality,
|
|
390
489
|
});
|
|
@@ -393,12 +492,11 @@ export async function runBoardExport(options) {
|
|
|
393
492
|
for (const warning of result.warnings) {
|
|
394
493
|
if (warning.kind === "images-missing" && options.withImages)
|
|
395
494
|
continue;
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
reported.add(text);
|
|
495
|
+
if (warning.kind === "shaders-unsupported" && options.renderer !== "canvas")
|
|
496
|
+
continue;
|
|
497
|
+
report(describeWarning(warning));
|
|
400
498
|
}
|
|
401
|
-
|
|
499
|
+
await emit({
|
|
402
500
|
time,
|
|
403
501
|
bytes: result.bytes,
|
|
404
502
|
width: result.plan.width,
|
|
@@ -406,25 +504,46 @@ export async function runBoardExport(options) {
|
|
|
406
504
|
scale: result.plan.scale,
|
|
407
505
|
itemCount: result.plan.items.length,
|
|
408
506
|
format: result.format,
|
|
409
|
-
};
|
|
410
|
-
if (options.onFrame)
|
|
411
|
-
await options.onFrame(frame, completed);
|
|
412
|
-
else
|
|
413
|
-
frames.push(frame);
|
|
414
|
-
completed++;
|
|
415
|
-
options.onProgress?.(completed, times.length);
|
|
507
|
+
});
|
|
416
508
|
trace("rendered", completed, result.plan);
|
|
417
509
|
}
|
|
418
|
-
return completed ? { frames, warnings } : null;
|
|
510
|
+
return completed ? { frames, warnings, media } : null;
|
|
419
511
|
}
|
|
420
512
|
finally {
|
|
513
|
+
await gpu?.close();
|
|
421
514
|
scriptHost?.destroy();
|
|
422
515
|
for (const texture of textureCache.textures.values())
|
|
423
516
|
texture.destroy(true);
|
|
424
517
|
backgroundTexture?.destroy(true);
|
|
518
|
+
for (const texture of frameTextures.values())
|
|
519
|
+
texture.destroy(true);
|
|
520
|
+
if (mediaDir)
|
|
521
|
+
await rm(mediaDir, { recursive: true, force: true });
|
|
425
522
|
renderer.destroy();
|
|
426
523
|
}
|
|
427
524
|
}
|
|
525
|
+
export function gpuResources(spaceId, document, framePicture = () => undefined) {
|
|
526
|
+
const client = createClient();
|
|
527
|
+
const scene = buildBoardScene(document);
|
|
528
|
+
return {
|
|
529
|
+
fonts: resolveBundledFonts().flatMap(({ path, family }) => (family ? [{ path, family }] : [])),
|
|
530
|
+
readModule: async (path) => new TextDecoder().decode((await readSpaceFileBytes(client, spaceId, path)).bytes),
|
|
531
|
+
async readPicture(request) {
|
|
532
|
+
if ("background" in request) {
|
|
533
|
+
const source = document.board.background.image && parseBoardMediaSource(document.board.background.image.src);
|
|
534
|
+
if (!source)
|
|
535
|
+
return null;
|
|
536
|
+
return source.kind === "url" ? downloadPublicImage(source.value) : readSpaceFileBytes(client, spaceId, source.value);
|
|
537
|
+
}
|
|
538
|
+
const frame = framePicture(request.key);
|
|
539
|
+
if (frame)
|
|
540
|
+
return { bytes: frame, mimeType: "image/jpeg" };
|
|
541
|
+
const source = boardAssetKeySource(request.key);
|
|
542
|
+
const item = scene.items.find((entry) => boardAssetKey(entry) === request.key);
|
|
543
|
+
return source && item ? fetchBoardPicture(client, spaceId, source, item, request.edge) : null;
|
|
544
|
+
},
|
|
545
|
+
};
|
|
546
|
+
}
|
|
428
547
|
function describeWarning(warning) {
|
|
429
548
|
if (warning.kind === "scale-clamped") {
|
|
430
549
|
return `Scale reduced from ${warning.requested}x to ${Number(warning.applied).toFixed(2)}x to stay within the size limit.`;
|
|
@@ -436,5 +555,5 @@ function describeWarning(warning) {
|
|
|
436
555
|
if (warning.kind === "many-items") {
|
|
437
556
|
return `${warning.count} items exported.`;
|
|
438
557
|
}
|
|
439
|
-
return warning
|
|
558
|
+
return describeBoardExportWarning(warning);
|
|
440
559
|
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import type { BoardGpuFrame, BoardGpuInit, BoardGpuResult, BoardGpuShader } from "./protocol.js";
|
|
2
|
+
export type { BoardGpuFrame, BoardGpuResult, BoardGpuShader } from "./protocol.js";
|
|
3
|
+
export declare function findBoardGpuBrowser(env?: NodeJS.ProcessEnv): string | null;
|
|
4
|
+
export type BoardGpuResources = {
|
|
5
|
+
readModule(path: string): Promise<string>;
|
|
6
|
+
readPicture(request: {
|
|
7
|
+
key: string;
|
|
8
|
+
edge: number;
|
|
9
|
+
} | {
|
|
10
|
+
background: true;
|
|
11
|
+
}): Promise<{
|
|
12
|
+
bytes: Uint8Array;
|
|
13
|
+
mimeType: string;
|
|
14
|
+
} | null>;
|
|
15
|
+
fonts: readonly {
|
|
16
|
+
family: string;
|
|
17
|
+
path: string;
|
|
18
|
+
}[];
|
|
19
|
+
};
|
|
20
|
+
export type BoardGpuExporter = {
|
|
21
|
+
readonly browser: string;
|
|
22
|
+
init(init: BoardGpuInit): Promise<void>;
|
|
23
|
+
render(frame: BoardGpuFrame): Promise<Omit<BoardGpuResult, "data"> & {
|
|
24
|
+
bytes: Uint8Array;
|
|
25
|
+
}>;
|
|
26
|
+
compile(shaders: BoardGpuShader[]): Promise<Record<string, string>>;
|
|
27
|
+
close(): Promise<void>;
|
|
28
|
+
};
|
|
29
|
+
export declare function launchBoardGpuExporter(resources: BoardGpuResources, options: {
|
|
30
|
+
browser: string;
|
|
31
|
+
signal?: AbortSignal;
|
|
32
|
+
}): Promise<BoardGpuExporter>;
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
import { spawn } from "node:child_process";
|
|
2
|
+
import { randomBytes } from "node:crypto";
|
|
3
|
+
import { existsSync } from "node:fs";
|
|
4
|
+
import { mkdtemp, readFile, rm } from "node:fs/promises";
|
|
5
|
+
import { createServer } from "node:http";
|
|
6
|
+
import { tmpdir } from "node:os";
|
|
7
|
+
import { basename, join } from "node:path";
|
|
8
|
+
import { fileURLToPath } from "node:url";
|
|
9
|
+
export function findBoardGpuBrowser(env = process.env) {
|
|
10
|
+
const named = env.COHUB_CHROME?.trim();
|
|
11
|
+
if (named)
|
|
12
|
+
return existsSync(named) ? named : null;
|
|
13
|
+
const candidates = process.platform === "darwin"
|
|
14
|
+
? ["/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", "/Applications/Chromium.app/Contents/MacOS/Chromium", "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge"]
|
|
15
|
+
: process.platform === "win32"
|
|
16
|
+
? [env.PROGRAMFILES, env["PROGRAMFILES(X86)"], env.LOCALAPPDATA].flatMap((root) => root ? [join(root, "Google/Chrome/Application/chrome.exe"), join(root, "Microsoft/Edge/Application/msedge.exe")] : [])
|
|
17
|
+
: ["google-chrome", "google-chrome-stable", "chromium", "chromium-browser", "microsoft-edge"].flatMap((name) => (env.PATH ?? "").split(":").filter(Boolean).map((dir) => join(dir, name)));
|
|
18
|
+
return candidates.find((path) => existsSync(path)) ?? null;
|
|
19
|
+
}
|
|
20
|
+
const CALL_TIMEOUT_MS = 120_000;
|
|
21
|
+
function devtools(input, output) {
|
|
22
|
+
let next = 0;
|
|
23
|
+
let buffer = "";
|
|
24
|
+
const pending = new Map();
|
|
25
|
+
const listeners = new Set();
|
|
26
|
+
output.setEncoding("utf8");
|
|
27
|
+
output.on("data", (chunk) => {
|
|
28
|
+
buffer += chunk;
|
|
29
|
+
for (let end = buffer.indexOf("\0"); end >= 0; end = buffer.indexOf("\0")) {
|
|
30
|
+
const raw = buffer.slice(0, end);
|
|
31
|
+
buffer = buffer.slice(end + 1);
|
|
32
|
+
let message;
|
|
33
|
+
try {
|
|
34
|
+
message = JSON.parse(raw);
|
|
35
|
+
}
|
|
36
|
+
catch {
|
|
37
|
+
continue;
|
|
38
|
+
}
|
|
39
|
+
const waiting = message.id === undefined ? undefined : pending.get(message.id);
|
|
40
|
+
if (waiting && message.id !== undefined) {
|
|
41
|
+
pending.delete(message.id);
|
|
42
|
+
if (message.error)
|
|
43
|
+
waiting.reject(new Error(message.error.message));
|
|
44
|
+
else
|
|
45
|
+
waiting.resolve(message.result ?? {});
|
|
46
|
+
}
|
|
47
|
+
for (const listener of listeners)
|
|
48
|
+
listener(message);
|
|
49
|
+
}
|
|
50
|
+
});
|
|
51
|
+
const fail = (error) => {
|
|
52
|
+
for (const waiting of pending.values())
|
|
53
|
+
waiting.reject(error);
|
|
54
|
+
pending.clear();
|
|
55
|
+
};
|
|
56
|
+
output.on("close", () => fail(new Error("The browser closed.")));
|
|
57
|
+
output.on("error", (error) => fail(error));
|
|
58
|
+
input.on("error", (error) => fail(error));
|
|
59
|
+
return {
|
|
60
|
+
send(method, params = {}, sessionId) {
|
|
61
|
+
const id = ++next;
|
|
62
|
+
return new Promise((resolve, reject) => {
|
|
63
|
+
pending.set(id, { resolve, reject });
|
|
64
|
+
input.write(`${JSON.stringify({ id, method, params, ...(sessionId ? { sessionId } : {}) })}\0`);
|
|
65
|
+
});
|
|
66
|
+
},
|
|
67
|
+
listen(listener) {
|
|
68
|
+
listeners.add(listener);
|
|
69
|
+
},
|
|
70
|
+
fail,
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
const PAGE = (fonts) => `<!doctype html><meta charset="utf-8"><style>${fonts
|
|
74
|
+
.map(({ family, file }) => `@font-face{font-family:"${family}";src:url("fonts/${file}") format("woff2");font-weight:100 900}`)
|
|
75
|
+
.join("")}</style><body>${fonts.map(({ family }) => `<span style="font-family:'${family}'">.</span>`).join("")}<script type="module" src="page.js"></script>`;
|
|
76
|
+
export async function launchBoardGpuExporter(resources, options) {
|
|
77
|
+
const bundle = fileURLToPath(new URL("./page.js", import.meta.url));
|
|
78
|
+
if (!existsSync(bundle))
|
|
79
|
+
throw new Error("This CLI build has no GPU exporter; reinstall @neta-art/cohub-cli.");
|
|
80
|
+
const token = randomBytes(16).toString("hex");
|
|
81
|
+
const fonts = resources.fonts.map(({ family, path }) => ({ family, path, file: basename(path) }));
|
|
82
|
+
const server = createServer(async (request, response) => {
|
|
83
|
+
const url = new URL(request.url ?? "/", "http://127.0.0.1");
|
|
84
|
+
const [, prefix, ...rest] = url.pathname.split("/");
|
|
85
|
+
const route = rest.join("/");
|
|
86
|
+
try {
|
|
87
|
+
if (prefix !== token)
|
|
88
|
+
throw Object.assign(new Error("Not found"), { status: 404 });
|
|
89
|
+
if (route === "")
|
|
90
|
+
return void response.writeHead(200, { "content-type": "text/html" }).end(PAGE(fonts));
|
|
91
|
+
if (route === "page.js")
|
|
92
|
+
return void response.writeHead(200, { "content-type": "text/javascript" }).end(await readFile(bundle));
|
|
93
|
+
const font = route.startsWith("fonts/") && fonts.find(({ file }) => file === route.slice(6));
|
|
94
|
+
if (font)
|
|
95
|
+
return void response.writeHead(200, { "content-type": "font/woff2" }).end(await readFile(font.path));
|
|
96
|
+
if (route === "module") {
|
|
97
|
+
const code = await resources.readModule(url.searchParams.get("path") ?? "");
|
|
98
|
+
return void response.writeHead(200, { "content-type": "text/javascript" }).end(code);
|
|
99
|
+
}
|
|
100
|
+
if (route === "asset") {
|
|
101
|
+
const picture = await resources.readPicture(url.searchParams.has("background")
|
|
102
|
+
? { background: true }
|
|
103
|
+
: { key: url.searchParams.get("key") ?? "", edge: Number(url.searchParams.get("edge")) || 4096 });
|
|
104
|
+
if (!picture)
|
|
105
|
+
throw Object.assign(new Error("Not found"), { status: 404 });
|
|
106
|
+
return void response.writeHead(200, { "content-type": picture.mimeType }).end(picture.bytes);
|
|
107
|
+
}
|
|
108
|
+
throw Object.assign(new Error("Not found"), { status: 404 });
|
|
109
|
+
}
|
|
110
|
+
catch (error) {
|
|
111
|
+
response.writeHead(error.status ?? 500).end(error instanceof Error ? error.message : String(error));
|
|
112
|
+
}
|
|
113
|
+
});
|
|
114
|
+
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
|
115
|
+
const port = server.address().port;
|
|
116
|
+
const profile = await mkdtemp(join(tmpdir(), "cohub-board-gpu-"));
|
|
117
|
+
let chrome = null;
|
|
118
|
+
const close = async () => {
|
|
119
|
+
chrome?.kill();
|
|
120
|
+
chrome = null;
|
|
121
|
+
await new Promise((resolve) => server.close(() => resolve()));
|
|
122
|
+
await rm(profile, { recursive: true, force: true }).catch(() => { });
|
|
123
|
+
};
|
|
124
|
+
const start = async (sandbox) => {
|
|
125
|
+
const child = spawn(options.browser, [
|
|
126
|
+
"--headless=new", "--remote-debugging-pipe", `--user-data-dir=${profile}`, "--no-first-run", "--no-default-browser-check",
|
|
127
|
+
"--disable-extensions", "--disable-background-networking", "--mute-audio", "--hide-scrollbars", "--enable-unsafe-swiftshader",
|
|
128
|
+
...(sandbox ? [] : ["--no-sandbox"]), "about:blank",
|
|
129
|
+
], { stdio: ["ignore", "ignore", "ignore", "pipe", "pipe"] });
|
|
130
|
+
const [, , , input, output] = child.stdio;
|
|
131
|
+
const protocol = devtools(input, output);
|
|
132
|
+
child.once("exit", () => protocol.fail(new Error("The browser stopped.")));
|
|
133
|
+
child.once("error", (error) => protocol.fail(error));
|
|
134
|
+
chrome = child;
|
|
135
|
+
return { protocol, created: await protocol.send("Target.createTarget", { url: "about:blank" }) };
|
|
136
|
+
};
|
|
137
|
+
try {
|
|
138
|
+
const { protocol, created } = await start(true).catch(() => {
|
|
139
|
+
chrome?.kill();
|
|
140
|
+
return start(false);
|
|
141
|
+
});
|
|
142
|
+
options.signal?.addEventListener("abort", () => void close(), { once: true });
|
|
143
|
+
const { targetId } = created;
|
|
144
|
+
const { sessionId } = await protocol.send("Target.attachToTarget", { targetId, flatten: true });
|
|
145
|
+
const page = (method, params) => protocol.send(method, params, sessionId);
|
|
146
|
+
let crashed = null;
|
|
147
|
+
protocol.listen((message) => {
|
|
148
|
+
if (message.sessionId === sessionId && message.method === "Runtime.exceptionThrown")
|
|
149
|
+
crashed ??= String(message.params?.exceptionDetails?.exception?.description ?? "The export page failed.");
|
|
150
|
+
});
|
|
151
|
+
await page("Runtime.enable");
|
|
152
|
+
await page("Page.enable");
|
|
153
|
+
await page("Page.navigate", { url: `http://127.0.0.1:${port}/${token}/` });
|
|
154
|
+
const call = async (expression, timeoutMs = CALL_TIMEOUT_MS) => {
|
|
155
|
+
let timer;
|
|
156
|
+
const expired = new Promise((_resolve, reject) => {
|
|
157
|
+
timer = setTimeout(() => reject(new Error("The GPU exporter stopped responding; a shader may be too heavy to render.")), timeoutMs);
|
|
158
|
+
});
|
|
159
|
+
const reply = await Promise.race([page("Runtime.evaluate", { expression, awaitPromise: true, returnByValue: true }), expired])
|
|
160
|
+
.finally(() => clearTimeout(timer));
|
|
161
|
+
if (reply.exceptionDetails)
|
|
162
|
+
throw new Error(reply.exceptionDetails.exception?.description?.split("\n")[0] ?? reply.exceptionDetails.text ?? "The export page failed.");
|
|
163
|
+
return reply.result?.value;
|
|
164
|
+
};
|
|
165
|
+
const loaded = () => call("Boolean(globalThis.cohubBoardGpu)", 5_000).catch(() => false);
|
|
166
|
+
for (const started = Date.now(); !(await loaded());) {
|
|
167
|
+
if (crashed || Date.now() - started > 20_000)
|
|
168
|
+
throw new Error(crashed ?? "The GPU exporter did not start.");
|
|
169
|
+
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
170
|
+
}
|
|
171
|
+
return {
|
|
172
|
+
browser: options.browser,
|
|
173
|
+
async init(init) {
|
|
174
|
+
await call(`cohubBoardGpu.init(${JSON.stringify(init)})`);
|
|
175
|
+
},
|
|
176
|
+
compile: (shaders) => call(`cohubBoardGpu.compile(${JSON.stringify(shaders)})`),
|
|
177
|
+
async render(frame) {
|
|
178
|
+
const { data, ...rest } = await call(`cohubBoardGpu.render(${JSON.stringify(frame)})`);
|
|
179
|
+
return { ...rest, bytes: new Uint8Array(Buffer.from(data, "base64")) };
|
|
180
|
+
},
|
|
181
|
+
close,
|
|
182
|
+
};
|
|
183
|
+
}
|
|
184
|
+
catch (error) {
|
|
185
|
+
await close();
|
|
186
|
+
throw error;
|
|
187
|
+
}
|
|
188
|
+
}
|