@needle-tools/gltf-build-pipeline 2.16.0-next.2b9edfd → 2.16.0-next.8ceb45e
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/cli/index.js +108 -2
- package/dist/transforms/needle_audio_registry.d.ts +4 -2
- package/dist/transforms/needle_audio_registry.js +4 -2
- package/dist/transforms/needle_pmrem.d.ts +27 -7
- package/dist/transforms/needle_pmrem.js +104 -95
- package/dist/utils/version.gen.d.ts +1 -1
- package/dist/utils/version.gen.js +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -49,7 +49,7 @@ The `gltf` value may also be a string shorthand for `usecase` — `"gltf": "prod
|
|
|
49
49
|
| `usecase` | `"default" \| "product" \| "world"` | `"default"` | High-level optimisation profile. |
|
|
50
50
|
| `textures.lods` | `boolean` | `true` | Generate progressive texture LODs. |
|
|
51
51
|
| `meshes.lods` | `boolean` | `true` | Generate progressive mesh LODs. |
|
|
52
|
-
| `audio.enabled` | `boolean` | `true` | Compress externally referenced
|
|
52
|
+
| `audio.enabled` | `boolean` | `true` | Compress externally referenced `.wav` audio files to Opus in an `.ogg` container, rewriting the URIs in the glTF. Lossy formats (`.mp3`, `.ogg`, `.m4a`, `.aac`) pass through unchanged to avoid compounding compression artifacts. Requires `ffmpeg` with `libopus` on `PATH`; when missing the build logs a warning and leaves the audio files untouched. |
|
|
53
53
|
| `audio.bitrate` | `number` (kbps) | `128` | Opus encoding bitrate. 96 is near-transparent for most music; 128 matches YouTube's music tier and is the safe "won't sound bad" default. Drop to 48–64 for known mono/UI-only assets. |
|
|
54
54
|
| `exr.enabled` | `boolean` | `true` | Convert EXR textures to KTX2 HDR via PMREM. When `false`, EXR textures (bundled and externally referenced) are left as-is so a downstream step can handle them. |
|
|
55
55
|
|
package/dist/cli/index.js
CHANGED
|
@@ -7,9 +7,10 @@ import { UsecaseOptions, createConfig, getConfig } from '../config/index.js';
|
|
|
7
7
|
import { isLOD, make_progressive } from '../transforms/needle_progressive.js';
|
|
8
8
|
import { cacheSizeLimit, clearCache, ensureHashReady, limitCacheSize } from '../cache/cache.js';
|
|
9
9
|
import { computeJobKey, saveJobResult, tryRestoreJobResult } from '../cache/job-cache.js';
|
|
10
|
-
import {
|
|
10
|
+
import { convertExrBytesToPmremKtx2, createPmremRuntime, disposePmremRuntime } from '../transforms/needle_pmrem.js';
|
|
11
|
+
import { existsSync, readFileSync, readdirSync, statSync, writeFileSync } from 'fs';
|
|
11
12
|
import { ensureIsDirectory, foreachGLTF, isDirectory } from '../utils/fileutils.js';
|
|
12
|
-
import path, { dirname, resolve } from 'path';
|
|
13
|
+
import path, { basename, dirname, resolve } from 'path';
|
|
13
14
|
import { ERROR_CODES } from '../constants.js';
|
|
14
15
|
import { trackPipelineStart, trackPipelineEnd, trackError } from '../utils/analytics.js';
|
|
15
16
|
// For testing / dev you can run `npm link` in the package directory
|
|
@@ -268,7 +269,112 @@ Each version will be compressed and written to the output directory.
|
|
|
268
269
|
});
|
|
269
270
|
if (useCache)
|
|
270
271
|
limitCacheSize();
|
|
272
|
+
})
|
|
273
|
+
.command("pmrem", "Convert EXR files to PMREM-processed KTX2 HDR (Needle PMREM)")
|
|
274
|
+
.help(`Runs the PMREM WASM + basisu encoder on .exr files, producing <name>.pmrem.ktx2.
|
|
275
|
+
This is the same processing the Needle build pipeline applies to EXR textures inside
|
|
276
|
+
GLB/GLTF files, exposed as a standalone command for raw .exr inputs.
|
|
277
|
+
`)
|
|
278
|
+
.argument('<input>', "Path to an .exr file OR a directory containing .exr files (recursive)", { validator: program.STRING })
|
|
279
|
+
.argument('<output>', "Output file (when input is a single .exr) or output directory. Defaults to writing next to the input file.", { default: null, validator: program.STRING })
|
|
280
|
+
.option("--cache", "Use \"--cache False\" to disable caching.", { validator: program.BOOLEAN, default: true })
|
|
281
|
+
.option("--debug", "Enable verbose output with \"--debug True\"", { validator: program.BOOLEAN, default: false })
|
|
282
|
+
.option("--verbose", "Enable verbose output with \"--verbose True\"", { validator: program.BOOLEAN, default: false })
|
|
283
|
+
.hide()
|
|
284
|
+
.action(async ({ logger, args, options }) => {
|
|
285
|
+
logger.level = (options.debug || options.verbose) ? "debug" : "info";
|
|
286
|
+
await ensureHashReady();
|
|
287
|
+
const input = args.input.toString();
|
|
288
|
+
if (!existsSync(input)) {
|
|
289
|
+
logger.error(`Input does not exist: ${input}`);
|
|
290
|
+
process.exit(ERROR_CODES.INVALID_ARGS);
|
|
291
|
+
}
|
|
292
|
+
const exrFiles = [];
|
|
293
|
+
const inputIsDirectory = isDirectory(input);
|
|
294
|
+
if (inputIsDirectory) {
|
|
295
|
+
collectExrFiles(input, exrFiles);
|
|
296
|
+
}
|
|
297
|
+
else if (input.toLowerCase().endsWith('.exr')) {
|
|
298
|
+
exrFiles.push(input);
|
|
299
|
+
}
|
|
300
|
+
else {
|
|
301
|
+
logger.error(`Input is not an .exr file or directory: ${input}`);
|
|
302
|
+
process.exit(ERROR_CODES.INVALID_ARGS);
|
|
303
|
+
}
|
|
304
|
+
if (exrFiles.length === 0) {
|
|
305
|
+
logger.warn(`No .exr files found in ${input}`);
|
|
306
|
+
return;
|
|
307
|
+
}
|
|
308
|
+
// caporal's program.cast(false) forces String(null) → "null" for the
|
|
309
|
+
// missing positional default, so treat that literal as "no output given".
|
|
310
|
+
const rawOutput = args.output?.toString() || null;
|
|
311
|
+
let outputArg = (rawOutput && rawOutput !== 'null') ? rawOutput : null;
|
|
312
|
+
if (outputArg && !path.isAbsolute(outputArg)) {
|
|
313
|
+
const base = inputIsDirectory ? input : dirname(input);
|
|
314
|
+
outputArg = resolve(base, outputArg);
|
|
315
|
+
}
|
|
316
|
+
const useCache = options.cache !== false;
|
|
317
|
+
logger.info(`[Needle Build Pipeline] v${getVersion()} — pmrem: ${exrFiles.length} file(s)`);
|
|
318
|
+
const runtime = await createPmremRuntime(logger);
|
|
319
|
+
if (!runtime) {
|
|
320
|
+
logger.error('pmrem runtime is not available (WASM or basisu binary missing). Run `npm run build:pmrem` in the package directory.');
|
|
321
|
+
process.exit(ERROR_CODES.PACKING_FAILED);
|
|
322
|
+
}
|
|
323
|
+
try {
|
|
324
|
+
for (const exrPath of exrFiles) {
|
|
325
|
+
const outPath = resolvePmremOutputPath(exrPath, input, inputIsDirectory, outputArg);
|
|
326
|
+
logger.info(`→ pmrem ${exrPath} → ${outPath}`);
|
|
327
|
+
const exrBytes = readFileSync(exrPath);
|
|
328
|
+
const ktx2Bytes = await convertExrBytesToPmremKtx2(runtime, new Uint8Array(exrBytes.buffer, exrBytes.byteOffset, exrBytes.byteLength), { useCache, logLabel: `pmrem ${basename(exrPath)}` });
|
|
329
|
+
writeFileSync(outPath, ktx2Bytes);
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
finally {
|
|
333
|
+
await disposePmremRuntime(runtime);
|
|
334
|
+
}
|
|
335
|
+
if (useCache)
|
|
336
|
+
limitCacheSize();
|
|
271
337
|
});
|
|
338
|
+
function collectExrFiles(dir, out) {
|
|
339
|
+
for (const entry of readdirSync(dir)) {
|
|
340
|
+
if (entry === 'node_modules' || entry.startsWith('.'))
|
|
341
|
+
continue;
|
|
342
|
+
const full = path.join(dir, entry);
|
|
343
|
+
const st = statSync(full);
|
|
344
|
+
if (st.isDirectory()) {
|
|
345
|
+
collectExrFiles(full, out);
|
|
346
|
+
}
|
|
347
|
+
else if (entry.toLowerCase().endsWith('.exr')) {
|
|
348
|
+
out.push(full);
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
function resolvePmremOutputPath(exrPath, input, inputIsDirectory, outputArg) {
|
|
353
|
+
const outName = basename(exrPath).replace(/\.exr$/i, '.pmrem.ktx2');
|
|
354
|
+
if (!outputArg) {
|
|
355
|
+
// Default: write next to the input file
|
|
356
|
+
return path.join(dirname(exrPath), outName);
|
|
357
|
+
}
|
|
358
|
+
if (inputIsDirectory) {
|
|
359
|
+
// Mirror the directory layout under outputArg, treating it as a directory
|
|
360
|
+
const rel = path.relative(input, exrPath);
|
|
361
|
+
const relDir = dirname(rel);
|
|
362
|
+
const outDir = relDir === '.' ? outputArg : path.join(outputArg, relDir);
|
|
363
|
+
ensureIsDirectory(outDir);
|
|
364
|
+
return path.join(outDir, outName);
|
|
365
|
+
}
|
|
366
|
+
// Single-file input. outputArg may be an existing directory, or an explicit file path.
|
|
367
|
+
if (existsSync(outputArg) && isDirectory(outputArg)) {
|
|
368
|
+
return path.join(outputArg, outName);
|
|
369
|
+
}
|
|
370
|
+
if (outputArg.toLowerCase().endsWith('.ktx2')) {
|
|
371
|
+
ensureIsDirectory(dirname(outputArg));
|
|
372
|
+
return outputArg;
|
|
373
|
+
}
|
|
374
|
+
// Treat as a directory that does not exist yet
|
|
375
|
+
ensureIsDirectory(outputArg);
|
|
376
|
+
return path.join(outputArg, outName);
|
|
377
|
+
}
|
|
272
378
|
program.run().catch(err => {
|
|
273
379
|
// Since we still invoke this package with paths when running npm scripts we need to ignore this error
|
|
274
380
|
// We might want to move this code here into an extra package.
|
|
@@ -28,8 +28,10 @@ export type AudioRef = {
|
|
|
28
28
|
export declare function registerAudioRef(ref: AudioRef): void;
|
|
29
29
|
export declare function listAudioRefs(): AudioRef[];
|
|
30
30
|
export declare function clearAudioRegistry(): void;
|
|
31
|
-
/** Audio extensions
|
|
32
|
-
|
|
31
|
+
/** Audio extensions the compression transform will re-encode. Lossless only —
|
|
32
|
+
* re-encoding lossy formats (mp3/m4a/aac/ogg) would compound artifacts. Those
|
|
33
|
+
* paths are still valid in source glTFs; they are simply passed through. */
|
|
34
|
+
export declare const AUDIO_EXTENSIONS: readonly [".wav"];
|
|
33
35
|
export declare function isAudioUri(value: string): boolean;
|
|
34
36
|
/** Returns true if the URI is a relative path we should attempt to resolve+compress.
|
|
35
37
|
* Absolute URLs (http(s), data, file, blob, leading slash, drive letter) are passed through unchanged. */
|
|
@@ -22,8 +22,10 @@ export function listAudioRefs() {
|
|
|
22
22
|
export function clearAudioRegistry() {
|
|
23
23
|
_refs = [];
|
|
24
24
|
}
|
|
25
|
-
/** Audio extensions
|
|
26
|
-
|
|
25
|
+
/** Audio extensions the compression transform will re-encode. Lossless only —
|
|
26
|
+
* re-encoding lossy formats (mp3/m4a/aac/ogg) would compound artifacts. Those
|
|
27
|
+
* paths are still valid in source glTFs; they are simply passed through. */
|
|
28
|
+
export const AUDIO_EXTENSIONS = ['.wav'];
|
|
27
29
|
const _audioRegex = new RegExp(`(${AUDIO_EXTENSIONS.map(e => '\\' + e).join('|')})$`, 'i');
|
|
28
30
|
export function isAudioUri(value) {
|
|
29
31
|
return _audioRegex.test(value);
|
|
@@ -1,13 +1,34 @@
|
|
|
1
|
-
import { Document, Texture } from '@gltf-transform/core';
|
|
1
|
+
import { Document, ILogger, Texture } from '@gltf-transform/core';
|
|
2
2
|
import { NEEDLE_compression_texture_schema } from '../extensions/index.js';
|
|
3
3
|
import { INeedleTextureTransformContext, NeedleTransformContext } from './needle_common.js';
|
|
4
|
+
export interface PmremRuntime {
|
|
5
|
+
wasmModule: any;
|
|
6
|
+
basisuExe: string;
|
|
7
|
+
pmremDir: string;
|
|
8
|
+
tmpDir: string;
|
|
9
|
+
logger: ILogger;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Load the PMREM WASM module and locate the basisu binary, returning a runtime that
|
|
13
|
+
* can be passed to {@link convertExrBytesToPmremKtx2}. Returns null if WASM or basisu
|
|
14
|
+
* is missing — callers can decide whether that is fatal.
|
|
15
|
+
*
|
|
16
|
+
* Each runtime owns a per-instance temp directory; call {@link disposePmremRuntime}
|
|
17
|
+
* to clean up.
|
|
18
|
+
*/
|
|
19
|
+
export declare function createPmremRuntime(logger: ILogger): Promise<PmremRuntime | null>;
|
|
20
|
+
export declare function disposePmremRuntime(runtime: PmremRuntime | null): Promise<void>;
|
|
21
|
+
/**
|
|
22
|
+
* Convert raw .exr bytes to KTX2 HDR bytes using PMREM + basisu.
|
|
23
|
+
* Result is cached on disk keyed by the EXR content hash when `useCache` is true.
|
|
24
|
+
*/
|
|
25
|
+
export declare function convertExrBytesToPmremKtx2(runtime: PmremRuntime, exrBytes: Uint8Array, opts?: {
|
|
26
|
+
useCache?: boolean;
|
|
27
|
+
logLabel?: string;
|
|
28
|
+
}): Promise<Uint8Array>;
|
|
4
29
|
export declare class NeedlePmremContext implements INeedleTextureTransformContext {
|
|
5
|
-
private
|
|
6
|
-
private pmremDir;
|
|
7
|
-
private basisuExe;
|
|
8
|
-
private tmpDir;
|
|
30
|
+
private runtime;
|
|
9
31
|
private processed;
|
|
10
|
-
private available;
|
|
11
32
|
private logger;
|
|
12
33
|
private readonly convertedTextures;
|
|
13
34
|
/** In-memory dedup: keyed by EXR content hash, resolves to KTX2 bytes */
|
|
@@ -16,6 +37,5 @@ export declare class NeedlePmremContext implements INeedleTextureTransformContex
|
|
|
16
37
|
process(index: number, texture: Texture, _settings: NEEDLE_compression_texture_schema | null, context: NeedleTransformContext): Promise<boolean>;
|
|
17
38
|
/** Get or process EXR bytes into KTX2, with in-memory dedup and disk caching */
|
|
18
39
|
private getOrProcessExr;
|
|
19
|
-
private processExrToKtx2;
|
|
20
40
|
finalize(document: Document): Promise<void>;
|
|
21
41
|
}
|
|
@@ -9,6 +9,100 @@ import { createHash } from 'crypto';
|
|
|
9
9
|
import tmp from 'tmp';
|
|
10
10
|
import { addToCache, tryGetFromCache } from '../cache/index.js';
|
|
11
11
|
import { testAssert } from '../utils/test-assert.js';
|
|
12
|
+
/**
|
|
13
|
+
* Load the PMREM WASM module and locate the basisu binary, returning a runtime that
|
|
14
|
+
* can be passed to {@link convertExrBytesToPmremKtx2}. Returns null if WASM or basisu
|
|
15
|
+
* is missing — callers can decide whether that is fatal.
|
|
16
|
+
*
|
|
17
|
+
* Each runtime owns a per-instance temp directory; call {@link disposePmremRuntime}
|
|
18
|
+
* to clean up.
|
|
19
|
+
*/
|
|
20
|
+
export async function createPmremRuntime(logger) {
|
|
21
|
+
const pmremDir = getPmremDir();
|
|
22
|
+
const wasmJsPath = join(pmremDir, 'pkg', 'pmrem_wasm.js');
|
|
23
|
+
const wasmBinPath = join(pmremDir, 'pkg', 'pmrem_wasm_bg.wasm');
|
|
24
|
+
if (!existsSync(wasmJsPath) || !existsSync(wasmBinPath)) {
|
|
25
|
+
logger.warn('pmrem: WASM not found — EXR textures will not be processed');
|
|
26
|
+
testAssert(false, 'pmrem WASM files are missing. Please run `npm run build:pmrem` to build the tools/pmrem/ WASM module.');
|
|
27
|
+
return null;
|
|
28
|
+
}
|
|
29
|
+
const basisuExe = resolveBasisuExe(pmremDir);
|
|
30
|
+
if (!existsSync(basisuExe)) {
|
|
31
|
+
logger.warn(`pmrem: basisu binary not found at ${basisuExe} — EXR textures will not be processed`);
|
|
32
|
+
testAssert(false, `basisu executable is missing for platform ${process.platform}. Please run \`npm run build:pmrem\` to build the tools/pmrem/ binaries.`);
|
|
33
|
+
return null;
|
|
34
|
+
}
|
|
35
|
+
if (process.platform !== 'win32') {
|
|
36
|
+
try {
|
|
37
|
+
chmodSync(basisuExe, 0o755);
|
|
38
|
+
}
|
|
39
|
+
catch { /* ignore */ }
|
|
40
|
+
}
|
|
41
|
+
let wasmModule;
|
|
42
|
+
try {
|
|
43
|
+
const mod = await import(pathToFileURL(wasmJsPath).href);
|
|
44
|
+
const wasmBytes = readFileSync(wasmBinPath);
|
|
45
|
+
await mod.default({ module_or_path: wasmBytes });
|
|
46
|
+
wasmModule = mod;
|
|
47
|
+
}
|
|
48
|
+
catch (err) {
|
|
49
|
+
logger.warn(`pmrem: Failed to load WASM — ${err.message}`);
|
|
50
|
+
testAssert(false, `Failed to load pmrem WASM module: ${err.message}`);
|
|
51
|
+
return null;
|
|
52
|
+
}
|
|
53
|
+
const tmpDir = join(tmp.tmpdir, 'needle-pmrem', uuid());
|
|
54
|
+
mkdirSync(tmpDir, { recursive: true });
|
|
55
|
+
logger.debug('pmrem: Ready (WASM + basisu loaded)');
|
|
56
|
+
return { wasmModule, basisuExe, pmremDir, tmpDir, logger };
|
|
57
|
+
}
|
|
58
|
+
export async function disposePmremRuntime(runtime) {
|
|
59
|
+
if (runtime?.tmpDir) {
|
|
60
|
+
await safeRmDir(runtime.tmpDir);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Convert raw .exr bytes to KTX2 HDR bytes using PMREM + basisu.
|
|
65
|
+
* Result is cached on disk keyed by the EXR content hash when `useCache` is true.
|
|
66
|
+
*/
|
|
67
|
+
export async function convertExrBytesToPmremKtx2(runtime, exrBytes, opts = {}) {
|
|
68
|
+
const useCache = opts.useCache !== false;
|
|
69
|
+
const label = opts.logLabel ?? 'pmrem';
|
|
70
|
+
const logger = runtime.logger;
|
|
71
|
+
const contentHash = createHash('md5').update(exrBytes).digest('hex');
|
|
72
|
+
const cacheKey = `pmrem-${contentHash}`;
|
|
73
|
+
if (useCache) {
|
|
74
|
+
const cached = tryGetFromCache(cacheKey);
|
|
75
|
+
if (cached && cached.length > 0) {
|
|
76
|
+
logger.debug(`${label}: loaded from cache`);
|
|
77
|
+
return cached;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
const inputSize = exrBytes.byteLength < 100 * 1024
|
|
81
|
+
? `${(exrBytes.byteLength / 1024).toFixed(0)} KB`
|
|
82
|
+
: `${(exrBytes.byteLength / (1024 * 1024)).toFixed(2)} MB`;
|
|
83
|
+
logger.debug(`${label}: Running PMREM WASM (${inputSize} EXR) — this may take a while...`);
|
|
84
|
+
await new Promise(r => setTimeout(r, 0));
|
|
85
|
+
const t0 = Date.now();
|
|
86
|
+
const pmremExrBytes = runtime.wasmModule.pmrem_exr(exrBytes);
|
|
87
|
+
const elapsed = ((Date.now() - t0) / 1000).toFixed(1);
|
|
88
|
+
logger.debug(`${label}: PMREM done in ${elapsed}s (${(pmremExrBytes.byteLength / 1024 / 1024).toFixed(1)} MB)`);
|
|
89
|
+
const id = uuid();
|
|
90
|
+
const tempExr = join(runtime.tmpDir, `${id}.pmrem.exr`);
|
|
91
|
+
const tempKtx2 = join(runtime.tmpDir, `${id}.ktx2`);
|
|
92
|
+
writeFileSync(tempExr, Buffer.from(pmremExrBytes));
|
|
93
|
+
logger.debug(`${label}: Encoding KTX2 HDR...`);
|
|
94
|
+
// 6x6 needs an update in Needle before we can enable it
|
|
95
|
+
const variant = 'hdr4x4';
|
|
96
|
+
await encodeKTX2WithBasisu(runtime.basisuExe, tempExr, tempKtx2, variant, true);
|
|
97
|
+
if (!existsSync(tempKtx2)) {
|
|
98
|
+
throw new Error(`${label}: basisu did not produce output`);
|
|
99
|
+
}
|
|
100
|
+
const ktx2Bytes = readFileSync(tempKtx2);
|
|
101
|
+
if (useCache) {
|
|
102
|
+
addToCache(cacheKey, ktx2Bytes);
|
|
103
|
+
}
|
|
104
|
+
return new Uint8Array(ktx2Bytes);
|
|
105
|
+
}
|
|
12
106
|
/** Resolve the tools/pmrem/ directory relative to this file.
|
|
13
107
|
* At runtime this file is at dist/transforms/needle_pmrem.js,
|
|
14
108
|
* so tools/pmrem/ is at ../../tools/pmrem/ */
|
|
@@ -43,12 +137,8 @@ function encodeKTX2WithBasisu(basisuExe, exrPath, outPath, variant, mipmaps) {
|
|
|
43
137
|
});
|
|
44
138
|
}
|
|
45
139
|
export class NeedlePmremContext {
|
|
46
|
-
|
|
47
|
-
pmremDir = '';
|
|
48
|
-
basisuExe = '';
|
|
49
|
-
tmpDir = '';
|
|
140
|
+
runtime = null;
|
|
50
141
|
processed = 0;
|
|
51
|
-
available = false;
|
|
52
142
|
logger;
|
|
53
143
|
convertedTextures = [];
|
|
54
144
|
/** In-memory dedup: keyed by EXR content hash, resolves to KTX2 bytes */
|
|
@@ -56,50 +146,11 @@ export class NeedlePmremContext {
|
|
|
56
146
|
async prepare(document) {
|
|
57
147
|
this.logger = document.getLogger();
|
|
58
148
|
this.processed = 0;
|
|
59
|
-
this.
|
|
60
|
-
|
|
61
|
-
const wasmJsPath = join(this.pmremDir, 'pkg', 'pmrem_wasm.js');
|
|
62
|
-
const wasmBinPath = join(this.pmremDir, 'pkg', 'pmrem_wasm_bg.wasm');
|
|
63
|
-
if (!existsSync(wasmJsPath) || !existsSync(wasmBinPath)) {
|
|
64
|
-
this.logger.warn('pmrem: WASM not found — EXR textures will not be processed');
|
|
65
|
-
testAssert(false, 'pmrem WASM files are missing. Please run `npm run build:pmrem` to build the tools/pmrem/ WASM module.');
|
|
66
|
-
return false;
|
|
67
|
-
}
|
|
68
|
-
// Check basisu binary
|
|
69
|
-
this.basisuExe = resolveBasisuExe(this.pmremDir);
|
|
70
|
-
if (!existsSync(this.basisuExe)) {
|
|
71
|
-
this.logger.warn(`pmrem: basisu binary not found at ${this.basisuExe} — EXR textures will not be processed`);
|
|
72
|
-
testAssert(false, `basisu executable is missing for platform ${process.platform}. Please run \`npm run build:pmrem\` to build the tools/pmrem/ binaries.`);
|
|
73
|
-
return false;
|
|
74
|
-
}
|
|
75
|
-
// Set executable permission
|
|
76
|
-
if (process.platform !== 'win32') {
|
|
77
|
-
try {
|
|
78
|
-
chmodSync(this.basisuExe, 0o755);
|
|
79
|
-
}
|
|
80
|
-
catch { /* ignore */ }
|
|
81
|
-
}
|
|
82
|
-
// Load WASM
|
|
83
|
-
try {
|
|
84
|
-
const mod = await import(pathToFileURL(wasmJsPath).href);
|
|
85
|
-
const wasmBytes = readFileSync(wasmBinPath);
|
|
86
|
-
await mod.default({ module_or_path: wasmBytes });
|
|
87
|
-
this.wasmModule = mod;
|
|
88
|
-
}
|
|
89
|
-
catch (err) {
|
|
90
|
-
this.logger.warn(`pmrem: Failed to load WASM — ${err.message}`);
|
|
91
|
-
testAssert(false, `Failed to load pmrem WASM module: ${err.message}`);
|
|
92
|
-
return false;
|
|
93
|
-
}
|
|
94
|
-
// Create temp directory (per-instance to avoid parallel cleanup races)
|
|
95
|
-
this.tmpDir = join(tmp.tmpdir, 'needle-pmrem', uuid());
|
|
96
|
-
mkdirSync(this.tmpDir, { recursive: true });
|
|
97
|
-
this.available = true;
|
|
98
|
-
this.logger.debug('pmrem: Ready (WASM + basisu loaded)');
|
|
99
|
-
return true;
|
|
149
|
+
this.runtime = await createPmremRuntime(this.logger);
|
|
150
|
+
return this.runtime !== null;
|
|
100
151
|
}
|
|
101
152
|
async process(index, texture, _settings, context) {
|
|
102
|
-
if (!this.
|
|
153
|
+
if (!this.runtime)
|
|
103
154
|
return false;
|
|
104
155
|
if (texture.getMimeType() !== 'image/exr')
|
|
105
156
|
return false;
|
|
@@ -163,64 +214,24 @@ export class NeedlePmremContext {
|
|
|
163
214
|
}
|
|
164
215
|
/** Get or process EXR bytes into KTX2, with in-memory dedup and disk caching */
|
|
165
216
|
getOrProcessExr(cacheKey, exrBytes, index, name, useCache) {
|
|
166
|
-
// In-memory dedup: if we're already processing identical EXR content, reuse the promise
|
|
167
217
|
const existing = this.processingPromises.get(cacheKey);
|
|
168
218
|
if (existing) {
|
|
169
219
|
this.logger.debug(`NEEDLE_pmrem[${index}] ${name}: reusing in-flight result (identical content)`);
|
|
170
220
|
return existing;
|
|
171
221
|
}
|
|
172
|
-
const promise = this.
|
|
222
|
+
const promise = convertExrBytesToPmremKtx2(this.runtime, exrBytes, {
|
|
223
|
+
useCache,
|
|
224
|
+
logLabel: `NEEDLE_pmrem[${index}] ${name}`,
|
|
225
|
+
});
|
|
173
226
|
this.processingPromises.set(cacheKey, promise);
|
|
174
227
|
return promise;
|
|
175
228
|
}
|
|
176
|
-
async processExrToKtx2(cacheKey, exrBytes, index, name, useCache) {
|
|
177
|
-
// Disk cache check
|
|
178
|
-
if (useCache) {
|
|
179
|
-
const cached = tryGetFromCache(cacheKey);
|
|
180
|
-
if (cached && cached.length > 0) {
|
|
181
|
-
this.logger.debug(`NEEDLE_pmrem[${index}] ${name}: loaded from cache`);
|
|
182
|
-
return cached;
|
|
183
|
-
}
|
|
184
|
-
}
|
|
185
|
-
// 1. Run PMREM on the EXR bytes (synchronous WASM — may take a while)
|
|
186
|
-
const inputSize = exrBytes.byteLength < 100 * 1024
|
|
187
|
-
? `${(exrBytes.byteLength / 1024).toFixed(0)} KB`
|
|
188
|
-
: `${(exrBytes.byteLength / (1024 * 1024)).toFixed(2)} MB`;
|
|
189
|
-
this.logger.debug(`NEEDLE_pmrem[${index}] ${name}: Running PMREM WASM (${inputSize} EXR) — this may take a while...`);
|
|
190
|
-
// Yield to allow log to flush before blocking WASM call
|
|
191
|
-
await new Promise(r => setTimeout(r, 0));
|
|
192
|
-
const t0 = Date.now();
|
|
193
|
-
const pmremExrBytes = this.wasmModule.pmrem_exr(exrBytes);
|
|
194
|
-
const elapsed = ((Date.now() - t0) / 1000).toFixed(1);
|
|
195
|
-
this.logger.debug(`NEEDLE_pmrem[${index}] ${name}: PMREM done in ${elapsed}s (${(pmremExrBytes.byteLength / 1024 / 1024).toFixed(1)} MB)`);
|
|
196
|
-
// 2. Write PMREM EXR to temp file
|
|
197
|
-
const id = uuid();
|
|
198
|
-
const tempExr = join(this.tmpDir, `${id}.pmrem.exr`);
|
|
199
|
-
const tempKtx2 = join(this.tmpDir, `${id}.ktx2`);
|
|
200
|
-
writeFileSync(tempExr, Buffer.from(pmremExrBytes));
|
|
201
|
-
// 3. Encode to KTX2 HDR via basisu
|
|
202
|
-
this.logger.debug(`NEEDLE_pmrem[${index}] ${name}: Encoding KTX2 HDR...`);
|
|
203
|
-
const variant = 'hdr4x4'; // 6x6 needs an update in Needle
|
|
204
|
-
await encodeKTX2WithBasisu(this.basisuExe, tempExr, tempKtx2, variant, true);
|
|
205
|
-
if (!existsSync(tempKtx2)) {
|
|
206
|
-
throw new Error(`NEEDLE_pmrem[${index}] ${name}: basisu did not produce output`);
|
|
207
|
-
}
|
|
208
|
-
// 4. Read KTX2 result
|
|
209
|
-
const ktx2Bytes = readFileSync(tempKtx2);
|
|
210
|
-
// Store in disk cache
|
|
211
|
-
if (useCache) {
|
|
212
|
-
addToCache(cacheKey, ktx2Bytes);
|
|
213
|
-
}
|
|
214
|
-
return new Uint8Array(ktx2Bytes);
|
|
215
|
-
}
|
|
216
229
|
async finalize(document) {
|
|
217
230
|
if (this.processed > 0) {
|
|
218
|
-
// Register NEEDLE_pmrem extension and mark converted textures
|
|
219
231
|
const pmremExt = document.createExtension(NEEDLE_pmrem);
|
|
220
232
|
for (const tex of this.convertedTextures) {
|
|
221
233
|
pmremExt.addTexture(tex);
|
|
222
234
|
}
|
|
223
|
-
// If no textures use EXR anymore, remove the EXT_texture_exr extension from the document
|
|
224
235
|
const hasExr = document.getRoot().listTextures().some(t => t.getMimeType() === 'image/exr');
|
|
225
236
|
if (!hasExr) {
|
|
226
237
|
const exrExt = document.getRoot().listExtensionsUsed()
|
|
@@ -230,9 +241,7 @@ export class NeedlePmremContext {
|
|
|
230
241
|
}
|
|
231
242
|
}
|
|
232
243
|
}
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
await safeRmDir(this.tmpDir);
|
|
236
|
-
}
|
|
244
|
+
await disposePmremRuntime(this.runtime);
|
|
245
|
+
this.runtime = null;
|
|
237
246
|
}
|
|
238
247
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
export declare const version = "2.16.0-next.
|
|
1
|
+
export declare const version = "2.16.0-next.8ceb45e";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
export const version = "2.16.0-next.
|
|
1
|
+
export const version = "2.16.0-next.8ceb45e";
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@needle-tools/gltf-build-pipeline",
|
|
3
|
-
"version": "2.16.0-next.
|
|
3
|
+
"version": "2.16.0-next.8ceb45e",
|
|
4
4
|
"description": "Pipeline and tools for optimizing gltf files using gltf-transform and compression settings within glTF extensions",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"type": "module",
|