@needle-tools/gltf-build-pipeline 2.16.0-alpha.bc3af72 → 2.16.0-alpha.eb5a640
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +29 -2
- package/dist/cache/cache.d.ts +1 -0
- package/dist/cache/cache.js +35 -15
- package/dist/cli/index.js +27 -4
- package/dist/config/index.d.ts +11 -0
- package/dist/config/index.js +9 -4
- package/dist/extensions/NEEDLE_opaque/NEEDLE_opaque.d.ts +2 -2
- package/dist/extensions/NEEDLE_opaque/NEEDLE_opaque.js +30 -0
- package/dist/extensions/NEEDLE_opaque/index.d.ts +1 -2
- package/dist/scripts/pack-gltf.js +10 -1
- package/dist/transforms/index.d.ts +3 -0
- package/dist/transforms/index.js +3 -0
- package/dist/transforms/needle_audio.d.ts +12 -0
- package/dist/transforms/needle_audio.js +104 -0
- package/dist/transforms/needle_audio_ffmpeg.d.ts +14 -0
- package/dist/transforms/needle_audio_ffmpeg.js +51 -0
- package/dist/transforms/needle_audio_registry.d.ts +38 -0
- package/dist/transforms/needle_audio_registry.js +47 -0
- package/dist/transforms/needle_progressive.d.ts +9 -1
- package/dist/transforms/needle_progressive.js +30 -14
- package/dist/transforms/needle_texture_transform.js +4 -0
- package/dist/transforms/toktx.js +23 -36
- package/dist/utils/index.d.ts +0 -1
- package/dist/utils/index.js +0 -1
- package/dist/utils/stats.js +12 -4
- package/dist/utils/version.gen.d.ts +1 -1
- package/dist/utils/version.gen.js +1 -1
- package/package.json +1 -1
- package/tools/pmrem/pkg/pmrem_wasm_bg.wasm +0 -0
- package/dist/utils/merge.d.ts +0 -12
- package/dist/utils/merge.js +0 -31
package/README.md
CHANGED
|
@@ -14,10 +14,10 @@ Includes support for generating LOD quality level for textures and meshes. Use w
|
|
|
14
14
|
In general all files that have been processed by this package are valid glTF/GLB files.
|
|
15
15
|
The glTF Build Pipeline package is meant to be used as a last step before delivery.
|
|
16
16
|
That means:
|
|
17
|
-
- files are meant to be loaded
|
|
17
|
+
- files are meant to be loaded by [Needle Engine](https://needle.tools) (and other three.js based runtimes)
|
|
18
18
|
- files are not to be opened in Blender again to be further edited
|
|
19
19
|
|
|
20
|
-
To load progressively enhanced glTF files in
|
|
20
|
+
To load progressively enhanced glTF files in Needle Engine or another three.js based runtime, see examples at [@needle-tools/gltf-progressive](https://www.npmjs.com/package/@needle-tools/gltf-progressive)
|
|
21
21
|
|
|
22
22
|
### Installation
|
|
23
23
|
|
|
@@ -26,6 +26,33 @@ To load progressively enhanced glTF files in any three.js based engine see examp
|
|
|
26
26
|
### Commands
|
|
27
27
|
- `help` - use help to view options
|
|
28
28
|
|
|
29
|
+
### Configuration
|
|
30
|
+
|
|
31
|
+
The pipeline reads an optional `needle.config.json` from the working directory (or any parent directory). Configure pipeline behaviour under the `gltf` key:
|
|
32
|
+
|
|
33
|
+
```json
|
|
34
|
+
{
|
|
35
|
+
"gltf": {
|
|
36
|
+
"usecase": "product",
|
|
37
|
+
"textures": { "lods": true },
|
|
38
|
+
"meshes": { "lods": true },
|
|
39
|
+
"audio": { "enabled": true, "bitrate": 64 },
|
|
40
|
+
"exr": { "enabled": true }
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
The `gltf` value may also be a string shorthand for `usecase` — `"gltf": "product"` is equivalent to `"gltf": { "usecase": "product" }`.
|
|
46
|
+
|
|
47
|
+
| Key | Type | Default | Description |
|
|
48
|
+
|-----|------|---------|-------------|
|
|
49
|
+
| `usecase` | `"default" \| "product" \| "world"` | `"default"` | High-level optimisation profile. |
|
|
50
|
+
| `textures.lods` | `boolean` | `true` | Generate progressive texture LODs. |
|
|
51
|
+
| `meshes.lods` | `boolean` | `true` | Generate progressive mesh LODs. |
|
|
52
|
+
| `audio.enabled` | `boolean` | `true` | Compress externally referenced `.wav` audio files to Opus in an `.ogg` container, rewriting the URIs in the glTF. Lossy formats (`.mp3`, `.ogg`, `.m4a`, `.aac`) pass through unchanged to avoid compounding compression artifacts. Requires `ffmpeg` with `libopus` on `PATH`; when missing the build logs a warning and leaves the audio files untouched. |
|
|
53
|
+
| `audio.bitrate` | `number` (kbps) | `128` | Opus encoding bitrate. 96 is near-transparent for most music; 128 matches YouTube's music tier and is the safe "won't sound bad" default. Drop to 48–64 for known mono/UI-only assets. |
|
|
54
|
+
| `exr.enabled` | `boolean` | `true` | Convert EXR textures to KTX2 HDR via PMREM. When `false`, EXR textures (bundled and externally referenced) are left as-is so a downstream step can handle them. |
|
|
55
|
+
|
|
29
56
|
# Contact
|
|
30
57
|
|
|
31
58
|
For licensing questions please contact us.
|
package/dist/cache/cache.d.ts
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
/// <reference types="node" resolution-mode="require"/>
|
|
3
3
|
import { ExtensibleProperty } from "@gltf-transform/core";
|
|
4
4
|
import { Logger } from "@donmccurdy/caporal";
|
|
5
|
+
export declare function ensureHashReady(): Promise<void>;
|
|
5
6
|
/**
|
|
6
7
|
* Clear the cache.
|
|
7
8
|
* This will remove all files in the cache directory.
|
package/dist/cache/cache.js
CHANGED
|
@@ -6,9 +6,16 @@ import { getVersion } from "../utils/version.js";
|
|
|
6
6
|
// import * as CHECKDISCSPACE from 'check-disk-space'
|
|
7
7
|
import xxhash from "xxhash-wasm";
|
|
8
8
|
let xxHashModule = null;
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
9
|
+
// Single shared promise that resolves once the WASM module is loaded (or has failed).
|
|
10
|
+
// Callers should `await ensureHashReady()` once before any synchronous hash call to
|
|
11
|
+
// avoid a race where early calls use the JS fallback and later calls use WASM —
|
|
12
|
+
// producing different cache keys for the same input within one process.
|
|
13
|
+
const xxHashReady = xxhash()
|
|
14
|
+
.then(module => { xxHashModule = module; })
|
|
15
|
+
.catch(() => { });
|
|
16
|
+
export function ensureHashReady() {
|
|
17
|
+
return xxHashReady;
|
|
18
|
+
}
|
|
12
19
|
/**
|
|
13
20
|
* Get the available space on the cache directory in bytes.
|
|
14
21
|
*/
|
|
@@ -179,6 +186,17 @@ export function getKeyWithHash(property) {
|
|
|
179
186
|
const key = `${prefix}-${hash}`;
|
|
180
187
|
return { key, hash };
|
|
181
188
|
}
|
|
189
|
+
/**
|
|
190
|
+
* Combine two 32-bit hashes with mixing. Plain `a + b` is commutative and
|
|
191
|
+
* has poor diffusion — swapping two attribute buffers collides. This uses a
|
|
192
|
+
* boost::hash_combine-style mix: xor in the new bits, then bit-shift mix so
|
|
193
|
+
* order matters and unrelated changes don't cancel out.
|
|
194
|
+
*/
|
|
195
|
+
function combineHash(seed, value) {
|
|
196
|
+
// 0x9e3779b9 = golden-ratio constant used by boost / fnv variants
|
|
197
|
+
seed = (seed ^ (value + 0x9e3779b9 + ((seed << 6) | 0) + (seed >>> 2))) | 0;
|
|
198
|
+
return seed;
|
|
199
|
+
}
|
|
182
200
|
/**
|
|
183
201
|
* Get the hash for a property or an object.
|
|
184
202
|
* @param property The property to get the hash for.
|
|
@@ -186,27 +204,26 @@ export function getKeyWithHash(property) {
|
|
|
186
204
|
* @returns The hash as a number.
|
|
187
205
|
*/
|
|
188
206
|
export function getHash(property, level = 0) {
|
|
189
|
-
let hashStr = "";
|
|
190
207
|
let hash = 0;
|
|
191
208
|
if (property instanceof ExtensibleProperty) {
|
|
192
209
|
const extensions = property.listExtensions();
|
|
193
210
|
const extras = property.getExtras();
|
|
194
|
-
hash
|
|
211
|
+
hash = combineHash(hash, hashObject({
|
|
195
212
|
extensions,
|
|
196
213
|
extras,
|
|
197
|
-
});
|
|
214
|
+
}));
|
|
198
215
|
if (property instanceof Texture) {
|
|
199
216
|
const image = property.getImage();
|
|
200
217
|
const bytes = image?.buffer;
|
|
201
218
|
if (bytes)
|
|
202
|
-
hash
|
|
219
|
+
hash = combineHash(hash, hashBuffer(bytes, image.byteOffset, image.byteLength));
|
|
203
220
|
}
|
|
204
221
|
else if (property instanceof Mesh) {
|
|
205
222
|
const weights = property.getWeights();
|
|
206
|
-
hash
|
|
223
|
+
hash = combineHash(hash, hashObject(weights));
|
|
207
224
|
const primitives = property.listPrimitives();
|
|
208
225
|
for (const prim of primitives) {
|
|
209
|
-
hash
|
|
226
|
+
hash = combineHash(hash, hashString(getHash(prim, level + 1)));
|
|
210
227
|
}
|
|
211
228
|
}
|
|
212
229
|
else if (property instanceof Primitive) {
|
|
@@ -220,32 +237,35 @@ export function getHash(property, level = 0) {
|
|
|
220
237
|
name: s,
|
|
221
238
|
};
|
|
222
239
|
});
|
|
223
|
-
hash
|
|
240
|
+
hash = combineHash(hash, hashObject({
|
|
224
241
|
indices,
|
|
225
242
|
attributes,
|
|
226
243
|
targets,
|
|
227
244
|
semantics: semanticsNames
|
|
228
|
-
});
|
|
245
|
+
}));
|
|
229
246
|
if (indices) {
|
|
230
247
|
const indexBuffer = indices.getArray();
|
|
231
248
|
if (indexBuffer) {
|
|
232
|
-
hash
|
|
249
|
+
hash = combineHash(hash, hashBuffer(indexBuffer.buffer, indexBuffer.byteOffset, indexBuffer.byteLength));
|
|
233
250
|
}
|
|
234
251
|
}
|
|
235
252
|
// We can not pass the whole buffer into hashObject since very large meshes will cause errors when trying to stringify
|
|
236
253
|
// See https://linear.app/needle/issue/NE-6414
|
|
254
|
+
// Mix the semantic name into each attribute hash so swapping two attributes (e.g. POSITION ↔ NORMAL)
|
|
255
|
+
// doesn't collide — without this, the commutative combine would erase attribute identity.
|
|
237
256
|
for (const sem of semantics) {
|
|
238
257
|
const arr = property.getAttribute(sem)?.getArray();
|
|
239
258
|
if (arr) {
|
|
240
|
-
hash
|
|
259
|
+
hash = combineHash(hash, hashString(sem));
|
|
260
|
+
hash = combineHash(hash, hashBuffer(arr.buffer, arr.byteOffset, arr.byteLength));
|
|
241
261
|
}
|
|
242
262
|
}
|
|
243
263
|
}
|
|
244
264
|
}
|
|
245
265
|
else {
|
|
246
|
-
hash
|
|
266
|
+
hash = combineHash(hash, hashObject(property));
|
|
247
267
|
}
|
|
248
|
-
return
|
|
268
|
+
return String(hash);
|
|
249
269
|
}
|
|
250
270
|
const cleanBuildPipelineVersion = getVersion(false).replace(/[^a-zA-Z0-9]/g, "_");
|
|
251
271
|
function getCacheDirectory(root) {
|
package/dist/cli/index.js
CHANGED
|
@@ -5,7 +5,7 @@ import { printStats, writeFileStatsToFile } from '../utils/stats.js';
|
|
|
5
5
|
import { getVersion } from '../utils/version.js';
|
|
6
6
|
import { UsecaseOptions, createConfig, getConfig } from '../config/index.js';
|
|
7
7
|
import { isLOD, make_progressive } from '../transforms/needle_progressive.js';
|
|
8
|
-
import { cacheSizeLimit, clearCache, limitCacheSize } from '../cache/cache.js';
|
|
8
|
+
import { cacheSizeLimit, clearCache, ensureHashReady, limitCacheSize } from '../cache/cache.js';
|
|
9
9
|
import { computeJobKey, saveJobResult, tryRestoreJobResult } from '../cache/job-cache.js';
|
|
10
10
|
import { existsSync, statSync } from 'fs';
|
|
11
11
|
import { ensureIsDirectory, foreachGLTF, isDirectory } from '../utils/fileutils.js';
|
|
@@ -14,6 +14,16 @@ import { ERROR_CODES } from '../constants.js';
|
|
|
14
14
|
import { trackPipelineStart, trackPipelineEnd, trackError } from '../utils/analytics.js';
|
|
15
15
|
// For testing / dev you can run `npm link` in the package directory
|
|
16
16
|
// For removing the link run `npm rm --global @needle-tools/gltf-build-pipeline`
|
|
17
|
+
// Disable caporal's autoCast for positional arguments.
|
|
18
|
+
//
|
|
19
|
+
// caporal's autoCast uses /^true|false$/ (precedence bug — should be /^(true|false)$/),
|
|
20
|
+
// which matches any string ending with "false" or starting with "true". An <output>
|
|
21
|
+
// path like ".../out-shellfalse" gets coerced to the boolean false, then stringified
|
|
22
|
+
// to "false", and the pipeline writes to "<inputDir>/false" instead of the requested
|
|
23
|
+
// directory. Same shape applies to paths ending with "on" or starting with "yes"/"1".
|
|
24
|
+
// Disabling autoCast at the program level forces positionals through String(...);
|
|
25
|
+
// options with explicit BOOLEAN/STRING validators are unaffected.
|
|
26
|
+
program.cast(false);
|
|
17
27
|
program
|
|
18
28
|
.command("version", "Print current version")
|
|
19
29
|
.action(async ({ logger }) => {
|
|
@@ -35,6 +45,9 @@ Caches are limited to ${cacheSizeLimit} MB disc space by default.
|
|
|
35
45
|
.option("--verbose", "Enable verbose output", { validator: program.BOOLEAN, default: false })
|
|
36
46
|
.action(async ({ logger, args, options }) => {
|
|
37
47
|
logger.level = options.verbose ? "debug" : "info";
|
|
48
|
+
// Wait for the xxHash WASM module so any hashing inside calculateStats sees the
|
|
49
|
+
// same hash function on every call (otherwise the first calls use the JS fallback).
|
|
50
|
+
await ensureHashReady();
|
|
38
51
|
logger.info("Calculating stats for \"" + args.input + "\"");
|
|
39
52
|
const input = args.input.toString();
|
|
40
53
|
const filestats = new Array();
|
|
@@ -60,8 +73,14 @@ Caches are limited to ${cacheSizeLimit} MB disc space by default.
|
|
|
60
73
|
This will produce multiple versions of the input file, each with a different level of optimization.
|
|
61
74
|
Each version will be compressed and written to the output directory.
|
|
62
75
|
`)
|
|
63
|
-
|
|
64
|
-
|
|
76
|
+
// Explicit STRING validators prevent caporal's autoCast() from silently coercing
|
|
77
|
+
// a path that ends with "false" / "on" (or starts with "true" / "yes" / "1") into
|
|
78
|
+
// a boolean — caporal's autoCast regex is missing parentheses and matches as
|
|
79
|
+
// /^true | false$/ rather than /^(true|false)$/. Without these validators, an
|
|
80
|
+
// output path like "out-shellfalse" gets cast to the boolean false → stringified
|
|
81
|
+
// → file written to "<inputDir>/false" instead of the named directory.
|
|
82
|
+
.argument('<input>', "The input glTF, GLB or VRM file that should be compressed OR a directory that contains files to be compressed", { validator: program.STRING })
|
|
83
|
+
.argument('<output>', "The output file or directory. It can be an absolute path or a relative path to the input file. If not provided, the input file will be overwritten.", { default: null, validator: program.STRING })
|
|
65
84
|
.option("--compress", `When enabled files will be compressed`, { validator: program.BOOLEAN, default: true })
|
|
66
85
|
.option("--progressive", `When enabled files will processed to be progressively loaded`, { validator: program.BOOLEAN, default: true })
|
|
67
86
|
.option("--usecase <usecase>", `The usecase for the compression. This will set the compression settings. Possible options: [${UsecaseOptions.join(", ")}]`, { validator: program.STRING })
|
|
@@ -76,6 +95,10 @@ Each version will be compressed and written to the output directory.
|
|
|
76
95
|
logger.error(`[Needle Build Pipeline] v${getVersion()} You need to enable at least one of the options: --compress or --progressive`);
|
|
77
96
|
return;
|
|
78
97
|
}
|
|
98
|
+
// Wait for the xxHash WASM module before any hashing happens. Without this the
|
|
99
|
+
// first hash calls in a process use the JS fallback while later ones use WASM,
|
|
100
|
+
// producing different cache keys for the same input within a single run.
|
|
101
|
+
await ensureHashReady();
|
|
79
102
|
logger.info(`[Needle Build Pipeline] v${getVersion()} — Transform '${args.input}'`);
|
|
80
103
|
const pipelineMode = [options.progressive && 'progressive', options.compress && 'compress'].filter(Boolean).join('+');
|
|
81
104
|
trackPipelineStart({ mode: pipelineMode, usecase: options.usecase?.toString() });
|
|
@@ -164,7 +187,7 @@ Each version will be compressed and written to the output directory.
|
|
|
164
187
|
stats.totalFileSizeInMBBefore += inputFileSize;
|
|
165
188
|
}
|
|
166
189
|
}
|
|
167
|
-
logger.info(`→ [CACHE] Restored ${restored.length} file(s)for job ${jobKey}`);
|
|
190
|
+
logger.info(`→ [CACHE] Restored ${restored.length} file(s) for job ${jobKey}`);
|
|
168
191
|
return;
|
|
169
192
|
}
|
|
170
193
|
}
|
package/dist/config/index.d.ts
CHANGED
|
@@ -13,6 +13,17 @@ export type Config = {
|
|
|
13
13
|
meshes?: {
|
|
14
14
|
lods: boolean;
|
|
15
15
|
};
|
|
16
|
+
audio?: {
|
|
17
|
+
/** Enable Opus/Ogg compression of externally referenced audio files. Default: true. */
|
|
18
|
+
enabled?: boolean;
|
|
19
|
+
/** Opus encoding bitrate in kbps. Default: 64. */
|
|
20
|
+
bitrate?: number;
|
|
21
|
+
};
|
|
22
|
+
exr?: {
|
|
23
|
+
/** Enable EXR -> KTX2 HDR conversion via PMREM. When false, EXR textures (bundled or
|
|
24
|
+
* externally referenced) are left untouched. Default: true. */
|
|
25
|
+
enabled?: boolean;
|
|
26
|
+
};
|
|
16
27
|
};
|
|
17
28
|
/**
|
|
18
29
|
* Create a config object with the given usecase.
|
package/dist/config/index.js
CHANGED
|
@@ -70,8 +70,11 @@ export function getConfig(configPath, fallbackSearchDirectory, opts) {
|
|
|
70
70
|
throw new Error(`Failed parsing config at \"${configPath}\"`);
|
|
71
71
|
}
|
|
72
72
|
}
|
|
73
|
+
// Always return a fresh shallow copy. Callers (notably the CLI) mutate the
|
|
74
|
+
// returned object (`config.usecase = ...`); returning the cache by reference
|
|
75
|
+
// would let those mutations poison subsequent getConfig() calls.
|
|
73
76
|
if (_cachedConfig !== undefined) {
|
|
74
|
-
return _cachedConfig;
|
|
77
|
+
return { ..._cachedConfig };
|
|
75
78
|
}
|
|
76
79
|
// Resolve the config once and cache it
|
|
77
80
|
// Check if a needle.config.json exists in the directory from which the script is run (or any parent directory)
|
|
@@ -82,12 +85,14 @@ export function getConfig(configPath, fallbackSearchDirectory, opts) {
|
|
|
82
85
|
logger.info(`Using config with usecase: ${_cachedConfig.usecase}`);
|
|
83
86
|
}
|
|
84
87
|
}
|
|
85
|
-
// If no config was found, use the default config
|
|
88
|
+
// If no config was found, use the default config. Store a copy in the cache
|
|
89
|
+
// so we never alias defaultConfig directly — that way even if a downstream
|
|
90
|
+
// bug ever returned the cache by reference, defaultConfig itself stays clean.
|
|
86
91
|
if (!_cachedConfig) {
|
|
87
92
|
logger.debug("No config found. Using default config.");
|
|
88
|
-
_cachedConfig = defaultConfig;
|
|
93
|
+
_cachedConfig = { ...defaultConfig };
|
|
89
94
|
}
|
|
90
|
-
return _cachedConfig;
|
|
95
|
+
return { ..._cachedConfig };
|
|
91
96
|
}
|
|
92
97
|
let _cachedConfig = undefined;
|
|
93
98
|
export function searchConfig(directory, logger) {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { WriterContext, PropertyType } from '@gltf-transform/core';
|
|
1
|
+
import { Extension, WriterContext, PropertyType } from '@gltf-transform/core';
|
|
2
2
|
declare interface IExtensibleProperty {
|
|
3
3
|
}
|
|
4
4
|
type DebugBreakPoint = boolean | "pointer:read";
|
|
@@ -23,5 +23,5 @@ export interface IExtensionWriter {
|
|
|
23
23
|
write(context: WriterContext, prop: IExtensibleProperty): void;
|
|
24
24
|
}
|
|
25
25
|
export declare let currentGenerator: string;
|
|
26
|
-
export declare function createOpaqueExtension(name: string, types: PropertyType | PropertyType[], opts?: Options):
|
|
26
|
+
export declare function createOpaqueExtension(name: string, types: PropertyType | PropertyType[], opts?: Options): typeof Extension;
|
|
27
27
|
export {};
|
|
@@ -5,6 +5,7 @@ import { NEEDLE_progressive_texture_settings } from '../NEEDLE_progressive_textu
|
|
|
5
5
|
import { testAssert } from '../../utils/test-assert.js';
|
|
6
6
|
import { existsSync, readFileSync } from 'fs';
|
|
7
7
|
import { dirname, join } from 'path';
|
|
8
|
+
import { isAudioUri, isRelativeAudioUri, listAudioRefs, registerAudioRef } from '../../transforms/needle_audio_registry.js';
|
|
8
9
|
const ALL_PROPERTY_TYPES = [];
|
|
9
10
|
for (const key in PropertyType) {
|
|
10
11
|
ALL_PROPERTY_TYPES.push(PropertyType[key]);
|
|
@@ -627,6 +628,25 @@ class JsonPointerHandler {
|
|
|
627
628
|
else
|
|
628
629
|
console.warn("WARN: failed registering pointer", fullPath, value);
|
|
629
630
|
}
|
|
631
|
+
// External audio reference (e.g. "click.wav" or "music.mp3" on a NEEDLE_components AudioSource).
|
|
632
|
+
// Audio is never embedded in the GLB — it stays as a sidecar file. We register the
|
|
633
|
+
// (obj, key) location so `needle_audio_transform` can compress the file and so the
|
|
634
|
+
// write-time pass can rewrite the JSON string to the new URI.
|
|
635
|
+
if (!isTexturePointer && isAudioUri(value) && isRelativeAudioUri(value)) {
|
|
636
|
+
const inputDir = getExtensionInputFile() ? dirname(getExtensionInputFile()) : '';
|
|
637
|
+
const audioPath = inputDir ? join(inputDir, value) : '';
|
|
638
|
+
if (audioPath && existsSync(audioPath)) {
|
|
639
|
+
registerAudioRef({ obj, key, sourcePath: audioPath, sourceUri: value });
|
|
640
|
+
if (this.debug)
|
|
641
|
+
console.log(`Registered external audio "${value}" at ${fullPath}`);
|
|
642
|
+
}
|
|
643
|
+
else if (audioPath) {
|
|
644
|
+
// Non-fatal: leave the URI untouched and warn. Mirrors EXR error path
|
|
645
|
+
// but uses warn instead of testAssert since missing audio shouldn't
|
|
646
|
+
// hard-fail the build.
|
|
647
|
+
this.document.getLogger().warn(`NEEDLE_opaque: external audio not found: ${audioPath}`);
|
|
648
|
+
}
|
|
649
|
+
}
|
|
630
650
|
// External EXR file reference (e.g. "StudioHDRI_ferndale_studio_04_1k.exr" in a ReflectionProbe)
|
|
631
651
|
// Inject the file into the document as a Texture so PMREM can process it
|
|
632
652
|
if (!isTexturePointer && value.match(/\.exr$/i)) {
|
|
@@ -816,6 +836,16 @@ class JsonPointerHandler {
|
|
|
816
836
|
if (ptr.resolve(context, step))
|
|
817
837
|
ptr.write();
|
|
818
838
|
});
|
|
839
|
+
// Resolve external audio pointers: rewrite the JSON string from the original URI
|
|
840
|
+
// (e.g. "click.wav") to the compressed URI (e.g. "click.opus.ogg"). Refs whose
|
|
841
|
+
// `newUri` was never set (ffmpeg missing, compression skipped) are left untouched.
|
|
842
|
+
for (const ref of listAudioRefs()) {
|
|
843
|
+
if (ref.newUri && ref.obj[ref.key] === ref.sourceUri) {
|
|
844
|
+
if (this.debug)
|
|
845
|
+
console.log(`< Resolved external audio: ${ref.sourceUri} → ${ref.newUri}`);
|
|
846
|
+
ref.obj[ref.key] = ref.newUri;
|
|
847
|
+
}
|
|
848
|
+
}
|
|
819
849
|
// Resolve external EXR pointers: update component data with the texture's current URI
|
|
820
850
|
for (const { obj, key, texture } of this.externalExrPointers) {
|
|
821
851
|
const uri = texture.getURI() || texture.getName() || '';
|
|
@@ -6,7 +6,7 @@ import { existsSync, readFileSync, statSync, writeFileSync } from 'fs';
|
|
|
6
6
|
import { MeshoptDecoder, MeshoptEncoder } from 'meshoptimizer';
|
|
7
7
|
import { dedup, metalRough, prune, resample } from '@gltf-transform/functions';
|
|
8
8
|
import { ALL_EXTENSIONS as NEEDLE_EXTENSIONS, NEEDLE_compression_texture, NEEDLE_mesh_compression, NEEDLE_pmrem, NEEDLE_lightmaps_ext, setExtensionInputFile } from '../extensions/index.js';
|
|
9
|
-
import { isLOD, isMeshLOD, needle_animation_transform, needle_asset, needle_mesh_transform, needle_texture_transform, } from '../transforms/index.js';
|
|
9
|
+
import { clearAudioRegistry, isLOD, isMeshLOD, listAudioRefs, needle_animation_transform, needle_asset, needle_audio_transform, needle_mesh_transform, needle_texture_transform, } from '../transforms/index.js';
|
|
10
10
|
import { copyExternalResources, getOutputPath, getVersion, isMaterialOnlyGLB, ioTryReadWithMissingResources, writeNodeIO } from "../utils/index.js";
|
|
11
11
|
import { addToCache, getHash, tryGetFromCache, } from "../cache/index.js";
|
|
12
12
|
import { TestAssertionError } from '../utils/test-assert.js';
|
|
@@ -70,6 +70,7 @@ export async function packGLTF(inputFile, outputFile, options) {
|
|
|
70
70
|
// Use sourceFile when available (e.g. when inputFile is a progressive output copy)
|
|
71
71
|
const assetSourceFile = options.sourceFile ?? inputFile;
|
|
72
72
|
setExtensionInputFile(assetSourceFile);
|
|
73
|
+
clearAudioRegistry();
|
|
73
74
|
const document = await ioTryReadWithMissingResources(io, inputFile);
|
|
74
75
|
if (options.logger) {
|
|
75
76
|
document.setLogger(options.logger);
|
|
@@ -96,6 +97,7 @@ export async function packGLTF(inputFile, outputFile, options) {
|
|
|
96
97
|
config,
|
|
97
98
|
}),
|
|
98
99
|
needle_mesh_transform({ file: inputFile, config }),
|
|
100
|
+
needle_audio_transform({ outfile: outputFile, useCache, config }),
|
|
99
101
|
needle_animation_transform({}),
|
|
100
102
|
resample({
|
|
101
103
|
tolerance: 1e-6, // NOTE: lowered from 1e-4 to 1e-6 to fix https://linear.app/needle/issue/NE-6872
|
|
@@ -148,6 +150,13 @@ export async function packGLTF(inputFile, outputFile, options) {
|
|
|
148
150
|
}
|
|
149
151
|
}
|
|
150
152
|
}
|
|
153
|
+
for (const ref of listAudioRefs()) {
|
|
154
|
+
const uri = ref.newUri ?? ref.sourceUri;
|
|
155
|
+
const absPath = path.resolve(outDir, uri);
|
|
156
|
+
if (existsSync(absPath)) {
|
|
157
|
+
options.externalFiles.push(absPath);
|
|
158
|
+
}
|
|
159
|
+
}
|
|
151
160
|
}
|
|
152
161
|
logger.debug(`← Writing to ${outputFile}`);
|
|
153
162
|
await writeNodeIO(io, outputFile, document);
|
|
@@ -7,3 +7,6 @@ export * from "./needle_mesh_transform.js";
|
|
|
7
7
|
export * from "./needle_common.js";
|
|
8
8
|
export * from "./needle_texture_transform.js";
|
|
9
9
|
export * from "./needle_animation_transform.js";
|
|
10
|
+
export * from "./needle_audio.js";
|
|
11
|
+
export * from "./needle_audio_registry.js";
|
|
12
|
+
export * from "./needle_audio_ffmpeg.js";
|
package/dist/transforms/index.js
CHANGED
|
@@ -7,3 +7,6 @@ export * from "./needle_mesh_transform.js";
|
|
|
7
7
|
export * from "./needle_common.js";
|
|
8
8
|
export * from "./needle_texture_transform.js";
|
|
9
9
|
export * from "./needle_animation_transform.js";
|
|
10
|
+
export * from "./needle_audio.js";
|
|
11
|
+
export * from "./needle_audio_registry.js";
|
|
12
|
+
export * from "./needle_audio_ffmpeg.js";
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { Transform } from '@gltf-transform/core';
|
|
2
|
+
import { Config } from '../config/index.js';
|
|
3
|
+
export type AudioTransformOptions = {
|
|
4
|
+
/** The path the GLB will be written to. Audio sidecars are placed alongside. */
|
|
5
|
+
outfile: string;
|
|
6
|
+
/** Disk cache toggle */
|
|
7
|
+
useCache: boolean;
|
|
8
|
+
config: Config;
|
|
9
|
+
};
|
|
10
|
+
export declare function needle_audio_transform(options: AudioTransformOptions): Transform;
|
|
11
|
+
/** "click.wav" → "click.opus.ogg". Preserves the basename casing. */
|
|
12
|
+
export declare function computeOpusOggUri(sourceUri: string): string;
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import { createTransform } from '@gltf-transform/functions';
|
|
2
|
+
import { createHash } from 'crypto';
|
|
3
|
+
import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'fs';
|
|
4
|
+
import { basename, dirname, extname, join } from 'path';
|
|
5
|
+
import { v4 as uuid } from 'uuid';
|
|
6
|
+
import tmp from 'tmp';
|
|
7
|
+
import { addToCache, tryGetFromCache } from '../cache/index.js';
|
|
8
|
+
import { safeRmDir } from '../utils/fileutils.js';
|
|
9
|
+
import { encodeOpusOgg, findFfmpeg } from './needle_audio_ffmpeg.js';
|
|
10
|
+
import { listAudioRefs } from './needle_audio_registry.js';
|
|
11
|
+
const DEFAULT_BITRATE_KBPS = 128;
|
|
12
|
+
export function needle_audio_transform(options) {
|
|
13
|
+
return createTransform('needle_audio_transform', async (document) => {
|
|
14
|
+
const refs = listAudioRefs();
|
|
15
|
+
if (refs.length === 0)
|
|
16
|
+
return;
|
|
17
|
+
const logger = document.getLogger();
|
|
18
|
+
const audioCfg = options.config.audio ?? {};
|
|
19
|
+
if (audioCfg.enabled === false) {
|
|
20
|
+
logger.debug(`audio: disabled in config — ${refs.length} external audio reference(s) left untouched`);
|
|
21
|
+
return;
|
|
22
|
+
}
|
|
23
|
+
const bitrate = audioCfg.bitrate ?? DEFAULT_BITRATE_KBPS;
|
|
24
|
+
const ffmpeg = await findFfmpeg();
|
|
25
|
+
if (!ffmpeg) {
|
|
26
|
+
logger.warn(`audio: ffmpeg not found on PATH — ${refs.length} audio file(s) will pass through uncompressed`);
|
|
27
|
+
return;
|
|
28
|
+
}
|
|
29
|
+
if (!ffmpeg.hasLibopus) {
|
|
30
|
+
logger.warn(`audio: ffmpeg is installed but libopus codec is missing — ${refs.length} audio file(s) will pass through uncompressed`);
|
|
31
|
+
return;
|
|
32
|
+
}
|
|
33
|
+
const outDir = dirname(options.outfile);
|
|
34
|
+
const bySource = new Map();
|
|
35
|
+
for (const ref of refs) {
|
|
36
|
+
const list = bySource.get(ref.sourcePath) ?? [];
|
|
37
|
+
list.push(ref);
|
|
38
|
+
bySource.set(ref.sourcePath, list);
|
|
39
|
+
}
|
|
40
|
+
const tmpDir = join(tmp.tmpdir, 'needle-audio', uuid());
|
|
41
|
+
mkdirSync(tmpDir, { recursive: true });
|
|
42
|
+
try {
|
|
43
|
+
for (const [sourcePath, refsForSource] of bySource) {
|
|
44
|
+
if (!existsSync(sourcePath)) {
|
|
45
|
+
logger.warn(`audio: source not found at write time, skipping: ${sourcePath}`);
|
|
46
|
+
continue;
|
|
47
|
+
}
|
|
48
|
+
const sourceUri = refsForSource[0].sourceUri;
|
|
49
|
+
const newUri = computeOpusOggUri(sourceUri);
|
|
50
|
+
const sourceBytes = readFileSync(sourcePath);
|
|
51
|
+
const contentHash = createHash('md5').update(sourceBytes).digest('hex');
|
|
52
|
+
const cacheKey = `audio-opus-${contentHash}-${bitrate}k`;
|
|
53
|
+
let oggBytes = null;
|
|
54
|
+
if (options.useCache) {
|
|
55
|
+
const cached = tryGetFromCache(cacheKey);
|
|
56
|
+
if (cached && cached.length > 0) {
|
|
57
|
+
logger.debug(`audio: cache hit for ${sourceUri} (${cacheKey})`);
|
|
58
|
+
oggBytes = cached;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
if (!oggBytes) {
|
|
62
|
+
const tmpOgg = join(tmpDir, `${uuid()}.ogg`);
|
|
63
|
+
const t0 = Date.now();
|
|
64
|
+
await encodeOpusOgg(ffmpeg.path, sourcePath, tmpOgg, bitrate);
|
|
65
|
+
if (!existsSync(tmpOgg)) {
|
|
66
|
+
logger.error(`audio: ffmpeg did not produce output for ${sourceUri}`);
|
|
67
|
+
continue;
|
|
68
|
+
}
|
|
69
|
+
oggBytes = new Uint8Array(readFileSync(tmpOgg));
|
|
70
|
+
const elapsed = ((Date.now() - t0) / 1000).toFixed(1);
|
|
71
|
+
const srcKb = sourceBytes.length / 1024;
|
|
72
|
+
const dstKb = oggBytes.length / 1024;
|
|
73
|
+
logger.info(`audio: ${sourceUri} (${srcKb.toFixed(0)} KB) → ${newUri} (${dstKb.toFixed(0)} KB) in ${elapsed}s`);
|
|
74
|
+
if (options.useCache)
|
|
75
|
+
addToCache(cacheKey, oggBytes);
|
|
76
|
+
}
|
|
77
|
+
const outPath = join(outDir, newUri);
|
|
78
|
+
mkdirSync(dirname(outPath), { recursive: true });
|
|
79
|
+
writeFileSync(outPath, oggBytes);
|
|
80
|
+
// Remove the source audio file from the output dir (mirrors PMREM at needle_pmrem.ts:161-164).
|
|
81
|
+
// The source bytes have already been read into memory above, so this is safe even in
|
|
82
|
+
// in-place mode (input dir == output dir).
|
|
83
|
+
const sourceInOutDir = join(outDir, sourceUri);
|
|
84
|
+
if (existsSync(sourceInOutDir)) {
|
|
85
|
+
rmSync(sourceInOutDir);
|
|
86
|
+
}
|
|
87
|
+
for (const ref of refsForSource) {
|
|
88
|
+
ref.newUri = newUri;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
finally {
|
|
93
|
+
await safeRmDir(tmpDir);
|
|
94
|
+
}
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
/** "click.wav" → "click.opus.ogg". Preserves the basename casing. */
|
|
98
|
+
export function computeOpusOggUri(sourceUri) {
|
|
99
|
+
const dir = dirname(sourceUri);
|
|
100
|
+
const ext = extname(sourceUri);
|
|
101
|
+
const stem = basename(sourceUri, ext);
|
|
102
|
+
const newName = `${stem}.opus.ogg`;
|
|
103
|
+
return dir === '.' || dir === '' ? newName : join(dir, newName);
|
|
104
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Soft-dependency probe for system `ffmpeg` with `libopus` encoder.
|
|
3
|
+
*
|
|
4
|
+
* Returns `{ path, hasLibopus }` if ffmpeg is on PATH, else `null`. Cached for the
|
|
5
|
+
* process lifetime — `findFfmpeg()` only spawns the probe once. Test code can
|
|
6
|
+
* reset the cache via `__resetFfmpegProbeForTests()`.
|
|
7
|
+
*/
|
|
8
|
+
export type FfmpegInfo = {
|
|
9
|
+
path: string;
|
|
10
|
+
hasLibopus: boolean;
|
|
11
|
+
};
|
|
12
|
+
export declare function findFfmpeg(): Promise<FfmpegInfo | null>;
|
|
13
|
+
export declare function __resetFfmpegProbeForTests(): void;
|
|
14
|
+
export declare function encodeOpusOgg(ffmpegPath: string, inputPath: string, outputPath: string, bitrateKbps: number): Promise<void>;
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Soft-dependency probe for system `ffmpeg` with `libopus` encoder.
|
|
3
|
+
*
|
|
4
|
+
* Returns `{ path, hasLibopus }` if ffmpeg is on PATH, else `null`. Cached for the
|
|
5
|
+
* process lifetime — `findFfmpeg()` only spawns the probe once. Test code can
|
|
6
|
+
* reset the cache via `__resetFfmpegProbeForTests()`.
|
|
7
|
+
*/
|
|
8
|
+
import { execFile } from 'child_process';
|
|
9
|
+
let _probe;
|
|
10
|
+
export function findFfmpeg() {
|
|
11
|
+
if (_probe === undefined) {
|
|
12
|
+
_probe = probe();
|
|
13
|
+
}
|
|
14
|
+
return _probe;
|
|
15
|
+
}
|
|
16
|
+
export function __resetFfmpegProbeForTests() {
|
|
17
|
+
_probe = undefined;
|
|
18
|
+
}
|
|
19
|
+
function probe() {
|
|
20
|
+
return new Promise(resolve => {
|
|
21
|
+
execFile('ffmpeg', ['-hide_banner', '-codecs'], { timeout: 5000, windowsHide: true }, (err, stdout) => {
|
|
22
|
+
if (err) {
|
|
23
|
+
resolve(null);
|
|
24
|
+
return;
|
|
25
|
+
}
|
|
26
|
+
const hasLibopus = /libopus/i.test(stdout) && /opus/i.test(stdout);
|
|
27
|
+
resolve({ path: 'ffmpeg', hasLibopus });
|
|
28
|
+
});
|
|
29
|
+
});
|
|
30
|
+
}
|
|
31
|
+
export function encodeOpusOgg(ffmpegPath, inputPath, outputPath, bitrateKbps) {
|
|
32
|
+
return new Promise((resolve, reject) => {
|
|
33
|
+
const args = [
|
|
34
|
+
'-hide_banner',
|
|
35
|
+
'-loglevel', 'error',
|
|
36
|
+
'-y',
|
|
37
|
+
'-i', inputPath,
|
|
38
|
+
'-vn',
|
|
39
|
+
'-c:a', 'libopus',
|
|
40
|
+
'-b:a', `${bitrateKbps}k`,
|
|
41
|
+
outputPath,
|
|
42
|
+
];
|
|
43
|
+
execFile(ffmpegPath, args, { windowsHide: true }, (err, _stdout, stderr) => {
|
|
44
|
+
if (err) {
|
|
45
|
+
reject(new Error(`ffmpeg failed (exit ${err.code}):\n${stderr || err.message}`));
|
|
46
|
+
return;
|
|
47
|
+
}
|
|
48
|
+
resolve();
|
|
49
|
+
});
|
|
50
|
+
});
|
|
51
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Module-global registry for external audio file references discovered during glTF read.
|
|
3
|
+
*
|
|
4
|
+
* Mirrors the `_extensionInputFile` module-global pattern used elsewhere in the codebase.
|
|
5
|
+
* The CLI processes files sequentially, so the registry is reset at the start of every
|
|
6
|
+
* `packGLTF` call via `clearAudioRegistry()`.
|
|
7
|
+
*
|
|
8
|
+
* Population happens in `JsonPointerHandler.read` when it encounters a string ending
|
|
9
|
+
* in a known audio extension. Consumption happens in two places:
|
|
10
|
+
* 1. `needle_audio_transform` — reads `listAudioRefs()`, compresses each unique
|
|
11
|
+
* source file, fills in `newUri` on every ref pointing at that source.
|
|
12
|
+
* 2. `JsonPointerHandler.resolveAndWrite` — overwrites `ref.obj[ref.key]` with
|
|
13
|
+
* `ref.newUri` so the JSON output points at the compressed file.
|
|
14
|
+
*/
|
|
15
|
+
export type AudioRef = {
|
|
16
|
+
/** The component-data object whose `[key]` field holds the audio path */
|
|
17
|
+
obj: object;
|
|
18
|
+
/** Key on `obj` that holds the original audio URI string */
|
|
19
|
+
key: string;
|
|
20
|
+
/** Absolute path to the source audio file on disk */
|
|
21
|
+
sourcePath: string;
|
|
22
|
+
/** Original URI as it appeared in the JSON (relative path) */
|
|
23
|
+
sourceUri: string;
|
|
24
|
+
/** Set by `needle_audio_transform` after compression. If unset, the JSON ref
|
|
25
|
+
* is left untouched (e.g. ffmpeg missing, compression skipped). */
|
|
26
|
+
newUri?: string;
|
|
27
|
+
};
|
|
28
|
+
export declare function registerAudioRef(ref: AudioRef): void;
|
|
29
|
+
export declare function listAudioRefs(): AudioRef[];
|
|
30
|
+
export declare function clearAudioRegistry(): void;
|
|
31
|
+
/** Audio extensions the compression transform will re-encode. Lossless only —
|
|
32
|
+
* re-encoding lossy formats (mp3/m4a/aac/ogg) would compound artifacts. Those
|
|
33
|
+
* paths are still valid in source glTFs; they are simply passed through. */
|
|
34
|
+
export declare const AUDIO_EXTENSIONS: readonly [".wav"];
|
|
35
|
+
export declare function isAudioUri(value: string): boolean;
|
|
36
|
+
/** Returns true if the URI is a relative path we should attempt to resolve+compress.
|
|
37
|
+
* Absolute URLs (http(s), data, file, blob, leading slash, drive letter) are passed through unchanged. */
|
|
38
|
+
export declare function isRelativeAudioUri(value: string): boolean;
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Module-global registry for external audio file references discovered during glTF read.
|
|
3
|
+
*
|
|
4
|
+
* Mirrors the `_extensionInputFile` module-global pattern used elsewhere in the codebase.
|
|
5
|
+
* The CLI processes files sequentially, so the registry is reset at the start of every
|
|
6
|
+
* `packGLTF` call via `clearAudioRegistry()`.
|
|
7
|
+
*
|
|
8
|
+
* Population happens in `JsonPointerHandler.read` when it encounters a string ending
|
|
9
|
+
* in a known audio extension. Consumption happens in two places:
|
|
10
|
+
* 1. `needle_audio_transform` — reads `listAudioRefs()`, compresses each unique
|
|
11
|
+
* source file, fills in `newUri` on every ref pointing at that source.
|
|
12
|
+
* 2. `JsonPointerHandler.resolveAndWrite` — overwrites `ref.obj[ref.key]` with
|
|
13
|
+
* `ref.newUri` so the JSON output points at the compressed file.
|
|
14
|
+
*/
|
|
15
|
+
let _refs = [];
|
|
16
|
+
export function registerAudioRef(ref) {
|
|
17
|
+
_refs.push(ref);
|
|
18
|
+
}
|
|
19
|
+
export function listAudioRefs() {
|
|
20
|
+
return _refs;
|
|
21
|
+
}
|
|
22
|
+
export function clearAudioRegistry() {
|
|
23
|
+
_refs = [];
|
|
24
|
+
}
|
|
25
|
+
/** Audio extensions the compression transform will re-encode. Lossless only —
|
|
26
|
+
* re-encoding lossy formats (mp3/m4a/aac/ogg) would compound artifacts. Those
|
|
27
|
+
* paths are still valid in source glTFs; they are simply passed through. */
|
|
28
|
+
export const AUDIO_EXTENSIONS = ['.wav'];
|
|
29
|
+
const _audioRegex = new RegExp(`(${AUDIO_EXTENSIONS.map(e => '\\' + e).join('|')})$`, 'i');
|
|
30
|
+
export function isAudioUri(value) {
|
|
31
|
+
return _audioRegex.test(value);
|
|
32
|
+
}
|
|
33
|
+
/** Returns true if the URI is a relative path we should attempt to resolve+compress.
|
|
34
|
+
* Absolute URLs (http(s), data, file, blob, leading slash, drive letter) are passed through unchanged. */
|
|
35
|
+
export function isRelativeAudioUri(value) {
|
|
36
|
+
if (!value)
|
|
37
|
+
return false;
|
|
38
|
+
if (/^[a-z]+:/i.test(value))
|
|
39
|
+
return false; // http:, https:, data:, file:, blob:, etc.
|
|
40
|
+
if (value.startsWith('//'))
|
|
41
|
+
return false; // protocol-relative
|
|
42
|
+
if (value.startsWith('/'))
|
|
43
|
+
return false; // POSIX absolute
|
|
44
|
+
if (/^[a-z]:[\\/]/i.test(value))
|
|
45
|
+
return false; // Windows drive (C:\ or C:/)
|
|
46
|
+
return true;
|
|
47
|
+
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { ILogger, Texture, Transform, vec2 } from '@gltf-transform/core';
|
|
1
|
+
import { ILogger, Primitive, Texture, Transform, TypedArray, vec2 } from '@gltf-transform/core';
|
|
2
2
|
import { Config } from '../config/index.js';
|
|
3
3
|
declare interface ProgressiveOptions {
|
|
4
4
|
path: string | Array<string>;
|
|
@@ -39,6 +39,14 @@ export declare enum TextureResizeFilter {
|
|
|
39
39
|
export declare const TEXTURE_RESIZE_DEFAULTS: TextureResizeOptions;
|
|
40
40
|
export declare function resizeTextureToMax(texture: Texture, maxSize: number, logger: ILogger): Promise<void>;
|
|
41
41
|
export declare function make_progressive_meshes(filePath: string, _options: any, opts?: ProgressiveOptions): Transform;
|
|
42
|
+
export declare type PrimitiveData = {
|
|
43
|
+
indices: TypedArray;
|
|
44
|
+
attributes: {
|
|
45
|
+
[key: string]: TypedArray;
|
|
46
|
+
};
|
|
47
|
+
};
|
|
48
|
+
export declare function clonePrimitiveData(prim: Primitive, logger: ILogger): PrimitiveData | null;
|
|
49
|
+
export declare function setPrimitiveData(prim: Primitive, data: PrimitiveData): void;
|
|
42
50
|
export declare function isMeshLOD(filepath: string): boolean;
|
|
43
51
|
export declare function isImageLOD(filepath: string): boolean;
|
|
44
52
|
export declare function isLOD(filepath: string): boolean;
|
|
@@ -304,10 +304,11 @@ export function make_progressive_textures(filePath, _options = TEXTURE_RESIZE_DE
|
|
|
304
304
|
// If any texture LOD was generated we can resize the texture here
|
|
305
305
|
// E.g. for specular/glossiness textures we might not generate any LODs and then we don't want to downsize the texture
|
|
306
306
|
if (results?.length) {
|
|
307
|
-
options
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
307
|
+
// Build a per-iteration resize options object — never mutate the shared `options`,
|
|
308
|
+
// otherwise the next texture's size check (line ~238) would see the previous texture's maxSize.
|
|
309
|
+
const resizeOpts = { ...options, size: [settings.maxSize, settings.maxSize] };
|
|
310
|
+
logger.debug(`[${NAME}] Resizing embedded texture[${index}] to ${resizeOpts.size[0]}x${resizeOpts.size[1]}`);
|
|
311
|
+
await resizeTexture(resizeOpts, texture, logger, uri || name, slots);
|
|
311
312
|
if (opts?.useCache === false) {
|
|
312
313
|
}
|
|
313
314
|
else
|
|
@@ -453,7 +454,7 @@ async function createTextureLod(args, opts) {
|
|
|
453
454
|
mkdirSync(basePath, { recursive: true });
|
|
454
455
|
}
|
|
455
456
|
logger.info(`[${NAME}] Write texture LOD ${level} to \"${filename}\"`);
|
|
456
|
-
newIo.write(outName, newDoc);
|
|
457
|
+
await newIo.write(outName, newDoc);
|
|
457
458
|
logger.info(`[${NAME}] Generate hash for texture LOD ${level}`);
|
|
458
459
|
const hash = getHash(newTexture);
|
|
459
460
|
return {
|
|
@@ -830,24 +831,30 @@ export function make_progressive_meshes(filePath, _options, opts) {
|
|
|
830
831
|
function deleteFile(filePath, _context) {
|
|
831
832
|
rmSync(filePath);
|
|
832
833
|
}
|
|
833
|
-
|
|
834
|
+
// Clone a TypedArray preserving its concrete type (Uint16Array stays Uint16Array etc.).
|
|
835
|
+
// Coercing to Float32Array breaks integer attributes (JOINTS, FEATURE_ID, indices) and changes
|
|
836
|
+
// the accessor's componentType when written back via setArray.
|
|
837
|
+
function cloneTypedArray(src) {
|
|
838
|
+
return src.slice();
|
|
839
|
+
}
|
|
840
|
+
export function clonePrimitiveData(prim, logger) {
|
|
834
841
|
const semantics = prim.listSemantics();
|
|
835
842
|
if (semantics.some(s => s.includes("JOINTS"))) {
|
|
836
843
|
return null;
|
|
837
844
|
}
|
|
838
|
-
const
|
|
839
|
-
const indices =
|
|
845
|
+
const prevIndicesArray = prim.getIndices()?.getArray();
|
|
846
|
+
const indices = prevIndicesArray ? cloneTypedArray(prevIndicesArray) : new Uint32Array();
|
|
840
847
|
const attributes = {};
|
|
841
848
|
for (const attr of semantics) {
|
|
842
849
|
logger.info(`[${NAME}] Clone attribute ${attr}`);
|
|
843
|
-
const
|
|
844
|
-
if (
|
|
845
|
-
attributes[attr] =
|
|
850
|
+
const src = prim.getAttribute(attr)?.getArray();
|
|
851
|
+
if (src) {
|
|
852
|
+
attributes[attr] = cloneTypedArray(src);
|
|
846
853
|
}
|
|
847
854
|
}
|
|
848
855
|
return { indices, attributes };
|
|
849
856
|
}
|
|
850
|
-
function setPrimitiveData(prim, data) {
|
|
857
|
+
export function setPrimitiveData(prim, data) {
|
|
851
858
|
const indices = prim.getIndices();
|
|
852
859
|
if (indices) {
|
|
853
860
|
indices.setArray(data.indices);
|
|
@@ -1090,11 +1097,20 @@ async function simplifyMesh(doc, mesh, prim, index, lodinfo) {
|
|
|
1090
1097
|
};
|
|
1091
1098
|
return compressPrimitive(doc, mesh, prim, index, weldOptions, simplifyOptions);
|
|
1092
1099
|
}
|
|
1100
|
+
// Match only files this pipeline emits: <basename>_<level>_<uuid>.glb
|
|
1101
|
+
// (uuid is generated via generateGuid → uuid v5, 36 chars with hyphens).
|
|
1102
|
+
// Anchoring against the basename + UUID format prevents false positives from
|
|
1103
|
+
// user assets like "image_hero.glb" or "/path/mesh_lod_test.glb", which the
|
|
1104
|
+
// previous substring check (filepath.includes(...)) misclassified as LODs,
|
|
1105
|
+
// causing the pipeline to silently skip them.
|
|
1106
|
+
const UUID_RE_SRC = '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}';
|
|
1107
|
+
const IMAGE_LOD_RE = new RegExp(`(?:^|[\\\\/])image_\\d+_${UUID_RE_SRC}\\.glb$`, 'i');
|
|
1108
|
+
const MESH_LOD_RE = new RegExp(`(?:^|[\\\\/])mesh_lod_\\d+_${UUID_RE_SRC}\\.glb$`, 'i');
|
|
1093
1109
|
export function isMeshLOD(filepath) {
|
|
1094
|
-
return
|
|
1110
|
+
return MESH_LOD_RE.test(filepath);
|
|
1095
1111
|
}
|
|
1096
1112
|
export function isImageLOD(filepath) {
|
|
1097
|
-
return
|
|
1113
|
+
return IMAGE_LOD_RE.test(filepath);
|
|
1098
1114
|
}
|
|
1099
1115
|
export function isLOD(filepath) {
|
|
1100
1116
|
return isMeshLOD(filepath) || isImageLOD(filepath);
|
|
@@ -96,6 +96,10 @@ async function processTexture(texture, index, document, logger, context) {
|
|
|
96
96
|
const originalMimeType = texture.getMimeType();
|
|
97
97
|
// Handle EXR textures via PMREM → KTX2 HDR
|
|
98
98
|
if (originalMimeType === "image/exr") {
|
|
99
|
+
if (context.config.exr?.enabled === false) {
|
|
100
|
+
logger.debug(`→ Skipping EXR -> KTX2 HDR for ${texture.getName()} [${index}] (config.exr.enabled = false)`);
|
|
101
|
+
return;
|
|
102
|
+
}
|
|
99
103
|
await context.pmremContext.process(index, texture, null, context);
|
|
100
104
|
return;
|
|
101
105
|
}
|
package/dist/transforms/toktx.js
CHANGED
|
@@ -4,8 +4,7 @@ import os from 'os';
|
|
|
4
4
|
import semver from 'semver';
|
|
5
5
|
import tmp from 'tmp';
|
|
6
6
|
import { TextureChannel } from '@gltf-transform/core';
|
|
7
|
-
import {
|
|
8
|
-
import { spawn, commandExists, waitExit, MICROMATCH_OPTIONS } from './util.js';
|
|
7
|
+
import { spawn, waitExit, MICROMATCH_OPTIONS } from './util.js';
|
|
9
8
|
import { TextureResizeFilter, getTextureColorSpace } from '@gltf-transform/functions';
|
|
10
9
|
import { ETC1S_DEFAULTS, UASTC_DEFAULTS } from './toktx_types.js';
|
|
11
10
|
tmp.setGracefulCleanup();
|
|
@@ -143,53 +142,41 @@ export function createParams(texture, slots, channels, size, logger, numTextures
|
|
|
143
142
|
}
|
|
144
143
|
return params;
|
|
145
144
|
}
|
|
146
|
-
let
|
|
145
|
+
let didLogKtxVersion = false;
|
|
147
146
|
export async function checkKTXSoftware(logger, env) {
|
|
148
|
-
|
|
149
|
-
|
|
147
|
+
// Probe the `ktx` binary because that's what the compression step actually runs
|
|
148
|
+
// (see needle_toktx.ts: spawn('ktx', params, ...)). Probing `toktx` would let a
|
|
149
|
+
// partial install pass the prerequisite check and then fail mid-compression
|
|
150
|
+
// when `ktx create` is invoked. `ktx` ships with KTX-Software 4.3+ alongside
|
|
151
|
+
// `toktx`, so this matches our KTX_SOFTWARE_VERSION_MIN.
|
|
152
|
+
let failedFindingKtx = false;
|
|
153
|
+
const proc = spawn('ktx', ['--version'], { env: env });
|
|
150
154
|
proc.on('error', (err) => {
|
|
151
155
|
if (err.message.endsWith("ENOENT")) {
|
|
152
|
-
|
|
156
|
+
failedFindingKtx = true;
|
|
153
157
|
}
|
|
154
158
|
});
|
|
155
159
|
const [status, stdout, stderr] = await waitExit(proc);
|
|
156
|
-
if (
|
|
157
|
-
console.log(`WARN: Unable to find "
|
|
160
|
+
if (failedFindingKtx) {
|
|
161
|
+
console.log(`WARN: Unable to find "ktx". Confirm KTX-Software is installed from: https://github.com/KhronosGroup/KTX-Software.`);
|
|
158
162
|
return null;
|
|
159
163
|
}
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
throw new Error('Unable to find "toktx" version. Confirm KTX-Software is installed from:\n\nhttps://github.com/KhronosGroup/KTX-Software.');
|
|
164
|
+
// Extract the X.Y.Z from outputs like "ktx version: v4.3.2~6" or "toktx v4.3.0~28".
|
|
165
|
+
const versionMatch = (stdout || stderr).match(/v?(\d+\.\d+\.\d+)/);
|
|
166
|
+
const version = versionMatch ? versionMatch[1] : null;
|
|
167
|
+
if (status !== 0 || !version || !semver.valid(version)) {
|
|
168
|
+
throw new Error('Unable to find "ktx" version. Confirm KTX-Software is installed from:\n\nhttps://github.com/KhronosGroup/KTX-Software.');
|
|
166
169
|
}
|
|
167
|
-
else if (semver.lt(
|
|
168
|
-
logger.warn(`
|
|
170
|
+
else if (semver.lt(version, KTX_SOFTWARE_VERSION_MIN)) {
|
|
171
|
+
logger.warn(`ktx: Expected KTX-Software >= v${KTX_SOFTWARE_VERSION_MIN}, found ${version}.`);
|
|
169
172
|
return null;
|
|
170
173
|
}
|
|
171
174
|
else {
|
|
172
|
-
if (!
|
|
173
|
-
logger.debug(`
|
|
174
|
-
|
|
175
|
+
if (!didLogKtxVersion)
|
|
176
|
+
logger.debug(`ktx: Found KTX-Software ${version}.`);
|
|
177
|
+
didLogKtxVersion = true;
|
|
175
178
|
}
|
|
176
|
-
return
|
|
177
|
-
}
|
|
178
|
-
async function toktxCommandExistsSync(env) {
|
|
179
|
-
if (env) {
|
|
180
|
-
// if env is defined use that to check if toktx is installed
|
|
181
|
-
try {
|
|
182
|
-
execSync('toktx --version', { encoding: 'utf-8', env: env });
|
|
183
|
-
return true;
|
|
184
|
-
}
|
|
185
|
-
catch (error) {
|
|
186
|
-
return false;
|
|
187
|
-
}
|
|
188
|
-
}
|
|
189
|
-
else if (!await commandExists('toktx')) {
|
|
190
|
-
return false;
|
|
191
|
-
}
|
|
192
|
-
return true;
|
|
179
|
+
return version;
|
|
193
180
|
}
|
|
194
181
|
function isPowerOfTwo(value) {
|
|
195
182
|
if (value <= 2)
|
package/dist/utils/index.d.ts
CHANGED
package/dist/utils/index.js
CHANGED
package/dist/utils/stats.js
CHANGED
|
@@ -88,6 +88,14 @@ export class FileStats extends Stats {
|
|
|
88
88
|
this.isLOD = isLOD(this.filename);
|
|
89
89
|
}
|
|
90
90
|
}
|
|
91
|
+
// Union the two arrays, tolerating undefined on either side. `new Set(undefined)`
|
|
92
|
+
// throws, which is what the prior implementation would do whenever `this.errors`
|
|
93
|
+
// was undefined but `other.errors` was non-empty.
|
|
94
|
+
function mergeUnique(a, b) {
|
|
95
|
+
if (!a && !b)
|
|
96
|
+
return undefined;
|
|
97
|
+
return Array.from(new Set([...(a ?? []), ...(b ?? [])]));
|
|
98
|
+
}
|
|
91
99
|
export class MeshInfo {
|
|
92
100
|
count = 0;
|
|
93
101
|
primitives = 0;
|
|
@@ -104,8 +112,8 @@ export class MeshInfo {
|
|
|
104
112
|
this.triangles += other.triangles;
|
|
105
113
|
this.memory += other.memory;
|
|
106
114
|
this.formats = Array.from(new Set(this.formats.concat(other.formats)));
|
|
107
|
-
this.errors =
|
|
108
|
-
this.warnings =
|
|
115
|
+
this.errors = mergeUnique(this.errors, other.errors);
|
|
116
|
+
this.warnings = mergeUnique(this.warnings, other.warnings);
|
|
109
117
|
}
|
|
110
118
|
}
|
|
111
119
|
export class TextureInfo {
|
|
@@ -120,8 +128,8 @@ export class TextureInfo {
|
|
|
120
128
|
this.memory += other.memory;
|
|
121
129
|
this.gpu_memory += other.gpu_memory;
|
|
122
130
|
this.formats = Array.from(new Set(this.formats.concat(other.formats)));
|
|
123
|
-
this.errors =
|
|
124
|
-
this.warnings =
|
|
131
|
+
this.errors = mergeUnique(this.errors, other.errors);
|
|
132
|
+
this.warnings = mergeUnique(this.warnings, other.warnings);
|
|
125
133
|
}
|
|
126
134
|
}
|
|
127
135
|
export class AnimationInfo {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
export declare const version = "2.16.0-alpha.
|
|
1
|
+
export declare const version = "2.16.0-alpha.eb5a640";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
export const version = "2.16.0-alpha.
|
|
1
|
+
export const version = "2.16.0-alpha.eb5a640";
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@needle-tools/gltf-build-pipeline",
|
|
3
|
-
"version": "2.16.0-alpha.
|
|
3
|
+
"version": "2.16.0-alpha.eb5a640",
|
|
4
4
|
"description": "Pipeline and tools for optimizing gltf files using gltf-transform and compression settings within glTF extensions",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"type": "module",
|
|
Binary file
|
package/dist/utils/merge.d.ts
DELETED
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Simple object check.
|
|
3
|
-
* @param item
|
|
4
|
-
* @returns {boolean}
|
|
5
|
-
*/
|
|
6
|
-
export declare function isObject(item: object): boolean;
|
|
7
|
-
/**
|
|
8
|
-
* Deep merge two objects.
|
|
9
|
-
* @param target
|
|
10
|
-
* @param ...sources
|
|
11
|
-
*/
|
|
12
|
-
export declare function mergeDeep(target: any, ...sources: any[]): any;
|
package/dist/utils/merge.js
DELETED
|
@@ -1,31 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Simple object check.
|
|
3
|
-
* @param item
|
|
4
|
-
* @returns {boolean}
|
|
5
|
-
*/
|
|
6
|
-
export function isObject(item) {
|
|
7
|
-
return (item && typeof item === 'object' && !Array.isArray(item));
|
|
8
|
-
}
|
|
9
|
-
/**
|
|
10
|
-
* Deep merge two objects.
|
|
11
|
-
* @param target
|
|
12
|
-
* @param ...sources
|
|
13
|
-
*/
|
|
14
|
-
export function mergeDeep(target, ...sources) {
|
|
15
|
-
if (!sources.length)
|
|
16
|
-
return target;
|
|
17
|
-
const source = sources.shift();
|
|
18
|
-
if (isObject(target) && isObject(source)) {
|
|
19
|
-
for (const key in source) {
|
|
20
|
-
if (isObject(source[key])) {
|
|
21
|
-
if (!target[key])
|
|
22
|
-
Object.assign(target, { [key]: {} });
|
|
23
|
-
mergeDeep(target[key], source[key]);
|
|
24
|
-
}
|
|
25
|
-
else {
|
|
26
|
-
Object.assign(target, { [key]: source[key] });
|
|
27
|
-
}
|
|
28
|
-
}
|
|
29
|
-
}
|
|
30
|
-
return mergeDeep(target, ...sources);
|
|
31
|
-
}
|