@needle-tools/gltf-build-pipeline 2.15.0-next.d34f97f → 2.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +5 -0
- package/dist/cache/cache.js +19 -15
- package/dist/cli/index.js +19 -0
- package/dist/extensions/NEEDLE_opaque/NEEDLE_opaque.d.ts +1 -0
- package/dist/extensions/NEEDLE_opaque/NEEDLE_opaque.js +59 -0
- package/dist/extensions/NEEDLE_pmrem/index.d.ts +4 -2
- package/dist/extensions/NEEDLE_pmrem/index.js +52 -1
- package/dist/extensions/index.js +5 -1
- package/dist/scripts/pack-gltf.js +3 -2
- package/dist/transforms/needle_toktx.js +2 -3
- package/dist/utils/analytics.d.ts +24 -0
- package/dist/utils/analytics.js +126 -0
- package/dist/utils/version.gen.d.ts +1 -1
- package/dist/utils/version.gen.js +1 -1
- package/package.json +1 -1
- package/tools/pmrem/basis/linux/SHA_d8f36fcbb09e79614e96baaed5b1e1cd5d2f5e73.txt +1 -0
- package/tools/pmrem/basis/linux/basisu +0 -0
- package/tools/pmrem/basis/osx/SHA_d8f36fcbb09e79614e96baaed5b1e1cd5d2f5e73.txt +1 -0
- package/tools/pmrem/basis/osx/basisu +0 -0
- package/tools/pmrem/basis/wasm/SHA_d8f36fcbb09e79614e96baaed5b1e1cd5d2f5e73.txt +1 -0
- package/tools/pmrem/basis/wasm/basis_encoder_threads.js +2 -6693
- package/tools/pmrem/basis/wasm/basis_encoder_threads.wasm +0 -0
- package/tools/pmrem/basis/win/SHA_d8f36fcbb09e79614e96baaed5b1e1cd5d2f5e73.txt +1 -0
- package/tools/pmrem/basis/win/basisu.exe +0 -0
- package/tools/pmrem/basis/win/ocl_kernels.cl +66 -90
- package/tools/pmrem/pkg/pmrem_wasm_bg.wasm +0 -0
package/CHANGELOG.md
CHANGED
|
@@ -4,6 +4,11 @@ All notable changes to this package will be documented in this file.
|
|
|
4
4
|
The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
|
|
5
5
|
and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
|
|
6
6
|
|
|
7
|
+
## [2.15.0] - 2026-03-20
|
|
8
|
+
- add: EXR environment map compression (PMREM / FASTHDR) for Needle Engine, supporting both Unity and Blender exports
|
|
9
|
+
- fix: cache hashing stability (circular reference handling)
|
|
10
|
+
- fix: progressive texture GUID generation now based on texture content hash
|
|
11
|
+
|
|
7
12
|
## [2.14.0] - 2026-01-30
|
|
8
13
|
- add: KHR_materials_pbrSpecularGlossiness extension support
|
|
9
14
|
- fix: reduce `resample` tolerance to `1e-6` to avoid animation issues with some assets
|
package/dist/cache/cache.js
CHANGED
|
@@ -191,8 +191,6 @@ export function getHash(property, level = 0) {
|
|
|
191
191
|
if (property instanceof ExtensibleProperty) {
|
|
192
192
|
const extensions = property.listExtensions();
|
|
193
193
|
const extras = property.getExtras();
|
|
194
|
-
// const slots = listTextureSlots(doc, texture);
|
|
195
|
-
// const channels = getTextureChannelMask(doc, texture);
|
|
196
194
|
hash += hashObject({
|
|
197
195
|
extensions,
|
|
198
196
|
extras,
|
|
@@ -269,38 +267,44 @@ function hashObject(obj, depth = 0) {
|
|
|
269
267
|
return hashString(String(obj));
|
|
270
268
|
}
|
|
271
269
|
try {
|
|
272
|
-
const
|
|
270
|
+
const seen = new WeakSet();
|
|
271
|
+
const str = JSON.stringify(obj, (_key, value) => {
|
|
272
|
+
if (typeof value === 'object' && value !== null) {
|
|
273
|
+
if (seen.has(value))
|
|
274
|
+
return undefined;
|
|
275
|
+
seen.add(value);
|
|
276
|
+
}
|
|
277
|
+
if (typeof value === 'function')
|
|
278
|
+
return undefined;
|
|
279
|
+
return value;
|
|
280
|
+
});
|
|
273
281
|
return hashString(str);
|
|
274
282
|
}
|
|
275
283
|
catch (err) {
|
|
276
284
|
console.warn("WARN: [Cache] error during hashing of object:\n" + err);
|
|
277
285
|
if (debugLogs)
|
|
278
286
|
debugger;
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
let fallbackHashj = 0;
|
|
282
|
-
if (typeof obj === 'object') {
|
|
283
|
-
if (obj === null)
|
|
284
|
-
return 0;
|
|
287
|
+
let fallbackHash = 0;
|
|
288
|
+
if (typeof obj === 'object' && obj !== null) {
|
|
285
289
|
for (const key in obj) {
|
|
286
290
|
if (Object.prototype.hasOwnProperty.call(obj, key)) {
|
|
287
291
|
const value = obj[key];
|
|
288
|
-
|
|
289
|
-
if (typeof value === 'object') {
|
|
292
|
+
fallbackHash += hashString(key);
|
|
293
|
+
if (typeof value === 'object' && value !== null) {
|
|
290
294
|
if (depth > 10) {
|
|
291
|
-
|
|
295
|
+
fallbackHash += hashString("[object]");
|
|
292
296
|
}
|
|
293
297
|
else {
|
|
294
|
-
|
|
298
|
+
fallbackHash += hashObject(value, depth + 1);
|
|
295
299
|
}
|
|
296
300
|
}
|
|
297
301
|
else {
|
|
298
|
-
|
|
302
|
+
fallbackHash += hashString(String(value));
|
|
299
303
|
}
|
|
300
304
|
}
|
|
301
305
|
}
|
|
302
306
|
}
|
|
303
|
-
return
|
|
307
|
+
return fallbackHash;
|
|
304
308
|
}
|
|
305
309
|
}
|
|
306
310
|
function hashBuffer(buffer, offset, length) {
|
package/dist/cli/index.js
CHANGED
|
@@ -10,6 +10,7 @@ import { existsSync } from 'fs';
|
|
|
10
10
|
import { ensureIsDirectory, foreachGLTF, isDirectory } from '../utils/fileutils.js';
|
|
11
11
|
import path, { resolve } from 'path';
|
|
12
12
|
import { ERROR_CODES } from '../constants.js';
|
|
13
|
+
import { trackPipelineStart, trackPipelineEnd, trackError } from '../utils/analytics.js';
|
|
13
14
|
// For testing / dev you can run `npm link` in the package directory
|
|
14
15
|
// For removing the link run `npm rm --global @needle-tools/gltf-build-pipeline`
|
|
15
16
|
program
|
|
@@ -73,6 +74,8 @@ Each version will be compressed and written to the output directory.
|
|
|
73
74
|
return;
|
|
74
75
|
}
|
|
75
76
|
logger.info(`[Needle Build Pipeline] v${getVersion()} — Transform '${args.input}'`);
|
|
77
|
+
const pipelineMode = [options.progressive && 'progressive', options.compress && 'compress'].filter(Boolean).join('+');
|
|
78
|
+
trackPipelineStart({ mode: pipelineMode, usecase: options.usecase?.toString() });
|
|
76
79
|
const stats = {
|
|
77
80
|
startTime: Date.now(),
|
|
78
81
|
totalGPUMemoryInMB: 0,
|
|
@@ -103,6 +106,7 @@ Each version will be compressed and written to the output directory.
|
|
|
103
106
|
const outputIsDirectory = output && isDirectory(output);
|
|
104
107
|
if (inputIsDirectory && output && !outputIsDirectory) {
|
|
105
108
|
logger.error("When <input> is a directory then <output> must also be a directory");
|
|
109
|
+
trackError(new Error('Invalid args: input is directory but output is not'), { mode: pipelineMode });
|
|
106
110
|
process.exit(ERROR_CODES.INVALID_ARGS);
|
|
107
111
|
}
|
|
108
112
|
// Handle the output bath being relative to the input path
|
|
@@ -126,6 +130,7 @@ Each version will be compressed and written to the output directory.
|
|
|
126
130
|
}
|
|
127
131
|
else {
|
|
128
132
|
logger.error(`ERR: Previously found file is now missing: \"${file}\"`);
|
|
133
|
+
trackError(new Error(`Lost file: ${file}`), { mode: pipelineMode, usecase: options.usecase?.toString(), file });
|
|
129
134
|
process.exit(ERROR_CODES.LOST_FILE);
|
|
130
135
|
}
|
|
131
136
|
}
|
|
@@ -148,8 +153,10 @@ Each version will be compressed and written to the output directory.
|
|
|
148
153
|
logger,
|
|
149
154
|
}).catch(err => {
|
|
150
155
|
logger.error(err);
|
|
156
|
+
trackError(err, { mode: pipelineMode, usecase: options.usecase?.toString(), file });
|
|
151
157
|
process.exit(ERROR_CODES.PROGRESSIVE_FAILED);
|
|
152
158
|
}) === false) {
|
|
159
|
+
trackError(new Error('Progressive processing returned false'), { mode: pipelineMode, usecase: options.usecase?.toString(), file });
|
|
153
160
|
process.exit(ERROR_CODES.PROGRESSIVE_FAILED);
|
|
154
161
|
}
|
|
155
162
|
if (verbose) {
|
|
@@ -170,6 +177,7 @@ Each version will be compressed and written to the output directory.
|
|
|
170
177
|
if (progressive_results.length > 0) {
|
|
171
178
|
for (const res of progressive_results) {
|
|
172
179
|
if (await packGLTF(res, res, opts) === false) {
|
|
180
|
+
trackError(new Error('Packing failed'), { mode: pipelineMode, usecase: options.usecase?.toString(), file: res });
|
|
173
181
|
process.exit(ERROR_CODES.PACKING_FAILED);
|
|
174
182
|
}
|
|
175
183
|
}
|
|
@@ -178,12 +186,22 @@ Each version will be compressed and written to the output directory.
|
|
|
178
186
|
// We can not just check if the file is in the list of results because the output path might have changed the name
|
|
179
187
|
else {
|
|
180
188
|
if (await packGLTF(file, output, opts) === false) {
|
|
189
|
+
trackError(new Error('Packing failed'), { mode: pipelineMode, usecase: options.usecase?.toString(), file });
|
|
181
190
|
process.exit(ERROR_CODES.PACKING_FAILED);
|
|
182
191
|
}
|
|
183
192
|
}
|
|
184
193
|
}
|
|
185
194
|
});
|
|
186
195
|
printStats(stats, logger, config);
|
|
196
|
+
trackPipelineEnd({
|
|
197
|
+
mode: pipelineMode,
|
|
198
|
+
usecase: config?.usecase?.toString(),
|
|
199
|
+
filesProcessed: stats.totalFilesProcessed,
|
|
200
|
+
sizeBefore_MB: stats.totalFileSizeInMBBefore,
|
|
201
|
+
sizeAfter_MB: stats.totalFileSizeInMB,
|
|
202
|
+
gpuMemory_MB: stats.totalGPUMemoryInMB,
|
|
203
|
+
duration_ms: Date.now() - stats.startTime,
|
|
204
|
+
});
|
|
187
205
|
if (useCache)
|
|
188
206
|
limitCacheSize();
|
|
189
207
|
});
|
|
@@ -194,5 +212,6 @@ program.run().catch(err => {
|
|
|
194
212
|
// ignore
|
|
195
213
|
return;
|
|
196
214
|
}
|
|
215
|
+
trackError(err);
|
|
197
216
|
throw err;
|
|
198
217
|
});
|
|
@@ -23,5 +23,6 @@ export interface IExtensionWriter {
|
|
|
23
23
|
write(context: WriterContext, prop: IExtensibleProperty): void;
|
|
24
24
|
}
|
|
25
25
|
export declare let currentGenerator: string;
|
|
26
|
+
export declare function setOpaqueInputFile(path: string): void;
|
|
26
27
|
export declare function createOpaqueExtension(name: string, types: PropertyType | PropertyType[], opts?: Options): IOpaqueExtension;
|
|
27
28
|
export {};
|
|
@@ -3,11 +3,16 @@ import { getTexture, tryGetInArray, writeExtension } from '../utils.js';
|
|
|
3
3
|
import { isLightmap } from '../../utils/texture.js';
|
|
4
4
|
import { NEEDLE_progressive_texture_settings } from '../NEEDLE_progressive_texture_settings/index.js';
|
|
5
5
|
import { testAssert } from '../../utils/test-assert.js';
|
|
6
|
+
import { existsSync, readFileSync } from 'fs';
|
|
7
|
+
import { dirname, join } from 'path';
|
|
6
8
|
const ALL_PROPERTY_TYPES = [];
|
|
7
9
|
for (const key in PropertyType) {
|
|
8
10
|
ALL_PROPERTY_TYPES.push(PropertyType[key]);
|
|
9
11
|
}
|
|
10
12
|
export let currentGenerator = "";
|
|
13
|
+
/** Set before io.read() to allow resolving external file paths (e.g. .exr) in component data */
|
|
14
|
+
let _opaqueInputFile = "";
|
|
15
|
+
export function setOpaqueInputFile(path) { _opaqueInputFile = path; }
|
|
11
16
|
// an opaque extension can be used to pass data through the gltf transform process
|
|
12
17
|
// it should also be able to re-write/fix json pointers after the transformation which might have shuffled them around
|
|
13
18
|
export function createOpaqueExtension(name, types, opts) {
|
|
@@ -539,6 +544,10 @@ class JsonPointerHandler {
|
|
|
539
544
|
debug = false;
|
|
540
545
|
breakPoint = false;
|
|
541
546
|
VRM_0_MaterialIndexRegex = new RegExp(/materialProperties\/(?<material_index>\d+)\//g);
|
|
547
|
+
/** Cache of external EXR files already injected as Textures (keyed by filename) */
|
|
548
|
+
externalExrTextures = new Map();
|
|
549
|
+
/** External EXR references that need URI resolution at write time */
|
|
550
|
+
externalExrPointers = [];
|
|
542
551
|
constructor(document, extension_name) {
|
|
543
552
|
this.document = document;
|
|
544
553
|
this.extension_name = extension_name;
|
|
@@ -620,6 +629,37 @@ class JsonPointerHandler {
|
|
|
620
629
|
else
|
|
621
630
|
console.warn("WARN: failed registering pointer", fullPath, value);
|
|
622
631
|
}
|
|
632
|
+
// External EXR file reference (e.g. "StudioHDRI_ferndale_studio_04_1k.exr" in a ReflectionProbe)
|
|
633
|
+
// Inject the file into the document as a Texture so PMREM can process it
|
|
634
|
+
if (!isTexturePointer && value.match(/\.exr$/i)) {
|
|
635
|
+
const inputDir = _opaqueInputFile ? dirname(_opaqueInputFile) : '';
|
|
636
|
+
const exrPath = inputDir ? join(inputDir, value) : '';
|
|
637
|
+
if (exrPath && existsSync(exrPath)) {
|
|
638
|
+
// Check if we already injected this file
|
|
639
|
+
let texture = this.externalExrTextures.get(value);
|
|
640
|
+
if (!texture) {
|
|
641
|
+
const exrBytes = readFileSync(exrPath);
|
|
642
|
+
texture = this.document.createTexture(value)
|
|
643
|
+
.setImage(new Uint8Array(exrBytes))
|
|
644
|
+
.setMimeType('image/exr')
|
|
645
|
+
.setURI(value);
|
|
646
|
+
this.externalExrTextures.set(value, texture);
|
|
647
|
+
if (this.debug)
|
|
648
|
+
console.log(`Injected external EXR "${value}" as Texture`);
|
|
649
|
+
}
|
|
650
|
+
// Set a reference so gltf-transform doesn't prune it
|
|
651
|
+
prop.setReference(`exr_${value}`, texture, { channels: TextureChannel.R | TextureChannel.G | TextureChannel.B | TextureChannel.A });
|
|
652
|
+
const buffer = this.document.getRoot().listBuffers()[0];
|
|
653
|
+
if (buffer)
|
|
654
|
+
prop.setReference(`exr_buffer_${value}`, buffer);
|
|
655
|
+
// At write time, update the property to the texture's current URI
|
|
656
|
+
// (PMREM will have updated it from "foo.exr" to "foo.pmrem.ktx2")
|
|
657
|
+
this.externalExrPointers.push({ obj, key, texture });
|
|
658
|
+
}
|
|
659
|
+
else if (exrPath) {
|
|
660
|
+
console.warn(`WARN: External EXR not found: ${exrPath}`);
|
|
661
|
+
}
|
|
662
|
+
}
|
|
623
663
|
if (value.startsWith("/materials/")) {
|
|
624
664
|
if (this.debug)
|
|
625
665
|
console.log("Read pointer", value);
|
|
@@ -777,6 +817,25 @@ class JsonPointerHandler {
|
|
|
777
817
|
if (ptr.resolve(context, step))
|
|
778
818
|
ptr.write();
|
|
779
819
|
});
|
|
820
|
+
// Resolve external EXR pointers: update component data with the texture's current URI
|
|
821
|
+
for (const { obj, key, texture } of this.externalExrPointers) {
|
|
822
|
+
const uri = texture.getURI() || texture.getName() || '';
|
|
823
|
+
if (uri) {
|
|
824
|
+
if (this.debug)
|
|
825
|
+
console.log(`< Resolved external EXR: ${obj[key]} → ${uri}`);
|
|
826
|
+
obj[key] = uri;
|
|
827
|
+
// Ensure the image definition in the output JSON has the URI set.
|
|
828
|
+
// When PMREM clears image data (setImage(null)) for external textures,
|
|
829
|
+
// gltf-transform's core writer won't set a uri or bufferView on the image.
|
|
830
|
+
const imageIndex = context.imageIndexMap.get(texture);
|
|
831
|
+
if (imageIndex !== undefined) {
|
|
832
|
+
const images = context.jsonDoc.json.images;
|
|
833
|
+
if (images && images[imageIndex] && !images[imageIndex].uri) {
|
|
834
|
+
images[imageIndex].uri = uri;
|
|
835
|
+
}
|
|
836
|
+
}
|
|
837
|
+
}
|
|
838
|
+
}
|
|
780
839
|
}
|
|
781
840
|
}
|
|
782
841
|
class JsonPointer {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { Extension, Texture, WriterContext } from '@gltf-transform/core';
|
|
1
|
+
import { Extension, PropertyType, ReaderContext, Texture, WriterContext } from '@gltf-transform/core';
|
|
2
2
|
/**
|
|
3
3
|
* NEEDLE_pmrem extension for gltf-transform.
|
|
4
4
|
* Marks textures that have been pre-processed with PMREM (EXR → KTX2 HDR).
|
|
@@ -13,7 +13,9 @@ export declare class NEEDLE_pmrem extends Extension {
|
|
|
13
13
|
/** Mark a texture as PMREM-converted */
|
|
14
14
|
addTexture(texture: Texture): void;
|
|
15
15
|
/** @hidden */
|
|
16
|
-
|
|
16
|
+
preread(context: ReaderContext, _propertyType: PropertyType): this;
|
|
17
|
+
/** @hidden */
|
|
18
|
+
read(context: ReaderContext): this;
|
|
17
19
|
/** @hidden */
|
|
18
20
|
write(context: WriterContext): this;
|
|
19
21
|
}
|
|
@@ -16,7 +16,44 @@ export class NEEDLE_pmrem extends Extension {
|
|
|
16
16
|
this.pmremTextures.add(texture);
|
|
17
17
|
}
|
|
18
18
|
/** @hidden */
|
|
19
|
-
|
|
19
|
+
preread(context, _propertyType) {
|
|
20
|
+
const json = context.jsonDoc.json;
|
|
21
|
+
// For each texture with NEEDLE_pmrem, ensure `source` is set so the core reader
|
|
22
|
+
// links the texture to its image (we removed KHR_texture_basisu.source at write time).
|
|
23
|
+
for (const textureDef of json.textures ?? []) {
|
|
24
|
+
const pmrem = textureDef.extensions?.[EXTENSION_NAME];
|
|
25
|
+
if (pmrem?.source !== undefined && textureDef.source === undefined) {
|
|
26
|
+
textureDef.source = pmrem.source;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
// Also ensure the image has the correct mime type
|
|
30
|
+
for (const textureDef of json.textures ?? []) {
|
|
31
|
+
const pmrem = textureDef.extensions?.[EXTENSION_NAME];
|
|
32
|
+
if (pmrem?.source !== undefined) {
|
|
33
|
+
const imageDef = json.images?.[pmrem.source];
|
|
34
|
+
if (imageDef && !imageDef.mimeType) {
|
|
35
|
+
imageDef.mimeType = 'image/ktx2';
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
return this;
|
|
40
|
+
}
|
|
41
|
+
/** @hidden */
|
|
42
|
+
read(context) {
|
|
43
|
+
const json = context.jsonDoc.json;
|
|
44
|
+
const textureDefs = json.textures ?? [];
|
|
45
|
+
const textures = this.document.getRoot().listTextures();
|
|
46
|
+
for (let i = 0; i < textureDefs.length; i++) {
|
|
47
|
+
const textureDef = textureDefs[i];
|
|
48
|
+
const pmrem = textureDef.extensions?.[EXTENSION_NAME];
|
|
49
|
+
if (pmrem?.source === undefined)
|
|
50
|
+
continue;
|
|
51
|
+
const texture = textures[i];
|
|
52
|
+
if (!texture)
|
|
53
|
+
continue;
|
|
54
|
+
texture.setMimeType('image/ktx2');
|
|
55
|
+
this.pmremTextures.add(texture);
|
|
56
|
+
}
|
|
20
57
|
return this;
|
|
21
58
|
}
|
|
22
59
|
/** @hidden */
|
|
@@ -39,6 +76,20 @@ export class NEEDLE_pmrem extends Extension {
|
|
|
39
76
|
}
|
|
40
77
|
}
|
|
41
78
|
}
|
|
79
|
+
// If no textures still use KHR_texture_basisu, remove it from extensionsUsed/Required
|
|
80
|
+
// to prevent KHRTextureBasisu.preread() from crashing when re-reading the GLB
|
|
81
|
+
const anyBasisu = (jsonDoc.json.textures ?? []).some((t) => t.extensions?.['KHR_texture_basisu']);
|
|
82
|
+
if (!anyBasisu) {
|
|
83
|
+
const remove = (arr) => {
|
|
84
|
+
if (!arr)
|
|
85
|
+
return;
|
|
86
|
+
const idx = arr.indexOf('KHR_texture_basisu');
|
|
87
|
+
if (idx >= 0)
|
|
88
|
+
arr.splice(idx, 1);
|
|
89
|
+
};
|
|
90
|
+
remove(jsonDoc.json.extensionsUsed);
|
|
91
|
+
remove(jsonDoc.json.extensionsRequired);
|
|
92
|
+
}
|
|
42
93
|
return this;
|
|
43
94
|
}
|
|
44
95
|
}
|
package/dist/extensions/index.js
CHANGED
|
@@ -10,10 +10,14 @@ export * from "./NEEDLE_lightmaps/index.js";
|
|
|
10
10
|
import { ALL_EXTENSIONS } from '@gltf-transform/extensions';
|
|
11
11
|
import { ALL_EXTENSIONS as NEEDLE_EXTENSIONS } from "./NEEDLE_opaque/index.js";
|
|
12
12
|
import { NEEDLE_progressive } from './NEEDLE_progressive/index.js';
|
|
13
|
+
import { NEEDLE_pmrem } from './NEEDLE_pmrem/index.js';
|
|
14
|
+
import { NEEDLE_lightmaps_ext } from './NEEDLE_lightmaps/index.js';
|
|
13
15
|
export function registerExtensions(node) {
|
|
14
16
|
node.registerExtensions(ALL_EXTENSIONS);
|
|
15
17
|
node.registerExtensions(NEEDLE_EXTENSIONS);
|
|
16
18
|
node.registerExtensions([
|
|
17
|
-
NEEDLE_progressive
|
|
19
|
+
NEEDLE_progressive,
|
|
20
|
+
NEEDLE_pmrem,
|
|
21
|
+
NEEDLE_lightmaps_ext,
|
|
18
22
|
]);
|
|
19
23
|
}
|
|
@@ -5,7 +5,7 @@ import draco3d from 'draco3dgltf';
|
|
|
5
5
|
import { existsSync, readFileSync, statSync, writeFileSync } from 'fs';
|
|
6
6
|
import { MeshoptDecoder, MeshoptEncoder } from 'meshoptimizer';
|
|
7
7
|
import { dedup, metalRough, prune, resample } from '@gltf-transform/functions';
|
|
8
|
-
import { ALL_EXTENSIONS as NEEDLE_EXTENSIONS, NEEDLE_compression_texture, NEEDLE_mesh_compression, NEEDLE_pmrem, NEEDLE_lightmaps_ext, } from '../extensions/index.js';
|
|
8
|
+
import { ALL_EXTENSIONS as NEEDLE_EXTENSIONS, NEEDLE_compression_texture, NEEDLE_mesh_compression, NEEDLE_pmrem, NEEDLE_lightmaps_ext, setOpaqueInputFile, } from '../extensions/index.js';
|
|
9
9
|
import { isLOD, isMeshLOD, needle_animation_transform, needle_asset, needle_mesh_transform, needle_texture_transform, } from '../transforms/index.js';
|
|
10
10
|
import { getOutputPath, getVersion, ioTryReadWithMissingResources, writeNodeIO } from "../utils/index.js";
|
|
11
11
|
import { addToCache, getHash, tryGetFromCache, } from "../cache/index.js";
|
|
@@ -66,8 +66,9 @@ export async function packGLTF(inputFile, outputFile, options) {
|
|
|
66
66
|
'meshopt.decoder': MeshoptDecoder,
|
|
67
67
|
'meshopt.encoder': MeshoptEncoder,
|
|
68
68
|
});
|
|
69
|
-
// Set input file path so
|
|
69
|
+
// Set input file path so extensions can resolve external EXR files
|
|
70
70
|
NEEDLE_lightmaps_ext.inputFile = inputFile;
|
|
71
|
+
setOpaqueInputFile(inputFile);
|
|
71
72
|
const document = await ioTryReadWithMissingResources(io, inputFile);
|
|
72
73
|
if (options.logger) {
|
|
73
74
|
document.setLogger(options.logger);
|
|
@@ -76,10 +76,9 @@ export class NeedleToktxContext {
|
|
|
76
76
|
return false;
|
|
77
77
|
}
|
|
78
78
|
this.basisuExtension = document.createExtension(KHRTextureBasisu).setRequired(true);
|
|
79
|
-
const tmpDir = `${tmp.tmpdir}/gltf-transform`;
|
|
79
|
+
const tmpDir = `${tmp.tmpdir}/gltf-transform/${uuid()}`;
|
|
80
80
|
this.cacheDirectory = tmpDir;
|
|
81
|
-
|
|
82
|
-
mkdirSync(tmpDir);
|
|
81
|
+
mkdirSync(tmpDir, { recursive: true });
|
|
83
82
|
this.tmpPathBase = tmpDir + "/" + uuid();
|
|
84
83
|
this.numTextures = document.getRoot().listTextures().length;
|
|
85
84
|
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
export interface PipelineAnalyticsInfo {
|
|
2
|
+
/** 'compress' | 'progressive' | 'compress+progressive' */
|
|
3
|
+
mode: string;
|
|
4
|
+
usecase?: string;
|
|
5
|
+
filesProcessed: number;
|
|
6
|
+
sizeBefore_MB: number;
|
|
7
|
+
sizeAfter_MB: number;
|
|
8
|
+
gpuMemory_MB: number;
|
|
9
|
+
duration_ms: number;
|
|
10
|
+
extensionsUsed?: string[];
|
|
11
|
+
}
|
|
12
|
+
/** Send a "pipeline_start" event. */
|
|
13
|
+
export declare function trackPipelineStart(info: {
|
|
14
|
+
mode: string;
|
|
15
|
+
usecase?: string;
|
|
16
|
+
}): void;
|
|
17
|
+
/** Send a pipeline error event. Deduplicates identical errors within 1 minute. */
|
|
18
|
+
export declare function trackError(error: unknown, context?: {
|
|
19
|
+
mode?: string;
|
|
20
|
+
usecase?: string;
|
|
21
|
+
file?: string;
|
|
22
|
+
}): void;
|
|
23
|
+
/** Send a "pipeline_end" event with size reduction and extension info. */
|
|
24
|
+
export declare function trackPipelineEnd(info: PipelineAnalyticsInfo): void;
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
import os from 'os';
|
|
2
|
+
import https from 'https';
|
|
3
|
+
import { getVersion } from './version.js';
|
|
4
|
+
const RYBBIT_URL = 'https://needle.tools/api/v1/rum/t';
|
|
5
|
+
const SITE_ID = '5c44f60eb2d7';
|
|
6
|
+
/** Build a User-Agent string that UA parsers recognize as desktop + correct OS. */
|
|
7
|
+
function getUserAgent() {
|
|
8
|
+
const v = getVersion();
|
|
9
|
+
const platform = os.platform();
|
|
10
|
+
const release = os.release();
|
|
11
|
+
const arch = os.arch();
|
|
12
|
+
const node = process.version;
|
|
13
|
+
// Map Node's os.platform() to a UA-parser-friendly OS token
|
|
14
|
+
let osToken;
|
|
15
|
+
if (platform === 'darwin') {
|
|
16
|
+
// macOS: "Macintosh; Intel Mac OS X 25_3_0"
|
|
17
|
+
const ver = release.replace(/\./g, '_');
|
|
18
|
+
osToken = `Macintosh; Intel Mac OS X ${ver}`;
|
|
19
|
+
}
|
|
20
|
+
else if (platform === 'win32') {
|
|
21
|
+
// Windows: "Windows NT 10.0; Win64; x64"
|
|
22
|
+
osToken = `Windows NT ${release}; Win64; ${arch}`;
|
|
23
|
+
}
|
|
24
|
+
else {
|
|
25
|
+
// Linux / other: "X11; Linux x86_64"
|
|
26
|
+
osToken = `X11; Linux ${arch}`;
|
|
27
|
+
}
|
|
28
|
+
return `Mozilla/5.0 (${osToken}) NeedleBuildPipeline/${v} Node/${node}`;
|
|
29
|
+
}
|
|
30
|
+
/** Fire-and-forget analytics event. Never throws, never blocks. */
|
|
31
|
+
function sendEvent(eventName, properties, type = 'custom_event') {
|
|
32
|
+
try {
|
|
33
|
+
const body = JSON.stringify({
|
|
34
|
+
site_id: SITE_ID,
|
|
35
|
+
type,
|
|
36
|
+
event_name: eventName,
|
|
37
|
+
properties: JSON.stringify(properties),
|
|
38
|
+
user_agent: getUserAgent(),
|
|
39
|
+
});
|
|
40
|
+
const url = new URL(RYBBIT_URL);
|
|
41
|
+
const req = https.request({
|
|
42
|
+
hostname: url.hostname,
|
|
43
|
+
path: url.pathname,
|
|
44
|
+
method: 'POST',
|
|
45
|
+
headers: {
|
|
46
|
+
'Content-Type': 'application/json',
|
|
47
|
+
'Content-Length': Buffer.byteLength(body),
|
|
48
|
+
},
|
|
49
|
+
timeout: 5000,
|
|
50
|
+
});
|
|
51
|
+
req.on('error', () => { });
|
|
52
|
+
req.on('timeout', () => { req.destroy(); });
|
|
53
|
+
req.end(body);
|
|
54
|
+
}
|
|
55
|
+
catch {
|
|
56
|
+
// silent
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
/** Send a "pipeline_start" event. */
|
|
60
|
+
export function trackPipelineStart(info) {
|
|
61
|
+
sendEvent('pipeline_start', {
|
|
62
|
+
version: getVersion(),
|
|
63
|
+
mode: info.mode,
|
|
64
|
+
usecase: info.usecase,
|
|
65
|
+
platform: process.platform,
|
|
66
|
+
arch: process.arch,
|
|
67
|
+
node: process.version,
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
/** Dedup cache to avoid logging the same error repeatedly in one session */
|
|
71
|
+
const recentErrors = new Map();
|
|
72
|
+
const ERROR_DEDUP_MS = 60_000;
|
|
73
|
+
/** Send a pipeline error event. Deduplicates identical errors within 1 minute. */
|
|
74
|
+
export function trackError(error, context) {
|
|
75
|
+
const err = error instanceof Error ? error : new Error(String(error));
|
|
76
|
+
const message = (err.message || 'Unknown error').substring(0, 500);
|
|
77
|
+
const stack = (err.stack || '').substring(0, 2000);
|
|
78
|
+
const errorName = err.name || 'Error';
|
|
79
|
+
// Dedup
|
|
80
|
+
const key = `${errorName}|${message}`;
|
|
81
|
+
const now = Date.now();
|
|
82
|
+
const cached = recentErrors.get(key);
|
|
83
|
+
if (cached && now - cached.time < ERROR_DEDUP_MS) {
|
|
84
|
+
cached.count++;
|
|
85
|
+
return;
|
|
86
|
+
}
|
|
87
|
+
recentErrors.set(key, { time: now, count: (cached?.count || 0) + 1 });
|
|
88
|
+
const properties = {
|
|
89
|
+
version: getVersion(),
|
|
90
|
+
error_name: errorName + (cached && cached.count > 1 ? ` x${cached.count}` : ''),
|
|
91
|
+
message,
|
|
92
|
+
platform: process.platform,
|
|
93
|
+
arch: process.arch,
|
|
94
|
+
node: process.version,
|
|
95
|
+
};
|
|
96
|
+
if (stack)
|
|
97
|
+
properties.stack = stack;
|
|
98
|
+
if (context?.mode)
|
|
99
|
+
properties.mode = context.mode;
|
|
100
|
+
if (context?.usecase)
|
|
101
|
+
properties.usecase = context.usecase;
|
|
102
|
+
if (context?.file)
|
|
103
|
+
properties.file = context.file;
|
|
104
|
+
sendEvent(errorName, properties, 'error');
|
|
105
|
+
}
|
|
106
|
+
/** Send a "pipeline_end" event with size reduction and extension info. */
|
|
107
|
+
export function trackPipelineEnd(info) {
|
|
108
|
+
const reductionPct = info.sizeBefore_MB > 0
|
|
109
|
+
? ((1 - info.sizeAfter_MB / info.sizeBefore_MB) * 100).toFixed(1)
|
|
110
|
+
: '0';
|
|
111
|
+
sendEvent('pipeline_end', {
|
|
112
|
+
version: getVersion(),
|
|
113
|
+
mode: info.mode,
|
|
114
|
+
usecase: info.usecase,
|
|
115
|
+
files: info.filesProcessed,
|
|
116
|
+
size_before_mb: +info.sizeBefore_MB.toFixed(2),
|
|
117
|
+
size_after_mb: +info.sizeAfter_MB.toFixed(2),
|
|
118
|
+
reduction_pct: +reductionPct,
|
|
119
|
+
gpu_memory_mb: +info.gpuMemory_MB.toFixed(2),
|
|
120
|
+
duration_ms: info.duration_ms,
|
|
121
|
+
extensions: info.extensionsUsed?.join(','),
|
|
122
|
+
platform: process.platform,
|
|
123
|
+
arch: process.arch,
|
|
124
|
+
node: process.version,
|
|
125
|
+
});
|
|
126
|
+
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
export declare const version = "2.15.0
|
|
1
|
+
export declare const version = "2.15.0";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
export const version = "2.15.0
|
|
1
|
+
export const version = "2.15.0";
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@needle-tools/gltf-build-pipeline",
|
|
3
|
-
"version": "2.15.0
|
|
3
|
+
"version": "2.15.0",
|
|
4
4
|
"description": "Pipeline and tools for optimizing gltf files using gltf-transform and compression settings within glTF extensions",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"type": "module",
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
d8f36fcbb09e79614e96baaed5b1e1cd5d2f5e73
|
|
Binary file
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
d8f36fcbb09e79614e96baaed5b1e1cd5d2f5e73
|
|
Binary file
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
d8f36fcbb09e79614e96baaed5b1e1cd5d2f5e73
|