@needle-tools/gltf-build-pipeline 1.5.0-alpha → 1.5.0-alpha.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +31 -0
- package/dist/cache/cache.d.ts +2 -2
- package/dist/cache/cache.js +36 -20
- package/dist/extensions/NEEDLE_compression_texture/NEEDLE_compression_texture.d.ts +1 -1
- package/dist/extensions/NEEDLE_opaque/NEEDLE_opaque.d.ts +2 -1
- package/dist/extensions/NEEDLE_opaque/NEEDLE_opaque.js +72 -15
- package/dist/extensions/NEEDLE_opaque/index.js +7 -0
- package/dist/extensions/NEEDLE_progressive_mesh_settings/index.d.ts +4 -1
- package/dist/extensions/NEEDLE_progressive_texture_settings/index.d.ts +4 -0
- package/dist/transforms/needle_common.d.ts +2 -0
- package/dist/transforms/needle_mesh_transform.d.ts +5 -1
- package/dist/transforms/needle_mesh_transform.js +13 -7
- package/dist/transforms/needle_progressive.js +32 -8
- package/dist/transforms/needle_texture_transform.js +46 -7
- package/dist/transforms/needle_toktx.d.ts +1 -1
- package/dist/transforms/needle_toktx.js +35 -5
- package/dist/transforms/needle_webp.d.ts +1 -1
- package/dist/utils/compression_utils.d.ts +2 -0
- package/dist/utils/compression_utils.js +6 -0
- package/dist/utils/fileutils.d.ts +1 -0
- package/dist/utils/fileutils.js +6 -1
- package/dist/utils/index.d.ts +2 -0
- package/dist/utils/index.js +2 -0
- package/dist/utils/nodeio.d.ts +5 -0
- package/dist/utils/nodeio.js +14 -0
- package/limit-caches.mjs +4 -0
- package/pack-gltf.mjs +16 -12
- package/package.json +2 -1
- package/dist/functions/gzip.d.ts +0 -1
- package/dist/functions/gzip.js +0 -16
- package/dist/functions/index.d.ts +0 -1
- package/dist/functions/index.js +0 -1
- package/dist/functions/list-texture-slots.d.ts +0 -1
- package/dist/functions/list-texture-slots.js +0 -35
- package/dist/transforms/needle_simplify.d.ts +0 -3
- package/dist/transforms/needle_simplify.js +0 -76
- package/dist/transforms/types/squoosh.d.ts +0 -31
- package/dist/transforms/types/squoosh.js +0 -33
package/CHANGELOG.md
CHANGED
|
@@ -4,6 +4,37 @@ All notable changes to this package will be documented in this file.
|
|
|
4
4
|
The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
|
|
5
5
|
and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
|
|
6
6
|
|
|
7
|
+
## [1.5.0-alpha.10] - 2023-05-17
|
|
8
|
+
- add: `generateLods` option to progressive texture settings
|
|
9
|
+
|
|
10
|
+
## [1.5.0-alpha.9] - 2023-05-16
|
|
11
|
+
- add: start of VRM 1.0 support. Not implemented is the `VRMC_materials_mtoon` extension
|
|
12
|
+
|
|
13
|
+
## [1.5.0-alpha.8] - 2023-05-16
|
|
14
|
+
- fix: issue where updating ktx compressed texture extension also flattened the path
|
|
15
|
+
|
|
16
|
+
## [1.5.0-alpha.7] - 2023-05-16
|
|
17
|
+
- fix: external resources that are KTX2 compressed are now saved with a `.ktx2` extension
|
|
18
|
+
- change: toktx now uses compression "auto" mode to select uastc or etc1s
|
|
19
|
+
- change: toktx now checks the slots and increases quality for `metallicRoughnessTexture` and `normalTexture` but keeps using ETC1S for smaller file size
|
|
20
|
+
|
|
21
|
+
## [1.5.0-alpha.6] - 2023-05-15
|
|
22
|
+
- fix: If not further specified then automatically use UASTC for textures that are in either `normalTexture` or `metallicRoughnessTexture` slots
|
|
23
|
+
|
|
24
|
+
## [1.5.0-alpha.5] - 2023-05-04
|
|
25
|
+
- fix: NEEDLE_opaque should handle if VRM texture index is negative
|
|
26
|
+
- change: exit with code 1 if any file packing fails
|
|
27
|
+
|
|
28
|
+
## [1.5.0-alpha.4] - 2023-05-03
|
|
29
|
+
- fix: issue in NEEDLE_opaque resolving of VRM textures, also resolve `meta.texture`
|
|
30
|
+
- fix: VRMs now compress meshes with mesh-opt if nothing is specified
|
|
31
|
+
- change: include version in cache directory
|
|
32
|
+
|
|
33
|
+
## [1.5.0-alpha.3] - 2023-05-03
|
|
34
|
+
- add: VRM progressive textures support
|
|
35
|
+
- fix: VRM format should be saved as GLB. See https://github.com/donmccurdy/glTF-Transform/issues/1390
|
|
36
|
+
- fix: VRM texture index handling
|
|
37
|
+
|
|
7
38
|
## [1.5.0-alpha] - 2023-05-01
|
|
8
39
|
- add: NEEDLE_opaque `node: <index>` retargeting
|
|
9
40
|
- add: passthrough VRM extension
|
package/dist/cache/cache.d.ts
CHANGED
|
@@ -8,7 +8,7 @@ export declare function clearCache(): void;
|
|
|
8
8
|
* Ensure that the cache size does not exceed the given maximum size in MB.
|
|
9
9
|
* @param maxSizeMB The maximum size in MB.
|
|
10
10
|
*/
|
|
11
|
-
export declare function
|
|
11
|
+
export declare function limitCacheSize(maxSizeMB?: number): void;
|
|
12
12
|
/**
|
|
13
13
|
* Add data to the cache.
|
|
14
14
|
* @param key The key to store the data under.
|
|
@@ -48,4 +48,4 @@ export declare function getKeyWithHash(property: ExtensibleProperty): {
|
|
|
48
48
|
* @param property The property to get the hash for.
|
|
49
49
|
* @returns The hash as a number.
|
|
50
50
|
*/
|
|
51
|
-
export declare function getHash(property: ExtensibleProperty | object): number;
|
|
51
|
+
export declare function getHash(property: ExtensibleProperty | object, level?: number): number;
|
package/dist/cache/cache.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { ExtensibleProperty, Texture, Primitive, Accessor, Mesh } from "@gltf-transform/core";
|
|
2
2
|
import tmp from 'tmp';
|
|
3
|
-
import fs from 'fs';
|
|
3
|
+
import fs, { statSync } from 'fs';
|
|
4
4
|
import path from 'path';
|
|
5
5
|
import { getVersion } from "../utils/version.js";
|
|
6
6
|
// import * as CHECKDISCSPACE from 'check-disk-space'
|
|
@@ -24,7 +24,7 @@ const buildPipelineVersion = getVersion(false);
|
|
|
24
24
|
* This will remove all files in the cache directory.
|
|
25
25
|
*/
|
|
26
26
|
export function clearCache() {
|
|
27
|
-
const dir = getCacheDirectory();
|
|
27
|
+
const dir = getCacheDirectory(true);
|
|
28
28
|
fs.rmSync(dir, { recursive: true });
|
|
29
29
|
console.log(`Cleared cache at ${dir}`);
|
|
30
30
|
}
|
|
@@ -32,16 +32,27 @@ export function clearCache() {
|
|
|
32
32
|
* Ensure that the cache size does not exceed the given maximum size in MB.
|
|
33
33
|
* @param maxSizeMB The maximum size in MB.
|
|
34
34
|
*/
|
|
35
|
-
export function
|
|
36
|
-
const dir = getCacheDirectory();
|
|
35
|
+
export function limitCacheSize(maxSizeMB = 1024) {
|
|
36
|
+
const dir = getCacheDirectory(true);
|
|
37
|
+
console.log(`Limit cache size to ${maxSizeMB.toFixed(2)} MB at ${dir}...`);
|
|
37
38
|
let totalSize = 0;
|
|
38
|
-
|
|
39
|
-
const
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
39
|
+
function recurseDirectories(dir) {
|
|
40
|
+
const files = fs.readdirSync(dir).map(file => {
|
|
41
|
+
const path = `${dir}/${file}`;
|
|
42
|
+
const stats = statSync(path);
|
|
43
|
+
if (stats.isDirectory()) {
|
|
44
|
+
recurseDirectories(path);
|
|
45
|
+
}
|
|
46
|
+
else {
|
|
47
|
+
totalSize += stats.size;
|
|
48
|
+
}
|
|
49
|
+
return { path, size: stats.size };
|
|
50
|
+
});
|
|
51
|
+
return files;
|
|
52
|
+
}
|
|
53
|
+
const files = recurseDirectories(dir);
|
|
44
54
|
const maxSizeInBytes = maxSizeMB * 1024 * 1024;
|
|
55
|
+
console.log(`Current cache size is ${(totalSize / (1024 * 1024)).toFixed(1)} MB`);
|
|
45
56
|
if (totalSize < maxSizeInBytes)
|
|
46
57
|
return;
|
|
47
58
|
files.sort((a, b) => {
|
|
@@ -55,6 +66,7 @@ export function ensureCacheSize(maxSizeMB) {
|
|
|
55
66
|
const file = files.shift();
|
|
56
67
|
if (!file)
|
|
57
68
|
break;
|
|
69
|
+
console.log(`Removing cache file ${file.path}...`);
|
|
58
70
|
fs.rmSync(file.path);
|
|
59
71
|
totalSize -= file.size;
|
|
60
72
|
}
|
|
@@ -153,7 +165,7 @@ export function getKeyWithHash(property) {
|
|
|
153
165
|
* @param property The property to get the hash for.
|
|
154
166
|
* @returns The hash as a number.
|
|
155
167
|
*/
|
|
156
|
-
export function getHash(property) {
|
|
168
|
+
export function getHash(property, level = 0) {
|
|
157
169
|
let hash = 0;
|
|
158
170
|
if (property instanceof ExtensibleProperty) {
|
|
159
171
|
const extensions = property.listExtensions();
|
|
@@ -175,7 +187,7 @@ export function getHash(property) {
|
|
|
175
187
|
hash += hashObject(weights);
|
|
176
188
|
const primitives = property.listPrimitives();
|
|
177
189
|
for (const prim of primitives)
|
|
178
|
-
hash += getHash(prim);
|
|
190
|
+
hash += getHash(prim, level + 1);
|
|
179
191
|
}
|
|
180
192
|
else if (property instanceof Primitive) {
|
|
181
193
|
const indices = property.getIndices();
|
|
@@ -198,8 +210,9 @@ export function getHash(property) {
|
|
|
198
210
|
}
|
|
199
211
|
return hash;
|
|
200
212
|
}
|
|
201
|
-
|
|
202
|
-
|
|
213
|
+
const cleanBuildPipelineVersion = buildPipelineVersion.replace(/[^a-zA-Z0-9]/g, "_");
|
|
214
|
+
function getCacheDirectory(root) {
|
|
215
|
+
const tmpDir = path.join(tmp.tmpdir, "Needle", "build-pipeline-cache", root ? "" : cleanBuildPipelineVersion);
|
|
203
216
|
if (!fs.existsSync(tmpDir)) {
|
|
204
217
|
fs.mkdirSync(tmpDir, { recursive: true });
|
|
205
218
|
}
|
|
@@ -211,12 +224,7 @@ function hashObject(obj) {
|
|
|
211
224
|
return hashBuffer(buffer);
|
|
212
225
|
}
|
|
213
226
|
const str = JSON.stringify(obj);
|
|
214
|
-
|
|
215
|
-
for (let i = 0; i < str.length; i++) {
|
|
216
|
-
hash = ((hash << 5) - hash) + str.charCodeAt(i);
|
|
217
|
-
hash |= 0;
|
|
218
|
-
}
|
|
219
|
-
return hash;
|
|
227
|
+
return hashString(str);
|
|
220
228
|
}
|
|
221
229
|
function hashBuffer(buffer, offset, length) {
|
|
222
230
|
let hash = 0;
|
|
@@ -227,3 +235,11 @@ function hashBuffer(buffer, offset, length) {
|
|
|
227
235
|
}
|
|
228
236
|
return hash;
|
|
229
237
|
}
|
|
238
|
+
function hashString(str) {
|
|
239
|
+
let hash = 0;
|
|
240
|
+
for (let i = 0; i < str.length; i++) {
|
|
241
|
+
hash = ((hash << 5) - hash) + str.charCodeAt(i);
|
|
242
|
+
hash |= 0;
|
|
243
|
+
}
|
|
244
|
+
return hash;
|
|
245
|
+
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { Extension, ExtensionProperty, PropertyType, ReaderContext, WriterContext } from "@gltf-transform/core";
|
|
2
2
|
export declare const EXTENSION_NAME = "NEEDLE_compression_texture";
|
|
3
3
|
export declare interface NEEDLE_compression_texture_schema {
|
|
4
|
-
mode:
|
|
4
|
+
mode: "none" | "auto" | "webp" | "UASTC" | "ETC1S";
|
|
5
5
|
slot: string;
|
|
6
6
|
/** quality in range of 0 - 1 */
|
|
7
7
|
quality: number;
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { WriterContext, PropertyType } from '@gltf-transform/core';
|
|
2
2
|
declare interface IExtensibleProperty {
|
|
3
3
|
}
|
|
4
|
+
type DebugBreakPoint = boolean | "pointer:read";
|
|
4
5
|
export type Options = {
|
|
5
6
|
debug?: boolean;
|
|
6
|
-
breakpoint?:
|
|
7
|
+
breakpoint?: DebugBreakPoint;
|
|
7
8
|
write?: boolean;
|
|
8
9
|
};
|
|
9
10
|
export interface IOpaqueExtension {
|
|
@@ -26,6 +26,7 @@ export function createOpaqueExtension(name, types, opts) {
|
|
|
26
26
|
console.log("READ " + EXTENSION_NAME);
|
|
27
27
|
this.jsonPointerHandler = new JsonPointerHandler(this.document, EXTENSION_NAME);
|
|
28
28
|
this.jsonPointerHandler.debug = debugLog;
|
|
29
|
+
this.jsonPointerHandler.breakPoint = opts.breakpoint;
|
|
29
30
|
for (const type of types) {
|
|
30
31
|
switch (type) {
|
|
31
32
|
case PropertyType.ACCESSOR:
|
|
@@ -514,6 +515,7 @@ class JsonPointerHandler {
|
|
|
514
515
|
document;
|
|
515
516
|
map = new Map();
|
|
516
517
|
debug = false;
|
|
518
|
+
breakPoint = false;
|
|
517
519
|
constructor(document, id) {
|
|
518
520
|
this.document = document;
|
|
519
521
|
this.id = id;
|
|
@@ -524,7 +526,11 @@ class JsonPointerHandler {
|
|
|
524
526
|
if (key !== undefined) {
|
|
525
527
|
fullPath ??= "";
|
|
526
528
|
fullPath += "/" + key;
|
|
529
|
+
if (this.breakPoint === "pointer:read") {
|
|
530
|
+
debugger;
|
|
531
|
+
}
|
|
527
532
|
}
|
|
533
|
+
// if(this.debug && key === "colliders") debugger;
|
|
528
534
|
switch (typeof value) {
|
|
529
535
|
case 'object':
|
|
530
536
|
if (value !== null && value !== undefined) {
|
|
@@ -556,7 +562,8 @@ class JsonPointerHandler {
|
|
|
556
562
|
console.log("Texture index", index, imageIndex, context.jsonDoc.json.textures.length, "Textures:" + context.textures.length);
|
|
557
563
|
const object = tryGetInArray(context.textures, imageIndex);
|
|
558
564
|
if (object) {
|
|
559
|
-
const res = new
|
|
565
|
+
const res = new TextureIndexResolver(this.document, object, definition);
|
|
566
|
+
res.debug = this.debug;
|
|
560
567
|
res.init(context); // < init here otherwise the fields defined in the TexturePointer will be undefined (this caused e.g. finding the previous texture filter to fail)
|
|
561
568
|
pointer = new JsonPointer(path, res, value);
|
|
562
569
|
pointer.debug = this.debug;
|
|
@@ -569,7 +576,7 @@ class JsonPointerHandler {
|
|
|
569
576
|
if (pointer) {
|
|
570
577
|
if (this.debug)
|
|
571
578
|
console.log("> Register pointer", this.id + "/" + fullPath + " : " + value);
|
|
572
|
-
pointer?.register(obj, key);
|
|
579
|
+
pointer?.register(obj, key, fullPath);
|
|
573
580
|
}
|
|
574
581
|
else
|
|
575
582
|
console.warn("WARN: failed registering pointer", fullPath, value);
|
|
@@ -589,33 +596,80 @@ class JsonPointerHandler {
|
|
|
589
596
|
let pointer = this.map.get(value);
|
|
590
597
|
if (!pointer) {
|
|
591
598
|
const definition = context.jsonDoc.json.materials[index];
|
|
592
|
-
const res = new
|
|
599
|
+
const res = new MaterialIndexResolver(this.document, mat, definition);
|
|
600
|
+
res.debug = this.debug;
|
|
593
601
|
res.init(context); // < init here otherwise the fields defined in the TexturePointer will be undefined (this caused e.g. finding the previous texture filter to fail)
|
|
594
602
|
pointer = new JsonPointer("/materials/", res, value);
|
|
595
603
|
pointer.debug = this.debug;
|
|
596
604
|
this.map.set(value, pointer);
|
|
597
605
|
}
|
|
598
|
-
pointer?.register(obj, key);
|
|
606
|
+
pointer?.register(obj, key, fullPath);
|
|
599
607
|
}
|
|
600
608
|
}
|
|
601
609
|
}
|
|
602
610
|
break;
|
|
603
611
|
case 'number':
|
|
604
|
-
if (key === "node"
|
|
612
|
+
if (key === "node" || key === "mesh"
|
|
613
|
+
// VRM 1.0 spring bone center https://github.com/vrm-c/vrm-specification/blob/master/specification/VRMC_springBone-1.0/schema/VRMC_springBone.spring.schema.json
|
|
614
|
+
|| fullPath.match(/springs\/\d+\/center/g)) {
|
|
605
615
|
const object = context.nodes[value];
|
|
606
616
|
if (object) {
|
|
607
617
|
const res = new NodeIndexResolver(this.document, object, context.jsonDoc.json.nodes[value]);
|
|
618
|
+
res.debug = this.debug;
|
|
608
619
|
res.init(context);
|
|
609
|
-
const dictKey =
|
|
620
|
+
const dictKey = `/${key}/${value}`;
|
|
610
621
|
let pointer = this.map.get(dictKey);
|
|
611
622
|
if (!pointer) {
|
|
612
623
|
pointer = new JsonPointer(null, res, value);
|
|
613
624
|
}
|
|
614
625
|
pointer.debug = this.debug;
|
|
615
|
-
pointer.register(obj, key);
|
|
626
|
+
pointer.register(obj, key, fullPath);
|
|
616
627
|
this.map.set(dictKey, pointer);
|
|
617
628
|
}
|
|
618
629
|
}
|
|
630
|
+
else if (
|
|
631
|
+
// key texture is matching for example for VRM meta/texture (AvatarSample_A)
|
|
632
|
+
key === "texture"
|
|
633
|
+
// VRM 0.0 material properties
|
|
634
|
+
|| fullPath.match(/materialProperties\/\d+\/textureProperties/g)
|
|
635
|
+
// VRM 1.0 meta
|
|
636
|
+
|| fullPath.match(/meta\/thumbnailImage/g)) {
|
|
637
|
+
const index = value;
|
|
638
|
+
// e.g. in meta.texture for VRM avatars it seems like we can have negative values
|
|
639
|
+
if (index < 0) {
|
|
640
|
+
console.warn(`WARN: texture index is negative ${index} at ${fullPath}`);
|
|
641
|
+
return;
|
|
642
|
+
}
|
|
643
|
+
const definition = context.jsonDoc.json.textures[index];
|
|
644
|
+
if (!definition) {
|
|
645
|
+
console.warn(`WARN: texture definition with index ${index} not found or does not exist. Set in \"${fullPath}\"`);
|
|
646
|
+
return;
|
|
647
|
+
}
|
|
648
|
+
const imageIndex = findSource(definition);
|
|
649
|
+
if (this.debug)
|
|
650
|
+
console.log("Texture index", index, imageIndex, context.jsonDoc.json.textures.length, "Textures:" + context.textures.length);
|
|
651
|
+
const object = tryGetInArray(context.textures, imageIndex);
|
|
652
|
+
if (object) {
|
|
653
|
+
if (this.debug)
|
|
654
|
+
console.log("Found texture index at " + fullPath + ": " + value);
|
|
655
|
+
const res = new TextureIndexResolver(this.document, object, definition);
|
|
656
|
+
res.debug = this.debug;
|
|
657
|
+
res.init(context);
|
|
658
|
+
const dictKey = `/textures/${value}`;
|
|
659
|
+
let pointer = this.map.get(dictKey);
|
|
660
|
+
if (!pointer) {
|
|
661
|
+
pointer = new JsonPointer(null, res, value);
|
|
662
|
+
}
|
|
663
|
+
pointer.debug = this.debug;
|
|
664
|
+
pointer.register(obj, key, fullPath);
|
|
665
|
+
this.map.set(dictKey, pointer);
|
|
666
|
+
if (ext) {
|
|
667
|
+
ext.setReference(dictKey, object);
|
|
668
|
+
}
|
|
669
|
+
}
|
|
670
|
+
}
|
|
671
|
+
break;
|
|
672
|
+
default:
|
|
619
673
|
break;
|
|
620
674
|
}
|
|
621
675
|
}
|
|
@@ -638,9 +692,11 @@ class JsonPointer {
|
|
|
638
692
|
}
|
|
639
693
|
_objects = [];
|
|
640
694
|
_props = [];
|
|
641
|
-
|
|
695
|
+
_debugInfo = [];
|
|
696
|
+
register(obj, prop, fullpath) {
|
|
642
697
|
this._objects.push(obj);
|
|
643
698
|
this._props.push(prop);
|
|
699
|
+
this._debugInfo.push(fullpath);
|
|
644
700
|
}
|
|
645
701
|
newPointer;
|
|
646
702
|
resolve(context) {
|
|
@@ -655,18 +711,19 @@ class JsonPointer {
|
|
|
655
711
|
this.newPointer = index;
|
|
656
712
|
}
|
|
657
713
|
if (this.debug)
|
|
658
|
-
console.log(`RESOLVED ${this.oldPointer} to → ${this.newPointer}`);
|
|
714
|
+
console.log(`RESOLVED pointer ${this.oldPointer} to → ${this.newPointer}`);
|
|
659
715
|
}
|
|
660
716
|
else {
|
|
661
|
-
console.log("ERR:
|
|
717
|
+
console.log("ERR: 🧨 FAILED to resolve pointer", this.pointerBase, ` used by ${this._debugInfo.join(", ")}`);
|
|
662
718
|
}
|
|
663
719
|
}
|
|
664
720
|
write() {
|
|
665
721
|
for (let i = 0; i < this._objects.length; i++) {
|
|
666
722
|
const obj = this._objects[i];
|
|
667
723
|
const prop = this._props[i];
|
|
668
|
-
if (this.debug)
|
|
669
|
-
console.log(
|
|
724
|
+
if (this.debug) {
|
|
725
|
+
console.log(`< Resolved pointer ${this._debugInfo[i]}: ${this.newPointer}`);
|
|
726
|
+
}
|
|
670
727
|
obj[prop] = this.newPointer;
|
|
671
728
|
}
|
|
672
729
|
}
|
|
@@ -684,7 +741,7 @@ class PointerResolver {
|
|
|
684
741
|
}
|
|
685
742
|
debug = false;
|
|
686
743
|
}
|
|
687
|
-
class
|
|
744
|
+
class TextureIndexResolver extends PointerResolver {
|
|
688
745
|
_debugName;
|
|
689
746
|
_samplerDef;
|
|
690
747
|
init(context) {
|
|
@@ -719,7 +776,7 @@ class TexturePointerResolver extends PointerResolver {
|
|
|
719
776
|
// the texture was not found - add it
|
|
720
777
|
// create a new texture def entry
|
|
721
778
|
if (this.debug)
|
|
722
|
-
console.log("
|
|
779
|
+
console.log("Could not find texture, creating a new one: " + this._debugName);
|
|
723
780
|
const hasExtensionWithSource = this.inputDefinition.extensions && Object.keys(this.inputDefinition.extensions).some(key => this.inputDefinition.extensions[key]["source"] !== undefined);
|
|
724
781
|
const textureDef = {
|
|
725
782
|
source: hasExtensionWithSource ? undefined : imgIndex,
|
|
@@ -774,7 +831,7 @@ function findSource(texDef) {
|
|
|
774
831
|
}
|
|
775
832
|
return undefined;
|
|
776
833
|
}
|
|
777
|
-
class
|
|
834
|
+
class MaterialIndexResolver extends PointerResolver {
|
|
778
835
|
init(context) {
|
|
779
836
|
}
|
|
780
837
|
resolve(context) {
|
|
@@ -17,5 +17,12 @@ export const ALL_EXTENSIONS = [
|
|
|
17
17
|
createOpaqueExtension("EXT_texture_exr", [PropertyType.TEXTURE], { ...opaqueOptions }),
|
|
18
18
|
createOpaqueExtension("NEEDLE_editor", [PropertyType.NODE, PropertyType.MATERIAL], { ...opaqueOptions }),
|
|
19
19
|
createOpaqueExtension("NEEDLE_gltf_dependencies", [PropertyType.ROOT], { ...opaqueOptions }),
|
|
20
|
+
// VRM 0.0
|
|
20
21
|
createOpaqueExtension("VRM", [PropertyType.ROOT], { ...opaqueOptions }),
|
|
22
|
+
// VRM 1.0 https://vrm.dev/en/vrm1/changed/
|
|
23
|
+
createOpaqueExtension("VRMC_vrm", [PropertyType.ROOT], { ...opaqueOptions }),
|
|
24
|
+
createOpaqueExtension("VRMC_springBone", [PropertyType.ROOT], { ...opaqueOptions }),
|
|
25
|
+
createOpaqueExtension("VRMC_node_constraint", [PropertyType.NODE], { ...opaqueOptions }),
|
|
26
|
+
// TODO: mtoon materials don't work yet
|
|
27
|
+
// createOpaqueExtension("VRMC_materials_mtoon", [PropertyType.MATERIAL], { ...opaqueOptions, debug: true, breakpoint: "pointer:read" },),
|
|
21
28
|
];
|
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
import { Extension, ExtensionProperty, PropertyType, ReaderContext, WriterContext } from "@gltf-transform/core";
|
|
2
2
|
export declare interface IProgressiveMeshSettings {
|
|
3
|
-
|
|
3
|
+
/**
|
|
4
|
+
* When false no LODs will be generated for this mesh
|
|
5
|
+
*/
|
|
4
6
|
generateLods: boolean;
|
|
7
|
+
guid: string;
|
|
5
8
|
}
|
|
6
9
|
export declare class NEEDLE_progressive_mesh_settings extends Extension {
|
|
7
10
|
static EXTENSION_NAME: string;
|
|
@@ -1,5 +1,9 @@
|
|
|
1
1
|
import { Extension, ExtensionProperty, PropertyType, ReaderContext, WriterContext } from "@gltf-transform/core";
|
|
2
2
|
export declare interface IProgressiveTextureSettings {
|
|
3
|
+
/**
|
|
4
|
+
* When false the texture no progressive texture will be generated
|
|
5
|
+
*/
|
|
6
|
+
generateLods?: boolean;
|
|
3
7
|
maxSize: number;
|
|
4
8
|
guid: string;
|
|
5
9
|
}
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { Document, Texture } from '@gltf-transform/core';
|
|
2
2
|
import { NEEDLE_compression_texture_schema } from '../extensions/index.js';
|
|
3
3
|
export interface NeedleTransformContext {
|
|
4
|
+
/** full path to the file being processed */
|
|
5
|
+
file: string;
|
|
4
6
|
debug?: boolean;
|
|
5
7
|
useCache: boolean;
|
|
6
8
|
noToktx: boolean;
|
|
@@ -1,4 +1,8 @@
|
|
|
1
1
|
import { Transform, Document, Primitive, Mesh } from "@gltf-transform/core";
|
|
2
2
|
import { SimplifyOptions, WeldOptions } from "@gltf-transform/functions";
|
|
3
|
-
|
|
3
|
+
declare type MeshTransformContext = {
|
|
4
|
+
file: string;
|
|
5
|
+
};
|
|
6
|
+
export declare const needle_mesh_transform: (context: MeshTransformContext, simplify: SimplifyOptions) => Transform;
|
|
4
7
|
export declare function compressPrimitive(document: Document, mesh: Mesh, prim: Primitive, index: number, weldOptions: Required<WeldOptions>, meshOptions: Required<SimplifyOptions>): Promise<Primitive>;
|
|
8
|
+
export {};
|
|
@@ -3,14 +3,14 @@ import { meshopt, simplifyPrimitive, SIMPLIFY_DEFAULTS, weldPrimitive, WELD_DEFA
|
|
|
3
3
|
import { MeshoptEncoder, MeshoptSimplifier } from "meshoptimizer";
|
|
4
4
|
import { NEEDLE_mesh_compression } from "../extensions/index.js";
|
|
5
5
|
const NAME = 'needle_mesh_transform';
|
|
6
|
-
export const needle_mesh_transform = function (
|
|
6
|
+
export const needle_mesh_transform = function (context, simplify) {
|
|
7
7
|
return async (document, _context) => {
|
|
8
8
|
// TODO: needs to be moved into NEEDLE_progressive
|
|
9
|
-
await simplifierTransform(document, _context,
|
|
10
|
-
await applyMeshCompression(document, _context);
|
|
9
|
+
await simplifierTransform(document, _context, simplify);
|
|
10
|
+
await applyMeshCompression(document, _context, context);
|
|
11
11
|
};
|
|
12
12
|
};
|
|
13
|
-
async function applyMeshCompression(document,
|
|
13
|
+
async function applyMeshCompression(document, transform_context, context) {
|
|
14
14
|
const rootExtension = document.getRoot().getExtension(NEEDLE_mesh_compression.EXTENSION_NAME);
|
|
15
15
|
if (rootExtension) {
|
|
16
16
|
const rootModel = rootExtension.model;
|
|
@@ -22,17 +22,23 @@ async function applyMeshCompression(document, _context) {
|
|
|
22
22
|
console.log(`[${NAME}]: ${rootModel.compression.toUpperCase()} → will apply ${rootModel.compression} mesh compression`);
|
|
23
23
|
if (rootModel.compression === "draco") {
|
|
24
24
|
const transform = draco();
|
|
25
|
-
return transform(document,
|
|
25
|
+
return transform(document, transform_context);
|
|
26
26
|
}
|
|
27
27
|
else if (rootModel.compression === "meshopt") {
|
|
28
28
|
const transform = meshopt({ encoder: MeshoptEncoder });
|
|
29
|
-
return transform(document,
|
|
29
|
+
return transform(document, transform_context);
|
|
30
30
|
}
|
|
31
31
|
}
|
|
32
32
|
}
|
|
33
|
+
// If nothing is configured apply meshopt compression as long as we're running gltf-transform 3.x
|
|
34
|
+
if (context.file.endsWith(".vrm")) {
|
|
35
|
+
console.log(`[${NAME}]: VRM → will apply meshopt mesh compression`);
|
|
36
|
+
const transform = meshopt({ encoder: MeshoptEncoder });
|
|
37
|
+
return transform(document, transform_context);
|
|
38
|
+
}
|
|
33
39
|
// by default we still want to run draco if the root doesnt have a compression model configured
|
|
34
40
|
console.log(`[${NAME}]: No compression model configured → will apply draco mesh compression`);
|
|
35
|
-
return draco()(document,
|
|
41
|
+
return draco()(document, transform_context);
|
|
36
42
|
}
|
|
37
43
|
async function simplifierTransform(document, _context, _options) {
|
|
38
44
|
const simplifier = MeshoptSimplifier;
|
|
@@ -7,13 +7,13 @@ import path, { basename } from 'path';
|
|
|
7
7
|
import { existsSync, readFileSync, rmSync, statSync, writeFileSync } from 'fs';
|
|
8
8
|
import { addMetaToCache, addToCache, getHash, getKeyWithHash, tryGetFromCache, tryGetMetaFromCache } from '../cache/index.js';
|
|
9
9
|
import sharp from "sharp";
|
|
10
|
-
import { foreachGLTF, generateGuid } from '../utils/index.js';
|
|
10
|
+
import { detectTextureCompressionMode, foreachGLTF, generateGuid, writeNodeIO } from '../utils/index.js';
|
|
11
11
|
import { needle_asset, writeNeedleAsset } from './needle_asset.js';
|
|
12
12
|
import { MeshoptDecoder, MeshoptSimplifier } from 'meshoptimizer';
|
|
13
13
|
import { compressPrimitive } from './needle_mesh_transform.js';
|
|
14
14
|
import { calculateMeshDensity, getMeshInformation } from "../utils/mesh.js";
|
|
15
15
|
export async function make_progressive(opts) {
|
|
16
|
-
console.log(
|
|
16
|
+
console.log(`INFO: Run progressive transform: ${opts.path}`);
|
|
17
17
|
const file = opts.path;
|
|
18
18
|
return foreachGLTF(file, async (f) => {
|
|
19
19
|
// Don't re-process files that are already LODs
|
|
@@ -38,7 +38,7 @@ function registerExtensions(io) {
|
|
|
38
38
|
]);
|
|
39
39
|
}
|
|
40
40
|
async function onProcess(file, opts) {
|
|
41
|
-
if (!file.endsWith(".glb") && !file.endsWith(".gltf")) {
|
|
41
|
+
if (!file.endsWith(".glb") && !file.endsWith(".gltf") && !file.endsWith(".vrm")) {
|
|
42
42
|
console.log("ERR: file is not gltf/glb");
|
|
43
43
|
return;
|
|
44
44
|
}
|
|
@@ -71,7 +71,19 @@ async function onProcess(file, opts) {
|
|
|
71
71
|
}
|
|
72
72
|
//@ts-ignore
|
|
73
73
|
document.getLogger().verbosity = 0;
|
|
74
|
-
|
|
74
|
+
const transforms = [
|
|
75
|
+
needle_asset(),
|
|
76
|
+
make_progressive_textures(file, { size: [256, 256] }, opts),
|
|
77
|
+
sparse(),
|
|
78
|
+
];
|
|
79
|
+
// TODO: progressive meshes aren't supported yet for VRMs
|
|
80
|
+
if (!file.endsWith(".vrm")) {
|
|
81
|
+
transforms.push(make_progressive_meshes(file, {}, opts));
|
|
82
|
+
}
|
|
83
|
+
else
|
|
84
|
+
console.warn("WARN: Progressive meshes are not supported for VRM files yet");
|
|
85
|
+
console.log("→ Transforming: " + file);
|
|
86
|
+
await document.transform(...transforms);
|
|
75
87
|
let filename = file;
|
|
76
88
|
if (opts.debug === true) {
|
|
77
89
|
// get extension
|
|
@@ -81,7 +93,8 @@ async function onProcess(file, opts) {
|
|
|
81
93
|
// add new extension
|
|
82
94
|
filename = filename + ".debug." + extension;
|
|
83
95
|
}
|
|
84
|
-
io
|
|
96
|
+
await writeNodeIO(io, filename, document);
|
|
97
|
+
console.log("← Transforming done: " + filename);
|
|
85
98
|
}
|
|
86
99
|
}
|
|
87
100
|
catch (err) {
|
|
@@ -130,7 +143,7 @@ export function make_progressive_textures(filePath, _options = TEXTURE_RESIZE_DE
|
|
|
130
143
|
if (!settings) {
|
|
131
144
|
const size = texture.getSize();
|
|
132
145
|
if (size?.length >= 2 && size[0] <= 128 && size[1] <= 128) {
|
|
133
|
-
logger.warn(`[${NAME}] Skipping texture[${textureIndex}], no settings found`);
|
|
146
|
+
logger.warn(`[${NAME}] Skipping texture[${textureIndex}], no settings found and the texture is already < 128x128`);
|
|
134
147
|
continue;
|
|
135
148
|
}
|
|
136
149
|
else {
|
|
@@ -141,6 +154,10 @@ export function make_progressive_textures(filePath, _options = TEXTURE_RESIZE_DE
|
|
|
141
154
|
logger.debug(`[${NAME}] No settings found for texture[${index}] - will use default settings (maxSize: ${settings.maxSize}, guid: ${settings.guid})`);
|
|
142
155
|
}
|
|
143
156
|
}
|
|
157
|
+
else if (settings.generateLods === false) {
|
|
158
|
+
logger.debug(`[${NAME}] Skipping texture[${textureIndex}]: progressive texture disabled`);
|
|
159
|
+
continue;
|
|
160
|
+
}
|
|
144
161
|
if (settings.maxSize == undefined || settings.maxSize <= 0) {
|
|
145
162
|
logger.warn(`[${NAME}] Skipping texture[${textureIndex}], settings are invalid (maxSize: ${settings.maxSize})`);
|
|
146
163
|
continue;
|
|
@@ -158,7 +175,7 @@ export function make_progressive_textures(filePath, _options = TEXTURE_RESIZE_DE
|
|
|
158
175
|
continue;
|
|
159
176
|
}
|
|
160
177
|
const textureGuid = settings.guid?.length ? settings.guid : generateGuid(path + "_texture_" + index.toString());
|
|
161
|
-
const outPath = createTextureLod(doc, textureLodBasePath, 0, texture, slots, textureGuid, textureIndex);
|
|
178
|
+
const outPath = createTextureLod(doc, filePath, textureLodBasePath, 0, texture, slots, textureGuid, textureIndex);
|
|
162
179
|
if (!outPath) {
|
|
163
180
|
continue;
|
|
164
181
|
}
|
|
@@ -194,7 +211,7 @@ export function make_progressive_textures(filePath, _options = TEXTURE_RESIZE_DE
|
|
|
194
211
|
logger.debug(`Progressive transform finished in ${((Date.now() - startTime) / 1000).toFixed(1)} sec.`);
|
|
195
212
|
});
|
|
196
213
|
}
|
|
197
|
-
function createTextureLod(doc, basePath, level, texture, slots, guid, textureIndex) {
|
|
214
|
+
function createTextureLod(doc, file, basePath, level, texture, slots, guid, textureIndex) {
|
|
198
215
|
// const imageBytes = texture.getImage();
|
|
199
216
|
// if (imageBytes) {
|
|
200
217
|
// // write bytes
|
|
@@ -245,6 +262,13 @@ function createTextureLod(doc, basePath, level, texture, slots, guid, textureInd
|
|
|
245
262
|
console.log("> Pass compression extension to new texture");
|
|
246
263
|
const newExt = newDoc.createExtension(NEEDLE_compression_texture);
|
|
247
264
|
const def = compressionSettings.getExtensionDefinition();
|
|
265
|
+
def.mode = detectTextureCompressionMode(slots);
|
|
266
|
+
def.slot = slots[0];
|
|
267
|
+
// for VRMS we just use webp
|
|
268
|
+
// if (file?.toLocaleLowerCase().endsWith(".vrm")) {
|
|
269
|
+
// def.mode = "webp";
|
|
270
|
+
// def.quality = 70;
|
|
271
|
+
// }
|
|
248
272
|
const newProp = newExt.create(def);
|
|
249
273
|
newTexture.setExtension(NEEDLE_compression_texture.EXTENSION_NAME, newProp);
|
|
250
274
|
}
|
|
@@ -5,6 +5,8 @@ import { addToCache, getKey, tryGetFromCache } from "../cache/index.js";
|
|
|
5
5
|
import { NeedleWebPContext } from "./needle_webp.js";
|
|
6
6
|
import { NeedleToktxContext } from "./needle_toktx.js";
|
|
7
7
|
import { createTransform } from "@gltf-transform/functions";
|
|
8
|
+
import { dirname } from "path";
|
|
9
|
+
import { existsSync, rmSync } from "fs";
|
|
8
10
|
export async function resizeTextureIfNecessary(texture) {
|
|
9
11
|
const ext = texture.getExtension(NEEDLE_compression_texture.EXTENSION_NAME);
|
|
10
12
|
if (ext?.model) {
|
|
@@ -74,8 +76,8 @@ export const needle_texture_transform = function (options) {
|
|
|
74
76
|
});
|
|
75
77
|
};
|
|
76
78
|
async function processTexture(texture, index, document, logger, context) {
|
|
77
|
-
const
|
|
78
|
-
const isSupported =
|
|
79
|
+
const originalMimeType = texture.getMimeType();
|
|
80
|
+
const isSupported = originalMimeType === "image/jpeg" || originalMimeType === "image/png" || originalMimeType === "image/webp";
|
|
79
81
|
// dont try to transform EXR or already processed textures
|
|
80
82
|
if (!isSupported)
|
|
81
83
|
return;
|
|
@@ -96,6 +98,7 @@ async function processTexture(texture, index, document, logger, context) {
|
|
|
96
98
|
}
|
|
97
99
|
let successFullyProcessed = false;
|
|
98
100
|
const canRunToktx = context.noToktx !== true;
|
|
101
|
+
const originalUri = texture.getURI();
|
|
99
102
|
if (compressionSettings && compressionSettings.mode !== "auto") {
|
|
100
103
|
switch (compressionSettings.mode) {
|
|
101
104
|
case "webp":
|
|
@@ -125,21 +128,46 @@ async function processTexture(texture, index, document, logger, context) {
|
|
|
125
128
|
}
|
|
126
129
|
}
|
|
127
130
|
else {
|
|
131
|
+
let mimeType = "image/ktx2";
|
|
132
|
+
// const slots = listTextureSlots(texture);
|
|
133
|
+
// if (slots.includes("metallicRoughnessTexture"))
|
|
134
|
+
// mimeType = "image/webp";
|
|
135
|
+
const isVRM = context.file?.toLowerCase().endsWith(".vrm");
|
|
136
|
+
// TODO: need to make this into a function
|
|
137
|
+
// const webpThreshold = 256; // this is the default threshold in progressive textures (needle_progressive.ts)
|
|
138
|
+
// console.log(context.file, isVRM);
|
|
139
|
+
// // if we're dealing with a VRM and the texture size is < 128x128 we can use webp (this is e.g. the case for progressive textures)
|
|
140
|
+
// if (isVRM) {
|
|
141
|
+
// const size = texture.getSize();
|
|
142
|
+
// console.log(isVRM, size)
|
|
143
|
+
// if (size && size[0] <= webpThreshold && size[1] <= webpThreshold) {
|
|
144
|
+
// mimetype = "image/webp";
|
|
145
|
+
// }
|
|
146
|
+
// }
|
|
128
147
|
// If we have a texture cached we can use it
|
|
129
148
|
// the default compression is toktx so we can safely use that for now
|
|
130
149
|
if (cached) {
|
|
131
|
-
texture.setImage(cached).setMimeType(
|
|
150
|
+
texture.setImage(cached).setMimeType(mimeType);
|
|
132
151
|
return;
|
|
133
152
|
}
|
|
134
153
|
await resizeTextureIfNecessary(texture);
|
|
135
|
-
if (
|
|
154
|
+
if (mimeType === "image/webp") {
|
|
155
|
+
successFullyProcessed = await context.webPContext.process(index, texture, {
|
|
156
|
+
quality: .9,
|
|
157
|
+
});
|
|
158
|
+
}
|
|
159
|
+
else if (canRunToktx) {
|
|
136
160
|
// if no specific compression settings are provided we just use the toktx default settings
|
|
137
161
|
// if we ever change what is used for default compression we need to teach the cache to save the mime type
|
|
138
|
-
|
|
139
|
-
|
|
162
|
+
// For VRMs we use ETC1S always
|
|
163
|
+
let options = null;
|
|
164
|
+
if (isVRM) {
|
|
165
|
+
options = { mode: "ETC1S" };
|
|
166
|
+
}
|
|
167
|
+
successFullyProcessed = await context.toktxContext.process(index, texture, options);
|
|
140
168
|
}
|
|
141
169
|
else {
|
|
142
|
-
console.log("
|
|
170
|
+
console.log("Toktx not found: Skipping texture compression.");
|
|
143
171
|
}
|
|
144
172
|
}
|
|
145
173
|
if (context.useCache && successFullyProcessed && key) {
|
|
@@ -147,4 +175,15 @@ async function processTexture(texture, index, document, logger, context) {
|
|
|
147
175
|
if (data)
|
|
148
176
|
addToCache(key, data);
|
|
149
177
|
}
|
|
178
|
+
if (successFullyProcessed) {
|
|
179
|
+
const newUri = texture.getURI();
|
|
180
|
+
if (originalUri && originalUri != newUri) {
|
|
181
|
+
const fileDirectory = dirname(context.file);
|
|
182
|
+
const originalPath = fileDirectory + "/" + originalUri;
|
|
183
|
+
if (existsSync(originalPath)) {
|
|
184
|
+
console.log(`INFO: Texture moved \"${originalUri}\" → \"${newUri}\"`);
|
|
185
|
+
rmSync(originalPath);
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
}
|
|
150
189
|
}
|
|
@@ -20,6 +20,6 @@ export declare class NeedleToktxContext implements INeedleTextureTransformContex
|
|
|
20
20
|
private _prepareFailed;
|
|
21
21
|
constructor(options: NeedleToktxOptions);
|
|
22
22
|
prepare(document: Document): Promise<boolean>;
|
|
23
|
-
process(index: number, texture: Texture, settings: NEEDLE_compression_texture_schema | null): Promise<boolean>;
|
|
23
|
+
process(index: number, texture: Texture, settings: Pick<NEEDLE_compression_texture_schema, "mode"> | null): Promise<boolean>;
|
|
24
24
|
finalize(document: Document): Promise<void>;
|
|
25
25
|
}
|
|
@@ -8,6 +8,7 @@ import fs from 'fs/promises';
|
|
|
8
8
|
import os from 'os';
|
|
9
9
|
import tmp from 'tmp';
|
|
10
10
|
import pLimit from 'p-limit';
|
|
11
|
+
import { replaceExtension } from "../utils/fileutils.js";
|
|
11
12
|
const NUM_CPUS = os.cpus().length || 1; // microsoft/vscode#112122
|
|
12
13
|
let didLogEnvironmentInfo = false;
|
|
13
14
|
export class NeedleToktxContext {
|
|
@@ -113,17 +114,39 @@ export class NeedleToktxContext {
|
|
|
113
114
|
const outPath = tmpName + '.ktx2';
|
|
114
115
|
// const inBytes = image.byteLength;
|
|
115
116
|
await fs.writeFile(inPath, Buffer.from(image));
|
|
117
|
+
let mode = Mode.ETC1S;
|
|
118
|
+
let quality = 128;
|
|
119
|
+
let compression = 1;
|
|
120
|
+
if (settings) {
|
|
121
|
+
if (settings.mode === "auto") {
|
|
122
|
+
const useUASTC = slots.includes("normalTexture") || slots.includes("metallicRoughnessTexture");
|
|
123
|
+
mode = useUASTC ? Mode.UASTC : Mode.ETC1S;
|
|
124
|
+
}
|
|
125
|
+
else {
|
|
126
|
+
mode = settings.mode === "ETC1S" ? Mode.ETC1S : Mode.UASTC;
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
else if (slots) {
|
|
130
|
+
const isNormal = slots.includes("normalTexture");
|
|
131
|
+
const isMetallicRoughness = slots.includes("metallicRoughnessTexture");
|
|
132
|
+
const betterQuality = isNormal || isMetallicRoughness;
|
|
133
|
+
// mode = useUASTC ? Mode.UASTC : Mode.ETC1S;
|
|
134
|
+
mode = Mode.ETC1S;
|
|
135
|
+
// if (isMetallicRoughness) mode = Mode.UASTC;
|
|
136
|
+
// https://github.khronos.org/KTX-Software/ktxtools/toktx.html
|
|
137
|
+
quality = betterQuality ? 255 : 128;
|
|
138
|
+
// ETC1S / BasisLZ compression level, an encoding speed vs. quality tradeoff. Range is [0,5], default is 1. Higher values are slower but give higher quality.
|
|
139
|
+
compression = betterQuality ? 4 : 1;
|
|
140
|
+
}
|
|
141
|
+
const defaults = mode === Mode.ETC1S ? ETC1S_DEFAULTS : UASTC_DEFAULTS;
|
|
116
142
|
//@ts-ignore
|
|
117
|
-
const
|
|
118
|
-
defaults.mode = (!settings || settings.mode === "ETC1S") ? Mode.ETC1S : Mode.UASTC;
|
|
119
|
-
//@ts-ignore
|
|
120
|
-
const toktxParams = createParams(slots, channels, size, logger, this.numTextures, { ...defaults }, this.version);
|
|
143
|
+
const toktxParams = createParams(slots, channels, size, logger, this.numTextures, { ...defaults, mode, quality, compression }, this.version);
|
|
121
144
|
const params = [
|
|
122
145
|
...toktxParams,
|
|
123
146
|
outPath,
|
|
124
147
|
inPath,
|
|
125
148
|
];
|
|
126
|
-
logger.debug(`${prefix}: Spawning → toktx ${params.join(' ')}`);
|
|
149
|
+
logger.debug(`${prefix}: Spawning → toktx ${mode.toUpperCase()} ${params.join(' ')}`);
|
|
127
150
|
try {
|
|
128
151
|
// COMPRESS: Run `toktx` CLI tool.
|
|
129
152
|
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
@@ -135,6 +158,13 @@ export class NeedleToktxContext {
|
|
|
135
158
|
else {
|
|
136
159
|
const result = await fs.readFile(outPath);
|
|
137
160
|
texture.setImage(result).setMimeType('image/ktx2');
|
|
161
|
+
// TODO: we also need to remove the old file when the extension has changed - this should probably be done outside of this method to also support webp
|
|
162
|
+
// If this is a texture that was loaded from a file, we need to update the URI
|
|
163
|
+
if (texture.getURI()) {
|
|
164
|
+
const oldUri = texture.getURI();
|
|
165
|
+
const newUri = replaceExtension(oldUri, 'ktx2');
|
|
166
|
+
texture.setURI(newUri);
|
|
167
|
+
}
|
|
138
168
|
return true;
|
|
139
169
|
}
|
|
140
170
|
}
|
|
@@ -8,6 +8,6 @@ export declare class NeedleWebPContext implements INeedleTextureTransformContext
|
|
|
8
8
|
logger: ILogger;
|
|
9
9
|
constructor(options: NeedleWebPOptions);
|
|
10
10
|
prepare(document: Document): Promise<void>;
|
|
11
|
-
process(index: number, texture: Texture, settings: NEEDLE_compression_texture_schema): Promise<boolean>;
|
|
11
|
+
process(index: number, texture: Texture, settings: Pick<NEEDLE_compression_texture_schema, "quality">): Promise<boolean>;
|
|
12
12
|
finalize(document: Document): Promise<void>;
|
|
13
13
|
}
|
|
@@ -1,2 +1,3 @@
|
|
|
1
1
|
export declare type GLTFFileCallback = (file: string) => void | Promise<void>;
|
|
2
2
|
export declare function foreachGLTF(args: string | string[], callback: GLTFFileCallback): Promise<void>;
|
|
3
|
+
export declare function replaceExtension(filepath: string, newExtension: string): string;
|
package/dist/utils/fileutils.js
CHANGED
|
@@ -89,7 +89,7 @@ async function testEntry(entry, callback, context) {
|
|
|
89
89
|
}
|
|
90
90
|
}
|
|
91
91
|
}
|
|
92
|
-
if (entry.endsWith(".glb") || entry.endsWith(".gltf")) {
|
|
92
|
+
if (entry.endsWith(".glb") || entry.endsWith(".gltf") || entry.endsWith(".vrm")) {
|
|
93
93
|
context.filesFound += 1;
|
|
94
94
|
const res = callback(entry);
|
|
95
95
|
if (res instanceof Promise) {
|
|
@@ -113,3 +113,8 @@ async function testEntry(entry, callback, context) {
|
|
|
113
113
|
}
|
|
114
114
|
}
|
|
115
115
|
}
|
|
116
|
+
export function replaceExtension(filepath, newExtension) {
|
|
117
|
+
const parts = filepath.split(".");
|
|
118
|
+
parts.pop();
|
|
119
|
+
return parts.join(".") + "." + newExtension;
|
|
120
|
+
}
|
package/dist/utils/index.d.ts
CHANGED
package/dist/utils/index.js
CHANGED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { writeFile } from "fs/promises";
|
|
2
|
+
/**
|
|
3
|
+
* Handle writing glTf, GLB and VRM files
|
|
4
|
+
*/
|
|
5
|
+
export async function writeNodeIO(nodeio, filename, document) {
|
|
6
|
+
// see https://github.com/donmccurdy/glTF-Transform/issues/1390
|
|
7
|
+
if (filename.endsWith(".vrm")) {
|
|
8
|
+
const bin = await nodeio.writeBinary(document);
|
|
9
|
+
await writeFile(filename, bin);
|
|
10
|
+
}
|
|
11
|
+
else
|
|
12
|
+
await nodeio.write(filename, document);
|
|
13
|
+
return filename;
|
|
14
|
+
}
|
package/limit-caches.mjs
ADDED
package/pack-gltf.mjs
CHANGED
|
@@ -12,7 +12,8 @@ import { needle_animation_transform } from "./dist/transforms/needle_animation_t
|
|
|
12
12
|
import { foreachGLTF } from './fileutils.mjs';
|
|
13
13
|
import { needle_asset } from './dist/transforms/needle_asset.js';
|
|
14
14
|
import { MeshoptDecoder, MeshoptEncoder } from 'meshoptimizer';
|
|
15
|
-
import { getVersion, getHash, addToCache, tryGetFromCache, isLOD,
|
|
15
|
+
import { getVersion, getHash, addToCache, tryGetFromCache, isLOD, limitCacheSize, isMeshLOD } from './dist/index.js';
|
|
16
|
+
import { writeNodeIO } from './dist/utils/nodeio.js';
|
|
16
17
|
|
|
17
18
|
console.log(`[Needle Build Pipeline] pack glTF — version ${getVersion()}`)
|
|
18
19
|
|
|
@@ -59,11 +60,9 @@ if (debug) {
|
|
|
59
60
|
|
|
60
61
|
let allowCaching = true;
|
|
61
62
|
if (args.includes("--no-caching")) {
|
|
62
|
-
console.log("Caching is disabled")
|
|
63
63
|
allowCaching = false;
|
|
64
64
|
}
|
|
65
|
-
else
|
|
66
|
-
console.log("Caching is enabled, use \"--no-caching\" to disable it");
|
|
65
|
+
else console.log("Use --no-caching to disable caching");
|
|
67
66
|
|
|
68
67
|
const noToktx = args.includes("no-toktx");
|
|
69
68
|
if (noToktx)
|
|
@@ -105,10 +104,11 @@ async function packFile(file) {
|
|
|
105
104
|
|
|
106
105
|
if (packedFiles.has(file)) return;
|
|
107
106
|
packedFiles.add(file);
|
|
108
|
-
console.log("→ Begin packing: " + file);
|
|
109
107
|
|
|
110
108
|
try {
|
|
111
109
|
const prevSizeInMB = statSync(file).size / 1024 / 1024;
|
|
110
|
+
console.log(`→ Compressing ${prevSizeInMB.toFixed(1)} MB: ${file}`);
|
|
111
|
+
|
|
112
112
|
const startTime = Date.now();
|
|
113
113
|
const io = new NodeIO()
|
|
114
114
|
.registerExtensions(ALL_EXTENSIONS)
|
|
@@ -129,17 +129,19 @@ async function packFile(file) {
|
|
|
129
129
|
await document.transform(
|
|
130
130
|
needle_asset(),
|
|
131
131
|
needle_texture_transform({
|
|
132
|
+
file,
|
|
132
133
|
useCache: allowCaching,
|
|
133
134
|
noToktx: noToktx,
|
|
134
135
|
debug: debug,
|
|
135
136
|
reportGPUMemory: val => totalGPUTextureMemmory += val
|
|
136
137
|
}),
|
|
137
|
-
needle_mesh_transform({ ratio: 0.75, error: 0.0005 }),
|
|
138
|
+
needle_mesh_transform({ file }, { ratio: 0.75, error: 0.0005 }),
|
|
138
139
|
needle_animation_transform(),
|
|
139
140
|
// draco(),
|
|
140
141
|
);
|
|
141
142
|
if (debug) file += ".debug.glb";
|
|
142
|
-
|
|
143
|
+
|
|
144
|
+
await writeNodeIO(io, file, document);
|
|
143
145
|
|
|
144
146
|
if (allowCaching && cacheKey != undefined && existsSync(file))
|
|
145
147
|
addToCache(cacheKey, readFileSync(file));
|
|
@@ -167,18 +169,20 @@ async function packFile(file) {
|
|
|
167
169
|
|
|
168
170
|
let startTime = Date.now();
|
|
169
171
|
const inputArgs = args.slice(2);
|
|
172
|
+
let failed = false;
|
|
170
173
|
foreachGLTF(inputArgs, async (file) => {
|
|
171
|
-
await packFile(file);
|
|
174
|
+
const res = await packFile(file);
|
|
175
|
+
if (!res) failed = true;
|
|
172
176
|
}).then(async () => {
|
|
173
177
|
const duration = Date.now() - startTime;
|
|
174
178
|
const durationInMinutes = duration / 1000 / 60;
|
|
175
179
|
if (allowCaching) {
|
|
176
|
-
|
|
177
|
-
// const freeSpaceInMB = (await getCacheAvailableSpaceOnDisc().catch(() => 0)) / 1024 / 1024;
|
|
178
|
-
// const maxCacheSize = freeSpaceInMB > 25000 ? 2000 : 1000;
|
|
179
|
-
ensureMaxCacheSize(1000);
|
|
180
|
+
limitCacheSize();
|
|
180
181
|
}
|
|
181
182
|
console.log(`[Needle Build Pipeline] v${getVersion()} — Finished packing ${totalFilesProcessed} files in ${durationInMinutes.toFixed(1)} minutes; total size ${totalFileSizeInMBBefore.toFixed(1)} MB → ${totalFileSizeInMB.toFixed(1)} MB, total GPU memory: ${totalGPUMemoryInMB.toFixed(1)} MB`);
|
|
183
|
+
if (failed) {
|
|
184
|
+
process.exit(1);
|
|
185
|
+
}
|
|
182
186
|
}).catch(err => {
|
|
183
187
|
console.error("ERR: packing failed: " + err.message);
|
|
184
188
|
if (debug) throw err;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@needle-tools/gltf-build-pipeline",
|
|
3
|
-
"version": "1.5.0-alpha",
|
|
3
|
+
"version": "1.5.0-alpha.10",
|
|
4
4
|
"description": "Pipeline and tools for optimizing gltf files using gltf-transform and compression settings within glTF extensions",
|
|
5
5
|
"main": "index.mjs",
|
|
6
6
|
"type": "module",
|
|
@@ -13,6 +13,7 @@
|
|
|
13
13
|
"pack-gltf": "node --no-experimental-fetch pack-gltf.mjs",
|
|
14
14
|
"dev:pack-gltf": "npm run compile && node pack-gltf.mjs",
|
|
15
15
|
"clear-caches": "node clear-caches.mjs",
|
|
16
|
+
"limit-caches": "node limit-caches.mjs",
|
|
16
17
|
"compile": "tsc || exit 1"
|
|
17
18
|
},
|
|
18
19
|
"dependencies": {
|
package/dist/functions/gzip.d.ts
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export declare function gzipFile(filepath: string): Promise<string>;
|
package/dist/functions/gzip.js
DELETED
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
import { promises as fs, existsSync } from 'fs';
|
|
2
|
-
import { gzip } from 'node-gzip';
|
|
3
|
-
export async function gzipFile(filepath) {
|
|
4
|
-
if (filepath.endsWith('.gz')) {
|
|
5
|
-
return filepath;
|
|
6
|
-
}
|
|
7
|
-
if (!existsSync(filepath)) {
|
|
8
|
-
console.log("File not found: " + filepath);
|
|
9
|
-
return filepath;
|
|
10
|
-
}
|
|
11
|
-
const inBuffer = await fs.readFile(filepath);
|
|
12
|
-
const outBuffer = await gzip(inBuffer);
|
|
13
|
-
const outPath = filepath;
|
|
14
|
-
await fs.writeFile(outPath, outBuffer);
|
|
15
|
-
return outPath;
|
|
16
|
-
}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export * from "./gzip.js";
|
package/dist/functions/index.js
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export * from "./gzip.js";
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export {};
|
|
@@ -1,35 +0,0 @@
|
|
|
1
|
-
export {};
|
|
2
|
-
// TODO: remove once this is resolved: https://github.com/donmccurdy/glTF-Transform/issues/855
|
|
3
|
-
// modified to take a document as a parameter
|
|
4
|
-
// export function listTextureSlots(texture: Texture, document: Document): string[] {
|
|
5
|
-
// const root = document.getRoot();
|
|
6
|
-
// const slots = texture
|
|
7
|
-
// .getGraph()
|
|
8
|
-
// .listParentEdges(texture)
|
|
9
|
-
// .filter((edge) => edge.getParent() !== root)
|
|
10
|
-
// .map((edge) => edge.getName());
|
|
11
|
-
// return Array.from(new Set(slots));
|
|
12
|
-
// }
|
|
13
|
-
// export function getTextureChannelMask(texture: Texture, document: Document): number {
|
|
14
|
-
// let mask = 0x0000;
|
|
15
|
-
// for (const edge of document.getGraph().listParentEdges(texture)) {
|
|
16
|
-
// const parent = edge.getParent();
|
|
17
|
-
// let { channels } = edge.getAttributes() as { channels: number | undefined };
|
|
18
|
-
// if (
|
|
19
|
-
// channels &&
|
|
20
|
-
// edge.getName() === 'baseColorTexture' &&
|
|
21
|
-
// parent instanceof Material &&
|
|
22
|
-
// parent.getAlphaMode() === Material.AlphaMode.OPAQUE
|
|
23
|
-
// ) {
|
|
24
|
-
// channels &= ~TextureChannel.A;
|
|
25
|
-
// }
|
|
26
|
-
// if (channels) {
|
|
27
|
-
// mask |= channels;
|
|
28
|
-
// continue;
|
|
29
|
-
// }
|
|
30
|
-
// if (parent.propertyType !== PropertyType.ROOT) {
|
|
31
|
-
// document.getLogger().warn(`Missing attribute ".channels" on edge, "${edge.getName()}".`);
|
|
32
|
-
// }
|
|
33
|
-
// }
|
|
34
|
-
// return mask;
|
|
35
|
-
// }
|
|
@@ -1,76 +0,0 @@
|
|
|
1
|
-
import { Primitive } from "@gltf-transform/core";
|
|
2
|
-
import { simplifyPrimitive, weldPrimitive } from "@gltf-transform/functions";
|
|
3
|
-
import { MeshoptSimplifier } from "meshoptimizer";
|
|
4
|
-
import { NEEDLE_mesh_compression } from "../extensions/index.js";
|
|
5
|
-
const NAME = 'needle_simplify';
|
|
6
|
-
export const needle_mesh_transform = function (options) {
|
|
7
|
-
const simplifier = MeshoptSimplifier;
|
|
8
|
-
if (!simplifier) {
|
|
9
|
-
throw new Error(`${NAME}: simplifier dependency required — install "meshoptimizer".`);
|
|
10
|
-
}
|
|
11
|
-
return (document, _context) => {
|
|
12
|
-
const logger = document.getLogger();
|
|
13
|
-
options = { ...options, simplifier: simplifier };
|
|
14
|
-
const weldOptions = {
|
|
15
|
-
tolerance: 0.00000000001,
|
|
16
|
-
overwrite: true
|
|
17
|
-
};
|
|
18
|
-
for (const mesh of document.getRoot().listMeshes()) {
|
|
19
|
-
for (const prim of mesh.listPrimitives()) {
|
|
20
|
-
// const key = getKey(prim);
|
|
21
|
-
// const cached = tryGetFromCache(key);
|
|
22
|
-
// if (cached) {
|
|
23
|
-
// logger.debug(`${NAME}: Skipping primitive of mesh "${mesh.getName()}": Already cached.`);
|
|
24
|
-
// continue;
|
|
25
|
-
// }
|
|
26
|
-
// updatePrimitive(prim, simplifier, null, prim.getIndices());
|
|
27
|
-
// continue;
|
|
28
|
-
const ext = prim.getExtension(NEEDLE_mesh_compression.EXTENSION_NAME);
|
|
29
|
-
if (!ext) {
|
|
30
|
-
// logger.warn("Skipping primitive of mesh " + mesh.getName() + ": No NEEDLE_compression_mesh extension.");
|
|
31
|
-
continue;
|
|
32
|
-
}
|
|
33
|
-
let meshOptions = { ...options };
|
|
34
|
-
if (ext.model) {
|
|
35
|
-
const model = ext.model;
|
|
36
|
-
if (!model.useSimplifier) {
|
|
37
|
-
logger.debug(`${NAME}: Skipping primitive of mesh "${mesh.getName()}": useSimplifier is false.`);
|
|
38
|
-
continue;
|
|
39
|
-
}
|
|
40
|
-
if (model.error >= 0)
|
|
41
|
-
meshOptions.error = model.error;
|
|
42
|
-
if (model.ratio >= 0)
|
|
43
|
-
meshOptions.ratio = model.ratio;
|
|
44
|
-
meshOptions.lockBorder = model.lockBorder;
|
|
45
|
-
}
|
|
46
|
-
if (prim.getMode() !== Primitive.Mode.TRIANGLES) {
|
|
47
|
-
logger.warn(`${NAME}: Skipping primitive of mesh "${mesh.getName()}": Requires TRIANGLES draw mode.`);
|
|
48
|
-
continue;
|
|
49
|
-
}
|
|
50
|
-
logger.debug(`${NAME}: Welding primitive of mesh "${mesh.getName()}" → tolerance: ${weldOptions.tolerance}`);
|
|
51
|
-
weldPrimitive(document, prim, weldOptions);
|
|
52
|
-
logger.debug(`${NAME}: Simplifying primitive of mesh "${mesh.getName()}" → ratio: ${meshOptions.ratio}, error: ${meshOptions.error}, lockBorder: ${meshOptions.lockBorder}`);
|
|
53
|
-
simplifyPrimitive(document, prim, meshOptions);
|
|
54
|
-
}
|
|
55
|
-
}
|
|
56
|
-
};
|
|
57
|
-
};
|
|
58
|
-
// function updatePrimitive(prim: Primitive, simplifier: any, positionArray, dstIndicesArray) {
|
|
59
|
-
// const position = prim.getAttribute('POSITION')!;
|
|
60
|
-
// const srcIndices = prim.getIndices()!;
|
|
61
|
-
// const srcVertexCount = position.getCount();
|
|
62
|
-
// const [remap, unique] = simplifier.compactMesh(dstIndicesArray);
|
|
63
|
-
// // logger.debug(`${NAME}: ${formatDeltaOp(position.getCount(), unique)} vertices, error: ${error.toFixed(4)}.`);
|
|
64
|
-
// // (3) Write vertex attributes.
|
|
65
|
-
// for (const srcAttribute of deepListAttributes(prim)) {
|
|
66
|
-
// const dstAttribute = srcAttribute.clone();
|
|
67
|
-
// remapAttribute(dstAttribute, remap, unique);
|
|
68
|
-
// deepSwapAttribute(prim, srcAttribute, dstAttribute);
|
|
69
|
-
// if (srcAttribute.listParents().length === 1) srcAttribute.dispose();
|
|
70
|
-
// }
|
|
71
|
-
// // (4) Write indices.
|
|
72
|
-
// const dstIndices = srcIndices.clone();
|
|
73
|
-
// dstIndices.setArray(srcVertexCount <= 65534 ? new Uint16Array(dstIndicesArray) : dstIndicesArray);
|
|
74
|
-
// prim.setIndices(dstIndices);
|
|
75
|
-
// if (srcIndices.listParents().length === 1) srcIndices.dispose();
|
|
76
|
-
// }
|
|
@@ -1,31 +0,0 @@
|
|
|
1
|
-
export declare class WebPEncoderOptions {
|
|
2
|
-
quality: number;
|
|
3
|
-
target_size: number;
|
|
4
|
-
target_PSNR: number;
|
|
5
|
-
method: number;
|
|
6
|
-
sns_strength: number;
|
|
7
|
-
filter_strength: number;
|
|
8
|
-
filter_sharpness: number;
|
|
9
|
-
/** 1 = strong filter */
|
|
10
|
-
filter_type: number;
|
|
11
|
-
partitions: number;
|
|
12
|
-
segments: number;
|
|
13
|
-
pass: number;
|
|
14
|
-
show_compressed: number;
|
|
15
|
-
preprocessing: number;
|
|
16
|
-
autofilter: number;
|
|
17
|
-
partition_limit: number;
|
|
18
|
-
alpha_compression: number;
|
|
19
|
-
alpha_filtering: number;
|
|
20
|
-
alpha_quality: number;
|
|
21
|
-
lossless: number;
|
|
22
|
-
exact: number;
|
|
23
|
-
image_hint: number;
|
|
24
|
-
emulate_jpeg_size: number;
|
|
25
|
-
thread_level: number;
|
|
26
|
-
low_memory: number;
|
|
27
|
-
near_lossless: number;
|
|
28
|
-
use_delta_palette: number;
|
|
29
|
-
/** probably sharp rgb -> YUV conversion */
|
|
30
|
-
use_sharp_yuv: number;
|
|
31
|
-
}
|
|
@@ -1,33 +0,0 @@
|
|
|
1
|
-
// client: https://github.com/GoogleChromeLabs/squoosh/blob/97d13de273b67ee01a80cb32161f1bb2103b73cb/src/features/encoders/webP/client/index.tsx#L292
|
|
2
|
-
// emscripten bindings:
|
|
3
|
-
export class WebPEncoderOptions {
|
|
4
|
-
quality = 75;
|
|
5
|
-
target_size = 0;
|
|
6
|
-
target_PSNR = 0;
|
|
7
|
-
method = 4;
|
|
8
|
-
sns_strength = 50;
|
|
9
|
-
filter_strength = 60;
|
|
10
|
-
filter_sharpness = 0;
|
|
11
|
-
/** 1 = strong filter */
|
|
12
|
-
filter_type = 1;
|
|
13
|
-
partitions = 0;
|
|
14
|
-
segments = 4;
|
|
15
|
-
pass = 1;
|
|
16
|
-
show_compressed = 0;
|
|
17
|
-
preprocessing = 0;
|
|
18
|
-
autofilter = 0;
|
|
19
|
-
partition_limit = 0;
|
|
20
|
-
alpha_compression = 1;
|
|
21
|
-
alpha_filtering = 1;
|
|
22
|
-
alpha_quality = 100;
|
|
23
|
-
lossless = 0;
|
|
24
|
-
exact = 0;
|
|
25
|
-
image_hint = 0;
|
|
26
|
-
emulate_jpeg_size = 0;
|
|
27
|
-
thread_level = 0;
|
|
28
|
-
low_memory = 0;
|
|
29
|
-
near_lossless = 100;
|
|
30
|
-
use_delta_palette = 0;
|
|
31
|
-
/** probably sharp rgb -> YUV conversion */
|
|
32
|
-
use_sharp_yuv = 0;
|
|
33
|
-
}
|