@needle-tools/gltf-build-pipeline 1.5.0-alpha → 1.5.0-alpha.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/CHANGELOG.md +31 -0
  2. package/dist/cache/cache.d.ts +2 -2
  3. package/dist/cache/cache.js +36 -20
  4. package/dist/extensions/NEEDLE_compression_texture/NEEDLE_compression_texture.d.ts +1 -1
  5. package/dist/extensions/NEEDLE_opaque/NEEDLE_opaque.d.ts +2 -1
  6. package/dist/extensions/NEEDLE_opaque/NEEDLE_opaque.js +72 -15
  7. package/dist/extensions/NEEDLE_opaque/index.js +7 -0
  8. package/dist/extensions/NEEDLE_progressive_mesh_settings/index.d.ts +4 -1
  9. package/dist/extensions/NEEDLE_progressive_texture_settings/index.d.ts +4 -0
  10. package/dist/transforms/needle_common.d.ts +2 -0
  11. package/dist/transforms/needle_mesh_transform.d.ts +5 -1
  12. package/dist/transforms/needle_mesh_transform.js +13 -7
  13. package/dist/transforms/needle_progressive.js +32 -8
  14. package/dist/transforms/needle_texture_transform.js +46 -7
  15. package/dist/transforms/needle_toktx.d.ts +1 -1
  16. package/dist/transforms/needle_toktx.js +35 -5
  17. package/dist/transforms/needle_webp.d.ts +1 -1
  18. package/dist/utils/compression_utils.d.ts +2 -0
  19. package/dist/utils/compression_utils.js +6 -0
  20. package/dist/utils/fileutils.d.ts +1 -0
  21. package/dist/utils/fileutils.js +6 -1
  22. package/dist/utils/index.d.ts +2 -0
  23. package/dist/utils/index.js +2 -0
  24. package/dist/utils/nodeio.d.ts +5 -0
  25. package/dist/utils/nodeio.js +14 -0
  26. package/limit-caches.mjs +4 -0
  27. package/pack-gltf.mjs +16 -12
  28. package/package.json +2 -1
  29. package/dist/functions/gzip.d.ts +0 -1
  30. package/dist/functions/gzip.js +0 -16
  31. package/dist/functions/index.d.ts +0 -1
  32. package/dist/functions/index.js +0 -1
  33. package/dist/functions/list-texture-slots.d.ts +0 -1
  34. package/dist/functions/list-texture-slots.js +0 -35
  35. package/dist/transforms/needle_simplify.d.ts +0 -3
  36. package/dist/transforms/needle_simplify.js +0 -76
  37. package/dist/transforms/types/squoosh.d.ts +0 -31
  38. package/dist/transforms/types/squoosh.js +0 -33
package/CHANGELOG.md CHANGED
@@ -4,6 +4,37 @@ All notable changes to this package will be documented in this file.
4
4
  The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
5
5
  and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
6
6
 
7
+ ## [1.5.0-alpha.10] - 2023-05-17
8
+ - add: `generateLods` option to progressive texture settings
9
+
10
+ ## [1.5.0-alpha.9] - 2023-05-16
11
+ - add: start of VRM 1.0 support. Not implemented is the `VRMC_materials_mtoon` extension
12
+
13
+ ## [1.5.0-alpha.8] - 2023-05-16
14
+ - fix: issue where updating ktx compressed texture extension also flattened the path
15
+
16
+ ## [1.5.0-alpha.7] - 2023-05-16
17
+ - fix: external resources that are KTX2 compressed are now saved with a `.ktx2` extension
18
+ - change: toktx now uses compression "auto" mode to select uastc or etc1s
19
+ - change: toktx now checks the slots and increases quality for `metallicRoughnessTexture` and `normalTexture` but keeps using ETC1S for smaller file size
20
+
21
+ ## [1.5.0-alpha.6] - 2023-05-15
22
+ - fix: If not further specified then automatically use UASTC for textures that are in either `normalTexture` or `metallicRoughnessTexture` slots
23
+
24
+ ## [1.5.0-alpha.5] - 2023-05-04
25
+ - fix: NEEDLE_opaque should handle if VRM texture index is negative
26
+ - change: exit with code 1 if any file packing fails
27
+
28
+ ## [1.5.0-alpha.4] - 2023-05-03
29
+ - fix: issue in NEEDLE_opaque resolving of VRM textures, also resolve `meta.texture`
30
+ - fix: VRMs now compress meshes with mesh-opt if nothing is specified
31
+ - change: include version in cache directory
32
+
33
+ ## [1.5.0-alpha.3] - 2023-05-03
34
+ - add: VRM progressive textures support
35
+ - fix: VRM format should be saved as GLB. See https://github.com/donmccurdy/glTF-Transform/issues/1390
36
+ - fix: VRM texture index handling
37
+
7
38
  ## [1.5.0-alpha] - 2023-05-01
8
39
  - add: NEEDLE_opaque `node: <index>` retargeting
9
40
  - add: passthrough VRM extension
@@ -8,7 +8,7 @@ export declare function clearCache(): void;
8
8
  * Ensure that the cache size does not exceed the given maximum size in MB.
9
9
  * @param maxSizeMB The maximum size in MB.
10
10
  */
11
- export declare function ensureCacheSize(maxSizeMB: number): void;
11
+ export declare function limitCacheSize(maxSizeMB?: number): void;
12
12
  /**
13
13
  * Add data to the cache.
14
14
  * @param key The key to store the data under.
@@ -48,4 +48,4 @@ export declare function getKeyWithHash(property: ExtensibleProperty): {
48
48
  * @param property The property to get the hash for.
49
49
  * @returns The hash as a number.
50
50
  */
51
- export declare function getHash(property: ExtensibleProperty | object): number;
51
+ export declare function getHash(property: ExtensibleProperty | object, level?: number): number;
@@ -1,6 +1,6 @@
1
1
  import { ExtensibleProperty, Texture, Primitive, Accessor, Mesh } from "@gltf-transform/core";
2
2
  import tmp from 'tmp';
3
- import fs from 'fs';
3
+ import fs, { statSync } from 'fs';
4
4
  import path from 'path';
5
5
  import { getVersion } from "../utils/version.js";
6
6
  // import * as CHECKDISCSPACE from 'check-disk-space'
@@ -24,7 +24,7 @@ const buildPipelineVersion = getVersion(false);
24
24
  * This will remove all files in the cache directory.
25
25
  */
26
26
  export function clearCache() {
27
- const dir = getCacheDirectory();
27
+ const dir = getCacheDirectory(true);
28
28
  fs.rmSync(dir, { recursive: true });
29
29
  console.log(`Cleared cache at ${dir}`);
30
30
  }
@@ -32,16 +32,27 @@ export function clearCache() {
32
32
  * Ensure that the cache size does not exceed the given maximum size in MB.
33
33
  * @param maxSizeMB The maximum size in MB.
34
34
  */
35
- export function ensureCacheSize(maxSizeMB) {
36
- const dir = getCacheDirectory();
35
+ export function limitCacheSize(maxSizeMB = 1024) {
36
+ const dir = getCacheDirectory(true);
37
+ console.log(`Limit cache size to ${maxSizeMB.toFixed(2)} MB at ${dir}...`);
37
38
  let totalSize = 0;
38
- const files = fs.readdirSync(dir).map(file => {
39
- const path = `${dir}/${file}`;
40
- const stats = fs.statSync(path);
41
- totalSize += stats.size;
42
- return { path, size: stats.size };
43
- });
39
+ function recurseDirectories(dir) {
40
+ const files = fs.readdirSync(dir).map(file => {
41
+ const path = `${dir}/${file}`;
42
+ const stats = statSync(path);
43
+ if (stats.isDirectory()) {
44
+ recurseDirectories(path);
45
+ }
46
+ else {
47
+ totalSize += stats.size;
48
+ }
49
+ return { path, size: stats.size };
50
+ });
51
+ return files;
52
+ }
53
+ const files = recurseDirectories(dir);
44
54
  const maxSizeInBytes = maxSizeMB * 1024 * 1024;
55
+ console.log(`Current cache size is ${(totalSize / (1024 * 1024)).toFixed(1)} MB`);
45
56
  if (totalSize < maxSizeInBytes)
46
57
  return;
47
58
  files.sort((a, b) => {
@@ -55,6 +66,7 @@ export function ensureCacheSize(maxSizeMB) {
55
66
  const file = files.shift();
56
67
  if (!file)
57
68
  break;
69
+ console.log(`Removing cache file ${file.path}...`);
58
70
  fs.rmSync(file.path);
59
71
  totalSize -= file.size;
60
72
  }
@@ -153,7 +165,7 @@ export function getKeyWithHash(property) {
153
165
  * @param property The property to get the hash for.
154
166
  * @returns The hash as a number.
155
167
  */
156
- export function getHash(property) {
168
+ export function getHash(property, level = 0) {
157
169
  let hash = 0;
158
170
  if (property instanceof ExtensibleProperty) {
159
171
  const extensions = property.listExtensions();
@@ -175,7 +187,7 @@ export function getHash(property) {
175
187
  hash += hashObject(weights);
176
188
  const primitives = property.listPrimitives();
177
189
  for (const prim of primitives)
178
- hash += getHash(prim);
190
+ hash += getHash(prim, level + 1);
179
191
  }
180
192
  else if (property instanceof Primitive) {
181
193
  const indices = property.getIndices();
@@ -198,8 +210,9 @@ export function getHash(property) {
198
210
  }
199
211
  return hash;
200
212
  }
201
- function getCacheDirectory() {
202
- const tmpDir = path.join(tmp.tmpdir, "Needle", "build-pipeline-cache");
213
+ const cleanBuildPipelineVersion = buildPipelineVersion.replace(/[^a-zA-Z0-9]/g, "_");
214
+ function getCacheDirectory(root) {
215
+ const tmpDir = path.join(tmp.tmpdir, "Needle", "build-pipeline-cache", root ? "" : cleanBuildPipelineVersion);
203
216
  if (!fs.existsSync(tmpDir)) {
204
217
  fs.mkdirSync(tmpDir, { recursive: true });
205
218
  }
@@ -211,12 +224,7 @@ function hashObject(obj) {
211
224
  return hashBuffer(buffer);
212
225
  }
213
226
  const str = JSON.stringify(obj);
214
- let hash = 0;
215
- for (let i = 0; i < str.length; i++) {
216
- hash = ((hash << 5) - hash) + str.charCodeAt(i);
217
- hash |= 0;
218
- }
219
- return hash;
227
+ return hashString(str);
220
228
  }
221
229
  function hashBuffer(buffer, offset, length) {
222
230
  let hash = 0;
@@ -227,3 +235,11 @@ function hashBuffer(buffer, offset, length) {
227
235
  }
228
236
  return hash;
229
237
  }
238
+ function hashString(str) {
239
+ let hash = 0;
240
+ for (let i = 0; i < str.length; i++) {
241
+ hash = ((hash << 5) - hash) + str.charCodeAt(i);
242
+ hash |= 0;
243
+ }
244
+ return hash;
245
+ }
@@ -1,7 +1,7 @@
1
1
  import { Extension, ExtensionProperty, PropertyType, ReaderContext, WriterContext } from "@gltf-transform/core";
2
2
  export declare const EXTENSION_NAME = "NEEDLE_compression_texture";
3
3
  export declare interface NEEDLE_compression_texture_schema {
4
- mode: string;
4
+ mode: "none" | "auto" | "webp" | "UASTC" | "ETC1S";
5
5
  slot: string;
6
6
  /** quality in range of 0 - 1 */
7
7
  quality: number;
@@ -1,9 +1,10 @@
1
1
  import { WriterContext, PropertyType } from '@gltf-transform/core';
2
2
  declare interface IExtensibleProperty {
3
3
  }
4
+ type DebugBreakPoint = boolean | "pointer:read";
4
5
  export type Options = {
5
6
  debug?: boolean;
6
- breakpoint?: boolean;
7
+ breakpoint?: DebugBreakPoint;
7
8
  write?: boolean;
8
9
  };
9
10
  export interface IOpaqueExtension {
@@ -26,6 +26,7 @@ export function createOpaqueExtension(name, types, opts) {
26
26
  console.log("READ " + EXTENSION_NAME);
27
27
  this.jsonPointerHandler = new JsonPointerHandler(this.document, EXTENSION_NAME);
28
28
  this.jsonPointerHandler.debug = debugLog;
29
+ this.jsonPointerHandler.breakPoint = opts.breakpoint;
29
30
  for (const type of types) {
30
31
  switch (type) {
31
32
  case PropertyType.ACCESSOR:
@@ -514,6 +515,7 @@ class JsonPointerHandler {
514
515
  document;
515
516
  map = new Map();
516
517
  debug = false;
518
+ breakPoint = false;
517
519
  constructor(document, id) {
518
520
  this.document = document;
519
521
  this.id = id;
@@ -524,7 +526,11 @@ class JsonPointerHandler {
524
526
  if (key !== undefined) {
525
527
  fullPath ??= "";
526
528
  fullPath += "/" + key;
529
+ if (this.breakPoint === "pointer:read") {
530
+ debugger;
531
+ }
527
532
  }
533
+ // if(this.debug && key === "colliders") debugger;
528
534
  switch (typeof value) {
529
535
  case 'object':
530
536
  if (value !== null && value !== undefined) {
@@ -556,7 +562,8 @@ class JsonPointerHandler {
556
562
  console.log("Texture index", index, imageIndex, context.jsonDoc.json.textures.length, "Textures:" + context.textures.length);
557
563
  const object = tryGetInArray(context.textures, imageIndex);
558
564
  if (object) {
559
- const res = new TexturePointerResolver(this.document, object, definition);
565
+ const res = new TextureIndexResolver(this.document, object, definition);
566
+ res.debug = this.debug;
560
567
  res.init(context); // < init here otherwise the fields defined in the TexturePointer will be undefined (this caused e.g. finding the previous texture filter to fail)
561
568
  pointer = new JsonPointer(path, res, value);
562
569
  pointer.debug = this.debug;
@@ -569,7 +576,7 @@ class JsonPointerHandler {
569
576
  if (pointer) {
570
577
  if (this.debug)
571
578
  console.log("> Register pointer", this.id + "/" + fullPath + " : " + value);
572
- pointer?.register(obj, key);
579
+ pointer?.register(obj, key, fullPath);
573
580
  }
574
581
  else
575
582
  console.warn("WARN: failed registering pointer", fullPath, value);
@@ -589,33 +596,80 @@ class JsonPointerHandler {
589
596
  let pointer = this.map.get(value);
590
597
  if (!pointer) {
591
598
  const definition = context.jsonDoc.json.materials[index];
592
- const res = new MaterialPointerResolver(this.document, mat, definition);
599
+ const res = new MaterialIndexResolver(this.document, mat, definition);
600
+ res.debug = this.debug;
593
601
  res.init(context); // < init here otherwise the fields defined in the TexturePointer will be undefined (this caused e.g. finding the previous texture filter to fail)
594
602
  pointer = new JsonPointer("/materials/", res, value);
595
603
  pointer.debug = this.debug;
596
604
  this.map.set(value, pointer);
597
605
  }
598
- pointer?.register(obj, key);
606
+ pointer?.register(obj, key, fullPath);
599
607
  }
600
608
  }
601
609
  }
602
610
  break;
603
611
  case 'number':
604
- if (key === "node") {
612
+ if (key === "node" || key === "mesh"
613
+ // VRM 1.0 spring bone center https://github.com/vrm-c/vrm-specification/blob/master/specification/VRMC_springBone-1.0/schema/VRMC_springBone.spring.schema.json
614
+ || fullPath.match(/springs\/\d+\/center/g)) {
605
615
  const object = context.nodes[value];
606
616
  if (object) {
607
617
  const res = new NodeIndexResolver(this.document, object, context.jsonDoc.json.nodes[value]);
618
+ res.debug = this.debug;
608
619
  res.init(context);
609
- const dictKey = "/nodes/" + value;
620
+ const dictKey = `/${key}/${value}`;
610
621
  let pointer = this.map.get(dictKey);
611
622
  if (!pointer) {
612
623
  pointer = new JsonPointer(null, res, value);
613
624
  }
614
625
  pointer.debug = this.debug;
615
- pointer.register(obj, key);
626
+ pointer.register(obj, key, fullPath);
616
627
  this.map.set(dictKey, pointer);
617
628
  }
618
629
  }
630
+ else if (
631
+ // key texture is matching for example for VRM meta/texture (AvatarSample_A)
632
+ key === "texture"
633
+ // VRM 0.0 material properties
634
+ || fullPath.match(/materialProperties\/\d+\/textureProperties/g)
635
+ // VRM 1.0 meta
636
+ || fullPath.match(/meta\/thumbnailImage/g)) {
637
+ const index = value;
638
+ // e.g. in meta.texture for VRM avatars it seems like we can have negative values
639
+ if (index < 0) {
640
+ console.warn(`WARN: texture index is negative ${index} at ${fullPath}`);
641
+ return;
642
+ }
643
+ const definition = context.jsonDoc.json.textures[index];
644
+ if (!definition) {
645
+ console.warn(`WARN: texture definition with index ${index} not found or does not exist. Set in \"${fullPath}\"`);
646
+ return;
647
+ }
648
+ const imageIndex = findSource(definition);
649
+ if (this.debug)
650
+ console.log("Texture index", index, imageIndex, context.jsonDoc.json.textures.length, "Textures:" + context.textures.length);
651
+ const object = tryGetInArray(context.textures, imageIndex);
652
+ if (object) {
653
+ if (this.debug)
654
+ console.log("Found texture index at " + fullPath + ": " + value);
655
+ const res = new TextureIndexResolver(this.document, object, definition);
656
+ res.debug = this.debug;
657
+ res.init(context);
658
+ const dictKey = `/textures/${value}`;
659
+ let pointer = this.map.get(dictKey);
660
+ if (!pointer) {
661
+ pointer = new JsonPointer(null, res, value);
662
+ }
663
+ pointer.debug = this.debug;
664
+ pointer.register(obj, key, fullPath);
665
+ this.map.set(dictKey, pointer);
666
+ if (ext) {
667
+ ext.setReference(dictKey, object);
668
+ }
669
+ }
670
+ }
671
+ break;
672
+ default:
619
673
  break;
620
674
  }
621
675
  }
@@ -638,9 +692,11 @@ class JsonPointer {
638
692
  }
639
693
  _objects = [];
640
694
  _props = [];
641
- register(obj, prop) {
695
+ _debugInfo = [];
696
+ register(obj, prop, fullpath) {
642
697
  this._objects.push(obj);
643
698
  this._props.push(prop);
699
+ this._debugInfo.push(fullpath);
644
700
  }
645
701
  newPointer;
646
702
  resolve(context) {
@@ -655,18 +711,19 @@ class JsonPointer {
655
711
  this.newPointer = index;
656
712
  }
657
713
  if (this.debug)
658
- console.log(`RESOLVED ${this.oldPointer} to → ${this.newPointer}`);
714
+ console.log(`RESOLVED pointer ${this.oldPointer} to → ${this.newPointer}`);
659
715
  }
660
716
  else {
661
- console.log("ERR: failed to resolve pointer", this.pointerBase);
717
+ console.log("ERR: 🧨 FAILED to resolve pointer", this.pointerBase, ` used by ${this._debugInfo.join(", ")}`);
662
718
  }
663
719
  }
664
720
  write() {
665
721
  for (let i = 0; i < this._objects.length; i++) {
666
722
  const obj = this._objects[i];
667
723
  const prop = this._props[i];
668
- if (this.debug)
669
- console.log("< Resolved pointer", this.newPointer, "in", prop);
724
+ if (this.debug) {
725
+ console.log(`< Resolved pointer ${this._debugInfo[i]}: ${this.newPointer}`);
726
+ }
670
727
  obj[prop] = this.newPointer;
671
728
  }
672
729
  }
@@ -684,7 +741,7 @@ class PointerResolver {
684
741
  }
685
742
  debug = false;
686
743
  }
687
- class TexturePointerResolver extends PointerResolver {
744
+ class TextureIndexResolver extends PointerResolver {
688
745
  _debugName;
689
746
  _samplerDef;
690
747
  init(context) {
@@ -719,7 +776,7 @@ class TexturePointerResolver extends PointerResolver {
719
776
  // the texture was not found - add it
720
777
  // create a new texture def entry
721
778
  if (this.debug)
722
- console.log("Creating new texture", this._debugName);
779
+ console.log("Could not find texture, creating a new one: " + this._debugName);
723
780
  const hasExtensionWithSource = this.inputDefinition.extensions && Object.keys(this.inputDefinition.extensions).some(key => this.inputDefinition.extensions[key]["source"] !== undefined);
724
781
  const textureDef = {
725
782
  source: hasExtensionWithSource ? undefined : imgIndex,
@@ -774,7 +831,7 @@ function findSource(texDef) {
774
831
  }
775
832
  return undefined;
776
833
  }
777
- class MaterialPointerResolver extends PointerResolver {
834
+ class MaterialIndexResolver extends PointerResolver {
778
835
  init(context) {
779
836
  }
780
837
  resolve(context) {
@@ -17,5 +17,12 @@ export const ALL_EXTENSIONS = [
17
17
  createOpaqueExtension("EXT_texture_exr", [PropertyType.TEXTURE], { ...opaqueOptions }),
18
18
  createOpaqueExtension("NEEDLE_editor", [PropertyType.NODE, PropertyType.MATERIAL], { ...opaqueOptions }),
19
19
  createOpaqueExtension("NEEDLE_gltf_dependencies", [PropertyType.ROOT], { ...opaqueOptions }),
20
+ // VRM 0.0
20
21
  createOpaqueExtension("VRM", [PropertyType.ROOT], { ...opaqueOptions }),
22
+ // VRM 1.0 https://vrm.dev/en/vrm1/changed/
23
+ createOpaqueExtension("VRMC_vrm", [PropertyType.ROOT], { ...opaqueOptions }),
24
+ createOpaqueExtension("VRMC_springBone", [PropertyType.ROOT], { ...opaqueOptions }),
25
+ createOpaqueExtension("VRMC_node_constraint", [PropertyType.NODE], { ...opaqueOptions }),
26
+ // TODO: mtoon materials don't work yet
27
+ // createOpaqueExtension("VRMC_materials_mtoon", [PropertyType.MATERIAL], { ...opaqueOptions, debug: true, breakpoint: "pointer:read" },),
21
28
  ];
@@ -1,7 +1,10 @@
1
1
  import { Extension, ExtensionProperty, PropertyType, ReaderContext, WriterContext } from "@gltf-transform/core";
2
2
  export declare interface IProgressiveMeshSettings {
3
- guid: string;
3
+ /**
4
+ * When false no LODs will be generated for this mesh
5
+ */
4
6
  generateLods: boolean;
7
+ guid: string;
5
8
  }
6
9
  export declare class NEEDLE_progressive_mesh_settings extends Extension {
7
10
  static EXTENSION_NAME: string;
@@ -1,5 +1,9 @@
1
1
  import { Extension, ExtensionProperty, PropertyType, ReaderContext, WriterContext } from "@gltf-transform/core";
2
2
  export declare interface IProgressiveTextureSettings {
3
+ /**
4
+ * When false the texture no progressive texture will be generated
5
+ */
6
+ generateLods?: boolean;
3
7
  maxSize: number;
4
8
  guid: string;
5
9
  }
@@ -1,6 +1,8 @@
1
1
  import { Document, Texture } from '@gltf-transform/core';
2
2
  import { NEEDLE_compression_texture_schema } from '../extensions/index.js';
3
3
  export interface NeedleTransformContext {
4
+ /** full path to the file being processed */
5
+ file: string;
4
6
  debug?: boolean;
5
7
  useCache: boolean;
6
8
  noToktx: boolean;
@@ -1,4 +1,8 @@
1
1
  import { Transform, Document, Primitive, Mesh } from "@gltf-transform/core";
2
2
  import { SimplifyOptions, WeldOptions } from "@gltf-transform/functions";
3
- export declare const needle_mesh_transform: (_options: SimplifyOptions) => Transform;
3
+ declare type MeshTransformContext = {
4
+ file: string;
5
+ };
6
+ export declare const needle_mesh_transform: (context: MeshTransformContext, simplify: SimplifyOptions) => Transform;
4
7
  export declare function compressPrimitive(document: Document, mesh: Mesh, prim: Primitive, index: number, weldOptions: Required<WeldOptions>, meshOptions: Required<SimplifyOptions>): Promise<Primitive>;
8
+ export {};
@@ -3,14 +3,14 @@ import { meshopt, simplifyPrimitive, SIMPLIFY_DEFAULTS, weldPrimitive, WELD_DEFA
3
3
  import { MeshoptEncoder, MeshoptSimplifier } from "meshoptimizer";
4
4
  import { NEEDLE_mesh_compression } from "../extensions/index.js";
5
5
  const NAME = 'needle_mesh_transform';
6
- export const needle_mesh_transform = function (_options) {
6
+ export const needle_mesh_transform = function (context, simplify) {
7
7
  return async (document, _context) => {
8
8
  // TODO: needs to be moved into NEEDLE_progressive
9
- await simplifierTransform(document, _context, _options);
10
- await applyMeshCompression(document, _context);
9
+ await simplifierTransform(document, _context, simplify);
10
+ await applyMeshCompression(document, _context, context);
11
11
  };
12
12
  };
13
- async function applyMeshCompression(document, _context) {
13
+ async function applyMeshCompression(document, transform_context, context) {
14
14
  const rootExtension = document.getRoot().getExtension(NEEDLE_mesh_compression.EXTENSION_NAME);
15
15
  if (rootExtension) {
16
16
  const rootModel = rootExtension.model;
@@ -22,17 +22,23 @@ async function applyMeshCompression(document, _context) {
22
22
  console.log(`[${NAME}]: ${rootModel.compression.toUpperCase()} → will apply ${rootModel.compression} mesh compression`);
23
23
  if (rootModel.compression === "draco") {
24
24
  const transform = draco();
25
- return transform(document, _context);
25
+ return transform(document, transform_context);
26
26
  }
27
27
  else if (rootModel.compression === "meshopt") {
28
28
  const transform = meshopt({ encoder: MeshoptEncoder });
29
- return transform(document, _context);
29
+ return transform(document, transform_context);
30
30
  }
31
31
  }
32
32
  }
33
+ // If nothing is configured apply meshopt compression as long as we're running gltf-transform 3.x
34
+ if (context.file.endsWith(".vrm")) {
35
+ console.log(`[${NAME}]: VRM → will apply meshopt mesh compression`);
36
+ const transform = meshopt({ encoder: MeshoptEncoder });
37
+ return transform(document, transform_context);
38
+ }
33
39
  // by default we still want to run draco if the root doesnt have a compression model configured
34
40
  console.log(`[${NAME}]: No compression model configured → will apply draco mesh compression`);
35
- return draco()(document, _context);
41
+ return draco()(document, transform_context);
36
42
  }
37
43
  async function simplifierTransform(document, _context, _options) {
38
44
  const simplifier = MeshoptSimplifier;
@@ -7,13 +7,13 @@ import path, { basename } from 'path';
7
7
  import { existsSync, readFileSync, rmSync, statSync, writeFileSync } from 'fs';
8
8
  import { addMetaToCache, addToCache, getHash, getKeyWithHash, tryGetFromCache, tryGetMetaFromCache } from '../cache/index.js';
9
9
  import sharp from "sharp";
10
- import { foreachGLTF, generateGuid } from '../utils/index.js';
10
+ import { detectTextureCompressionMode, foreachGLTF, generateGuid, writeNodeIO } from '../utils/index.js';
11
11
  import { needle_asset, writeNeedleAsset } from './needle_asset.js';
12
12
  import { MeshoptDecoder, MeshoptSimplifier } from 'meshoptimizer';
13
13
  import { compressPrimitive } from './needle_mesh_transform.js';
14
14
  import { calculateMeshDensity, getMeshInformation } from "../utils/mesh.js";
15
15
  export async function make_progressive(opts) {
16
- console.log("INFO: Run progressive transform: " + opts.path);
16
+ console.log(`INFO: Run progressive transform: ${opts.path}`);
17
17
  const file = opts.path;
18
18
  return foreachGLTF(file, async (f) => {
19
19
  // Don't re-process files that are already LODs
@@ -38,7 +38,7 @@ function registerExtensions(io) {
38
38
  ]);
39
39
  }
40
40
  async function onProcess(file, opts) {
41
- if (!file.endsWith(".glb") && !file.endsWith(".gltf")) {
41
+ if (!file.endsWith(".glb") && !file.endsWith(".gltf") && !file.endsWith(".vrm")) {
42
42
  console.log("ERR: file is not gltf/glb");
43
43
  return;
44
44
  }
@@ -71,7 +71,19 @@ async function onProcess(file, opts) {
71
71
  }
72
72
  //@ts-ignore
73
73
  document.getLogger().verbosity = 0;
74
- await document.transform(needle_asset(), make_progressive_textures(file, { size: [256, 256] }, opts), make_progressive_meshes(file, {}, opts), sparse());
74
+ const transforms = [
75
+ needle_asset(),
76
+ make_progressive_textures(file, { size: [256, 256] }, opts),
77
+ sparse(),
78
+ ];
79
+ // TODO: progressive meshes aren't supported yet for VRMs
80
+ if (!file.endsWith(".vrm")) {
81
+ transforms.push(make_progressive_meshes(file, {}, opts));
82
+ }
83
+ else
84
+ console.warn("WARN: Progressive meshes are not supported for VRM files yet");
85
+ console.log("→ Transforming: " + file);
86
+ await document.transform(...transforms);
75
87
  let filename = file;
76
88
  if (opts.debug === true) {
77
89
  // get extension
@@ -81,7 +93,8 @@ async function onProcess(file, opts) {
81
93
  // add new extension
82
94
  filename = filename + ".debug." + extension;
83
95
  }
84
- io.write(filename, document);
96
+ await writeNodeIO(io, filename, document);
97
+ console.log("← Transforming done: " + filename);
85
98
  }
86
99
  }
87
100
  catch (err) {
@@ -130,7 +143,7 @@ export function make_progressive_textures(filePath, _options = TEXTURE_RESIZE_DE
130
143
  if (!settings) {
131
144
  const size = texture.getSize();
132
145
  if (size?.length >= 2 && size[0] <= 128 && size[1] <= 128) {
133
- logger.warn(`[${NAME}] Skipping texture[${textureIndex}], no settings found`);
146
+ logger.warn(`[${NAME}] Skipping texture[${textureIndex}], no settings found and the texture is already < 128x128`);
134
147
  continue;
135
148
  }
136
149
  else {
@@ -141,6 +154,10 @@ export function make_progressive_textures(filePath, _options = TEXTURE_RESIZE_DE
141
154
  logger.debug(`[${NAME}] No settings found for texture[${index}] - will use default settings (maxSize: ${settings.maxSize}, guid: ${settings.guid})`);
142
155
  }
143
156
  }
157
+ else if (settings.generateLods === false) {
158
+ logger.debug(`[${NAME}] Skipping texture[${textureIndex}]: progressive texture disabled`);
159
+ continue;
160
+ }
144
161
  if (settings.maxSize == undefined || settings.maxSize <= 0) {
145
162
  logger.warn(`[${NAME}] Skipping texture[${textureIndex}], settings are invalid (maxSize: ${settings.maxSize})`);
146
163
  continue;
@@ -158,7 +175,7 @@ export function make_progressive_textures(filePath, _options = TEXTURE_RESIZE_DE
158
175
  continue;
159
176
  }
160
177
  const textureGuid = settings.guid?.length ? settings.guid : generateGuid(path + "_texture_" + index.toString());
161
- const outPath = createTextureLod(doc, textureLodBasePath, 0, texture, slots, textureGuid, textureIndex);
178
+ const outPath = createTextureLod(doc, filePath, textureLodBasePath, 0, texture, slots, textureGuid, textureIndex);
162
179
  if (!outPath) {
163
180
  continue;
164
181
  }
@@ -194,7 +211,7 @@ export function make_progressive_textures(filePath, _options = TEXTURE_RESIZE_DE
194
211
  logger.debug(`Progressive transform finished in ${((Date.now() - startTime) / 1000).toFixed(1)} sec.`);
195
212
  });
196
213
  }
197
- function createTextureLod(doc, basePath, level, texture, slots, guid, textureIndex) {
214
+ function createTextureLod(doc, file, basePath, level, texture, slots, guid, textureIndex) {
198
215
  // const imageBytes = texture.getImage();
199
216
  // if (imageBytes) {
200
217
  // // write bytes
@@ -245,6 +262,13 @@ function createTextureLod(doc, basePath, level, texture, slots, guid, textureInd
245
262
  console.log("> Pass compression extension to new texture");
246
263
  const newExt = newDoc.createExtension(NEEDLE_compression_texture);
247
264
  const def = compressionSettings.getExtensionDefinition();
265
+ def.mode = detectTextureCompressionMode(slots);
266
+ def.slot = slots[0];
267
+ // for VRMS we just use webp
268
+ // if (file?.toLocaleLowerCase().endsWith(".vrm")) {
269
+ // def.mode = "webp";
270
+ // def.quality = 70;
271
+ // }
248
272
  const newProp = newExt.create(def);
249
273
  newTexture.setExtension(NEEDLE_compression_texture.EXTENSION_NAME, newProp);
250
274
  }
@@ -5,6 +5,8 @@ import { addToCache, getKey, tryGetFromCache } from "../cache/index.js";
5
5
  import { NeedleWebPContext } from "./needle_webp.js";
6
6
  import { NeedleToktxContext } from "./needle_toktx.js";
7
7
  import { createTransform } from "@gltf-transform/functions";
8
+ import { dirname } from "path";
9
+ import { existsSync, rmSync } from "fs";
8
10
  export async function resizeTextureIfNecessary(texture) {
9
11
  const ext = texture.getExtension(NEEDLE_compression_texture.EXTENSION_NAME);
10
12
  if (ext?.model) {
@@ -74,8 +76,8 @@ export const needle_texture_transform = function (options) {
74
76
  });
75
77
  };
76
78
  async function processTexture(texture, index, document, logger, context) {
77
- const mimeType = texture.getMimeType();
78
- const isSupported = mimeType === "image/jpeg" || mimeType === "image/png" || mimeType === "image/webp";
79
+ const originalMimeType = texture.getMimeType();
80
+ const isSupported = originalMimeType === "image/jpeg" || originalMimeType === "image/png" || originalMimeType === "image/webp";
79
81
  // dont try to transform EXR or already processed textures
80
82
  if (!isSupported)
81
83
  return;
@@ -96,6 +98,7 @@ async function processTexture(texture, index, document, logger, context) {
96
98
  }
97
99
  let successFullyProcessed = false;
98
100
  const canRunToktx = context.noToktx !== true;
101
+ const originalUri = texture.getURI();
99
102
  if (compressionSettings && compressionSettings.mode !== "auto") {
100
103
  switch (compressionSettings.mode) {
101
104
  case "webp":
@@ -125,21 +128,46 @@ async function processTexture(texture, index, document, logger, context) {
125
128
  }
126
129
  }
127
130
  else {
131
+ let mimeType = "image/ktx2";
132
+ // const slots = listTextureSlots(texture);
133
+ // if (slots.includes("metallicRoughnessTexture"))
134
+ // mimeType = "image/webp";
135
+ const isVRM = context.file?.toLowerCase().endsWith(".vrm");
136
+ // TODO: need to make this into a function
137
+ // const webpThreshold = 256; // this is the default threshold in progressive textures (needle_progressive.ts)
138
+ // console.log(context.file, isVRM);
139
+ // // if we're dealing with a VRM and the texture size is < 128x128 we can use webp (this is e.g. the case for progressive textures)
140
+ // if (isVRM) {
141
+ // const size = texture.getSize();
142
+ // console.log(isVRM, size)
143
+ // if (size && size[0] <= webpThreshold && size[1] <= webpThreshold) {
144
+ // mimetype = "image/webp";
145
+ // }
146
+ // }
128
147
  // If we have a texture cached we can use it
129
148
  // the default compression is toktx so we can safely use that for now
130
149
  if (cached) {
131
- texture.setImage(cached).setMimeType("image/ktx2");
150
+ texture.setImage(cached).setMimeType(mimeType);
132
151
  return;
133
152
  }
134
153
  await resizeTextureIfNecessary(texture);
135
- if (canRunToktx) {
154
+ if (mimeType === "image/webp") {
155
+ successFullyProcessed = await context.webPContext.process(index, texture, {
156
+ quality: .9,
157
+ });
158
+ }
159
+ else if (canRunToktx) {
136
160
  // if no specific compression settings are provided we just use the toktx default settings
137
161
  // if we ever change what is used for default compression we need to teach the cache to save the mime type
138
- successFullyProcessed = await context.toktxContext.process(index, texture, null);
139
- // don't add textures to the cache when toktx doesnt run
162
+ // For VRMs we use ETC1S always
163
+ let options = null;
164
+ if (isVRM) {
165
+ options = { mode: "ETC1S" };
166
+ }
167
+ successFullyProcessed = await context.toktxContext.process(index, texture, options);
140
168
  }
141
169
  else {
142
- console.log("toktx is disallowed. Skipping texture compression.");
170
+ console.log("Toktx not found: Skipping texture compression.");
143
171
  }
144
172
  }
145
173
  if (context.useCache && successFullyProcessed && key) {
@@ -147,4 +175,15 @@ async function processTexture(texture, index, document, logger, context) {
147
175
  if (data)
148
176
  addToCache(key, data);
149
177
  }
178
+ if (successFullyProcessed) {
179
+ const newUri = texture.getURI();
180
+ if (originalUri && originalUri != newUri) {
181
+ const fileDirectory = dirname(context.file);
182
+ const originalPath = fileDirectory + "/" + originalUri;
183
+ if (existsSync(originalPath)) {
184
+ console.log(`INFO: Texture moved \"${originalUri}\" → \"${newUri}\"`);
185
+ rmSync(originalPath);
186
+ }
187
+ }
188
+ }
150
189
  }
@@ -20,6 +20,6 @@ export declare class NeedleToktxContext implements INeedleTextureTransformContex
20
20
  private _prepareFailed;
21
21
  constructor(options: NeedleToktxOptions);
22
22
  prepare(document: Document): Promise<boolean>;
23
- process(index: number, texture: Texture, settings: NEEDLE_compression_texture_schema | null): Promise<boolean>;
23
+ process(index: number, texture: Texture, settings: Pick<NEEDLE_compression_texture_schema, "mode"> | null): Promise<boolean>;
24
24
  finalize(document: Document): Promise<void>;
25
25
  }
@@ -8,6 +8,7 @@ import fs from 'fs/promises';
8
8
  import os from 'os';
9
9
  import tmp from 'tmp';
10
10
  import pLimit from 'p-limit';
11
+ import { replaceExtension } from "../utils/fileutils.js";
11
12
  const NUM_CPUS = os.cpus().length || 1; // microsoft/vscode#112122
12
13
  let didLogEnvironmentInfo = false;
13
14
  export class NeedleToktxContext {
@@ -113,17 +114,39 @@ export class NeedleToktxContext {
113
114
  const outPath = tmpName + '.ktx2';
114
115
  // const inBytes = image.byteLength;
115
116
  await fs.writeFile(inPath, Buffer.from(image));
117
+ let mode = Mode.ETC1S;
118
+ let quality = 128;
119
+ let compression = 1;
120
+ if (settings) {
121
+ if (settings.mode === "auto") {
122
+ const useUASTC = slots.includes("normalTexture") || slots.includes("metallicRoughnessTexture");
123
+ mode = useUASTC ? Mode.UASTC : Mode.ETC1S;
124
+ }
125
+ else {
126
+ mode = settings.mode === "ETC1S" ? Mode.ETC1S : Mode.UASTC;
127
+ }
128
+ }
129
+ else if (slots) {
130
+ const isNormal = slots.includes("normalTexture");
131
+ const isMetallicRoughness = slots.includes("metallicRoughnessTexture");
132
+ const betterQuality = isNormal || isMetallicRoughness;
133
+ // mode = useUASTC ? Mode.UASTC : Mode.ETC1S;
134
+ mode = Mode.ETC1S;
135
+ // if (isMetallicRoughness) mode = Mode.UASTC;
136
+ // https://github.khronos.org/KTX-Software/ktxtools/toktx.html
137
+ quality = betterQuality ? 255 : 128;
138
+ // ETC1S / BasisLZ compression level, an encoding speed vs. quality tradeoff. Range is [0,5], default is 1. Higher values are slower but give higher quality.
139
+ compression = betterQuality ? 4 : 1;
140
+ }
141
+ const defaults = mode === Mode.ETC1S ? ETC1S_DEFAULTS : UASTC_DEFAULTS;
116
142
  //@ts-ignore
117
- const defaults = (!settings || settings.mode === "ETC1S") ? ETC1S_DEFAULTS : UASTC_DEFAULTS;
118
- defaults.mode = (!settings || settings.mode === "ETC1S") ? Mode.ETC1S : Mode.UASTC;
119
- //@ts-ignore
120
- const toktxParams = createParams(slots, channels, size, logger, this.numTextures, { ...defaults }, this.version);
143
+ const toktxParams = createParams(slots, channels, size, logger, this.numTextures, { ...defaults, mode, quality, compression }, this.version);
121
144
  const params = [
122
145
  ...toktxParams,
123
146
  outPath,
124
147
  inPath,
125
148
  ];
126
- logger.debug(`${prefix}: Spawning → toktx ${params.join(' ')}`);
149
+ logger.debug(`${prefix}: Spawning → toktx ${mode.toUpperCase()} ${params.join(' ')}`);
127
150
  try {
128
151
  // COMPRESS: Run `toktx` CLI tool.
129
152
  // eslint-disable-next-line @typescript-eslint/no-unused-vars
@@ -135,6 +158,13 @@ export class NeedleToktxContext {
135
158
  else {
136
159
  const result = await fs.readFile(outPath);
137
160
  texture.setImage(result).setMimeType('image/ktx2');
161
+ // TODO: we also need to remove the old file when the extension has changed - this should probably be done outside of this method to also support webp
162
+ // If this is a texture that was loaded from a file, we need to update the URI
163
+ if (texture.getURI()) {
164
+ const oldUri = texture.getURI();
165
+ const newUri = replaceExtension(oldUri, 'ktx2');
166
+ texture.setURI(newUri);
167
+ }
138
168
  return true;
139
169
  }
140
170
  }
@@ -8,6 +8,6 @@ export declare class NeedleWebPContext implements INeedleTextureTransformContext
8
8
  logger: ILogger;
9
9
  constructor(options: NeedleWebPOptions);
10
10
  prepare(document: Document): Promise<void>;
11
- process(index: number, texture: Texture, settings: NEEDLE_compression_texture_schema): Promise<boolean>;
11
+ process(index: number, texture: Texture, settings: Pick<NEEDLE_compression_texture_schema, "quality">): Promise<boolean>;
12
12
  finalize(document: Document): Promise<void>;
13
13
  }
@@ -0,0 +1,2 @@
1
+ export declare type TextureCompressionMode = "webp" | "ETC1S" | "UASTC";
2
+ export declare function detectTextureCompressionMode(slots: string[]): "ETC1S" | "UASTC";
@@ -0,0 +1,6 @@
1
+ export function detectTextureCompressionMode(slots) {
2
+ if (slots.includes("normalTexture") || slots.includes("metallicRoughnessTexture")) {
3
+ return "UASTC";
4
+ }
5
+ return "ETC1S";
6
+ }
@@ -1,2 +1,3 @@
1
1
  export declare type GLTFFileCallback = (file: string) => void | Promise<void>;
2
2
  export declare function foreachGLTF(args: string | string[], callback: GLTFFileCallback): Promise<void>;
3
+ export declare function replaceExtension(filepath: string, newExtension: string): string;
@@ -89,7 +89,7 @@ async function testEntry(entry, callback, context) {
89
89
  }
90
90
  }
91
91
  }
92
- if (entry.endsWith(".glb") || entry.endsWith(".gltf")) {
92
+ if (entry.endsWith(".glb") || entry.endsWith(".gltf") || entry.endsWith(".vrm")) {
93
93
  context.filesFound += 1;
94
94
  const res = callback(entry);
95
95
  if (res instanceof Promise) {
@@ -113,3 +113,8 @@ async function testEntry(entry, callback, context) {
113
113
  }
114
114
  }
115
115
  }
116
+ export function replaceExtension(filepath, newExtension) {
117
+ const parts = filepath.split(".");
118
+ parts.pop();
119
+ return parts.join(".") + "." + newExtension;
120
+ }
@@ -1,4 +1,6 @@
1
+ export * from "./compression_utils.js";
1
2
  export * from "./fileutils.js";
2
3
  export * from "./version.js";
3
4
  export * from "./mesh.js";
4
5
  export * from "./guid.js";
6
+ export * from "./nodeio.js";
@@ -1,4 +1,6 @@
1
+ export * from "./compression_utils.js";
1
2
  export * from "./fileutils.js";
2
3
  export * from "./version.js";
3
4
  export * from "./mesh.js";
4
5
  export * from "./guid.js";
6
+ export * from "./nodeio.js";
@@ -0,0 +1,5 @@
1
+ import { NodeIO, Document } from "@gltf-transform/core";
2
+ /**
3
+ * Handle writing glTf, GLB and VRM files
4
+ */
5
+ export declare function writeNodeIO(nodeio: NodeIO, filename: string, document: Document): Promise<string>;
@@ -0,0 +1,14 @@
1
+ import { writeFile } from "fs/promises";
2
+ /**
3
+ * Handle writing glTf, GLB and VRM files
4
+ */
5
+ export async function writeNodeIO(nodeio, filename, document) {
6
+ // see https://github.com/donmccurdy/glTF-Transform/issues/1390
7
+ if (filename.endsWith(".vrm")) {
8
+ const bin = await nodeio.writeBinary(document);
9
+ await writeFile(filename, bin);
10
+ }
11
+ else
12
+ await nodeio.write(filename, document);
13
+ return filename;
14
+ }
@@ -0,0 +1,4 @@
1
+
2
+ import { limitCacheSize } from './dist/cache/index.js';
3
+
4
+ limitCacheSize(512);
package/pack-gltf.mjs CHANGED
@@ -12,7 +12,8 @@ import { needle_animation_transform } from "./dist/transforms/needle_animation_t
12
12
  import { foreachGLTF } from './fileutils.mjs';
13
13
  import { needle_asset } from './dist/transforms/needle_asset.js';
14
14
  import { MeshoptDecoder, MeshoptEncoder } from 'meshoptimizer';
15
- import { getVersion, getHash, addToCache, tryGetFromCache, isLOD, ensureCacheSize as ensureMaxCacheSize, isMeshLOD } from './dist/index.js';
15
+ import { getVersion, getHash, addToCache, tryGetFromCache, isLOD, limitCacheSize, isMeshLOD } from './dist/index.js';
16
+ import { writeNodeIO } from './dist/utils/nodeio.js';
16
17
 
17
18
  console.log(`[Needle Build Pipeline] pack glTF — version ${getVersion()}`)
18
19
 
@@ -59,11 +60,9 @@ if (debug) {
59
60
 
60
61
  let allowCaching = true;
61
62
  if (args.includes("--no-caching")) {
62
- console.log("Caching is disabled")
63
63
  allowCaching = false;
64
64
  }
65
- else if (showHelp)
66
- console.log("Caching is enabled, use \"--no-caching\" to disable it");
65
+ else console.log("Use --no-caching to disable caching");
67
66
 
68
67
  const noToktx = args.includes("no-toktx");
69
68
  if (noToktx)
@@ -105,10 +104,11 @@ async function packFile(file) {
105
104
 
106
105
  if (packedFiles.has(file)) return;
107
106
  packedFiles.add(file);
108
- console.log("→ Begin packing: " + file);
109
107
 
110
108
  try {
111
109
  const prevSizeInMB = statSync(file).size / 1024 / 1024;
110
+ console.log(`→ Compressing ${prevSizeInMB.toFixed(1)} MB: ${file}`);
111
+
112
112
  const startTime = Date.now();
113
113
  const io = new NodeIO()
114
114
  .registerExtensions(ALL_EXTENSIONS)
@@ -129,17 +129,19 @@ async function packFile(file) {
129
129
  await document.transform(
130
130
  needle_asset(),
131
131
  needle_texture_transform({
132
+ file,
132
133
  useCache: allowCaching,
133
134
  noToktx: noToktx,
134
135
  debug: debug,
135
136
  reportGPUMemory: val => totalGPUTextureMemmory += val
136
137
  }),
137
- needle_mesh_transform({ ratio: 0.75, error: 0.0005 }),
138
+ needle_mesh_transform({ file }, { ratio: 0.75, error: 0.0005 }),
138
139
  needle_animation_transform(),
139
140
  // draco(),
140
141
  );
141
142
  if (debug) file += ".debug.glb";
142
- await io.write(file, document);
143
+
144
+ await writeNodeIO(io, file, document);
143
145
 
144
146
  if (allowCaching && cacheKey != undefined && existsSync(file))
145
147
  addToCache(cacheKey, readFileSync(file));
@@ -167,18 +169,20 @@ async function packFile(file) {
167
169
 
168
170
  let startTime = Date.now();
169
171
  const inputArgs = args.slice(2);
172
+ let failed = false;
170
173
  foreachGLTF(inputArgs, async (file) => {
171
- await packFile(file);
174
+ const res = await packFile(file);
175
+ if (!res) failed = true;
172
176
  }).then(async () => {
173
177
  const duration = Date.now() - startTime;
174
178
  const durationInMinutes = duration / 1000 / 60;
175
179
  if (allowCaching) {
176
- // if the user's cache directory has more than 25GB free space, we allow a cache of 2 GB - otherwise 1 GB
177
- // const freeSpaceInMB = (await getCacheAvailableSpaceOnDisc().catch(() => 0)) / 1024 / 1024;
178
- // const maxCacheSize = freeSpaceInMB > 25000 ? 2000 : 1000;
179
- ensureMaxCacheSize(1000);
180
+ limitCacheSize();
180
181
  }
181
182
  console.log(`[Needle Build Pipeline] v${getVersion()} — Finished packing ${totalFilesProcessed} files in ${durationInMinutes.toFixed(1)} minutes; total size ${totalFileSizeInMBBefore.toFixed(1)} MB → ${totalFileSizeInMB.toFixed(1)} MB, total GPU memory: ${totalGPUMemoryInMB.toFixed(1)} MB`);
183
+ if (failed) {
184
+ process.exit(1);
185
+ }
182
186
  }).catch(err => {
183
187
  console.error("ERR: packing failed: " + err.message);
184
188
  if (debug) throw err;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@needle-tools/gltf-build-pipeline",
3
- "version": "1.5.0-alpha",
3
+ "version": "1.5.0-alpha.10",
4
4
  "description": "Pipeline and tools for optimizing gltf files using gltf-transform and compression settings within glTF extensions",
5
5
  "main": "index.mjs",
6
6
  "type": "module",
@@ -13,6 +13,7 @@
13
13
  "pack-gltf": "node --no-experimental-fetch pack-gltf.mjs",
14
14
  "dev:pack-gltf": "npm run compile && node pack-gltf.mjs",
15
15
  "clear-caches": "node clear-caches.mjs",
16
+ "limit-caches": "node limit-caches.mjs",
16
17
  "compile": "tsc || exit 1"
17
18
  },
18
19
  "dependencies": {
@@ -1 +0,0 @@
1
- export declare function gzipFile(filepath: string): Promise<string>;
@@ -1,16 +0,0 @@
1
- import { promises as fs, existsSync } from 'fs';
2
- import { gzip } from 'node-gzip';
3
- export async function gzipFile(filepath) {
4
- if (filepath.endsWith('.gz')) {
5
- return filepath;
6
- }
7
- if (!existsSync(filepath)) {
8
- console.log("File not found: " + filepath);
9
- return filepath;
10
- }
11
- const inBuffer = await fs.readFile(filepath);
12
- const outBuffer = await gzip(inBuffer);
13
- const outPath = filepath;
14
- await fs.writeFile(outPath, outBuffer);
15
- return outPath;
16
- }
@@ -1 +0,0 @@
1
- export * from "./gzip.js";
@@ -1 +0,0 @@
1
- export * from "./gzip.js";
@@ -1 +0,0 @@
1
- export {};
@@ -1,35 +0,0 @@
1
- export {};
2
- // TODO: remove once this is resolved: https://github.com/donmccurdy/glTF-Transform/issues/855
3
- // modified to take a document as a parameter
4
- // export function listTextureSlots(texture: Texture, document: Document): string[] {
5
- // const root = document.getRoot();
6
- // const slots = texture
7
- // .getGraph()
8
- // .listParentEdges(texture)
9
- // .filter((edge) => edge.getParent() !== root)
10
- // .map((edge) => edge.getName());
11
- // return Array.from(new Set(slots));
12
- // }
13
- // export function getTextureChannelMask(texture: Texture, document: Document): number {
14
- // let mask = 0x0000;
15
- // for (const edge of document.getGraph().listParentEdges(texture)) {
16
- // const parent = edge.getParent();
17
- // let { channels } = edge.getAttributes() as { channels: number | undefined };
18
- // if (
19
- // channels &&
20
- // edge.getName() === 'baseColorTexture' &&
21
- // parent instanceof Material &&
22
- // parent.getAlphaMode() === Material.AlphaMode.OPAQUE
23
- // ) {
24
- // channels &= ~TextureChannel.A;
25
- // }
26
- // if (channels) {
27
- // mask |= channels;
28
- // continue;
29
- // }
30
- // if (parent.propertyType !== PropertyType.ROOT) {
31
- // document.getLogger().warn(`Missing attribute ".channels" on edge, "${edge.getName()}".`);
32
- // }
33
- // }
34
- // return mask;
35
- // }
@@ -1,3 +0,0 @@
1
- import { Transform } from "@gltf-transform/core";
2
- import { SimplifyOptions } from "@gltf-transform/functions";
3
- export declare const needle_mesh_transform: (options: SimplifyOptions) => Transform;
@@ -1,76 +0,0 @@
1
- import { Primitive } from "@gltf-transform/core";
2
- import { simplifyPrimitive, weldPrimitive } from "@gltf-transform/functions";
3
- import { MeshoptSimplifier } from "meshoptimizer";
4
- import { NEEDLE_mesh_compression } from "../extensions/index.js";
5
- const NAME = 'needle_simplify';
6
- export const needle_mesh_transform = function (options) {
7
- const simplifier = MeshoptSimplifier;
8
- if (!simplifier) {
9
- throw new Error(`${NAME}: simplifier dependency required — install "meshoptimizer".`);
10
- }
11
- return (document, _context) => {
12
- const logger = document.getLogger();
13
- options = { ...options, simplifier: simplifier };
14
- const weldOptions = {
15
- tolerance: 0.00000000001,
16
- overwrite: true
17
- };
18
- for (const mesh of document.getRoot().listMeshes()) {
19
- for (const prim of mesh.listPrimitives()) {
20
- // const key = getKey(prim);
21
- // const cached = tryGetFromCache(key);
22
- // if (cached) {
23
- // logger.debug(`${NAME}: Skipping primitive of mesh "${mesh.getName()}": Already cached.`);
24
- // continue;
25
- // }
26
- // updatePrimitive(prim, simplifier, null, prim.getIndices());
27
- // continue;
28
- const ext = prim.getExtension(NEEDLE_mesh_compression.EXTENSION_NAME);
29
- if (!ext) {
30
- // logger.warn("Skipping primitive of mesh " + mesh.getName() + ": No NEEDLE_compression_mesh extension.");
31
- continue;
32
- }
33
- let meshOptions = { ...options };
34
- if (ext.model) {
35
- const model = ext.model;
36
- if (!model.useSimplifier) {
37
- logger.debug(`${NAME}: Skipping primitive of mesh "${mesh.getName()}": useSimplifier is false.`);
38
- continue;
39
- }
40
- if (model.error >= 0)
41
- meshOptions.error = model.error;
42
- if (model.ratio >= 0)
43
- meshOptions.ratio = model.ratio;
44
- meshOptions.lockBorder = model.lockBorder;
45
- }
46
- if (prim.getMode() !== Primitive.Mode.TRIANGLES) {
47
- logger.warn(`${NAME}: Skipping primitive of mesh "${mesh.getName()}": Requires TRIANGLES draw mode.`);
48
- continue;
49
- }
50
- logger.debug(`${NAME}: Welding primitive of mesh "${mesh.getName()}" → tolerance: ${weldOptions.tolerance}`);
51
- weldPrimitive(document, prim, weldOptions);
52
- logger.debug(`${NAME}: Simplifying primitive of mesh "${mesh.getName()}" → ratio: ${meshOptions.ratio}, error: ${meshOptions.error}, lockBorder: ${meshOptions.lockBorder}`);
53
- simplifyPrimitive(document, prim, meshOptions);
54
- }
55
- }
56
- };
57
- };
58
- // function updatePrimitive(prim: Primitive, simplifier: any, positionArray, dstIndicesArray) {
59
- // const position = prim.getAttribute('POSITION')!;
60
- // const srcIndices = prim.getIndices()!;
61
- // const srcVertexCount = position.getCount();
62
- // const [remap, unique] = simplifier.compactMesh(dstIndicesArray);
63
- // // logger.debug(`${NAME}: ${formatDeltaOp(position.getCount(), unique)} vertices, error: ${error.toFixed(4)}.`);
64
- // // (3) Write vertex attributes.
65
- // for (const srcAttribute of deepListAttributes(prim)) {
66
- // const dstAttribute = srcAttribute.clone();
67
- // remapAttribute(dstAttribute, remap, unique);
68
- // deepSwapAttribute(prim, srcAttribute, dstAttribute);
69
- // if (srcAttribute.listParents().length === 1) srcAttribute.dispose();
70
- // }
71
- // // (4) Write indices.
72
- // const dstIndices = srcIndices.clone();
73
- // dstIndices.setArray(srcVertexCount <= 65534 ? new Uint16Array(dstIndicesArray) : dstIndicesArray);
74
- // prim.setIndices(dstIndices);
75
- // if (srcIndices.listParents().length === 1) srcIndices.dispose();
76
- // }
@@ -1,31 +0,0 @@
1
- export declare class WebPEncoderOptions {
2
- quality: number;
3
- target_size: number;
4
- target_PSNR: number;
5
- method: number;
6
- sns_strength: number;
7
- filter_strength: number;
8
- filter_sharpness: number;
9
- /** 1 = strong filter */
10
- filter_type: number;
11
- partitions: number;
12
- segments: number;
13
- pass: number;
14
- show_compressed: number;
15
- preprocessing: number;
16
- autofilter: number;
17
- partition_limit: number;
18
- alpha_compression: number;
19
- alpha_filtering: number;
20
- alpha_quality: number;
21
- lossless: number;
22
- exact: number;
23
- image_hint: number;
24
- emulate_jpeg_size: number;
25
- thread_level: number;
26
- low_memory: number;
27
- near_lossless: number;
28
- use_delta_palette: number;
29
- /** probably sharp rgb -> YUV conversion */
30
- use_sharp_yuv: number;
31
- }
@@ -1,33 +0,0 @@
1
- // client: https://github.com/GoogleChromeLabs/squoosh/blob/97d13de273b67ee01a80cb32161f1bb2103b73cb/src/features/encoders/webP/client/index.tsx#L292
2
- // emscripten bindings:
3
- export class WebPEncoderOptions {
4
- quality = 75;
5
- target_size = 0;
6
- target_PSNR = 0;
7
- method = 4;
8
- sns_strength = 50;
9
- filter_strength = 60;
10
- filter_sharpness = 0;
11
- /** 1 = strong filter */
12
- filter_type = 1;
13
- partitions = 0;
14
- segments = 4;
15
- pass = 1;
16
- show_compressed = 0;
17
- preprocessing = 0;
18
- autofilter = 0;
19
- partition_limit = 0;
20
- alpha_compression = 1;
21
- alpha_filtering = 1;
22
- alpha_quality = 100;
23
- lossless = 0;
24
- exact = 0;
25
- image_hint = 0;
26
- emulate_jpeg_size = 0;
27
- thread_level = 0;
28
- low_memory = 0;
29
- near_lossless = 100;
30
- use_delta_palette = 0;
31
- /** probably sharp rgb -> YUV conversion */
32
- use_sharp_yuv = 0;
33
- }