@needle-tools/gltf-build-pipeline 1.5.0-alpha.5 → 1.5.0-alpha.6

package/CHANGELOG.md

@@ -4,6 +4,9 @@ All notable changes to this package will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [1.5.0-alpha.6] - 2023-05-15
+- fix: If not further specified then automatically use UASTC for textures that are in either `normalTexture` or `metallicOcclusionTexture` slots
+
 ## [1.5.0-alpha.5] - 2023-05-04
 - fix: NEEDLE_opaque should handle if VRM texture index is negative
 - change: exit with code 1 if any file packing fails

package/dist/extensions/NEEDLE_compression_texture/NEEDLE_compression_texture.d.ts

@@ -1,7 +1,7 @@
 import { Extension, ExtensionProperty, PropertyType, ReaderContext, WriterContext } from "@gltf-transform/core";
 export declare const EXTENSION_NAME = "NEEDLE_compression_texture";
 export declare interface NEEDLE_compression_texture_schema {
-    mode: string;
+    mode: "none" | "auto" | "webp" | "UASTC" | "ETC1S";
     slot: string;
     /** quality in range of 0 - 1 */
     quality: number;

package/dist/transforms/needle_common.d.ts

@@ -1,6 +1,7 @@
 import { Document, Texture } from '@gltf-transform/core';
 import { NEEDLE_compression_texture_schema } from '../extensions/index.js';
 export interface NeedleTransformContext {
+    file: string;
     debug?: boolean;
     useCache: boolean;
     noToktx: boolean;

package/dist/transforms/needle_progressive.js

@@ -7,13 +7,13 @@ import path, { basename } from 'path';
 import { existsSync, readFileSync, rmSync, statSync, writeFileSync } from 'fs';
 import { addMetaToCache, addToCache, getHash, getKeyWithHash, tryGetFromCache, tryGetMetaFromCache } from '../cache/index.js';
 import sharp from "sharp";
-import { foreachGLTF, generateGuid, writeNodeIO } from '../utils/index.js';
+import { detectTextureCompressionMode, foreachGLTF, generateGuid, writeNodeIO } from '../utils/index.js';
 import { needle_asset, writeNeedleAsset } from './needle_asset.js';
 import { MeshoptDecoder, MeshoptSimplifier } from 'meshoptimizer';
 import { compressPrimitive } from './needle_mesh_transform.js';
 import { calculateMeshDensity, getMeshInformation } from "../utils/mesh.js";
 export async function make_progressive(opts) {
-    console.log("INFO: Run progressive transform: " + opts.path);
+    console.log(`INFO: Run progressive transform: ${opts.path}`);
     const file = opts.path;
     return foreachGLTF(file, async (f) => {
         // Don't re-process files that are already LODs
@@ -82,6 +82,7 @@ async function onProcess(file, opts) {
             }
             else
                 console.warn("WARN: Progressive meshes are not supported for VRM files yet");
+            console.log("→ Transforming: " + file);
             await document.transform(...transforms);
             let filename = file;
             if (opts.debug === true) {
@@ -93,6 +94,7 @@ async function onProcess(file, opts) {
                 filename = filename + ".debug." + extension;
             }
             await writeNodeIO(io, filename, document);
+            console.log("← Transforming done: " + filename);
         }
     }
     catch (err) {
@@ -169,7 +171,7 @@ export function make_progressive_textures(filePath, _options = TEXTURE_RESIZE_DE
                 continue;
             }
             const textureGuid = settings.guid?.length ? settings.guid : generateGuid(path + "_texture_" + index.toString());
-            const outPath = createTextureLod(doc, textureLodBasePath, 0, texture, slots, textureGuid, textureIndex);
+            const outPath = createTextureLod(doc, filePath, textureLodBasePath, 0, texture, slots, textureGuid, textureIndex);
             if (!outPath) {
                 continue;
             }
@@ -205,7 +207,7 @@ export function make_progressive_textures(filePath, _options = TEXTURE_RESIZE_DE
         logger.debug(`Progressive transform finished in ${((Date.now() - startTime) / 1000).toFixed(1)} sec.`);
     });
 }
-function createTextureLod(doc, basePath, level, texture, slots, guid, textureIndex) {
+function createTextureLod(doc, file, basePath, level, texture, slots, guid, textureIndex) {
     // const imageBytes = texture.getImage();
     // if (imageBytes) {
     // 	// write bytes
@@ -256,6 +258,13 @@ function createTextureLod(doc, basePath, level, texture, slots, guid, textureInd
         console.log("> Pass compression extension to new texture");
         const newExt = newDoc.createExtension(NEEDLE_compression_texture);
         const def = compressionSettings.getExtensionDefinition();
+        def.mode = detectTextureCompressionMode(slots);
+        def.slot = slots[0];
+        // for VRMS we just use webp
+        // if (file?.toLocaleLowerCase().endsWith(".vrm")) {
+        // 	def.mode = "webp";
+        // 	def.quality = 70;
+        // }
         const newProp = newExt.create(def);
         newTexture.setExtension(NEEDLE_compression_texture.EXTENSION_NAME, newProp);
     }

package/dist/transforms/needle_texture_transform.js

@@ -125,21 +125,41 @@ async function processTexture(texture, index, document, logger, context) {
         }
     }
     else {
+        let mimetype = "image/ktx2";
+        const isVRM = context.file?.toLowerCase().endsWith(".vrm");
+        // TODO: need to make this into a function
+        // const webpThreshold = 256; // this is the default threshold in progressive textures (needle_progressive.ts)
+        // console.log(context.file, isVRM);
+        // // if we're dealing with a VRM and the texture size is < 128x128 we can use webp (this is e.g. the case for progressive textures)
+        // if (isVRM) {
+        //     const size = texture.getSize();
+        //     console.log(isVRM, size)
+        //     if (size && size[0] <= webpThreshold && size[1] <= webpThreshold) {
+        //         mimetype = "image/webp";
+        //     }
+        // }
         // If we have a texture cached we can use it
         // the default compression is toktx so we can safely use that for now
         if (cached) {
-            texture.setImage(cached).setMimeType("image/ktx2");
+            texture.setImage(cached).setMimeType(mimetype);
             return;
         }
         await resizeTextureIfNecessary(texture);
-        if (canRunToktx) {
+        if (mimeType === "image/webp") {
+            successFullyProcessed = await context.webPContext.process(index, texture, null);
+        }
+        else if (canRunToktx) {
             // if no specific compression settings are provided we just use the toktx default settings
             // if we ever change what is used for default compression we need to teach the cache to save the mime type
-            successFullyProcessed = await context.toktxContext.process(index, texture, null);
-            // don't add textures to the cache when toktx doesnt run
+            // For VRMs we use ETC1S always
+            let options = null;
+            if (isVRM) {
+                options = { mode: "ETC1S" };
+            }
+            successFullyProcessed = await context.toktxContext.process(index, texture, options);
         }
         else {
-            console.log("toktx is disallowed. Skipping texture compression.");
+            console.log("Toktx not found: Skipping texture compression.");
         }
     }
     if (context.useCache && successFullyProcessed && key) {

package/dist/transforms/needle_toktx.d.ts

@@ -20,6 +20,6 @@ export declare class NeedleToktxContext implements INeedleTextureTransformContex
     private _prepareFailed;
     constructor(options: NeedleToktxOptions);
     prepare(document: Document): Promise<boolean>;
-    process(index: number, texture: Texture, settings: NEEDLE_compression_texture_schema | null): Promise<boolean>;
+    process(index: number, texture: Texture, settings: Pick<NEEDLE_compression_texture_schema, "mode"> | null): Promise<boolean>;
     finalize(document: Document): Promise<void>;
 }

package/dist/transforms/needle_toktx.js

@@ -113,17 +113,23 @@ export class NeedleToktxContext {
             const outPath = tmpName + '.ktx2';
             // const inBytes = image.byteLength;
             await fs.writeFile(inPath, Buffer.from(image));
+            let mode = Mode.ETC1S;
+            if (settings) {
+                mode = settings.mode === "ETC1S" ? Mode.ETC1S : Mode.UASTC;
+            }
+            else if (slots) {
+                const useUASTC = slots.includes("normalTexture") || slots.includes("metallicRoughnessTexture");
+                mode = useUASTC ? Mode.UASTC : Mode.ETC1S;
+            }
+            const defaults = mode === Mode.ETC1S ? ETC1S_DEFAULTS : UASTC_DEFAULTS;
             //@ts-ignore
-            const defaults = (!settings || settings.mode === "ETC1S") ? ETC1S_DEFAULTS : UASTC_DEFAULTS;
-            defaults.mode = (!settings || settings.mode === "ETC1S") ? Mode.ETC1S : Mode.UASTC;
-            //@ts-ignore
-            const toktxParams = createParams(slots, channels, size, logger, this.numTextures, { ...defaults }, this.version);
+            const toktxParams = createParams(slots, channels, size, logger, this.numTextures, { ...defaults, mode }, this.version);
             const params = [
                 ...toktxParams,
                 outPath,
                 inPath,
             ];
-            logger.debug(`${prefix}: Spawning → toktx ${params.join(' ')}`);
+            logger.debug(`${prefix}: Spawning → toktx ${mode.toUpperCase()} ${params.join(' ')}`);
             try {
                 // COMPRESS: Run `toktx` CLI tool.
                 // eslint-disable-next-line @typescript-eslint/no-unused-vars

package/dist/utils/compression_utils.d.ts

@@ -0,0 +1,2 @@
+export declare type TextureCompressionMode = "webp" | "ETC1S" | "UASTC";
+export declare function detectTextureCompressionMode(slots: string[]): "ETC1S" | "UASTC";

package/dist/utils/compression_utils.js

@@ -0,0 +1,6 @@
+export function detectTextureCompressionMode(slots) {
+    if (slots.includes("normalTexture") || slots.includes("metallicRoughnessTexture")) {
+        return "UASTC";
+    }
+    return "ETC1S";
+}

package/dist/utils/index.d.ts

@@ -1,3 +1,4 @@
+export * from "./compression_utils.js";
 export * from "./fileutils.js";
 export * from "./version.js";
 export * from "./mesh.js";

package/dist/utils/index.js

@@ -1,3 +1,4 @@
+export * from "./compression_utils.js";
 export * from "./fileutils.js";
 export * from "./version.js";
 export * from "./mesh.js";

package/pack-gltf.mjs

@@ -60,11 +60,9 @@ if (debug) {
 
 let allowCaching = true;
 if (args.includes("--no-caching")) {
-    console.log("Caching is disabled")
     allowCaching = false;
 }
-else if (showHelp)
-    console.log("Caching is enabled, use \"--no-caching\" to disable it");
+else console.log("Use --no-caching to disable caching");
 
 const noToktx = args.includes("no-toktx");
 if (noToktx)
@@ -106,10 +104,11 @@ async function packFile(file) {
 
     if (packedFiles.has(file)) return;
     packedFiles.add(file);
-    console.log("→ Begin packing: " + file);
 
     try {
         const prevSizeInMB = statSync(file).size / 1024 / 1024;
+        console.log(`→ Compressing ${prevSizeInMB.toFixed(1)} MB: ${file}`);
+
         const startTime = Date.now();
         const io = new NodeIO()
             .registerExtensions(ALL_EXTENSIONS)
@@ -130,6 +129,7 @@ async function packFile(file) {
         await document.transform(
             needle_asset(),
             needle_texture_transform({
+                file,
                 useCache: allowCaching,
                 noToktx: noToktx,
                 debug: debug,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@needle-tools/gltf-build-pipeline",
-  "version": "1.5.0-alpha.5",
+  "version": "1.5.0-alpha.6",
   "description": "Pipeline and tools for optimizing gltf files using gltf-transform and compression settings within glTF extensions",
   "main": "index.mjs",
   "type": "module",