@needle-tools/gltf-build-pipeline 2.2.1-alpha.5 → 2.2.1-alpha.7

package/CHANGELOG.md

@@ -4,6 +4,10 @@ All notable changes to this package will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [2.2.1-alpha.7] - 2023-06-19
+- fix: WARN if image validation can not run
+- change: compress lightmap textures with webp
+
 ## [2.2.1-alpha.5] - 2023-06-18
 - add: `stats` now includes json and animation size info
 - change: VRM's are compressed with meshopt again

package/dist/cache/cache.js

@@ -41,8 +41,9 @@ export function limitCacheSize(maxSizeMB = cacheSizeLimit) {
     const dir = getCacheDirectory(true);
     console.log(`Limit cache size to ${maxSizeMB.toFixed(2)} MB at ${dir}...`);
     let totalSize = 0;
+    const files = new Array();
     function recurseDirectories(dir) {
-        const files = fs.readdirSync(dir).map(file => {
+        fs.readdirSync(dir).forEach(file => {
             const path = `${dir}/${file}`;
             const stats = statSync(path);
             if (stats.isDirectory()) {
@@ -50,12 +51,11 @@ export function limitCacheSize(maxSizeMB = cacheSizeLimit) {
             }
             else {
                 totalSize += stats.size;
+                files.push({ path, size: stats.size });
             }
-            return { path, size: stats.size };
         });
-        return files;
     }
-    const files = recurseDirectories(dir);
+    recurseDirectories(dir);
     const maxSizeInBytes = maxSizeMB * 1024 * 1024;
     console.log(`Current cache size is ${(totalSize / (1024 * 1024)).toFixed(1)} MB`);
     if (totalSize < maxSizeInBytes)
@@ -67,17 +67,19 @@ export function limitCacheSize(maxSizeMB = cacheSizeLimit) {
     });
     const targetSize = maxSizeInBytes * 0.9;
     // remove oldest files first
+    let filesRemoved = 0;
     while (totalSize > targetSize) {
         const file = files.shift();
         if (!file)
             break;
-        if (fs.statSync(file.path).isDirectory())
+        if (fs.statSync(file.path).isDirectory()) {
             continue;
-        console.log(`Removing cache file ${file.path}...`);
+        }
+        filesRemoved++;
         fs.rmSync(file.path);
         totalSize -= file.size;
     }
-    console.log(`Cache size reduced to ${(totalSize / (1024 * 1024)).toFixed(1)} MB`);
+    console.log(`Cache size reduced to ${(totalSize / (1024 * 1024)).toFixed(1)} MB (${filesRemoved} files removed)`);
 }
 /**
  * Add data to the cache.

package/dist/cli/index.js

@@ -5,7 +5,7 @@ import { printStats, writeFileStatsToFile } from '../utils/stats.js';
 import { getVersion } from '../utils/version.js';
 import { UsecaseOptions, createConfig, getConfig } from '../config/index.js';
 import { make_progressive } from '../transforms/needle_progressive.js';
-import { cacheSizeLimit, clearCache } from '../cache/cache.js';
+import { cacheSizeLimit, clearCache, limitCacheSize } from '../cache/cache.js';
 import { statSync } from 'fs';
 import { foreachGLTF, isDirectory } from '../utils/fileutils.js';
 import path, { dirname } from 'path';
@@ -135,6 +135,7 @@ Each version will be compressed and written to the output directory.
         }
     });
     printStats(stats, logger, config);
+    limitCacheSize();
 });
 program.run().catch(err => {
     // Since we still invoke this package with paths when running npm scripts we need to ignore this error

package/dist/extensions/NEEDLE_opaque/NEEDLE_opaque.js

@@ -1,5 +1,6 @@
 import { Extension, ExtensionProperty, PropertyType, TextureChannel } from '@gltf-transform/core';
 import { getTexture, tryGetInArray, writeExtension } from '../utils.js';
+import { isLightmap } from '../../utils/texture.js';
 const ALL_PROPERTY_TYPES = [];
 for (const key in PropertyType) {
     ALL_PROPERTY_TYPES.push(PropertyType[key]);
@@ -163,11 +164,11 @@ export function createOpaqueExtension(name, types, opts) {
                             continue;
                         const ext = sourceArray[EXTENSION_NAME];
                         if (ext) {
-                            this.lightmapWorkaround(context);
                             const root = this.document.getRoot();
                             const prop = this.createExtension();
                             this.jsonPointerHandler.read(context, root, ext, prop);
                             root.setExtension(EXTENSION_NAME, prop);
+                            this.lightmapWorkaround(context, prop, this.document.getLogger());
                             prop.add(ctx => {
                                 ctx.jsonDoc.json.extensions = ctx.jsonDoc.json.extensions || {};
                                 ctx.jsonDoc.json.extensions[EXTENSION_NAME] = ext;
@@ -180,7 +181,7 @@ export function createOpaqueExtension(name, types, opts) {
         }
         // this is temporarely here to prevent compression on textures named Lightmap
         // until we have better control over the compression process
-        lightmapWorkaround(context) {
+        lightmapWorkaround(context, prop, logger) {
             const textureDefinitions = context.jsonDoc.json.textures;
             if (!textureDefinitions)
                 return;
@@ -189,10 +190,9 @@ export function createOpaqueExtension(name, types, opts) {
                 const tex = textures[i];
                 const texDef = textureDefinitions[i];
                 const name = texDef?.name;
-                if (name?.startsWith("Lightmap") && tex.getMimeType() !== "image/exr") {
-                    if (debugLog)
-                        console.log("Found lightmap texture", name, tex.getMimeType(), EXTENSION_NAME);
-                    tex.setMimeType("image/lightmap");
+                if (isLightmap(tex) && tex.getMimeType() !== "image/exr") {
+                    prop.setReference("lightmapTexture", tex, { channels: TextureChannel.R | TextureChannel.G | TextureChannel.B | TextureChannel.A });
+                    logger.info(`Found lightmap texture ${name}: ${tex.getMimeType()} in ${EXTENSION_NAME}`);
                 }
             }
         }

package/dist/transforms/needle_progressive.js

@@ -7,7 +7,7 @@ import path, { basename, resolve } from 'path';
 import { existsSync, readFileSync, rmSync, statSync, writeFileSync } from 'fs';
 import { addMetaToCache, addToCache, getHash, getKey, tryGetFromCache, tryGetMetaFromCache } from '../cache/index.js';
 import sharp from "sharp";
-import { detectTextureCompressionMode, foreachGLTF, generateGuid, validateImageMimeType, writeNodeIO } from '../utils/index.js';
+import { detectTextureCompressionMode, foreachGLTF, generateGuid, isLightmap, setLightmapName, validateImageMimeType, writeNodeIO } from '../utils/index.js';
 import { needle_asset, writeNeedleAsset } from './needle_asset.js';
 import { MeshoptDecoder, MeshoptSimplifier } from 'meshoptimizer';
 import { compressPrimitive } from './needle_mesh_transform.js';
@@ -193,7 +193,9 @@ export function make_progressive_textures(filePath, _options = TEXTURE_RESIZE_DE
             // generateLods ||= opts.config?.textures?.lods === true;
             // // Ensure that texture lods are not disabled by { textures: { lods: false } }
             // generateLods &&= opts.config?.textures?.lods !== false;
-            if (generateLods) {
+            const _isLightmap = slots.includes("lightmapTexture") || isLightmap(texture);
+            // Lightmaps don't have multiple levels right now
+            if (generateLods && !_isLightmap) {
                 const LEVEL = 6;
                 let lastSize = currentSize;
                 for (let i = 1; i < LEVEL; i++) {
@@ -273,6 +275,7 @@ async function createTextureLod(args, opts) {
     newTexture.setImage(img);
     newTexture.setURI(texture.getURI());
     newTexture.setMimeType(texture.getMimeType());
+    newTexture.setName(texture.getName() || ("LOD_" + textureIndex));
     if (args.maxSize && args.maxSize >= 16) {
         const currentSize = newTexture.getSize();
         if (currentSize[0] > args.maxSize && currentSize[1] > args.maxSize) {
@@ -308,6 +311,10 @@ async function createTextureLod(args, opts) {
             case "emissiveTexture":
                 newMaterial.setEmissiveTexture(newTexture);
                 break;
+            case "lightmapTexture":
+                newMaterial.setBaseColorTexture(newTexture);
+                setLightmapName(newTexture);
+                break;
             default:
                 if (slot.startsWith("/textures/")) {
                     // Ensure the texture is using the alpha channel since we don't know what slots the original texture was used for
@@ -322,6 +329,7 @@ async function createTextureLod(args, opts) {
         // Make sure the material is used
         createMeshForMaterial(newDoc, newMaterial, slot_index++);
     }
+    // Pass compression settings to the new texture
     const compressionSettingsExt = texture.getExtension(NEEDLE_compression_texture.EXTENSION_NAME);
     if (compressionSettingsExt !== undefined && compressionSettingsExt !== null && compressionSettingsExt.getExtensionDefinition) {
         logger.info("> Pass compression extension to new texture");

package/dist/transforms/needle_texture_transform.js

@@ -4,10 +4,11 @@ import { NEEDLE_compression_texture } from '../extensions/index.js';
 import { addToCache, getHash, getKey, tryGetFromCache } from "../cache/index.js";
 import { NeedleWebPContext } from "./needle_webp.js";
 import { NeedleToktxContext } from "./needle_toktx.js";
-import { createTransform } from "@gltf-transform/functions";
+import { createTransform, listTextureSlots } from "@gltf-transform/functions";
 import { dirname } from "path";
 import { existsSync, rmSync } from "fs";
 import { validateImageMimeType } from "../utils/validate.js";
+import { isLightmap } from "../utils/texture.js";
 export async function resizeTextureIfNecessary(texture) {
     const ext = texture.getExtension(NEEDLE_compression_texture.EXTENSION_NAME);
     if (ext?.model) {
@@ -84,7 +85,7 @@ export const needle_texture_transform = function (options) {
 async function processTexture(texture, index, document, logger, context) {
     const validationResult = validateImageMimeType(texture.getMimeType(), texture.getImage());
     if (validationResult?.error) {
-        logger.warn(`Texture ${texture.getName()} [${index}] has an incorrect mime type: ${validationResult.mimeType}: auto-correcting.`);
+        logger.warn(`WARN: Texture ${texture.getName()} [${index}] has an incorrect mime type \"${texture.getMimeType()}\" → auto-correcting to ${validationResult.mimeType}`);
         texture.setMimeType(validationResult.mimeType);
     }
     const originalMimeType = texture.getMimeType();
@@ -142,13 +143,16 @@ async function processTexture(texture, index, document, logger, context) {
     }
     else {
         let mimeType = "image/ktx2";
-        // const slots = listTextureSlots(texture);
+        const slots = listTextureSlots(texture);
         // if (slots.includes("metallicRoughnessTexture"))
         //     mimeType = "image/webp";
         const isVRM = context.file?.toLowerCase().endsWith(".vrm");
         if (context.config?.usecase === "product") {
             mimeType = "image/webp";
         }
+        else if (slots.includes("lightmapTexture") || isLightmap(texture)) {
+            mimeType = "image/webp";
+        }
         // TODO: need to make this into a function
         // const webpThreshold = 256; // this is the default threshold in progressive textures (needle_progressive.ts)
         // console.log(context.file, isVRM);

package/dist/transforms/needle_toktx.d.ts

@@ -11,6 +11,7 @@ export declare class NeedleToktxContext implements INeedleTextureTransformContex
     limit: any;
     basisuExtension: KHRTextureBasisu;
     version: string;
+    cacheDirectory: string;
     tmpPathBase: string;
     document: Document;
     env: undefined | {

package/dist/transforms/needle_toktx.js

@@ -2,7 +2,7 @@ import { BufferUtils, FileUtils, ImageUtils, uuid } from "@gltf-transform/core";
 import { listTextureSlots, getTextureChannelMask, fitPowerOfTwo, TextureResizeFilter } from "@gltf-transform/functions";
 import { KHRTextureBasisu } from "@gltf-transform/extensions";
 import { checkKTXSoftware, createParams, KTX_SOFTWARE_VERSION_MIN, Mode } from "./toktx.js";
-import { existsSync, mkdirSync } from "fs";
+import { existsSync, mkdirSync, rmdirSync } from "fs";
 import { formatBytes, spawn, waitExit } from './util.js';
 import { ETC1S_DEFAULTS, UASTC_DEFAULTS } from "./toktx_types.js";
 import fs from 'fs/promises';
@@ -10,7 +10,7 @@ import os from 'os';
 import tmp from 'tmp';
 import pLimit from 'p-limit';
 import { replaceExtension } from "../utils/fileutils.js";
-import { ceilMultipleOfFour, isMultipleOfFour } from "../utils/index.js";
+import { ceilMultipleOfFour, isLightmap, isMultipleOfFour } from "../utils/index.js";
 import sharp from "sharp";
 const NUM_CPUS = os.cpus().length || 1; // microsoft/vscode#112122
 let didLogEnvironmentInfo = false;
@@ -18,6 +18,7 @@ export class NeedleToktxContext {
     limit;
     basisuExtension;
     version;
+    cacheDirectory;
     tmpPathBase;
     document;
     env;
@@ -76,6 +77,7 @@ export class NeedleToktxContext {
             }
             this.basisuExtension = document.createExtension(KHRTextureBasisu).setRequired(true);
             const tmpDir = `${tmp.tmpdir}/gltf-transform`;
+            this.cacheDirectory = tmpDir;
             if (!(existsSync(tmpDir)))
                 mkdirSync(tmpDir);
             this.tmpPathBase = tmpDir + "/" + uuid();
@@ -173,6 +175,10 @@ export class NeedleToktxContext {
                     quality = 255;
                     compression = 4;
                 }
+                else if (slots.includes("lightmapTexture") || isLightmap(texture)) {
+                    quality = 255;
+                    compression = 4;
+                }
             }
             const defaults = mode === Mode.ETC1S ? ETC1S_DEFAULTS : UASTC_DEFAULTS;
             // TODO: should we ensure that the texture is using all channels? Unused channels are discarded anyways. The issue we have here is that textures that are solely used from pointers have unknown channels. We register them with RGBA when generating progressive assets as well as assign texture json pointers /textures/ to
@@ -226,5 +232,8 @@ export class NeedleToktxContext {
         if (!usesKTX2) {
             this.basisuExtension?.dispose();
         }
+        if (this.cacheDirectory && existsSync(this.cacheDirectory)) {
+            rmdirSync(this.cacheDirectory, { recursive: true });
+        }
     }
 }

package/dist/utils/texture.d.ts

@@ -1,2 +1,5 @@
+import { Texture } from "@gltf-transform/core";
 export declare function isMultipleOfFour(value: number): boolean;
 export declare function ceilMultipleOfFour(value: number): number;
+export declare function isLightmap(texture: Texture): boolean;
+export declare function setLightmapName(texture: Texture): void;

package/dist/utils/texture.js

@@ -6,3 +6,9 @@ export function ceilMultipleOfFour(value) {
         return 4;
     return value % 4 ? value + 4 - (value % 4) : value;
 }
+export function isLightmap(texture) {
+    return texture.getName()?.startsWith("Lightmap");
+}
+export function setLightmapName(texture) {
+    texture.setName("Lightmap");
+}

package/dist/utils/validate.js

@@ -2,15 +2,16 @@ import { ImageUtils } from "@gltf-transform/core";
 export function validateImageMimeType(mimetype, image) {
     const actualMimeType = ImageUtils.getMimeType(image);
     if (actualMimeType != null && actualMimeType != mimetype) {
-        console.error(`Expected image to be of type ${mimetype}, but was ${actualMimeType}`);
+        const msg = `Image reported wrong mime type \"${mimetype}\" but is actually \"${actualMimeType}\"`;
+        console.error(`WARN: ${msg}`);
         return {
             error: true,
-            message: `Expected image to be of type ${mimetype}, but was ${actualMimeType}`,
+            message: msg,
             mimeType: actualMimeType
         };
     }
     else if (actualMimeType === null) {
-        console.warn("Could not determine image type");
+        console.warn(`WARN: Could not validate image type for \"${mimetype}\"`);
     }
     return null;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@needle-tools/gltf-build-pipeline",
-  "version": "2.2.1-alpha.5",
+  "version": "2.2.1-alpha.7",
   "description": "Pipeline and tools for optimizing gltf files using gltf-transform and compression settings within glTF extensions",
   "main": "dist/index.js",
   "type": "module",