@needle-tools/gltf-build-pipeline 2.13.0 → 2.13.1

package/dist/transforms/needle_texture_transform.d.ts

@@ -1,13 +1,13 @@
-import { Texture } from "@gltf-transform/core";
-import { Transform } from '@gltf-transform/core';
-import { NeedleWebPContext } from "./needle_webp.js";
-import { NeedleTransformContext } from "./needle_common.js";
-import { NeedleToktxContext } from "./needle_toktx.js";
-export declare function resizeTextureIfNecessary(texture: Texture): Promise<void>;
-interface NeedleTextureTransformContext extends NeedleTransformContext {
-    webPContext: NeedleWebPContext;
-    toktxContext: NeedleToktxContext;
-    reportGPUMemory?: (bytes: number) => void;
-}
-export declare const needle_texture_transform: (options: NeedleTransformContext & Pick<NeedleTextureTransformContext, "reportGPUMemory">) => Transform;
-export {};
+import { Texture } from "@gltf-transform/core";
+import { Transform } from '@gltf-transform/core';
+import { NeedleWebPContext } from "./needle_webp.js";
+import { NeedleTransformContext } from "./needle_common.js";
+import { NeedleToktxContext } from "./needle_toktx.js";
+export declare function resizeTextureIfNecessary(texture: Texture): Promise<void>;
+interface NeedleTextureTransformContext extends NeedleTransformContext {
+    webPContext: NeedleWebPContext;
+    toktxContext: NeedleToktxContext;
+    reportGPUMemory?: (bytes: number) => void;
+}
+export declare const needle_texture_transform: (options: NeedleTransformContext & Pick<NeedleTextureTransformContext, "reportGPUMemory">) => Transform;
+export {};

package/dist/transforms/needle_texture_transform.js

@@ -1,257 +1,257 @@
-import { ImageUtils } from "@gltf-transform/core";
-import { resizeTextureToMax } from "./needle_progressive.js";
-import { NEEDLE_compression_texture } from '../extensions/index.js';
-import { addToCache, getHash, getKey, tryGetFromCache } from "../cache/index.js";
-import { NeedleWebPContext } from "./needle_webp.js";
-import { NeedleToktxContext } from "./needle_toktx.js";
-import { createTransform, listTextureSlots } from "@gltf-transform/functions";
-import { dirname } from "path";
-import { existsSync, rmSync } from "fs";
-import { validateImageMimeType } from "../utils/validate.js";
-import { isLightmap } from "../utils/texture.js";
-export async function resizeTextureIfNecessary(texture) {
-    const ext = texture.getExtension(NEEDLE_compression_texture.EXTENSION_NAME);
-    if (ext?.model) {
-        const mimeType = texture.getMimeType();
-        const isSupported = mimeType === "image/jpeg" || mimeType === "image/png";
-        if (isSupported) {
-            const settings = ext.model;
-            const size = texture.getSize();
-            if (size) {
-                if (settings.maxSize > 4 && (size[0] > settings.maxSize || size[1] > settings.maxSize)) {
-                    console.log(`> Will resize texture ${texture.getName()} from ${size[0]}x${size[1]} to ${settings.maxSize}x${settings.maxSize}`);
-                    await resizeTextureToMax(texture, settings.maxSize, console);
-                }
-            }
-        }
-    }
-    return undefined;
-}
-export const needle_texture_transform = function (options) {
-    const webPContext = new NeedleWebPContext({});
-    const toktxContext = new NeedleToktxContext({
-        jobs: undefined,
-        debug: options.debug,
-    });
-    const context = {
-        config: options.config,
-        ...options,
-        webPContext,
-        toktxContext
-    };
-    return createTransform("needle_texture_transform", async (document) => {
-        await webPContext.prepare(document);
-        if (context.noToktx !== true) {
-            const validToktx = await toktxContext.prepare(document);
-            if (!validToktx) {
-                context.noToktx = true;
-            }
-        }
-        try {
-            const logger = document.getLogger();
-            const textures = document.getRoot().listTextures();
-            let totalGpuSize = 0;
-            await Promise.all(textures.map((texture, textureIndex) => {
-                return processTexture(texture, textureIndex, document, logger, context).then(() => {
-                    try {
-                        const size = ImageUtils.getVRAMByteLength(texture.getImage(), texture.getMimeType());
-                        totalGpuSize += size;
-                    }
-                    catch {
-                        // Ignore
-                    }
-                });
-            }));
-            if (totalGpuSize > 0) {
-                // TODO: report this to the parent
-                const gpuMemoryEstimate = totalGpuSize / 1024 / 1024;
-                if (context.reportGPUMemory)
-                    context.reportGPUMemory(totalGpuSize);
-                else {
-                    console.log(`> Total GPU size: ${(gpuMemoryEstimate).toFixed(2)} MB of ${textures.length} textures`);
-                }
-                if (gpuMemoryEstimate > 1000) {
-                    console.log(`WARN: Total estimated GPU size is very large - this may cause issues on some devices. Consider changing the compression settings or reducing texture size.`);
-                }
-            }
-        }
-        finally {
-            await webPContext.finalize(document);
-            if (context.noToktx !== true)
-                await toktxContext.finalize(document);
-        }
-    });
-};
-async function processTexture(texture, index, document, logger, context) {
-    const validationResult = validateImageMimeType(texture.getMimeType(), texture.getImage());
-    if (validationResult?.error) {
-        logger.warn(`WARN: Texture ${texture.getName()} [${index}] has an incorrect mime type \"${texture.getMimeType()}\" → auto-correcting to ${validationResult.mimeType}`);
-        texture.setMimeType(validationResult.mimeType);
-    }
-    const originalMimeType = texture.getMimeType();
-    const isSupported = originalMimeType === "image/jpeg" || originalMimeType === "image/png" || originalMimeType === "image/webp";
-    // dont try to transform EXR or already processed textures
-    if (!isSupported)
-        return;
-    const ext = texture.getExtension(NEEDLE_compression_texture.EXTENSION_NAME);
-    const compressionSettings = ext?.model;
-    // is a user has configured a texture explictly to NOT be compressed
-    // we should respect that
-    if (compressionSettings?.mode === "none") {
-        logger.info(`→ Skipping texture compression for ${texture.getName()} [${index}] as configured in the texture extension`);
-        return;
-    }
-    // generate the cache key from the file data and settings
-    const key = context.useCache ? (getKey(texture) + getHash(context.config)) : undefined;
-    let cached = context.useCache && key ? tryGetFromCache(key) : undefined;
-    if (cached) {
-        if (cached.length <= 0) {
-            logger.warn(`WARN: Cached texture ${texture.getName()} [${index}] is empty. Will reprocess.`);
-            cached = undefined;
-        }
-        else {
-            logger.info(`→ Using cached texture ${texture.getName()} [${index}] – ${(cached.length / 1024).toFixed(0)} KB`);
-        }
-    }
-    let successfullyProcessed = false;
-    let useDefaultCompression = true;
-    const canRunToktx = context.noToktx !== true;
-    const originalUri = texture.getURI();
-    if (compressionSettings && compressionSettings.mode !== "auto") {
-        // Handle highlevel modes
-        switch (compressionSettings.mode) {
-            case "world":
-                compressionSettings.mode = "ETC1S";
-                break;
-            case "product":
-                switch (compressionSettings.slot) {
-                    default:
-                        compressionSettings.mode = "ETC1S";
-                        break;
-                    case "normalTexture":
-                        compressionSettings.mode = "UASTC";
-                        break;
-                    case "baseColorTexture":
-                    case "metallicRoughnessTexture":
-                    case "lightmapTexture":
-                        compressionSettings.mode = "webp";
-                        compressionSettings.quality = 0.8;
-                        break;
-                }
-                break;
-        }
-        switch (compressionSettings.mode) {
-            case "webp":
-                if (cached) {
-                    texture.setImage(cached).setMimeType("image/webp");
-                    return;
-                }
-                await resizeTextureIfNecessary(texture);
-                useDefaultCompression = false;
-                successfullyProcessed = await context.webPContext.process(index, texture, compressionSettings, context);
-                break;
-            case "ETC1S":
-                if (cached) {
-                    texture.setImage(cached).setMimeType("image/ktx2");
-                    return;
-                }
-                await resizeTextureIfNecessary(texture);
-                useDefaultCompression = false;
-                successfullyProcessed = await context.toktxContext.process(index, texture, compressionSettings, context);
-                break;
-            case "UASTC":
-                if (cached) {
-                    texture.setImage(cached).setMimeType("image/ktx2");
-                    return;
-                }
-                await resizeTextureIfNecessary(texture);
-                useDefaultCompression = false;
-                successfullyProcessed = await context.toktxContext.process(index, texture, compressionSettings, context);
-                break;
-        }
-    }
-    if (useDefaultCompression) {
-        let mimeType = "image/ktx2";
-        let ktx2Format = undefined;
-        const slots = listTextureSlots(texture);
-        const size = texture.getSize();
-        const isVRM = context.file?.toLowerCase().endsWith(".vrm");
-        // TODO: need to make this into a function
-        // const webpThreshold = 256; // this is the default threshold in progressive textures (needle_progressive.ts)
-        // console.log(context.file, isVRM);
-        // // if we're dealing with a VRM and the texture size is < 128x128 we can use webp (this is e.g. the case for progressive textures)
-        // if (isVRM) {
-        //     const size = texture.getSize();
-        //     console.log(isVRM, size)
-        //     if (size && size[0] <= webpThreshold && size[1] <= webpThreshold) {
-        //         mimetype = "image/webp";
-        //     }
-        // }
-        const webpOptions = {};
-        if (slots.includes("lightmapTexture") || isLightmap(texture)) {
-            mimeType = "image/webp";
-        }
-        // for https://linear.app/needle/issue/NE-6464
-        else if (size[0] <= 32 || size[1] <= 32) {
-            // can not use UASTC because it's ALSO shitty, see issue above
-            mimeType = "image/webp";
-        }
-        // For product compression we automatically select webP if the texture is used in a PBR material
-        // https://linear.app/needle/issue/NE-6458
-        // TODO: this should probably moved to the top and we should create a compression settings object
-        else if (context.config.usecase === "product") {
-            const shouldUseWebp = slots.includes("normalTexture") || slots.includes("metallicRoughnessTexture");
-            if (shouldUseWebp) {
-                mimeType = "image/webp";
-            }
-        }
-        // If we have a texture cached we can use it
-        // the default compression is toktx so we can safely use that for now
-        if (cached) {
-            texture.setImage(cached).setMimeType(mimeType);
-            return;
-        }
-        await resizeTextureIfNecessary(texture);
-        if (mimeType === "image/webp") {
-            successfullyProcessed = await context.webPContext.process(index, texture, {
-                ...webpOptions,
-                quality: .95,
-                slot: slots.join(","),
-            }, context);
-        }
-        else if (canRunToktx) {
-            // if no specific compression settings are provided we just use the toktx default settings
-            // if we ever change what is used for default compression we need to teach the cache to save the mime type
-            // For VRMs we use ETC1S always
-            let options = null;
-            if (ktx2Format || isVRM) {
-                options = { mode: ktx2Format || "ETC1S" };
-            }
-            successfullyProcessed = await context.toktxContext.process(index, texture, options, context);
-        }
-        else {
-            logger.warn("Toktx not found: Skipping texture compression: " + originalMimeType);
-        }
-    }
-    if (context.useCache && successfullyProcessed && key) {
-        const data = texture.getImage();
-        if (data)
-            addToCache(key, data);
-    }
-    if (successfullyProcessed) {
-        const newUri = texture.getURI();
-        if (originalUri && originalUri != newUri) {
-            const inDirectory = dirname(context.file);
-            const outDirectory = dirname(context.outfile);
-            const originalPath = inDirectory + "/" + originalUri;
-            // if the file output directory is the same as the input directory
-            // (meaning compression runs in the same directory)
-            // we remove the original file (since it effectively gets replaced)
-            // otherwise we dont touch the original file
-            if (inDirectory == outDirectory && existsSync(originalPath)) {
-                logger.info(`Removing old texture \"${originalUri}\" → \"${newUri}\"`);
-                rmSync(originalPath);
-            }
-        }
-    }
-}
+import { ImageUtils } from "@gltf-transform/core";
+import { resizeTextureToMax } from "./needle_progressive.js";
+import { NEEDLE_compression_texture } from '../extensions/index.js';
+import { addToCache, getHash, getKey, tryGetFromCache } from "../cache/index.js";
+import { NeedleWebPContext } from "./needle_webp.js";
+import { NeedleToktxContext } from "./needle_toktx.js";
+import { createTransform, listTextureSlots } from "@gltf-transform/functions";
+import { dirname } from "path";
+import { existsSync, rmSync } from "fs";
+import { validateImageMimeType } from "../utils/validate.js";
+import { isLightmap } from "../utils/texture.js";
+export async function resizeTextureIfNecessary(texture) {
+    const ext = texture.getExtension(NEEDLE_compression_texture.EXTENSION_NAME);
+    if (ext?.model) {
+        const mimeType = texture.getMimeType();
+        const isSupported = mimeType === "image/jpeg" || mimeType === "image/png";
+        if (isSupported) {
+            const settings = ext.model;
+            const size = texture.getSize();
+            if (size) {
+                if (settings.maxSize > 4 && (size[0] > settings.maxSize || size[1] > settings.maxSize)) {
+                    console.log(`> Will resize texture ${texture.getName()} from ${size[0]}x${size[1]} to ${settings.maxSize}x${settings.maxSize}`);
+                    await resizeTextureToMax(texture, settings.maxSize, console);
+                }
+            }
+        }
+    }
+    return undefined;
+}
+export const needle_texture_transform = function (options) {
+    const webPContext = new NeedleWebPContext({});
+    const toktxContext = new NeedleToktxContext({
+        jobs: undefined,
+        debug: options.debug,
+    });
+    const context = {
+        config: options.config,
+        ...options,
+        webPContext,
+        toktxContext
+    };
+    return createTransform("needle_texture_transform", async (document) => {
+        await webPContext.prepare(document);
+        if (context.noToktx !== true) {
+            const validToktx = await toktxContext.prepare(document);
+            if (!validToktx) {
+                context.noToktx = true;
+            }
+        }
+        try {
+            const logger = document.getLogger();
+            const textures = document.getRoot().listTextures();
+            let totalGpuSize = 0;
+            await Promise.all(textures.map((texture, textureIndex) => {
+                return processTexture(texture, textureIndex, document, logger, context).then(() => {
+                    try {
+                        const size = ImageUtils.getVRAMByteLength(texture.getImage(), texture.getMimeType());
+                        totalGpuSize += size;
+                    }
+                    catch {
+                        // Ignore
+                    }
+                });
+            }));
+            if (totalGpuSize > 0) {
+                // TODO: report this to the parent
+                const gpuMemoryEstimate = totalGpuSize / 1024 / 1024;
+                if (context.reportGPUMemory)
+                    context.reportGPUMemory(totalGpuSize);
+                else {
+                    console.log(`> Total GPU size: ${(gpuMemoryEstimate).toFixed(2)} MB of ${textures.length} textures`);
+                }
+                if (gpuMemoryEstimate > 1000) {
+                    console.log(`WARN: Total estimated GPU size is very large - this may cause issues on some devices. Consider changing the compression settings or reducing texture size.`);
+                }
+            }
+        }
+        finally {
+            await webPContext.finalize(document);
+            if (context.noToktx !== true)
+                await toktxContext.finalize(document);
+        }
+    });
+};
+async function processTexture(texture, index, document, logger, context) {
+    const validationResult = validateImageMimeType(texture.getMimeType(), texture.getImage());
+    if (validationResult?.error) {
+        logger.warn(`WARN: Texture ${texture.getName()} [${index}] has an incorrect mime type \"${texture.getMimeType()}\" → auto-correcting to ${validationResult.mimeType}`);
+        texture.setMimeType(validationResult.mimeType);
+    }
+    const originalMimeType = texture.getMimeType();
+    const isSupported = originalMimeType === "image/jpeg" || originalMimeType === "image/png" || originalMimeType === "image/webp";
+    // dont try to transform EXR or already processed textures
+    if (!isSupported)
+        return;
+    const ext = texture.getExtension(NEEDLE_compression_texture.EXTENSION_NAME);
+    const compressionSettings = ext?.model;
+    // is a user has configured a texture explictly to NOT be compressed
+    // we should respect that
+    if (compressionSettings?.mode === "none") {
+        logger.info(`→ Skipping texture compression for ${texture.getName()} [${index}] as configured in the texture extension`);
+        return;
+    }
+    // generate the cache key from the file data and settings
+    const key = context.useCache ? (getKey(texture) + getHash(context.config)) : undefined;
+    let cached = context.useCache && key ? tryGetFromCache(key) : undefined;
+    if (cached) {
+        if (cached.length <= 0) {
+            logger.warn(`WARN: Cached texture ${texture.getName()} [${index}] is empty. Will reprocess.`);
+            cached = undefined;
+        }
+        else {
+            logger.info(`→ Using cached texture ${texture.getName()} [${index}] – ${(cached.length / 1024).toFixed(0)} KB`);
+        }
+    }
+    let successfullyProcessed = false;
+    let useDefaultCompression = true;
+    const canRunToktx = context.noToktx !== true;
+    const originalUri = texture.getURI();
+    if (compressionSettings && compressionSettings.mode !== "auto") {
+        // Handle highlevel modes
+        switch (compressionSettings.mode) {
+            case "world":
+                compressionSettings.mode = "ETC1S";
+                break;
+            case "product":
+                switch (compressionSettings.slot) {
+                    default:
+                        compressionSettings.mode = "ETC1S";
+                        break;
+                    case "normalTexture":
+                        compressionSettings.mode = "UASTC";
+                        break;
+                    case "baseColorTexture":
+                    case "metallicRoughnessTexture":
+                    case "lightmapTexture":
+                        compressionSettings.mode = "webp";
+                        compressionSettings.quality = 0.8;
+                        break;
+                }
+                break;
+        }
+        switch (compressionSettings.mode) {
+            case "webp":
+                if (cached) {
+                    texture.setImage(cached).setMimeType("image/webp");
+                    return;
+                }
+                await resizeTextureIfNecessary(texture);
+                useDefaultCompression = false;
+                successfullyProcessed = await context.webPContext.process(index, texture, compressionSettings, context);
+                break;
+            case "ETC1S":
+                if (cached) {
+                    texture.setImage(cached).setMimeType("image/ktx2");
+                    return;
+                }
+                await resizeTextureIfNecessary(texture);
+                useDefaultCompression = false;
+                successfullyProcessed = await context.toktxContext.process(index, texture, compressionSettings, context);
+                break;
+            case "UASTC":
+                if (cached) {
+                    texture.setImage(cached).setMimeType("image/ktx2");
+                    return;
+                }
+                await resizeTextureIfNecessary(texture);
+                useDefaultCompression = false;
+                successfullyProcessed = await context.toktxContext.process(index, texture, compressionSettings, context);
+                break;
+        }
+    }
+    if (useDefaultCompression) {
+        let mimeType = "image/ktx2";
+        let ktx2Format = undefined;
+        const slots = listTextureSlots(texture);
+        const size = texture.getSize();
+        const isVRM = context.file?.toLowerCase().endsWith(".vrm");
+        // TODO: need to make this into a function
+        // const webpThreshold = 256; // this is the default threshold in progressive textures (needle_progressive.ts)
+        // console.log(context.file, isVRM);
+        // // if we're dealing with a VRM and the texture size is < 128x128 we can use webp (this is e.g. the case for progressive textures)
+        // if (isVRM) {
+        //     const size = texture.getSize();
+        //     console.log(isVRM, size)
+        //     if (size && size[0] <= webpThreshold && size[1] <= webpThreshold) {
+        //         mimetype = "image/webp";
+        //     }
+        // }
+        const webpOptions = {};
+        if (slots.includes("lightmapTexture") || isLightmap(texture)) {
+            mimeType = "image/webp";
+        }
+        // for https://linear.app/needle/issue/NE-6464
+        else if (size[0] <= 32 || size[1] <= 32) {
+            // can not use UASTC because it's ALSO shitty, see issue above
+            mimeType = "image/webp";
+        }
+        // For product compression we automatically select webP if the texture is used in a PBR material
+        // https://linear.app/needle/issue/NE-6458
+        // TODO: this should probably moved to the top and we should create a compression settings object
+        else if (context.config.usecase === "product") {
+            const shouldUseWebp = slots.includes("normalTexture") || slots.includes("metallicRoughnessTexture");
+            if (shouldUseWebp) {
+                mimeType = "image/webp";
+            }
+        }
+        // If we have a texture cached we can use it
+        // the default compression is toktx so we can safely use that for now
+        if (cached) {
+            texture.setImage(cached).setMimeType(mimeType);
+            return;
+        }
+        await resizeTextureIfNecessary(texture);
+        if (mimeType === "image/webp") {
+            successfullyProcessed = await context.webPContext.process(index, texture, {
+                ...webpOptions,
+                quality: .95,
+                slot: slots.join(","),
+            }, context);
+        }
+        else if (canRunToktx) {
+            // if no specific compression settings are provided we just use the toktx default settings
+            // if we ever change what is used for default compression we need to teach the cache to save the mime type
+            // For VRMs we use ETC1S always
+            let options = null;
+            if (ktx2Format || isVRM) {
+                options = { mode: ktx2Format || "ETC1S" };
+            }
+            successfullyProcessed = await context.toktxContext.process(index, texture, options, context);
+        }
+        else {
+            logger.warn("Toktx not found: Skipping texture compression: " + originalMimeType);
+        }
+    }
+    if (context.useCache && successfullyProcessed && key) {
+        const data = texture.getImage();
+        if (data)
+            addToCache(key, data);
+    }
+    if (successfullyProcessed) {
+        const newUri = texture.getURI();
+        if (originalUri && originalUri != newUri) {
+            const inDirectory = dirname(context.file);
+            const outDirectory = dirname(context.outfile);
+            const originalPath = inDirectory + "/" + originalUri;
+            // if the file output directory is the same as the input directory
+            // (meaning compression runs in the same directory)
+            // we remove the original file (since it effectively gets replaced)
+            // otherwise we dont touch the original file
+            if (inDirectory == outDirectory && existsSync(originalPath)) {
+                logger.info(`Removing old texture \"${originalUri}\" → \"${newUri}\"`);
+                rmSync(originalPath);
+            }
+        }
+    }
+}

package/dist/transforms/needle_toktx.d.ts

@@ -1,26 +1,26 @@
-import { Document, Texture } from "@gltf-transform/core";
-import { KHRTextureBasisu } from "@gltf-transform/extensions";
-import { NEEDLE_compression_texture_schema } from "../extensions/index.js";
-import { INeedleTextureTransformContext, NeedleTransformContext } from "./needle_common.js";
-export interface NeedleToktxOptions {
-    debug?: boolean;
-    jobs?: number;
-    toktxPath?: string;
-}
-export declare class NeedleToktxContext implements INeedleTextureTransformContext {
-    limit: any;
-    basisuExtension: KHRTextureBasisu;
-    version: string;
-    cacheDirectory: string;
-    tmpPathBase: string;
-    document: Document;
-    env: undefined | {
-        PATH: string;
-    };
-    numTextures: number;
-    private _prepareFailed;
-    constructor(options: NeedleToktxOptions);
-    prepare(document: Document): Promise<boolean>;
-    process(index: number, texture: Texture, settings: Pick<NEEDLE_compression_texture_schema, "mode"> | null, context: NeedleTransformContext): Promise<boolean>;
-    finalize(document: Document): Promise<void>;
-}
+import { Document, Texture } from "@gltf-transform/core";
+import { KHRTextureBasisu } from "@gltf-transform/extensions";
+import { NEEDLE_compression_texture_schema } from "../extensions/index.js";
+import { INeedleTextureTransformContext, NeedleTransformContext } from "./needle_common.js";
+export interface NeedleToktxOptions {
+    debug?: boolean;
+    jobs?: number;
+    toktxPath?: string;
+}
+export declare class NeedleToktxContext implements INeedleTextureTransformContext {
+    limit: any;
+    basisuExtension: KHRTextureBasisu;
+    version: string;
+    cacheDirectory: string;
+    tmpPathBase: string;
+    document: Document;
+    env: undefined | {
+        PATH: string;
+    };
+    numTextures: number;
+    private _prepareFailed;
+    constructor(options: NeedleToktxOptions);
+    prepare(document: Document): Promise<boolean>;
+    process(index: number, texture: Texture, settings: Pick<NEEDLE_compression_texture_schema, "mode"> | null, context: NeedleTransformContext): Promise<boolean>;
+    finalize(document: Document): Promise<void>;
+}