@needle-tools/gltf-build-pipeline 2.5.0 → 2.6.0-alpha.1

package/CHANGELOG.md

@@ -4,32 +4,39 @@ All notable changes to this package will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
-## [2.4.0] - 2023-09-11
+## [2.6.0-alpha] - 2024-12-16
+- add: create placeholder textures for missing jpg, png, webp, gif, tif on disc
+- fix: compressing a gltf file to a different output directory does not falsely delete old external textures on disc in the input directory
+
+## [2.5.0] - 2024-12-16
+- release
+
+## [2.4.0] - 2024-09-11
 - release
 
-## [2.4.0-beta.5] - 2023-09-06
+## [2.4.0-beta.5] - 2024-09-06
 - fix: issue where mesh LODs would in some cases be compressed with "draco" while the root file would be compressed with "mesh-opt" if not explicitly specified in the root file. This would cause invalid rendering and randomly scaled LOD meshes. Now mesh LODs will always use the same mesh compression. [NE 5297](https://linear.app/needle/issue/NE-5297)
 - change: update gltf-transform dependency to 4.0.8
 
-## [2.4.0-beta] - 2023-08-13
+## [2.4.0-beta] - 2024-08-13
 - fix: issue where a mesh lod is removed during second processing of a directory that contains existing mesh lods
 
-## [2.4.0-alpha.3] - 2023-08-11
+## [2.4.0-alpha.3] - 2024-08-11
 - fix: issue where files would not correctly be overriden in subdirectories when no output directory was specified 
 
-## [2.4.0-alpha.2] - 2023-06-29
+## [2.4.0-alpha.2] - 2024-06-29
 - fix: prevent joining/flattening of progressive assets
 
-## [2.4.0-alpha.1] - 2023-06-26
+## [2.4.0-alpha.1] - 2024-06-26
 - add: heuristic for choosing mesh compression if not explictly configured
 
-## [2.3.0-alpha.2] - 2023-06-26
+## [2.3.0-alpha.2] - 2024-06-26
 - change: usecase `product` does compress with UASTC again (for certain slots)
 
-## [2.3.0-alpha.1] - 2023-06-23
+## [2.3.0-alpha.1] - 2024-06-23
 - change: bump gltf-transform to 4.0.2 to fix meshopt skinning regression [1439](https://github.com/donmccurdy/glTF-Transform/pull/1439)
 
-## [2.3.0-alpha] - 2023-06-22
+## [2.3.0-alpha] - 2024-06-22
 - add: flatten and join for `product` usecase
 
 ## [2.2.1-alpha.8] - 2023-06-22

package/dist/scripts/pack-gltf.js

@@ -7,7 +7,7 @@ import { MeshoptDecoder, MeshoptEncoder } from 'meshoptimizer';
 import { dedup, flatten, join, prune, resample } from '@gltf-transform/functions';
 import { ALL_EXTENSIONS as NEEDLE_EXTENSIONS, NEEDLE_compression_texture, NEEDLE_mesh_compression, } from '../extensions/index.js';
 import { isLOD, isMeshLOD, needle_animation_transform, needle_asset, needle_mesh_transform, needle_texture_transform, } from '../transforms/index.js';
-import { getOutputPath, writeNodeIO } from "../utils/index.js";
+import { getOutputPath, ioTryReadWithMissingResources, writeNodeIO } from "../utils/index.js";
 import { addToCache, getHash, tryGetFromCache, } from "../cache/index.js";
 /** Compress a glTF file
  */
@@ -42,6 +42,11 @@ export async function packGLTF(inputFile, outputFile, options) {
     }
     try {
         const config = options.config;
+        if (!outputFile)
+            outputFile = inputFile;
+        else {
+            outputFile = getOutputPath(inputFile, outputFile);
+        }
         const prevSizeInMB = statSync(inputFile).size / 1024 / 1024;
         logger.info(`→ Compressing ${prevSizeInMB.toFixed(1)} MB: ${inputFile}`);
         const startTime = Date.now();
@@ -58,7 +63,7 @@ export async function packGLTF(inputFile, outputFile, options) {
             'meshopt.decoder': MeshoptDecoder,
             'meshopt.encoder': MeshoptEncoder,
         });
-        const document = await io.read(inputFile);
+        const document = await ioTryReadWithMissingResources(io, inputFile); // await io.read(inputFile);
         /** @ts-ignore */
         document.getLogger().verbosity = options.verbose ? Verbosity.DEBUG : options.debug ? Verbosity.INFO : Verbosity.WARN;
         let totalGPUTextureMemory = 0;
@@ -67,6 +72,7 @@ export async function packGLTF(inputFile, outputFile, options) {
             needle_asset(),
             needle_texture_transform({
                 file: inputFile,
+                outfile: outputFile,
                 useCache,
                 noToktx: false,
                 debug: debug,
@@ -96,12 +102,7 @@ export async function packGLTF(inputFile, outputFile, options) {
             transforms.push(join({ keepNamed: false }));
         }
         await document.transform(...transforms);
-        if (!outputFile)
-            outputFile = inputFile;
-        else {
-            outputFile = getOutputPath(inputFile, outputFile);
-            logger.info(`⇄ Writing to ${outputFile}`);
-        }
+        logger.info(`⇄ Writing to ${outputFile}`);
         await writeNodeIO(io, outputFile, document);
         if (useCache && cacheKey != undefined && existsSync(outputFile))
             addToCache(cacheKey, readFileSync(outputFile));
@@ -119,7 +120,7 @@ export async function packGLTF(inputFile, outputFile, options) {
         return true;
     }
     catch (err) {
-        logger.error(`packing ${path.basename(inputFile)} failed: ${err}`);
+        logger.error(`Packing \"${path.basename(inputFile)}\" failed. Error: ${err.message}`);
         if (debug)
             throw err;
         return false;

package/dist/transforms/needle_common.d.ts

@@ -4,6 +4,7 @@ import { Config } from '../config/index.js';
 export interface NeedleTransformContext {
     /** full path to the file being processed */
     file: string;
+    outfile: string;
     config: Config;
     debug?: boolean;
     useCache: boolean;

package/dist/transforms/needle_progressive.js

@@ -7,7 +7,7 @@ import path, { basename, resolve } from 'path';
 import { existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'fs';
 import { addMetaToCache, addToCache, getHash, getKey, tryGetFromCache, tryGetMetaFromCache } from '../cache/index.js';
 import sharp from "sharp";
-import { detectTextureCompressionMode, ensureIsDirectory, foreachGLTF, generateGuid, getOutputPath, isLightmap, setLightmapName, validateImageMimeType, writeNodeIO } from '../utils/index.js';
+import { detectTextureCompressionMode, ensureIsDirectory, foreachGLTF, generateGuid, getOutputPath, isLightmap, setLightmapName, ioTryReadWithMissingResources, validateImageMimeType, writeNodeIO } from '../utils/index.js';
 import { needle_asset, writeNeedleAsset } from './needle_asset.js';
 import { MeshoptDecoder, MeshoptSimplifier } from 'meshoptimizer';
 import { compressPrimitive, determineMeshCompression } from './needle_mesh_transform.js';
@@ -61,7 +61,7 @@ async function onProcess(file, opts) {
             'draco3d.decoder': await draco3d.createDecoderModule(),
             'draco3d.encoder': await draco3d.createEncoderModule(),
         });
-        let document = await io.read(file);
+        let document = await ioTryReadWithMissingResources(io, file); // await io.read(file);
         if (document) {
             const logger = document.getLogger();
             //@ts-ignore

package/dist/transforms/needle_texture_transform.js

@@ -195,10 +195,15 @@ async function processTexture(texture, index, document, logger, context) {
     if (successFullyProcessed) {
         const newUri = texture.getURI();
         if (originalUri && originalUri != newUri) {
-            const fileDirectory = dirname(context.file);
-            const originalPath = fileDirectory + "/" + originalUri;
-            if (existsSync(originalPath)) {
-                console.log(`INFO: Texture moved \"${originalUri}\" → \"${newUri}\"`);
+            const inDirectory = dirname(context.file);
+            const outDirectory = dirname(context.outfile);
+            const originalPath = inDirectory + "/" + originalUri;
+            // if the file output directory is the same as the input directory
+            // (meaning compression runs in the same directory)
+            // we remove the original file (since it effectively gets replaced)
+            // otherwise we dont touch the original file
+            if (inDirectory == outDirectory && existsSync(originalPath)) {
+                console.log(`INFO: Remove old texture \"${originalUri}\" → \"${newUri}\"`);
                 rmSync(originalPath);
             }
         }

package/dist/utils/nodeio.d.ts

@@ -1,5 +1,6 @@
-import { NodeIO, Document } from "@gltf-transform/core";
+import { NodeIO, Document, PlatformIO } from "@gltf-transform/core";
 /**
  * Handle writing glTf, GLB and VRM files
  */
 export declare function writeNodeIO(nodeio: NodeIO, filename: string, document: Document): Promise<string>;
+export declare function ioTryReadWithMissingResources(io: PlatformIO, filepath: string, iteration?: number): Promise<Document>;

package/dist/utils/nodeio.js

@@ -1,4 +1,5 @@
 import { writeFile } from "fs/promises";
+import sharp from "sharp";
 /**
  * Handle writing glTf, GLB and VRM files
  */
@@ -12,3 +13,68 @@ export async function writeNodeIO(nodeio, filename, document) {
         await nodeio.write(filename, document);
     return filename;
 }
+const openFileRegex = /open \'(?<path>.+?)'/;
+const createdFiles = new Set();
+export async function ioTryReadWithMissingResources(io, filepath, iteration = 0) {
+    const document = await io.read(filepath)
+        .catch(err => { return err; });
+    if (document instanceof Error) {
+        const err = document;
+        // Handle glTF files where resource are missing
+        if (iteration < 100 && err.message?.includes("ENOENT")) {
+            try {
+                const match = err.message.match(openFileRegex);
+                if (match.groups?.path) {
+                    let missingPath = match.groups.path;
+                    // check if it's an URL
+                    if (missingPath.startsWith("http")) {
+                        throw err;
+                    }
+                    if (createdFiles.has(missingPath)) {
+                        throw err;
+                    }
+                    createdFiles.add(missingPath);
+                    console.warn(`Creating empty file for missing resource: ${missingPath}`);
+                    const ext = missingPath.split('.').pop();
+                    if (!ext) {
+                        console.warn(`Could not determine extension for ${missingPath}`);
+                        throw err;
+                    }
+                    const extLower = ext?.toLowerCase();
+                    let retry = false;
+                    // https://sharp.pixelplumbing.com/api-output#tofile
+                    switch (extLower) {
+                        case "jpeg":
+                        case "jpg":
+                        case "png":
+                        case "webp":
+                        case "tiff":
+                        case "gif":
+                            {
+                                await sharp({
+                                    create: {
+                                        width: 64,
+                                        height: 64,
+                                        channels: 4,
+                                        background: { r: 255, g: 255, b: 255, alpha: 1 }
+                                    }
+                                })
+                                    .toFormat(ext)
+                                    .toFile(missingPath);
+                                retry = true;
+                            }
+                    }
+                    if (retry) {
+                        console.warn(`Created empty file for missing resource: ${missingPath}`);
+                        return ioTryReadWithMissingResources(io, filepath, iteration + 1);
+                    }
+                }
+            }
+            catch (internal_error) {
+                console.warn(internal_error);
+            }
+        }
+        throw err;
+    }
+    return document;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@needle-tools/gltf-build-pipeline",
-  "version": "2.5.0",
+  "version": "2.6.0-alpha.1",
   "description": "Pipeline and tools for optimizing gltf files using gltf-transform and compression settings within glTF extensions",
   "main": "dist/index.js",
   "type": "module",