@needle-tools/gltf-build-pipeline 1.6.0-alpha.2 → 1.6.0-alpha.3

package/CHANGELOG.md

@@ -4,6 +4,9 @@ All notable changes to this package will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [1.6.0-alpha.3] - 2023-05-28
+- fix: determine build pipeline version
+
 ## [1.6.0-alpha.2] - 2023-05-27
 - add: `gltf-build-pipeline` commandline support with `transform` and `compress` (pack-gltf) options.
 

package/README.md

@@ -0,0 +1,17 @@
+# glTF Build Pipeline
+
+Creating optimized glTF, GLB and VRM files for the web.   
+Includes support for generating LOD quality level for textures and meshes that can be used with [@needle-tools/gltf-progressive](https://www.npmjs.com/package/@needle-tools/gltf-progressive)
+
+## Examples
+
+- [Example on Github](https://github.com/needle-engine/progressive-meshes-sample)
+
+
+
+# Contact ✒️
+<b>[🌵 needle — tools for creators](https://needle.tools)</b> • 
+[Twitter](https://twitter.com/NeedleTools) • 
+[Discord](https://discord.needle.tools) • 
+[Forum](https://forum.needle.tools)
+

package/dist/cache/cache.js

@@ -214,7 +214,7 @@ export function getHash(property, level = 0) {
     }
     return hash;
 }
-const cleanBuildPipelineVersion = buildPipelineVersion.replace(/[^a-zA-Z0-9]/g, "_");
+const cleanBuildPipelineVersion = getVersion(false).replace(/[^a-zA-Z0-9]/g, "_");
 function getCacheDirectory(root) {
     const tmpDir = path.join(tmp.tmpdir, "Needle", "build-pipeline-cache", root ? "" : cleanBuildPipelineVersion);
     if (!fs.existsSync(tmpDir)) {

package/dist/cli/index.js

@@ -1,12 +1,14 @@
 #! /usr/bin/env node
 import { program } from '@donmccurdy/caporal';
 import { packGLTF } from "../scripts/index.js";
-import { printStats } from '../utils/stats.js';
+import { printStats, writeFileStatsToFile } from '../utils/stats.js';
 import { getVersion } from '../utils/version.js';
 import { UsecaseOptions, createConfig, getConfig } from '../config/index.js';
 import { make_progressive } from '../transforms/needle_progressive.js';
 import { cacheSizeLimit, clearCache } from '../cache/cache.js';
+import { existsSync, statSync } from 'fs';
 import { foreachGLTF } from '../utils/fileutils.js';
+import { dirname } from 'path';
 // For testing / dev you can run `npm link` in the package directory
 // For removing the link run `npm rm --global @needle-tools/gltf-build-pipeline`
 program
@@ -22,7 +24,7 @@ Caches are limited to ${cacheSizeLimit} MB disc space by default.
     logger.info("Clearing caches");
     clearCache(logger);
 })
-    .command("compress", "Compress a glTF file")
+    .command("compress", "Compress glTF, GLB or VRM files")
     .help(`
 Compress a single glTF file. The input file is compressed and written to the output file.  
 `)
@@ -30,6 +32,7 @@ Compress a single glTF file. The input file is compressed and written to the out
     .option("--usecase <usecase>", `The usecase for the compression. This will set the compression settings. Possible options: [${UsecaseOptions.join(", ")}] e.g. \`--usecase ${UsecaseOptions[0]}\"`, { validator: program.STRING })
     .option("--config <config>", "The path to a JSON file containing the compression settings. This can also be a needle.config.json file with a \"compression\" object or usecase string.", { validator: program.STRING })
     .option("--output <output>", "The output file. If not provided, the input file will be overwritten.", { validator: program.STRING })
+    .option("--output-stats", "The path to a JSON file where the stats will be written to.", { validator: program.STRING })
     .action(async ({ logger, args, options }) => {
     logger.info("Transforming glTF file: " + args.input);
     const stats = {
@@ -51,20 +54,46 @@ Compress a single glTF file. The input file is compressed and written to the out
         }
     }
     const file = args.input.toString();
+    const outDir = statSync(file).isDirectory() ? file : dirname(file);
+    let outputIsDirectory = false;
+    if (statSync(file).isDirectory() && options.output) {
+        if (statSync(options.output.toString()).isDirectory()) {
+            outputIsDirectory = true;
+            if (!existsSync(options.output.toString())) {
+                logger.error("Output directory does not exist: " + options.output.toString());
+                return;
+            }
+        }
+        else {
+            logger.error("--output must be a directory if input is a directory");
+            return;
+        }
+    }
+    const file_stats = [];
     const set = new Set();
     await foreachGLTF(file, async (file) => {
-        await packGLTF(file, options.output?.toString() || file, {
+        let outputFile = options.output?.toString() || file;
+        if (outputIsDirectory) {
+            const filename = file.replaceAll("\\", "/").split("/").pop();
+            outputFile = options.output?.toString() + "/" + filename;
+        }
+        await packGLTF(file, outputFile, {
             config,
             packedFiles: set,
             stats,
             debug: false,
             useCache: true,
             logger: logger,
+            file_stats: file_stats,
         });
     });
     printStats(stats);
+    console.log("Stats: ", options);
+    if (options.outputStats) {
+        writeFileStatsToFile(file_stats, options.outputStats.toString());
+    }
 })
-    .command("transform", "Transform and optimize glTF, GLB or VRM files")
+    .command("transform", "Transform (progressive loading) and compress glTF, GLB or VRM files")
     .help(`
 This will produce multiple versions of the input file, each with a different level of optimization.   
 Each version will be compressed and written to the output directory.
@@ -72,6 +101,8 @@ Each version will be compressed and written to the output directory.
     .argument('<input>', "The input glTF, GLB or VRM file that should be compressed OR a directory that contains files to be compressed")
     .option("--usecase <usecase>", `The usecase for the compression. This will set the compression settings. Possible options: [${UsecaseOptions.join(", ")}]`, { validator: program.STRING })
     .option("--config <config>", "The path to a JSON file containing the compression settings. This can also be a needle.config.json file with a \"compression\" object or usecase string.", { validator: program.STRING })
+    // .option("--output <output>", "The output directory. If not provided, the input files will be overwritten.", { validator: program.STRING })
+    .option("--output-stats", "The path to a JSON file where the stats will be written to.", { validator: program.STRING })
     .action(async ({ logger, args, options }) => {
     logger.info("Transforming glTF file: " + args.input);
     const stats = {
@@ -93,6 +124,8 @@ Each version will be compressed and written to the output directory.
         }
     }
     const file = args.input.toString();
+    const outDir = statSync(file).isDirectory() ? file : dirname(file);
+    const file_stats = [];
     const set = new Set();
     await foreachGLTF(file, async (file) => {
         logger.info("> Transform " + file);
@@ -111,6 +144,7 @@ Each version will be compressed and written to the output directory.
             debug: false,
             useCache: true,
             logger: logger,
+            file_stats,
         };
         for (const res of progressive_results) {
             logger.info("> Compress " + res);
@@ -120,5 +154,8 @@ Each version will be compressed and written to the output directory.
         await packGLTF(file, file, opts);
     });
     printStats(stats);
+    if (options.outputStats) {
+        writeFileStatsToFile(file_stats, options.outputStats.toString());
+    }
 });
 program.run();

package/dist/config/index.js

@@ -101,25 +101,38 @@ function internalResolveConfig() {
 function resolveConfig(filename, content) {
     // Handle config being part of `needle.config.json`
     if (filename.endsWith("needle.config.json")) {
-        const config = JSON.parse(content);
-        if (config.optimization) {
-            console.log("INFO: Found optimization config at \"" + filename + "\".");
-            // If the optimization config is a string, it's the usecase
-            if (typeof config.optimization === "string") {
-                return createConfig(config.optimization);
+        try {
+            const config = JSON.parse(content);
+            if (config.optimization) {
+                console.log("INFO: Found optimization config at \"" + filename + "\".");
+                // If the optimization config is a string, it's the usecase
+                if (typeof config.optimization === "string") {
+                    return createConfig(config.optimization);
+                }
+                return config.optimization;
+            }
+            else {
+                console.log("INFO: No optimization config found in needle.config.json. Using default config.");
+                return null;
             }
-            return config.optimization;
         }
-        else {
-            console.log("INFO: No optimization config found in needle.config.json. Using default config.");
-            return null;
+        catch (err) {
+            console.error(`ERR: Failed parsing config \"${err.message}\" at ${filename}`);
         }
     }
     // Handle config being the content of the file
     else if (content.trimStart().startsWith("{")) {
-        return JSON.parse(content);
+        try {
+            return JSON.parse(content);
+        }
+        catch (err) {
+            if (err instanceof Error)
+                console.error(`ERR: Failed parsing config \"${err.message}\" at ${filename}`);
+            else
+                console.error(`ERR: Failed parsing config \"${err}\" at ${filename}`);
+        }
     }
     else {
-        throw new Error("Invalid config file format. Expected JSON object.");
+        console.error("ERR: Invalid config file format. Expected JSON object at " + filename);
     }
 }

package/dist/scripts/pack-gltf.d.ts

@@ -1,11 +1,12 @@
 import { ILogger } from '@gltf-transform/core';
-import { Stats } from "../utils/index.js";
+import { FileStats, Stats } from "../utils/index.js";
 import { Config } from '../config/index.js';
 export type PackGLTFOptions = {
     debug: boolean;
     useCache: boolean;
     packedFiles?: Set<string>;
     stats?: Stats;
+    file_stats: FileStats[] | null;
     config?: Config;
     logger?: ILogger;
 };

package/dist/scripts/pack-gltf.js

@@ -7,7 +7,7 @@ import { MeshoptDecoder, MeshoptEncoder } from 'meshoptimizer';
 import { resample } from '@gltf-transform/functions';
 import { ALL_EXTENSIONS as NEEDLE_EXTENSIONS, NEEDLE_compression_texture, NEEDLE_mesh_compression, } from '../extensions/index.js';
 import { isMeshLOD, needle_animation_transform, needle_asset, needle_mesh_transform, needle_texture_transform, } from '../transforms/index.js';
-import { writeNodeIO } from "../utils/index.js";
+import { createNewFilestats, writeNodeIO } from "../utils/index.js";
 import { addToCache, getHash, tryGetFromCache, } from "../cache/index.js";
 import { getConfig } from '../config/index.js';
 /** Compress a glTF file
@@ -66,7 +66,7 @@ export async function packGLTF(inputFile, outputFile, options) {
         const document = await io.read(inputFile);
         /** @ts-ignore */
         document.getLogger().verbosity = Verbosity.ERROR;
-        let totalGPUTextureMemmory = 0;
+        let totalGPUTextureMemory = 0;
         /** @type {Array<import('@gltf-transform/core').Transform>} */
         const transforms = [
             needle_asset(),
@@ -76,8 +76,7 @@ export async function packGLTF(inputFile, outputFile, options) {
                 noToktx: false,
                 debug: debug,
                 reportGPUMemory: val => {
-                    if (stats)
-                        stats.totalGPUMemoryInMB += val;
+                    totalGPUTextureMemory += val;
                 },
                 config,
             }),
@@ -102,7 +101,7 @@ export async function packGLTF(inputFile, outputFile, options) {
         const duration = Date.now() - startTime;
         const sizeInMB = statSync(outputFile).size / 1024 / 1024;
         const filename = path.basename(outputFile);
-        const gpuMemoryInMB = totalGPUTextureMemmory / 1024 / 1024;
+        const gpuMemoryInMB = totalGPUTextureMemory / 1024 / 1024;
         logger.info(`← Compressing done in ${(duration / 1000).toFixed(1)} sec, ${prevSizeInMB.toFixed(1)} MB → ${sizeInMB.toFixed(1)} MB (GPU: ${gpuMemoryInMB.toFixed(1)} MB textures): ${filename} at ${outputFile}`);
         if (stats) {
             stats.totalFilesProcessed++;
@@ -110,6 +109,10 @@ export async function packGLTF(inputFile, outputFile, options) {
             stats.totalFileSizeInMBBefore += prevSizeInMB;
             stats.totalGPUMemoryInMB += gpuMemoryInMB;
         }
+        const filestats = createNewFilestats(outputFile, document, {
+            gpu_memory: totalGPUTextureMemory,
+        });
+        options.file_stats?.push(filestats);
         return true;
     }
     catch (err) {

package/dist/utils/fileutils.d.ts

@@ -1,3 +1,4 @@
+export declare function getFileName(filepath: string): string;
 export declare type GLTFFileCallback = (file: string) => void | Promise<void>;
 export declare function foreachGLTF(args: string | string[], callback: GLTFFileCallback): Promise<void>;
 export declare function replaceExtension(filepath: string, newExtension: string): string;

package/dist/utils/fileutils.js

@@ -1,23 +1,27 @@
 import { existsSync, statSync, readdirSync, readFileSync } from 'fs';
 import { isAbsolute } from 'path';
+export function getFileName(filepath) {
+    const parts = filepath.replaceAll("\\", "/").split("/");
+    return parts[parts.length - 1];
+}
 export async function foreachGLTF(args, callback) {
     // get original directory this run was invoked from
     const initDir = process.env.INIT_CWD;
     const context = {
         baseDirectory: initDir,
-        filesFound: 0
+        files: [],
     };
     // traverse recursively through all files and directories
     if (Array.isArray(args)) {
         for (const arg of args) {
-            await testEntry(arg, callback, context);
+            await testEntry(arg, context);
         }
     }
     else if (typeof args === "string") {
-        await testEntry(args, callback, context);
+        await testEntry(args, context);
     }
     // try to run with needle config
-    if (context.filesFound <= 0 && context.baseDirectory) {
+    if (context.files?.length <= 0 && context.baseDirectory) {
         const needleConfigPath = context.baseDirectory + "/needle.config.json";
         if (existsSync(needleConfigPath)) {
             const config = JSON.parse(readFileSync(needleConfigPath, "utf8"));
@@ -26,13 +30,13 @@ export async function foreachGLTF(args, callback) {
                 // If it's an absolute path
                 if (existsSync(buildDirectory)) {
                     console.log("INFO: Using build directory from needle.config.json: " + buildDirectory);
-                    await testEntry(buildDirectory, callback, context);
+                    await testEntry(buildDirectory, context);
                 }
                 else { // if its a relative path
                     const newPath = context.baseDirectory + "/" + buildDirectory;
                     if (existsSync(newPath)) {
                         console.log("INFO: Using build directory from needle.config.json: " + newPath);
-                        await testEntry(newPath, callback, context);
+                        await testEntry(newPath, context);
                     }
                 }
             }
@@ -43,11 +47,16 @@ export async function foreachGLTF(args, callback) {
             // await testEntry(context.baseDirectory + "/dist", callback, context);
         }
     }
-    if (context.filesFound <= 0) {
+    if (context.files.length <= 0) {
         console.log("WARN: No glTF files found. Make sure to invoke with a valid path to a glTF file or directory. This path may be relative to " + initDir);
     }
+    else {
+        for (const file of context.files) {
+            await callback(file);
+        }
+    }
 }
-async function testEntry(entry, callback, context) {
+async function testEntry(entry, context) {
     // ignore argument names (e.g. --debug)
     if (entry.startsWith("--")) {
         return false;
@@ -90,11 +99,7 @@ async function testEntry(entry, callback, context) {
         }
     }
     if (entry.endsWith(".glb") || entry.endsWith(".gltf") || entry.endsWith(".vrm")) {
-        context.filesFound += 1;
-        const res = callback(entry);
-        if (res instanceof Promise) {
-            await res;
-        }
+        context.files.push(entry);
     }
     else {
         const stat = statSync(entry);
@@ -105,7 +110,7 @@ async function testEntry(entry, callback, context) {
             }
             const files = readdirSync(entry);
             for (const file of files) {
-                await testEntry(entry + "/" + file, callback, context);
+                await testEntry(entry + "/" + file, context);
             }
         }
         else {

package/dist/utils/stats.d.ts

@@ -1,4 +1,5 @@
 import { Logger } from "@donmccurdy/caporal";
+import { Document } from "@gltf-transform/core";
 export type Stats = {
     startTime: number;
     totalGPUMemoryInMB: number;
@@ -7,3 +8,16 @@ export type Stats = {
     totalFileSizeInMBBefore: number;
 };
 export declare function printStats(stats: Stats, logger?: Logger): void;
+export type FileStats = {
+    filename: string;
+    /** Size in bytes */
+    size: number;
+    gpu_memory: number;
+    vertex_count: number;
+    nodes: number;
+    materials: number;
+};
+export declare function createNewFilestats(file: string, document: Document, stats: {
+    gpu_memory: number;
+}): FileStats;
+export declare function writeFileStatsToFile(filestats: FileStats[], output_path: string): void;

package/dist/utils/stats.js

@@ -1,4 +1,6 @@
 import { getVersion } from "./version.js";
+import { getFileName } from "./fileutils.js";
+import { statSync, writeFileSync } from "fs";
 export function printStats(stats, logger) {
     const duration = Date.now() - stats.startTime;
     const durationInMinutes = duration / 1000 / 60;
@@ -8,3 +10,30 @@ export function printStats(stats, logger) {
     else
         console.log(msg);
 }
+export function createNewFilestats(file, document, stats) {
+    const filename = getFileName(file);
+    const root = document.getRoot();
+    const filestats = {
+        filename,
+        size: statSync(file).size,
+        gpu_memory: stats.gpu_memory,
+        nodes: root.listNodes().length,
+        materials: root.listMaterials().length,
+        vertex_count: 0,
+    };
+    // Count vertices
+    for (const mesh of root.listMeshes()) {
+        filestats.vertex_count += mesh.listPrimitives().reduce((acc, p) => acc + p.getAttribute('POSITION').getCount(), 0);
+    }
+    filestats.vertex_count = Math.round(filestats.vertex_count / 3);
+    return filestats;
+}
+export function writeFileStatsToFile(filestats, output_path) {
+    if (output_path == undefined) {
+        console.error("No stats output path provided");
+        return;
+    }
+    console.log("Writing file stats to: " + output_path);
+    const json = JSON.stringify(filestats, null, 2);
+    writeFileSync(output_path, json);
+}

package/dist/utils/version.js

@@ -3,12 +3,14 @@ let version = undefined;
 let isLocal = false;
 export function getVersion(marklocal = true) {
     if (version === undefined) {
+        console.log(process.argv);
         const scriptLocation = process.argv[1].replaceAll("\\", "/");
+        console.log("Determining version from " + scriptLocation);
         const path = scriptLocation.split('/');
         let packageJsonPath = null;
         while (path.length) {
             const dirname = path.join('/');
-            if (statSync(dirname || '/').isDirectory()) {
+            if (statSync(dirname).isDirectory()) {
                 const filename = dirname + '/package.json';
                 if (existsSync(filename)) {
                     packageJsonPath = filename;
@@ -27,6 +29,9 @@ export function getVersion(marklocal = true) {
             else
                 console.log("Could not find package.json at " + packageJsonPath);
         }
+        if (!version) {
+            throw new Error("Could not determine version");
+        }
     }
     let res = version;
     if (marklocal && isLocal)

package/package.json

@@ -1,9 +1,10 @@
 {
   "name": "@needle-tools/gltf-build-pipeline",
-  "version": "1.6.0-alpha.2",
+  "version": "1.6.0-alpha.3",
   "description": "Pipeline and tools for optimizing gltf files using gltf-transform and compression settings within glTF extensions",
   "main": "index.mjs",
   "type": "module",
+  "readme": "README.md",
   "bin": {
     "gltf-build-pipeline": "./dist/cli/index.js"
   },

package/scripts/pack-gltf.mjs

@@ -79,6 +79,7 @@ async function packFile(file) {
         packedFiles,
         useCache: allowCaching,
         stats,
+        file_stats: null
     })
 }