@needle-tools/gltf-build-pipeline 1.6.0-alpha.2 → 1.6.0-alpha.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/CHANGELOG.md +3 -0
  2. package/README.md +17 -0
  3. package/dist/package.json +66 -0
  4. package/dist/{cache → src/cache}/cache.js +1 -1
  5. package/dist/{cli → src/cli}/index.js +41 -4
  6. package/dist/{config → src/config}/index.js +25 -12
  7. package/dist/{scripts → src/scripts}/pack-gltf.d.ts +2 -1
  8. package/dist/{scripts → src/scripts}/pack-gltf.js +8 -5
  9. package/dist/{utils → src/utils}/fileutils.d.ts +1 -0
  10. package/dist/{utils → src/utils}/fileutils.js +19 -14
  11. package/dist/src/utils/stats.d.ts +23 -0
  12. package/dist/src/utils/stats.js +39 -0
  13. package/dist/src/utils/version.d.ts +1 -0
  14. package/dist/src/utils/version.js +5 -0
  15. package/package.json +4 -3
  16. package/scripts/pack-gltf.mjs +1 -0
  17. package/tsconfig.json +2 -1
  18. package/dist/utils/stats.d.ts +0 -9
  19. package/dist/utils/stats.js +0 -10
  20. package/dist/utils/version.d.ts +0 -1
  21. package/dist/utils/version.js +0 -35
  22. /package/dist/{cache → src/cache}/cache.d.ts +0 -0
  23. /package/dist/{cache → src/cache}/index.d.ts +0 -0
  24. /package/dist/{cache → src/cache}/index.js +0 -0
  25. /package/dist/{cli → src/cli}/index.d.ts +0 -0
  26. /package/dist/{config → src/config}/index.d.ts +0 -0
  27. /package/dist/{extensions → src/extensions}/NEEDLE_compression_texture/NEEDLE_compression_texture.d.ts +0 -0
  28. /package/dist/{extensions → src/extensions}/NEEDLE_compression_texture/NEEDLE_compression_texture.js +0 -0
  29. /package/dist/{extensions → src/extensions}/NEEDLE_compression_texture/index.d.ts +0 -0
  30. /package/dist/{extensions → src/extensions}/NEEDLE_compression_texture/index.js +0 -0
  31. /package/dist/{extensions → src/extensions}/NEEDLE_mesh_compression/NEEDLE_mesh_compression.d.ts +0 -0
  32. /package/dist/{extensions → src/extensions}/NEEDLE_mesh_compression/NEEDLE_mesh_compression.js +0 -0
  33. /package/dist/{extensions → src/extensions}/NEEDLE_mesh_compression/index.d.ts +0 -0
  34. /package/dist/{extensions → src/extensions}/NEEDLE_mesh_compression/index.js +0 -0
  35. /package/dist/{extensions → src/extensions}/NEEDLE_opaque/NEEDLE_opaque.d.ts +0 -0
  36. /package/dist/{extensions → src/extensions}/NEEDLE_opaque/NEEDLE_opaque.js +0 -0
  37. /package/dist/{extensions → src/extensions}/NEEDLE_opaque/index.d.ts +0 -0
  38. /package/dist/{extensions → src/extensions}/NEEDLE_opaque/index.js +0 -0
  39. /package/dist/{extensions → src/extensions}/NEEDLE_progressive/NEEDLE_progressive.d.ts +0 -0
  40. /package/dist/{extensions → src/extensions}/NEEDLE_progressive/NEEDLE_progressive.js +0 -0
  41. /package/dist/{extensions → src/extensions}/NEEDLE_progressive/index.d.ts +0 -0
  42. /package/dist/{extensions → src/extensions}/NEEDLE_progressive/index.js +0 -0
  43. /package/dist/{extensions → src/extensions}/NEEDLE_progressive_mesh_settings/index.d.ts +0 -0
  44. /package/dist/{extensions → src/extensions}/NEEDLE_progressive_mesh_settings/index.js +0 -0
  45. /package/dist/{extensions → src/extensions}/NEEDLE_progressive_texture_settings/index.d.ts +0 -0
  46. /package/dist/{extensions → src/extensions}/NEEDLE_progressive_texture_settings/index.js +0 -0
  47. /package/dist/{extensions → src/extensions}/index.d.ts +0 -0
  48. /package/dist/{extensions → src/extensions}/index.js +0 -0
  49. /package/dist/{extensions → src/extensions}/utils.d.ts +0 -0
  50. /package/dist/{extensions → src/extensions}/utils.js +0 -0
  51. /package/dist/{index.d.ts → src/index.d.ts} +0 -0
  52. /package/dist/{index.js → src/index.js} +0 -0
  53. /package/dist/{scripts → src/scripts}/index.d.ts +0 -0
  54. /package/dist/{scripts → src/scripts}/index.js +0 -0
  55. /package/dist/{transforms → src/transforms}/index.d.ts +0 -0
  56. /package/dist/{transforms → src/transforms}/index.js +0 -0
  57. /package/dist/{transforms → src/transforms}/needle_animation_transform.d.ts +0 -0
  58. /package/dist/{transforms → src/transforms}/needle_animation_transform.js +0 -0
  59. /package/dist/{transforms → src/transforms}/needle_asset.d.ts +0 -0
  60. /package/dist/{transforms → src/transforms}/needle_asset.js +0 -0
  61. /package/dist/{transforms → src/transforms}/needle_common.d.ts +0 -0
  62. /package/dist/{transforms → src/transforms}/needle_common.js +0 -0
  63. /package/dist/{transforms → src/transforms}/needle_mesh_transform.d.ts +0 -0
  64. /package/dist/{transforms → src/transforms}/needle_mesh_transform.js +0 -0
  65. /package/dist/{transforms → src/transforms}/needle_progressive.d.ts +0 -0
  66. /package/dist/{transforms → src/transforms}/needle_progressive.js +0 -0
  67. /package/dist/{transforms → src/transforms}/needle_texture_transform.d.ts +0 -0
  68. /package/dist/{transforms → src/transforms}/needle_texture_transform.js +0 -0
  69. /package/dist/{transforms → src/transforms}/needle_toktx.d.ts +0 -0
  70. /package/dist/{transforms → src/transforms}/needle_toktx.js +0 -0
  71. /package/dist/{transforms → src/transforms}/needle_webp.d.ts +0 -0
  72. /package/dist/{transforms → src/transforms}/needle_webp.js +0 -0
  73. /package/dist/{transforms → src/transforms}/toktx.d.ts +0 -0
  74. /package/dist/{transforms → src/transforms}/toktx.js +0 -0
  75. /package/dist/{transforms → src/transforms}/util.d.ts +0 -0
  76. /package/dist/{transforms → src/transforms}/util.js +0 -0
  77. /package/dist/{utils → src/utils}/compression_utils.d.ts +0 -0
  78. /package/dist/{utils → src/utils}/compression_utils.js +0 -0
  79. /package/dist/{utils → src/utils}/guid.d.ts +0 -0
  80. /package/dist/{utils → src/utils}/guid.js +0 -0
  81. /package/dist/{utils → src/utils}/index.d.ts +0 -0
  82. /package/dist/{utils → src/utils}/index.js +0 -0
  83. /package/dist/{utils → src/utils}/mesh.d.ts +0 -0
  84. /package/dist/{utils → src/utils}/mesh.js +0 -0
  85. /package/dist/{utils → src/utils}/nodeio.d.ts +0 -0
  86. /package/dist/{utils → src/utils}/nodeio.js +0 -0
package/CHANGELOG.md CHANGED
@@ -4,6 +4,9 @@ All notable changes to this package will be documented in this file.
4
4
  The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
5
5
  and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
6
6
 
7
+ ## [1.6.0-alpha.4] - 2023-05-28
8
+ - fix: determine build pipeline version
9
+
7
10
  ## [1.6.0-alpha.2] - 2023-05-27
8
11
  - add: `gltf-build-pipeline` commandline support with `transform` and `compress` (pack-gltf) options.
9
12
 
package/README.md ADDED
@@ -0,0 +1,17 @@
1
+ # glTF Build Pipeline
2
+
3
+ Creating optimized glTF, GLB and VRM files for the web.
4
+ Includes support for generating LOD quality level for textures and meshes that can be used with [@needle-tools/gltf-progressive](https://www.npmjs.com/package/@needle-tools/gltf-progressive)
5
+
6
+ ## Examples
7
+
8
+ - [Example on Github](https://github.com/needle-engine/progressive-meshes-sample)
9
+
10
+
11
+
12
+ # Contact ✒️
13
+ <b>[🌵 needle — tools for creators](https://needle.tools)</b> •
14
+ [Twitter](https://twitter.com/NeedleTools) •
15
+ [Discord](https://discord.needle.tools) •
16
+ [Forum](https://forum.needle.tools)
17
+
@@ -0,0 +1,66 @@
1
+ {
2
+ "name": "@needle-tools/gltf-build-pipeline",
3
+ "version": "1.6.0-alpha.4",
4
+ "description": "Pipeline and tools for optimizing gltf files using gltf-transform and compression settings within glTF extensions",
5
+ "main": "index.mjs",
6
+ "type": "module",
7
+ "readme": "README.md",
8
+ "bin": {
9
+ "gltf-build-pipeline": "./dist/src/cli/index.js"
10
+ },
11
+ "scripts": {
12
+ "start": "node scripts/index.mjs",
13
+ "dev": "npm-watch compile",
14
+ "transform": "node scripts/index.mjs",
15
+ "transform:make-progressive": "node scripts/make-progressive.mjs",
16
+ "transform:pack-gltf": "node --no-experimental-fetch scripts/pack-gltf.mjs",
17
+ "pack-gltf": "node --no-experimental-fetch scripts/pack-gltf.mjs",
18
+ "dev:pack-gltf": "npm run compile && node scripts/pack-gltf.mjs",
19
+ "clear-caches": "node scripts/clear-caches.mjs",
20
+ "limit-caches": "node scripts/limit-caches.mjs",
21
+ "compile": "tsc || exit 1"
22
+ },
23
+ "dependencies": {
24
+ "@donmccurdy/caporal": "~0.0.10",
25
+ "@gltf-transform/core": "^3.10.1",
26
+ "@gltf-transform/extensions": "^3.10.1",
27
+ "@gltf-transform/functions": "^3.10.1",
28
+ "command-exists": "^1.2.9",
29
+ "command-line-args": "^5.2.1",
30
+ "draco3dgltf": "^1.5.3",
31
+ "glob": "^7.2.0",
32
+ "meshoptimizer": "^0.18.1",
33
+ "micromatch": "^4.0.5",
34
+ "ndarray": "^1.0.19",
35
+ "ndarray-lanczos": "^0.3.0",
36
+ "ndarray-pixels": "^2.0.1",
37
+ "p-limit": "3.1.0",
38
+ "semver": "^7.3.7",
39
+ "sharp": "^0.32.6",
40
+ "tmp": "^0.2.1",
41
+ "uuid": "^9.0.1"
42
+ },
43
+ "watch": {
44
+ "compile": {
45
+ "patterns": [
46
+ "src"
47
+ ],
48
+ "extensions": "ts",
49
+ "quiet": false
50
+ }
51
+ },
52
+ "devDependencies": {
53
+ "@types/node": "^18.7.23",
54
+ "npm-watch": "^0.11.0",
55
+ "typescript": "^4.9.5"
56
+ },
57
+ "author": {
58
+ "name": "Needle",
59
+ "email": "help@needle.tools",
60
+ "url": "https://needle.tools/"
61
+ },
62
+ "repository": {
63
+ "type": "git",
64
+ "url": "git+https://github.com/needle-tools/needle-gltf-extensions.git"
65
+ }
66
+ }
@@ -214,7 +214,7 @@ export function getHash(property, level = 0) {
214
214
  }
215
215
  return hash;
216
216
  }
217
- const cleanBuildPipelineVersion = buildPipelineVersion.replace(/[^a-zA-Z0-9]/g, "_");
217
+ const cleanBuildPipelineVersion = getVersion(false).replace(/[^a-zA-Z0-9]/g, "_");
218
218
  function getCacheDirectory(root) {
219
219
  const tmpDir = path.join(tmp.tmpdir, "Needle", "build-pipeline-cache", root ? "" : cleanBuildPipelineVersion);
220
220
  if (!fs.existsSync(tmpDir)) {
@@ -1,12 +1,14 @@
1
1
  #! /usr/bin/env node
2
2
  import { program } from '@donmccurdy/caporal';
3
3
  import { packGLTF } from "../scripts/index.js";
4
- import { printStats } from '../utils/stats.js';
4
+ import { printStats, writeFileStatsToFile } from '../utils/stats.js';
5
5
  import { getVersion } from '../utils/version.js';
6
6
  import { UsecaseOptions, createConfig, getConfig } from '../config/index.js';
7
7
  import { make_progressive } from '../transforms/needle_progressive.js';
8
8
  import { cacheSizeLimit, clearCache } from '../cache/cache.js';
9
+ import { existsSync, statSync } from 'fs';
9
10
  import { foreachGLTF } from '../utils/fileutils.js';
11
+ import { dirname } from 'path';
10
12
  // For testing / dev you can run `npm link` in the package directory
11
13
  // For removing the link run `npm rm --global @needle-tools/gltf-build-pipeline`
12
14
  program
@@ -22,7 +24,7 @@ Caches are limited to ${cacheSizeLimit} MB disc space by default.
22
24
  logger.info("Clearing caches");
23
25
  clearCache(logger);
24
26
  })
25
- .command("compress", "Compress a glTF file")
27
+ .command("compress", "Compress glTF, GLB or VRM files")
26
28
  .help(`
27
29
  Compress a single glTF file. The input file is compressed and written to the output file.
28
30
  `)
@@ -30,6 +32,7 @@ Compress a single glTF file. The input file is compressed and written to the out
30
32
  .option("--usecase <usecase>", `The usecase for the compression. This will set the compression settings. Possible options: [${UsecaseOptions.join(", ")}] e.g. \`--usecase ${UsecaseOptions[0]}\"`, { validator: program.STRING })
31
33
  .option("--config <config>", "The path to a JSON file containing the compression settings. This can also be a needle.config.json file with a \"compression\" object or usecase string.", { validator: program.STRING })
32
34
  .option("--output <output>", "The output file. If not provided, the input file will be overwritten.", { validator: program.STRING })
35
+ .option("--output-stats", "The path to a JSON file where the stats will be written to.", { validator: program.STRING })
33
36
  .action(async ({ logger, args, options }) => {
34
37
  logger.info("Transforming glTF file: " + args.input);
35
38
  const stats = {
@@ -51,20 +54,46 @@ Compress a single glTF file. The input file is compressed and written to the out
51
54
  }
52
55
  }
53
56
  const file = args.input.toString();
57
+ const outDir = statSync(file).isDirectory() ? file : dirname(file);
58
+ let outputIsDirectory = false;
59
+ if (statSync(file).isDirectory() && options.output) {
60
+ if (statSync(options.output.toString()).isDirectory()) {
61
+ outputIsDirectory = true;
62
+ if (!existsSync(options.output.toString())) {
63
+ logger.error("Output directory does not exist: " + options.output.toString());
64
+ return;
65
+ }
66
+ }
67
+ else {
68
+ logger.error("--output must be a directory if input is a directory");
69
+ return;
70
+ }
71
+ }
72
+ const file_stats = [];
54
73
  const set = new Set();
55
74
  await foreachGLTF(file, async (file) => {
56
- await packGLTF(file, options.output?.toString() || file, {
75
+ let outputFile = options.output?.toString() || file;
76
+ if (outputIsDirectory) {
77
+ const filename = file.replaceAll("\\", "/").split("/").pop();
78
+ outputFile = options.output?.toString() + "/" + filename;
79
+ }
80
+ await packGLTF(file, outputFile, {
57
81
  config,
58
82
  packedFiles: set,
59
83
  stats,
60
84
  debug: false,
61
85
  useCache: true,
62
86
  logger: logger,
87
+ file_stats: file_stats,
63
88
  });
64
89
  });
65
90
  printStats(stats);
91
+ console.log("Stats: ", options);
92
+ if (options.outputStats) {
93
+ writeFileStatsToFile(file_stats, options.outputStats.toString());
94
+ }
66
95
  })
67
- .command("transform", "Transform and optimize glTF, GLB or VRM files")
96
+ .command("transform", "Transform (progressive loading) and compress glTF, GLB or VRM files")
68
97
  .help(`
69
98
  This will produce multiple versions of the input file, each with a different level of optimization.
70
99
  Each version will be compressed and written to the output directory.
@@ -72,6 +101,8 @@ Each version will be compressed and written to the output directory.
72
101
  .argument('<input>', "The input glTF, GLB or VRM file that should be compressed OR a directory that contains files to be compressed")
73
102
  .option("--usecase <usecase>", `The usecase for the compression. This will set the compression settings. Possible options: [${UsecaseOptions.join(", ")}]`, { validator: program.STRING })
74
103
  .option("--config <config>", "The path to a JSON file containing the compression settings. This can also be a needle.config.json file with a \"compression\" object or usecase string.", { validator: program.STRING })
104
+ // .option("--output <output>", "The output directory. If not provided, the input files will be overwritten.", { validator: program.STRING })
105
+ .option("--output-stats", "The path to a JSON file where the stats will be written to.", { validator: program.STRING })
75
106
  .action(async ({ logger, args, options }) => {
76
107
  logger.info("Transforming glTF file: " + args.input);
77
108
  const stats = {
@@ -93,6 +124,8 @@ Each version will be compressed and written to the output directory.
93
124
  }
94
125
  }
95
126
  const file = args.input.toString();
127
+ const outDir = statSync(file).isDirectory() ? file : dirname(file);
128
+ const file_stats = [];
96
129
  const set = new Set();
97
130
  await foreachGLTF(file, async (file) => {
98
131
  logger.info("> Transform " + file);
@@ -111,6 +144,7 @@ Each version will be compressed and written to the output directory.
111
144
  debug: false,
112
145
  useCache: true,
113
146
  logger: logger,
147
+ file_stats,
114
148
  };
115
149
  for (const res of progressive_results) {
116
150
  logger.info("> Compress " + res);
@@ -120,5 +154,8 @@ Each version will be compressed and written to the output directory.
120
154
  await packGLTF(file, file, opts);
121
155
  });
122
156
  printStats(stats);
157
+ if (options.outputStats) {
158
+ writeFileStatsToFile(file_stats, options.outputStats.toString());
159
+ }
123
160
  });
124
161
  program.run();
@@ -101,25 +101,38 @@ function internalResolveConfig() {
101
101
  function resolveConfig(filename, content) {
102
102
  // Handle config being part of `needle.config.json`
103
103
  if (filename.endsWith("needle.config.json")) {
104
- const config = JSON.parse(content);
105
- if (config.optimization) {
106
- console.log("INFO: Found optimization config at \"" + filename + "\".");
107
- // If the optimization config is a string, it's the usecase
108
- if (typeof config.optimization === "string") {
109
- return createConfig(config.optimization);
104
+ try {
105
+ const config = JSON.parse(content);
106
+ if (config.optimization) {
107
+ console.log("INFO: Found optimization config at \"" + filename + "\".");
108
+ // If the optimization config is a string, it's the usecase
109
+ if (typeof config.optimization === "string") {
110
+ return createConfig(config.optimization);
111
+ }
112
+ return config.optimization;
113
+ }
114
+ else {
115
+ console.log("INFO: No optimization config found in needle.config.json. Using default config.");
116
+ return null;
110
117
  }
111
- return config.optimization;
112
118
  }
113
- else {
114
- console.log("INFO: No optimization config found in needle.config.json. Using default config.");
115
- return null;
119
+ catch (err) {
120
+ console.error(`ERR: Failed parsing config \"${err.message}\" at ${filename}`);
116
121
  }
117
122
  }
118
123
  // Handle config being the content of the file
119
124
  else if (content.trimStart().startsWith("{")) {
120
- return JSON.parse(content);
125
+ try {
126
+ return JSON.parse(content);
127
+ }
128
+ catch (err) {
129
+ if (err instanceof Error)
130
+ console.error(`ERR: Failed parsing config \"${err.message}\" at ${filename}`);
131
+ else
132
+ console.error(`ERR: Failed parsing config \"${err}\" at ${filename}`);
133
+ }
121
134
  }
122
135
  else {
123
- throw new Error("Invalid config file format. Expected JSON object.");
136
+ console.error("ERR: Invalid config file format. Expected JSON object at " + filename);
124
137
  }
125
138
  }
@@ -1,11 +1,12 @@
1
1
  import { ILogger } from '@gltf-transform/core';
2
- import { Stats } from "../utils/index.js";
2
+ import { FileStats, Stats } from "../utils/index.js";
3
3
  import { Config } from '../config/index.js';
4
4
  export type PackGLTFOptions = {
5
5
  debug: boolean;
6
6
  useCache: boolean;
7
7
  packedFiles?: Set<string>;
8
8
  stats?: Stats;
9
+ file_stats: FileStats[] | null;
9
10
  config?: Config;
10
11
  logger?: ILogger;
11
12
  };
@@ -7,7 +7,7 @@ import { MeshoptDecoder, MeshoptEncoder } from 'meshoptimizer';
7
7
  import { resample } from '@gltf-transform/functions';
8
8
  import { ALL_EXTENSIONS as NEEDLE_EXTENSIONS, NEEDLE_compression_texture, NEEDLE_mesh_compression, } from '../extensions/index.js';
9
9
  import { isMeshLOD, needle_animation_transform, needle_asset, needle_mesh_transform, needle_texture_transform, } from '../transforms/index.js';
10
- import { writeNodeIO } from "../utils/index.js";
10
+ import { createNewFilestats, writeNodeIO } from "../utils/index.js";
11
11
  import { addToCache, getHash, tryGetFromCache, } from "../cache/index.js";
12
12
  import { getConfig } from '../config/index.js';
13
13
  /** Compress a glTF file
@@ -66,7 +66,7 @@ export async function packGLTF(inputFile, outputFile, options) {
66
66
  const document = await io.read(inputFile);
67
67
  /** @ts-ignore */
68
68
  document.getLogger().verbosity = Verbosity.ERROR;
69
- let totalGPUTextureMemmory = 0;
69
+ let totalGPUTextureMemory = 0;
70
70
  /** @type {Array<import('@gltf-transform/core').Transform>} */
71
71
  const transforms = [
72
72
  needle_asset(),
@@ -76,8 +76,7 @@ export async function packGLTF(inputFile, outputFile, options) {
76
76
  noToktx: false,
77
77
  debug: debug,
78
78
  reportGPUMemory: val => {
79
- if (stats)
80
- stats.totalGPUMemoryInMB += val;
79
+ totalGPUTextureMemory += val;
81
80
  },
82
81
  config,
83
82
  }),
@@ -102,7 +101,7 @@ export async function packGLTF(inputFile, outputFile, options) {
102
101
  const duration = Date.now() - startTime;
103
102
  const sizeInMB = statSync(outputFile).size / 1024 / 1024;
104
103
  const filename = path.basename(outputFile);
105
- const gpuMemoryInMB = totalGPUTextureMemmory / 1024 / 1024;
104
+ const gpuMemoryInMB = totalGPUTextureMemory / 1024 / 1024;
106
105
  logger.info(`← Compressing done in ${(duration / 1000).toFixed(1)} sec, ${prevSizeInMB.toFixed(1)} MB → ${sizeInMB.toFixed(1)} MB (GPU: ${gpuMemoryInMB.toFixed(1)} MB textures): ${filename} at ${outputFile}`);
107
106
  if (stats) {
108
107
  stats.totalFilesProcessed++;
@@ -110,6 +109,10 @@ export async function packGLTF(inputFile, outputFile, options) {
110
109
  stats.totalFileSizeInMBBefore += prevSizeInMB;
111
110
  stats.totalGPUMemoryInMB += gpuMemoryInMB;
112
111
  }
112
+ const filestats = createNewFilestats(outputFile, document, {
113
+ gpu_memory: totalGPUTextureMemory,
114
+ });
115
+ options.file_stats?.push(filestats);
113
116
  return true;
114
117
  }
115
118
  catch (err) {
@@ -1,3 +1,4 @@
1
+ export declare function getFileName(filepath: string): string;
1
2
  export declare type GLTFFileCallback = (file: string) => void | Promise<void>;
2
3
  export declare function foreachGLTF(args: string | string[], callback: GLTFFileCallback): Promise<void>;
3
4
  export declare function replaceExtension(filepath: string, newExtension: string): string;
@@ -1,23 +1,27 @@
1
1
  import { existsSync, statSync, readdirSync, readFileSync } from 'fs';
2
2
  import { isAbsolute } from 'path';
3
+ export function getFileName(filepath) {
4
+ const parts = filepath.replaceAll("\\", "/").split("/");
5
+ return parts[parts.length - 1];
6
+ }
3
7
  export async function foreachGLTF(args, callback) {
4
8
  // get original directory this run was invoked from
5
9
  const initDir = process.env.INIT_CWD;
6
10
  const context = {
7
11
  baseDirectory: initDir,
8
- filesFound: 0
12
+ files: [],
9
13
  };
10
14
  // traverse recursively through all files and directories
11
15
  if (Array.isArray(args)) {
12
16
  for (const arg of args) {
13
- await testEntry(arg, callback, context);
17
+ await testEntry(arg, context);
14
18
  }
15
19
  }
16
20
  else if (typeof args === "string") {
17
- await testEntry(args, callback, context);
21
+ await testEntry(args, context);
18
22
  }
19
23
  // try to run with needle config
20
- if (context.filesFound <= 0 && context.baseDirectory) {
24
+ if (context.files?.length <= 0 && context.baseDirectory) {
21
25
  const needleConfigPath = context.baseDirectory + "/needle.config.json";
22
26
  if (existsSync(needleConfigPath)) {
23
27
  const config = JSON.parse(readFileSync(needleConfigPath, "utf8"));
@@ -26,13 +30,13 @@ export async function foreachGLTF(args, callback) {
26
30
  // If it's an absolute path
27
31
  if (existsSync(buildDirectory)) {
28
32
  console.log("INFO: Using build directory from needle.config.json: " + buildDirectory);
29
- await testEntry(buildDirectory, callback, context);
33
+ await testEntry(buildDirectory, context);
30
34
  }
31
35
  else { // if its a relative path
32
36
  const newPath = context.baseDirectory + "/" + buildDirectory;
33
37
  if (existsSync(newPath)) {
34
38
  console.log("INFO: Using build directory from needle.config.json: " + newPath);
35
- await testEntry(newPath, callback, context);
39
+ await testEntry(newPath, context);
36
40
  }
37
41
  }
38
42
  }
@@ -43,11 +47,16 @@ export async function foreachGLTF(args, callback) {
43
47
  // await testEntry(context.baseDirectory + "/dist", callback, context);
44
48
  }
45
49
  }
46
- if (context.filesFound <= 0) {
50
+ if (context.files.length <= 0) {
47
51
  console.log("WARN: No glTF files found. Make sure to invoke with a valid path to a glTF file or directory. This path may be relative to " + initDir);
48
52
  }
53
+ else {
54
+ for (const file of context.files) {
55
+ await callback(file);
56
+ }
57
+ }
49
58
  }
50
- async function testEntry(entry, callback, context) {
59
+ async function testEntry(entry, context) {
51
60
  // ignore argument names (e.g. --debug)
52
61
  if (entry.startsWith("--")) {
53
62
  return false;
@@ -90,11 +99,7 @@ async function testEntry(entry, callback, context) {
90
99
  }
91
100
  }
92
101
  if (entry.endsWith(".glb") || entry.endsWith(".gltf") || entry.endsWith(".vrm")) {
93
- context.filesFound += 1;
94
- const res = callback(entry);
95
- if (res instanceof Promise) {
96
- await res;
97
- }
102
+ context.files.push(entry);
98
103
  }
99
104
  else {
100
105
  const stat = statSync(entry);
@@ -105,7 +110,7 @@ async function testEntry(entry, callback, context) {
105
110
  }
106
111
  const files = readdirSync(entry);
107
112
  for (const file of files) {
108
- await testEntry(entry + "/" + file, callback, context);
113
+ await testEntry(entry + "/" + file, context);
109
114
  }
110
115
  }
111
116
  else {
@@ -0,0 +1,23 @@
1
+ import { Logger } from "@donmccurdy/caporal";
2
+ import { Document } from "@gltf-transform/core";
3
+ export type Stats = {
4
+ startTime: number;
5
+ totalGPUMemoryInMB: number;
6
+ totalFilesProcessed: number;
7
+ totalFileSizeInMB: number;
8
+ totalFileSizeInMBBefore: number;
9
+ };
10
+ export declare function printStats(stats: Stats, logger?: Logger): void;
11
+ export type FileStats = {
12
+ filename: string;
13
+ /** Size in bytes */
14
+ size: number;
15
+ gpu_memory: number;
16
+ vertex_count: number;
17
+ nodes: number;
18
+ materials: number;
19
+ };
20
+ export declare function createNewFilestats(file: string, document: Document, stats: {
21
+ gpu_memory: number;
22
+ }): FileStats;
23
+ export declare function writeFileStatsToFile(filestats: FileStats[], output_path: string): void;
@@ -0,0 +1,39 @@
1
+ import { getVersion } from "./version.js";
2
+ import { getFileName } from "./fileutils.js";
3
+ import { statSync, writeFileSync } from "fs";
4
+ export function printStats(stats, logger) {
5
+ const duration = Date.now() - stats.startTime;
6
+ const durationInMinutes = duration / 1000 / 60;
7
+ const msg = `[Needle Build Pipeline] v${getVersion()} — Finished packing ${stats.totalFilesProcessed} files in ${durationInMinutes.toFixed(1)} minutes; total size ${stats.totalFileSizeInMBBefore.toFixed(1)} MB → ${stats.totalFileSizeInMB.toFixed(1)} MB, total GPU memory: ${stats.totalGPUMemoryInMB.toFixed(1)} MB`;
8
+ if (logger)
9
+ logger.info(msg);
10
+ else
11
+ console.log(msg);
12
+ }
13
+ export function createNewFilestats(file, document, stats) {
14
+ const filename = getFileName(file);
15
+ const root = document.getRoot();
16
+ const filestats = {
17
+ filename,
18
+ size: statSync(file).size,
19
+ gpu_memory: stats.gpu_memory,
20
+ nodes: root.listNodes().length,
21
+ materials: root.listMaterials().length,
22
+ vertex_count: 0,
23
+ };
24
+ // Count vertices
25
+ for (const mesh of root.listMeshes()) {
26
+ filestats.vertex_count += mesh.listPrimitives().reduce((acc, p) => acc + p.getAttribute('POSITION').getCount(), 0);
27
+ }
28
+ filestats.vertex_count = Math.round(filestats.vertex_count / 3);
29
+ return filestats;
30
+ }
31
+ export function writeFileStatsToFile(filestats, output_path) {
32
+ if (output_path == undefined) {
33
+ console.error("No stats output path provided");
34
+ return;
35
+ }
36
+ console.log("Writing file stats to: " + output_path);
37
+ const json = JSON.stringify(filestats, null, 2);
38
+ writeFileSync(output_path, json);
39
+ }
@@ -0,0 +1 @@
1
+ export declare function getVersion(_marklocal?: boolean): string;
@@ -0,0 +1,5 @@
1
+ import packageJson from "../../package.json" assert { type: "json" };
2
+ export function getVersion(_marklocal = true) {
3
+ let res = packageJson.version;
4
+ return res;
5
+ }
package/package.json CHANGED
@@ -1,11 +1,12 @@
1
1
  {
2
2
  "name": "@needle-tools/gltf-build-pipeline",
3
- "version": "1.6.0-alpha.2",
3
+ "version": "1.6.0-alpha.4",
4
4
  "description": "Pipeline and tools for optimizing gltf files using gltf-transform and compression settings within glTF extensions",
5
5
  "main": "index.mjs",
6
6
  "type": "module",
7
+ "readme": "README.md",
7
8
  "bin": {
8
- "gltf-build-pipeline": "./dist/cli/index.js"
9
+ "gltf-build-pipeline": "./dist/src/cli/index.js"
9
10
  },
10
11
  "scripts": {
11
12
  "start": "node scripts/index.mjs",
@@ -62,4 +63,4 @@
62
63
  "type": "git",
63
64
  "url": "git+https://github.com/needle-tools/needle-gltf-extensions.git"
64
65
  }
65
- }
66
+ }
@@ -79,6 +79,7 @@ async function packFile(file) {
79
79
  packedFiles,
80
80
  useCache: allowCaching,
81
81
  stats,
82
+ file_stats: null
82
83
  })
83
84
  }
84
85
 
package/tsconfig.json CHANGED
@@ -6,7 +6,8 @@
6
6
  "moduleResolution": "NodeNext",
7
7
  "module": "ESNext",
8
8
  "esModuleInterop": true,
9
- "skipLibCheck": true
9
+ "skipLibCheck": true,
10
+ "resolveJsonModule": true
10
11
 
11
12
  },
12
13
  "include": [
@@ -1,9 +0,0 @@
1
- import { Logger } from "@donmccurdy/caporal";
2
- export type Stats = {
3
- startTime: number;
4
- totalGPUMemoryInMB: number;
5
- totalFilesProcessed: number;
6
- totalFileSizeInMB: number;
7
- totalFileSizeInMBBefore: number;
8
- };
9
- export declare function printStats(stats: Stats, logger?: Logger): void;
@@ -1,10 +0,0 @@
1
- import { getVersion } from "./version.js";
2
- export function printStats(stats, logger) {
3
- const duration = Date.now() - stats.startTime;
4
- const durationInMinutes = duration / 1000 / 60;
5
- const msg = `[Needle Build Pipeline] v${getVersion()} — Finished packing ${stats.totalFilesProcessed} files in ${durationInMinutes.toFixed(1)} minutes; total size ${stats.totalFileSizeInMBBefore.toFixed(1)} MB → ${stats.totalFileSizeInMB.toFixed(1)} MB, total GPU memory: ${stats.totalGPUMemoryInMB.toFixed(1)} MB`;
6
- if (logger)
7
- logger.info(msg);
8
- else
9
- console.log(msg);
10
- }
@@ -1 +0,0 @@
1
- export declare function getVersion(marklocal?: boolean): string;
@@ -1,35 +0,0 @@
1
- import { existsSync, readFileSync, statSync } from 'fs';
2
- let version = undefined;
3
- let isLocal = false;
4
- export function getVersion(marklocal = true) {
5
- if (version === undefined) {
6
- const scriptLocation = process.argv[1].replaceAll("\\", "/");
7
- const path = scriptLocation.split('/');
8
- let packageJsonPath = null;
9
- while (path.length) {
10
- const dirname = path.join('/');
11
- if (statSync(dirname || '/').isDirectory()) {
12
- const filename = dirname + '/package.json';
13
- if (existsSync(filename)) {
14
- packageJsonPath = filename;
15
- break;
16
- }
17
- }
18
- path.pop();
19
- }
20
- if (packageJsonPath) {
21
- isLocal = !path.includes("node_modules");
22
- if (existsSync(packageJsonPath)) {
23
- const packageJson = readFileSync(packageJsonPath, "utf8");
24
- const packageVersion = JSON.parse(packageJson).version;
25
- version = packageVersion ?? null;
26
- }
27
- else
28
- console.log("Could not find package.json at " + packageJsonPath);
29
- }
30
- }
31
- let res = version;
32
- if (marklocal && isLocal)
33
- res += " (local)";
34
- return res;
35
- }
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes