@needle-tools/gltf-build-pipeline 1.5.0-alpha.9 → 1.6.0-alpha
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +10 -0
- package/dist/cache/cache.d.ts +2 -1
- package/dist/cache/cache.js +8 -2
- package/dist/cli/index.d.ts +2 -0
- package/dist/cli/index.js +102 -0
- package/dist/config/index.d.ts +43 -0
- package/dist/config/index.js +125 -0
- package/dist/extensions/NEEDLE_opaque/NEEDLE_opaque.js +3 -3
- package/dist/extensions/NEEDLE_opaque/index.d.ts +2 -2
- package/dist/extensions/NEEDLE_progressive_mesh_settings/index.d.ts +4 -1
- package/dist/extensions/NEEDLE_progressive_texture_settings/index.d.ts +4 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.js +3 -0
- package/dist/scripts/index.d.ts +1 -0
- package/dist/scripts/index.js +1 -0
- package/dist/scripts/pack-gltf.d.ts +12 -0
- package/dist/scripts/pack-gltf.js +119 -0
- package/dist/transforms/needle_common.d.ts +3 -1
- package/dist/transforms/needle_mesh_transform.d.ts +3 -1
- package/dist/transforms/needle_progressive.d.ts +3 -0
- package/dist/transforms/needle_progressive.js +9 -2
- package/dist/transforms/needle_texture_transform.d.ts +9 -1
- package/dist/transforms/needle_texture_transform.js +8 -7
- package/dist/transforms/needle_toktx.d.ts +2 -2
- package/dist/transforms/needle_toktx.js +12 -3
- package/dist/transforms/needle_webp.d.ts +2 -2
- package/dist/transforms/needle_webp.js +1 -1
- package/dist/utils/compression_utils.d.ts +1 -1
- package/dist/utils/index.d.ts +1 -0
- package/dist/utils/index.js +1 -0
- package/dist/utils/stats.d.ts +8 -0
- package/dist/utils/stats.js +6 -0
- package/package.json +13 -9
- package/scripts/clear-caches.mjs +4 -0
- package/scripts/defaults.mjs +12 -0
- package/{fileutils.mjs → scripts/fileutils.mjs} +1 -1
- package/{index.mjs → scripts/index.mjs} +3 -3
- package/scripts/limit-caches.mjs +4 -0
- package/{make-progressive.mjs → scripts/make-progressive.mjs} +4 -2
- package/scripts/pack-gltf.mjs +101 -0
- package/clear-caches.mjs +0 -4
- package/defaults.mjs +0 -12
- package/limit-caches.mjs +0 -4
- package/pack-gltf.mjs +0 -189
package/CHANGELOG.md
CHANGED
|
@@ -4,6 +4,16 @@ All notable changes to this package will be documented in this file.
|
|
|
4
4
|
The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
|
|
5
5
|
and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
|
|
6
6
|
|
|
7
|
+
## [1.6.0-alpha] - 2023-05-27
|
|
8
|
+
- add: `gltf-build-pipeline` commandline support with `transform` and `compress` (pack-gltf) options.
|
|
9
|
+
|
|
10
|
+
## [1.5.0-alpha.12] - 2023-05-20
|
|
11
|
+
- add: pack-gltf now also runs `resample` for animations
|
|
12
|
+
- fix: nullreference error in accessing opaque debug options
|
|
13
|
+
|
|
14
|
+
## [1.5.0-alpha.10] - 2023-05-17
|
|
15
|
+
- add: `generateLods` option to progressive texture settings
|
|
16
|
+
|
|
7
17
|
## [1.5.0-alpha.9] - 2023-05-16
|
|
8
18
|
- add: start of VRM 1.0 support. Not implemented is the `VRMC_materials_mtoon` extension
|
|
9
19
|
|
package/dist/cache/cache.d.ts
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { ExtensibleProperty } from "@gltf-transform/core";
|
|
2
|
+
import { Logger } from "@donmccurdy/caporal";
|
|
2
3
|
/**
|
|
3
4
|
* Clear the cache.
|
|
4
5
|
* This will remove all files in the cache directory.
|
|
5
6
|
*/
|
|
6
|
-
export declare function clearCache(): void;
|
|
7
|
+
export declare function clearCache(logger?: Logger): void;
|
|
7
8
|
/**
|
|
8
9
|
* Ensure that the cache size does not exceed the given maximum size in MB.
|
|
9
10
|
* @param maxSizeMB The maximum size in MB.
|
package/dist/cache/cache.js
CHANGED
|
@@ -23,10 +23,13 @@ const buildPipelineVersion = getVersion(false);
|
|
|
23
23
|
* Clear the cache.
|
|
24
24
|
* This will remove all files in the cache directory.
|
|
25
25
|
*/
|
|
26
|
-
export function clearCache() {
|
|
26
|
+
export function clearCache(logger) {
|
|
27
27
|
const dir = getCacheDirectory(true);
|
|
28
28
|
fs.rmSync(dir, { recursive: true });
|
|
29
|
-
|
|
29
|
+
if (!logger)
|
|
30
|
+
console.log(`Cleared cache at ${dir}`);
|
|
31
|
+
else
|
|
32
|
+
logger.info(`Cleared cache at ${dir}`);
|
|
30
33
|
}
|
|
31
34
|
/**
|
|
32
35
|
* Ensure that the cache size does not exceed the given maximum size in MB.
|
|
@@ -223,6 +226,9 @@ function hashObject(obj) {
|
|
|
223
226
|
const buffer = obj.getArray();
|
|
224
227
|
return hashBuffer(buffer);
|
|
225
228
|
}
|
|
229
|
+
else if (obj == null) {
|
|
230
|
+
return 0;
|
|
231
|
+
}
|
|
226
232
|
const str = JSON.stringify(obj);
|
|
227
233
|
return hashString(str);
|
|
228
234
|
}
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
#! /usr/bin/env node
|
|
2
|
+
import { program } from '@donmccurdy/caporal';
|
|
3
|
+
import { packGLTF } from "../scripts/index.js";
|
|
4
|
+
import { printStats } from '../utils/stats.js';
|
|
5
|
+
import { getVersion } from '../utils/version.js';
|
|
6
|
+
import { UsecaseOptions, createConfig, getConfig } from '../config/index.js';
|
|
7
|
+
import { make_progressive } from '../transforms/needle_progressive.js';
|
|
8
|
+
import { clearCache } from '../cache/cache.js';
|
|
9
|
+
// For testing / dev you can run `npm link` in the package directory
|
|
10
|
+
// For removing the link run `npm rm --global @needle-tools/gltf-build-pipeline`
|
|
11
|
+
program
|
|
12
|
+
.command("version", "Print current verison")
|
|
13
|
+
.action(async ({ logger }) => {
|
|
14
|
+
logger.info(getVersion());
|
|
15
|
+
})
|
|
16
|
+
.command("clear-caches", "Clear all caches")
|
|
17
|
+
.help(`
|
|
18
|
+
Clear all caches.
|
|
19
|
+
`)
|
|
20
|
+
.action(async ({ logger }) => {
|
|
21
|
+
logger.info("Clearing caches");
|
|
22
|
+
clearCache(logger);
|
|
23
|
+
})
|
|
24
|
+
.command("compress", "Compress a glTF file")
|
|
25
|
+
.help(`
|
|
26
|
+
Compress a single glTF file. The input file is compressed and written to the output file.
|
|
27
|
+
`)
|
|
28
|
+
.argument('<input>', "The input glTF, GLB or VRM file that should be compressed")
|
|
29
|
+
.option("--usecase <usecase>", `The usecase for the compression. This will set the compression settings. Options are ${UsecaseOptions.join(", ")}`, { validator: program.STRING })
|
|
30
|
+
.option("--config <config>", "The path to a JSON file containing the compression settings. This can also be a needle.config.json file with a \"compression\" object or usecase string.", { validator: program.STRING })
|
|
31
|
+
.option("--output <output>", "The output file. If not provided, the input file will be overwritten.", { validator: program.STRING })
|
|
32
|
+
.action(async ({ logger, args, options }) => {
|
|
33
|
+
logger.info("Transforming glTF file: " + args.input);
|
|
34
|
+
const stats = {
|
|
35
|
+
startTime: Date.now(),
|
|
36
|
+
totalGPUMemoryInMB: 0,
|
|
37
|
+
totalFilesProcessed: 0,
|
|
38
|
+
totalFileSizeInMB: 0,
|
|
39
|
+
totalFileSizeInMBBefore: 0,
|
|
40
|
+
};
|
|
41
|
+
let config = getConfig(options.config?.toString());
|
|
42
|
+
if (options.usecase) {
|
|
43
|
+
logger.info("Using usecase: " + options.usecase);
|
|
44
|
+
config = { ...config, ...createConfig(options.usecase.toString()) };
|
|
45
|
+
}
|
|
46
|
+
const file = args.input.toString();
|
|
47
|
+
const set = new Set();
|
|
48
|
+
await packGLTF(file, options.output?.toString() || file, {
|
|
49
|
+
config,
|
|
50
|
+
packedFiles: set,
|
|
51
|
+
stats,
|
|
52
|
+
debug: false,
|
|
53
|
+
useCache: true,
|
|
54
|
+
});
|
|
55
|
+
printStats(stats);
|
|
56
|
+
})
|
|
57
|
+
.command("transform", "Transform a glTF file")
|
|
58
|
+
.help(`
|
|
59
|
+
This will produce multiple versions of the input file, each with a different level of optimization.
|
|
60
|
+
Each version will be compressed and written to the output directory.
|
|
61
|
+
`)
|
|
62
|
+
.argument('<input>', "The input glTF, GLB or VRM file that should be compressed")
|
|
63
|
+
.option("--usecase <usecase>", `The usecase for the compression. This will set the compression settings. Options are ${UsecaseOptions.join(", ")}`, { validator: program.STRING })
|
|
64
|
+
.option("--config <config>", "The path to a JSON file containing the compression settings. This can also be a needle.config.json file with a \"compression\" object or usecase string.", { validator: program.STRING })
|
|
65
|
+
// .option("--output <output>", "The output file. If not provided, the input file will be overwritten.", { validator: program.STRING })
|
|
66
|
+
.action(async ({ logger, args, options }) => {
|
|
67
|
+
logger.info("Transforming glTF file: " + args.input);
|
|
68
|
+
const stats = {
|
|
69
|
+
startTime: Date.now(),
|
|
70
|
+
totalGPUMemoryInMB: 0,
|
|
71
|
+
totalFilesProcessed: 0,
|
|
72
|
+
totalFileSizeInMB: 0,
|
|
73
|
+
totalFileSizeInMBBefore: 0,
|
|
74
|
+
};
|
|
75
|
+
let config = getConfig(options.config?.toString());
|
|
76
|
+
if (options.usecase) {
|
|
77
|
+
logger.info("Using usecase: " + options.usecase);
|
|
78
|
+
config = { ...config, ...createConfig(options.usecase.toString()) };
|
|
79
|
+
}
|
|
80
|
+
const file = args.input.toString();
|
|
81
|
+
const progressive_results = new Array();
|
|
82
|
+
await make_progressive({
|
|
83
|
+
path: file,
|
|
84
|
+
results: progressive_results,
|
|
85
|
+
config,
|
|
86
|
+
debug: false,
|
|
87
|
+
useCache: true,
|
|
88
|
+
});
|
|
89
|
+
console.log("Progressive results: ", progressive_results);
|
|
90
|
+
const opts = {
|
|
91
|
+
config,
|
|
92
|
+
stats,
|
|
93
|
+
debug: false,
|
|
94
|
+
useCache: true,
|
|
95
|
+
};
|
|
96
|
+
for (const prog of progressive_results) {
|
|
97
|
+
await packGLTF(prog, prog, opts);
|
|
98
|
+
}
|
|
99
|
+
await packGLTF(file, file, opts);
|
|
100
|
+
printStats(stats);
|
|
101
|
+
});
|
|
102
|
+
program.run();
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
declare type Usecase = "unknown" | "product" | "game" | "metaverse";
|
|
2
|
+
export declare const UsecaseOptions: string[];
|
|
3
|
+
/**
|
|
4
|
+
* The config object contains information about how a file should be processed.
|
|
5
|
+
*/
|
|
6
|
+
export type Config = {
|
|
7
|
+
/** High-level configuration */
|
|
8
|
+
usecase: Usecase;
|
|
9
|
+
};
|
|
10
|
+
/**
|
|
11
|
+
* Create a config object with the given usecase.
|
|
12
|
+
*/
|
|
13
|
+
export declare function createConfig(usecase: string): {
|
|
14
|
+
usecase: Usecase;
|
|
15
|
+
};
|
|
16
|
+
/**
|
|
17
|
+
* Get the config object either provided by a CLI path or a needle.config.json file in the parent directory.
|
|
18
|
+
*
|
|
19
|
+
* @example CLI argument where the path points to a json file containing the config as `Config`
|
|
20
|
+
* ```sh
|
|
21
|
+
* node script.js --config path/to/my-optimization-config.json
|
|
22
|
+
* ```
|
|
23
|
+
*
|
|
24
|
+
* @example needle.config.json - the usecase can be provided as a string
|
|
25
|
+
* ```json
|
|
26
|
+
* {
|
|
27
|
+
* "optimization": "product"
|
|
28
|
+
* }
|
|
29
|
+
* ```
|
|
30
|
+
*
|
|
31
|
+
* @example needle.config.json - the whole config object can be provided
|
|
32
|
+
* ```json
|
|
33
|
+
* {
|
|
34
|
+
* "optimization": {
|
|
35
|
+
* "usecase": "product"
|
|
36
|
+
* }
|
|
37
|
+
* }
|
|
38
|
+
* ```
|
|
39
|
+
*
|
|
40
|
+
*
|
|
41
|
+
*/
|
|
42
|
+
export declare function getConfig(path?: string): Config;
|
|
43
|
+
export {};
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
import { existsSync, readFileSync } from "fs";
|
|
2
|
+
export const UsecaseOptions = ["product", "game", "metaverse"];
|
|
3
|
+
const defaultConfig = {
|
|
4
|
+
usecase: "unknown",
|
|
5
|
+
};
|
|
6
|
+
/**
|
|
7
|
+
* Create a config object with the given usecase.
|
|
8
|
+
*/
|
|
9
|
+
export function createConfig(usecase) {
|
|
10
|
+
if (!UsecaseOptions.includes(usecase)) {
|
|
11
|
+
throw new Error(`Invalid usecase: \"${usecase}\". Valid usecases: ${UsecaseOptions.join(", ")}`);
|
|
12
|
+
}
|
|
13
|
+
return {
|
|
14
|
+
usecase: usecase
|
|
15
|
+
};
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Get the config object either provided by a CLI path or a needle.config.json file in the parent directory.
|
|
19
|
+
*
|
|
20
|
+
* @example CLI argument where the path points to a json file containing the config as `Config`
|
|
21
|
+
* ```sh
|
|
22
|
+
* node script.js --config path/to/my-optimization-config.json
|
|
23
|
+
* ```
|
|
24
|
+
*
|
|
25
|
+
* @example needle.config.json - the usecase can be provided as a string
|
|
26
|
+
* ```json
|
|
27
|
+
* {
|
|
28
|
+
* "optimization": "product"
|
|
29
|
+
* }
|
|
30
|
+
* ```
|
|
31
|
+
*
|
|
32
|
+
* @example needle.config.json - the whole config object can be provided
|
|
33
|
+
* ```json
|
|
34
|
+
* {
|
|
35
|
+
* "optimization": {
|
|
36
|
+
* "usecase": "product"
|
|
37
|
+
* }
|
|
38
|
+
* }
|
|
39
|
+
* ```
|
|
40
|
+
*
|
|
41
|
+
*
|
|
42
|
+
*/
|
|
43
|
+
export function getConfig(path) {
|
|
44
|
+
if (path != null) {
|
|
45
|
+
if (!existsSync(path)) {
|
|
46
|
+
throw new Error(`No config found at \"${path}\"`);
|
|
47
|
+
}
|
|
48
|
+
return resolveConfig(path, readFileSync(path, "utf8"));
|
|
49
|
+
}
|
|
50
|
+
if (_cachedConfig !== undefined) {
|
|
51
|
+
return _cachedConfig;
|
|
52
|
+
}
|
|
53
|
+
// Resolve the config once and cache it
|
|
54
|
+
const result = internalResolveConfig();
|
|
55
|
+
if (result) {
|
|
56
|
+
console.log("INFO: Using optimization config with usecase: " + result.usecase);
|
|
57
|
+
_cachedConfig = result;
|
|
58
|
+
}
|
|
59
|
+
else {
|
|
60
|
+
console.log("INFO: No optimization config found. Using default config.");
|
|
61
|
+
_cachedConfig = defaultConfig;
|
|
62
|
+
}
|
|
63
|
+
return _cachedConfig;
|
|
64
|
+
}
|
|
65
|
+
let _cachedConfig = undefined;
|
|
66
|
+
function internalResolveConfig() {
|
|
67
|
+
// Check if a config path is passed as an argument
|
|
68
|
+
const args = process.argv.slice(2);
|
|
69
|
+
for (let i = 0; i < args.length; i++) {
|
|
70
|
+
if (args[i] === "--config") {
|
|
71
|
+
if (i + 1 < args.length) {
|
|
72
|
+
const configPath = args[i + 1];
|
|
73
|
+
if (existsSync(configPath)) {
|
|
74
|
+
const content = readFileSync(configPath, "utf8");
|
|
75
|
+
return resolveConfig(configPath, content);
|
|
76
|
+
}
|
|
77
|
+
throw new Error(`Config file not found at path configured by CLI --config argument: \"${configPath}\"`);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
// Check if a needle.config.json exists in the directory from which the script is run (or any parent directory)
|
|
82
|
+
const workingDir = process.cwd();
|
|
83
|
+
let directory = workingDir.replaceAll("\\", "/");
|
|
84
|
+
while (directory.length > 0) {
|
|
85
|
+
const configPath = `${directory}/needle.config.json`;
|
|
86
|
+
if (existsSync(configPath)) {
|
|
87
|
+
const content = readFileSync(configPath, "utf8");
|
|
88
|
+
const config = resolveConfig(configPath, content);
|
|
89
|
+
if (config) {
|
|
90
|
+
return config;
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
const lastSlash = directory.lastIndexOf("/");
|
|
94
|
+
if (lastSlash < 0) {
|
|
95
|
+
break;
|
|
96
|
+
}
|
|
97
|
+
directory = directory.slice(0, lastSlash);
|
|
98
|
+
}
|
|
99
|
+
return null;
|
|
100
|
+
}
|
|
101
|
+
function resolveConfig(filename, content) {
|
|
102
|
+
// Handle config being part of `needle.config.json`
|
|
103
|
+
if (filename.endsWith("needle.config.json")) {
|
|
104
|
+
const config = JSON.parse(content);
|
|
105
|
+
if (config.optimization) {
|
|
106
|
+
console.log("INFO: Found optimization config at \"" + filename + "\".");
|
|
107
|
+
// If the optimization config is a string, it's the usecase
|
|
108
|
+
if (typeof config.optimization === "string") {
|
|
109
|
+
return createConfig(config.optimization);
|
|
110
|
+
}
|
|
111
|
+
return config.optimization;
|
|
112
|
+
}
|
|
113
|
+
else {
|
|
114
|
+
console.log("INFO: No optimization config found in needle.config.json. Using default config.");
|
|
115
|
+
return null;
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
// Handle config being the content of the file
|
|
119
|
+
else if (content.trimStart().startsWith("{")) {
|
|
120
|
+
return JSON.parse(content);
|
|
121
|
+
}
|
|
122
|
+
else {
|
|
123
|
+
throw new Error("Invalid config file format. Expected JSON object.");
|
|
124
|
+
}
|
|
125
|
+
}
|
|
@@ -19,14 +19,14 @@ export function createOpaqueExtension(name, types, opts) {
|
|
|
19
19
|
// private roots: Array<OpaqueProperty> = [];
|
|
20
20
|
jsonPointerHandler;
|
|
21
21
|
read(context) {
|
|
22
|
-
if (opts
|
|
22
|
+
if (opts?.breakpoint === true)
|
|
23
23
|
debugger;
|
|
24
24
|
currentGenerator = context.jsonDoc.json.asset?.generator;
|
|
25
25
|
if (debugLog)
|
|
26
26
|
console.log("READ " + EXTENSION_NAME);
|
|
27
27
|
this.jsonPointerHandler = new JsonPointerHandler(this.document, EXTENSION_NAME);
|
|
28
28
|
this.jsonPointerHandler.debug = debugLog;
|
|
29
|
-
this.jsonPointerHandler.breakPoint = opts
|
|
29
|
+
this.jsonPointerHandler.breakPoint = opts?.breakpoint;
|
|
30
30
|
for (const type of types) {
|
|
31
31
|
switch (type) {
|
|
32
32
|
case PropertyType.ACCESSOR:
|
|
@@ -225,7 +225,7 @@ export function createOpaqueExtension(name, types, opts) {
|
|
|
225
225
|
}
|
|
226
226
|
}
|
|
227
227
|
write(context) {
|
|
228
|
-
if (opts
|
|
228
|
+
if (opts?.breakpoint === true)
|
|
229
229
|
debugger;
|
|
230
230
|
if (opts?.write === false) {
|
|
231
231
|
return this;
|
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { Extension } from "@gltf-transform/core";
|
|
2
2
|
export * from "./NEEDLE_opaque.js";
|
|
3
|
-
export declare const ALL_EXTENSIONS:
|
|
3
|
+
export declare const ALL_EXTENSIONS: (typeof Extension)[];
|
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
import { Extension, ExtensionProperty, PropertyType, ReaderContext, WriterContext } from "@gltf-transform/core";
|
|
2
2
|
export declare interface IProgressiveMeshSettings {
|
|
3
|
-
|
|
3
|
+
/**
|
|
4
|
+
* When false no LODs will be generated for this mesh
|
|
5
|
+
*/
|
|
4
6
|
generateLods: boolean;
|
|
7
|
+
guid: string;
|
|
5
8
|
}
|
|
6
9
|
export declare class NEEDLE_progressive_mesh_settings extends Extension {
|
|
7
10
|
static EXTENSION_NAME: string;
|
|
@@ -1,5 +1,9 @@
|
|
|
1
1
|
import { Extension, ExtensionProperty, PropertyType, ReaderContext, WriterContext } from "@gltf-transform/core";
|
|
2
2
|
export declare interface IProgressiveTextureSettings {
|
|
3
|
+
/**
|
|
4
|
+
* When false the texture no progressive texture will be generated
|
|
5
|
+
*/
|
|
6
|
+
generateLods?: boolean;
|
|
3
7
|
maxSize: number;
|
|
4
8
|
guid: string;
|
|
5
9
|
}
|
package/dist/index.d.ts
CHANGED
package/dist/index.js
CHANGED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from "./pack-gltf.js";
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from "./pack-gltf.js";
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { Stats } from "../utils/index.js";
|
|
2
|
+
import { Config } from '../config/index.js';
|
|
3
|
+
export type PackGLTFOptions = {
|
|
4
|
+
debug: boolean;
|
|
5
|
+
useCache: boolean;
|
|
6
|
+
packedFiles?: Set<string>;
|
|
7
|
+
stats?: Stats;
|
|
8
|
+
config?: Config;
|
|
9
|
+
};
|
|
10
|
+
/** Compress a glTF file
|
|
11
|
+
*/
|
|
12
|
+
export declare function packGLTF(inputFile: string, outputFile: string, options: PackGLTFOptions): Promise<boolean>;
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
import path from 'path';
|
|
2
|
+
import { NodeIO } from '@gltf-transform/core';
|
|
3
|
+
import { ALL_EXTENSIONS } from '@gltf-transform/extensions';
|
|
4
|
+
import draco3d from 'draco3dgltf';
|
|
5
|
+
import { existsSync, readFileSync, statSync, writeFileSync } from 'fs';
|
|
6
|
+
import { MeshoptDecoder, MeshoptEncoder } from 'meshoptimizer';
|
|
7
|
+
import { resample } from '@gltf-transform/functions';
|
|
8
|
+
import { ALL_EXTENSIONS as NEEDLE_EXTENSIONS, NEEDLE_compression_texture, NEEDLE_mesh_compression, } from '../extensions/index.js';
|
|
9
|
+
import { isMeshLOD, needle_animation_transform, needle_asset, needle_mesh_transform, needle_texture_transform, } from '../transforms/index.js';
|
|
10
|
+
import { writeNodeIO, getVersion } from "../utils/index.js";
|
|
11
|
+
import { addToCache, getHash, tryGetFromCache, } from "../cache/index.js";
|
|
12
|
+
import { getConfig } from '../config/index.js';
|
|
13
|
+
console.log(`[Needle Build Pipeline] pack glTF — version ${getVersion()}`);
|
|
14
|
+
/** Compress a glTF file
|
|
15
|
+
*/
|
|
16
|
+
export async function packGLTF(inputFile, outputFile, options) {
|
|
17
|
+
if (!existsSync(inputFile)) {
|
|
18
|
+
console.error("WARN: can not pack, file does not exist: " + inputFile);
|
|
19
|
+
return;
|
|
20
|
+
}
|
|
21
|
+
const { debug, stats, packedFiles, useCache } = options;
|
|
22
|
+
if (debug && inputFile.includes(".debug.")) {
|
|
23
|
+
return;
|
|
24
|
+
}
|
|
25
|
+
// check if it's in the cache.
|
|
26
|
+
// We only need to check mesh LOD files because image LODs are cache via our toktx transform
|
|
27
|
+
// We can not cache the main glTF files because the hash always changes for them
|
|
28
|
+
let cacheKey = undefined;
|
|
29
|
+
if (useCache && isMeshLOD(inputFile)) {
|
|
30
|
+
const existing = readFileSync(inputFile);
|
|
31
|
+
const hash = getHash(existing);
|
|
32
|
+
cacheKey = "packed-" + hash;
|
|
33
|
+
const cached = tryGetFromCache(cacheKey);
|
|
34
|
+
if (cached) {
|
|
35
|
+
console.log(`→ Loaded compressed file from cache: \"${path.basename(inputFile)}\"`);
|
|
36
|
+
writeFileSync(inputFile, cached);
|
|
37
|
+
return;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
if (packedFiles) {
|
|
41
|
+
if (packedFiles.has(inputFile))
|
|
42
|
+
return;
|
|
43
|
+
packedFiles.add(inputFile);
|
|
44
|
+
}
|
|
45
|
+
try {
|
|
46
|
+
const config = options.config || getConfig();
|
|
47
|
+
const prevSizeInMB = statSync(inputFile).size / 1024 / 1024;
|
|
48
|
+
console.log(`→ Compressing ${prevSizeInMB.toFixed(1)} MB: ${inputFile}`);
|
|
49
|
+
// const isProgressiveAsset = isLOD(file);
|
|
50
|
+
const startTime = Date.now();
|
|
51
|
+
const io = new NodeIO()
|
|
52
|
+
.registerExtensions(ALL_EXTENSIONS)
|
|
53
|
+
.registerExtensions(NEEDLE_EXTENSIONS)
|
|
54
|
+
.registerExtensions([
|
|
55
|
+
NEEDLE_compression_texture,
|
|
56
|
+
NEEDLE_mesh_compression,
|
|
57
|
+
])
|
|
58
|
+
.registerDependencies({
|
|
59
|
+
'draco3d.decoder': await draco3d.createDecoderModule(),
|
|
60
|
+
'draco3d.encoder': await draco3d.createEncoderModule(),
|
|
61
|
+
'meshopt.decoder': MeshoptDecoder,
|
|
62
|
+
'meshopt.encoder': MeshoptEncoder,
|
|
63
|
+
});
|
|
64
|
+
const document = await io.read(inputFile);
|
|
65
|
+
/** @ts-ignore */
|
|
66
|
+
document.getLogger().verbosity = 0;
|
|
67
|
+
let totalGPUTextureMemmory = 0;
|
|
68
|
+
/** @type {Array<import('@gltf-transform/core').Transform>} */
|
|
69
|
+
const transforms = [
|
|
70
|
+
needle_asset(),
|
|
71
|
+
needle_texture_transform({
|
|
72
|
+
file: inputFile,
|
|
73
|
+
useCache,
|
|
74
|
+
noToktx: false,
|
|
75
|
+
debug: debug,
|
|
76
|
+
reportGPUMemory: val => {
|
|
77
|
+
if (stats)
|
|
78
|
+
stats.totalGPUMemoryInMB += val;
|
|
79
|
+
},
|
|
80
|
+
config,
|
|
81
|
+
}),
|
|
82
|
+
needle_mesh_transform({ file: inputFile, config }, { ratio: 0.75, error: 0.0005 }),
|
|
83
|
+
needle_animation_transform({}),
|
|
84
|
+
resample(),
|
|
85
|
+
// TODO: dedup removes skybox texture
|
|
86
|
+
// dedup(),
|
|
87
|
+
];
|
|
88
|
+
// TODO: progressive assets need to make sure that e.g. textures are used
|
|
89
|
+
// if (!isProgressiveAsset) {
|
|
90
|
+
// transforms.push(prune());
|
|
91
|
+
// }
|
|
92
|
+
await document.transform(...transforms);
|
|
93
|
+
if (!outputFile)
|
|
94
|
+
outputFile = inputFile;
|
|
95
|
+
if (debug)
|
|
96
|
+
outputFile += ".debug.glb";
|
|
97
|
+
await writeNodeIO(io, outputFile, document);
|
|
98
|
+
if (useCache && cacheKey != undefined && existsSync(outputFile))
|
|
99
|
+
addToCache(cacheKey, readFileSync(outputFile));
|
|
100
|
+
const duration = Date.now() - startTime;
|
|
101
|
+
const sizeInMB = statSync(outputFile).size / 1024 / 1024;
|
|
102
|
+
const filename = path.basename(outputFile);
|
|
103
|
+
const gpuMemoryInMB = totalGPUTextureMemmory / 1024 / 1024;
|
|
104
|
+
console.log(`← Compressing done in ${(duration / 1000).toFixed(1)} sec, ${prevSizeInMB.toFixed(1)} MB → ${sizeInMB.toFixed(1)} MB (GPU: ${gpuMemoryInMB.toFixed(1)} MB textures): ${filename} at ${outputFile}`);
|
|
105
|
+
if (stats) {
|
|
106
|
+
stats.totalFilesProcessed++;
|
|
107
|
+
stats.totalFileSizeInMB += sizeInMB;
|
|
108
|
+
stats.totalFileSizeInMBBefore += prevSizeInMB;
|
|
109
|
+
stats.totalGPUMemoryInMB += gpuMemoryInMB;
|
|
110
|
+
}
|
|
111
|
+
return true;
|
|
112
|
+
}
|
|
113
|
+
catch (err) {
|
|
114
|
+
console.error("ERR: packing " + inputFile + " failed: " + err.message);
|
|
115
|
+
if (debug)
|
|
116
|
+
throw err;
|
|
117
|
+
return false;
|
|
118
|
+
}
|
|
119
|
+
}
|
|
@@ -1,14 +1,16 @@
|
|
|
1
1
|
import { Document, Texture } from '@gltf-transform/core';
|
|
2
2
|
import { NEEDLE_compression_texture_schema } from '../extensions/index.js';
|
|
3
|
+
import { Config } from '../config/index.js';
|
|
3
4
|
export interface NeedleTransformContext {
|
|
4
5
|
/** full path to the file being processed */
|
|
5
6
|
file: string;
|
|
7
|
+
config: Config;
|
|
6
8
|
debug?: boolean;
|
|
7
9
|
useCache: boolean;
|
|
8
10
|
noToktx: boolean;
|
|
9
11
|
}
|
|
10
12
|
export interface INeedleTextureTransformContext {
|
|
11
13
|
prepare(document: Document): Promise<void | boolean>;
|
|
12
|
-
process(index: number, texture: Texture, settings: NEEDLE_compression_texture_schema): Promise<boolean>;
|
|
14
|
+
process(index: number, texture: Texture, settings: NEEDLE_compression_texture_schema, context: NeedleTransformContext): Promise<boolean>;
|
|
13
15
|
finalize(document: Document): Promise<void>;
|
|
14
16
|
}
|
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import { Transform, Document, Primitive, Mesh } from "@gltf-transform/core";
|
|
2
2
|
import { SimplifyOptions, WeldOptions } from "@gltf-transform/functions";
|
|
3
|
+
import { Config } from "../config/index.js";
|
|
3
4
|
declare type MeshTransformContext = {
|
|
4
5
|
file: string;
|
|
6
|
+
config: Config;
|
|
5
7
|
};
|
|
6
|
-
export declare const needle_mesh_transform: (context: MeshTransformContext, simplify: SimplifyOptions) => Transform;
|
|
8
|
+
export declare const needle_mesh_transform: (context: MeshTransformContext, simplify: Omit<SimplifyOptions, "simplifier">) => Transform;
|
|
7
9
|
export declare function compressPrimitive(document: Document, mesh: Mesh, prim: Primitive, index: number, weldOptions: Required<WeldOptions>, meshOptions: Required<SimplifyOptions>): Promise<Primitive>;
|
|
8
10
|
export {};
|
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
import { ILogger, Texture, Transform, vec2 } from '@gltf-transform/core';
|
|
2
|
+
import { Config } from '../config/index.js';
|
|
2
3
|
declare interface ProgressiveOptions {
|
|
3
4
|
path: string | Array<string>;
|
|
5
|
+
results: Array<string>;
|
|
6
|
+
config: Config;
|
|
4
7
|
debug?: boolean;
|
|
5
8
|
useCache?: boolean;
|
|
6
9
|
}
|
|
@@ -3,7 +3,7 @@ import { SIMPLIFY_DEFAULTS, WELD_DEFAULTS, listTextureSlots, sparse } from '@glt
|
|
|
3
3
|
import draco3d from 'draco3dgltf';
|
|
4
4
|
import { NEEDLE_PROGRESSIVE_EXTENSION_NAME, NEEDLE_progressive } from "../extensions/NEEDLE_progressive/index.js";
|
|
5
5
|
import { createOpaqueExtension, currentGenerator, NEEDLE_compression_texture, NEEDLE_mesh_compression, NEEDLE_progressive_mesh_settings, NEEDLE_progressive_texture_settings, registerExtensions as registerDefaultExtensions } from '../extensions/index.js';
|
|
6
|
-
import path, { basename } from 'path';
|
|
6
|
+
import path, { basename, resolve } from 'path';
|
|
7
7
|
import { existsSync, readFileSync, rmSync, statSync, writeFileSync } from 'fs';
|
|
8
8
|
import { addMetaToCache, addToCache, getHash, getKeyWithHash, tryGetFromCache, tryGetMetaFromCache } from '../cache/index.js';
|
|
9
9
|
import sharp from "sharp";
|
|
@@ -94,6 +94,7 @@ async function onProcess(file, opts) {
|
|
|
94
94
|
filename = filename + ".debug." + extension;
|
|
95
95
|
}
|
|
96
96
|
await writeNodeIO(io, filename, document);
|
|
97
|
+
opts.results.push(filename);
|
|
97
98
|
console.log("← Transforming done: " + filename);
|
|
98
99
|
}
|
|
99
100
|
}
|
|
@@ -143,7 +144,7 @@ export function make_progressive_textures(filePath, _options = TEXTURE_RESIZE_DE
|
|
|
143
144
|
if (!settings) {
|
|
144
145
|
const size = texture.getSize();
|
|
145
146
|
if (size?.length >= 2 && size[0] <= 128 && size[1] <= 128) {
|
|
146
|
-
logger.warn(`[${NAME}] Skipping texture[${textureIndex}], no settings found`);
|
|
147
|
+
logger.warn(`[${NAME}] Skipping texture[${textureIndex}], no settings found and the texture is already < 128x128`);
|
|
147
148
|
continue;
|
|
148
149
|
}
|
|
149
150
|
else {
|
|
@@ -154,6 +155,10 @@ export function make_progressive_textures(filePath, _options = TEXTURE_RESIZE_DE
|
|
|
154
155
|
logger.debug(`[${NAME}] No settings found for texture[${index}] - will use default settings (maxSize: ${settings.maxSize}, guid: ${settings.guid})`);
|
|
155
156
|
}
|
|
156
157
|
}
|
|
158
|
+
else if (settings.generateLods === false) {
|
|
159
|
+
logger.debug(`[${NAME}] Skipping texture[${textureIndex}]: progressive texture disabled`);
|
|
160
|
+
continue;
|
|
161
|
+
}
|
|
157
162
|
if (settings.maxSize == undefined || settings.maxSize <= 0) {
|
|
158
163
|
logger.warn(`[${NAME}] Skipping texture[${textureIndex}], settings are invalid (maxSize: ${settings.maxSize})`);
|
|
159
164
|
continue;
|
|
@@ -175,6 +180,7 @@ export function make_progressive_textures(filePath, _options = TEXTURE_RESIZE_DE
|
|
|
175
180
|
if (!outPath) {
|
|
176
181
|
continue;
|
|
177
182
|
}
|
|
183
|
+
opts.results.push(outPath);
|
|
178
184
|
const { key, hash } = getKeyWithHash(texture);
|
|
179
185
|
const relName = path.relative(directory, outPath);
|
|
180
186
|
// TODO: this path should be added to the NEEDLE dependencies extension
|
|
@@ -548,6 +554,7 @@ export function make_progressive_meshes(filePath, _options, opts) {
|
|
|
548
554
|
const model = progressiveExt.createMesh({
|
|
549
555
|
guid: settings.guid,
|
|
550
556
|
lods: generatedLODInfos.map(l => {
|
|
557
|
+
opts.results.push(resolve(outputDirectory, l.path));
|
|
551
558
|
return {
|
|
552
559
|
path: l.path,
|
|
553
560
|
hash: l.content_hash.toString(),
|
|
@@ -1,5 +1,13 @@
|
|
|
1
1
|
import { Texture } from "@gltf-transform/core";
|
|
2
2
|
import { Transform } from '@gltf-transform/core';
|
|
3
|
+
import { NeedleWebPContext } from "./needle_webp.js";
|
|
3
4
|
import { NeedleTransformContext } from "./needle_common.js";
|
|
5
|
+
import { NeedleToktxContext } from "./needle_toktx.js";
|
|
4
6
|
export declare function resizeTextureIfNecessary(texture: Texture): Promise<void>;
|
|
5
|
-
|
|
7
|
+
interface NeedleTextureTransformContext extends NeedleTransformContext {
|
|
8
|
+
webPContext: NeedleWebPContext;
|
|
9
|
+
toktxContext: NeedleToktxContext;
|
|
10
|
+
reportGPUMemory?: (bytes: number) => void;
|
|
11
|
+
}
|
|
12
|
+
export declare const needle_texture_transform: (options: NeedleTransformContext & Pick<NeedleTextureTransformContext, "reportGPUMemory">) => Transform;
|
|
13
|
+
export {};
|