@needle-tools/gltf-build-pipeline 2.2.1-alpha.2 → 2.2.1-alpha.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.temp/Go2.bin +0 -0
- package/.temp/textures/0_1.exr +0 -0
- package/.temp/textures/1_1.jpg +0 -0
- package/CHANGELOG.md +395 -394
- package/README.md +35 -35
- package/dist/cache/cache.d.ts +54 -54
- package/dist/cache/cache.js +257 -257
- package/dist/cache/index.d.ts +1 -1
- package/dist/cache/index.js +1 -1
- package/dist/cli/index.d.ts +2 -2
- package/dist/cli/index.js +154 -154
- package/dist/config/index.d.ts +53 -53
- package/dist/config/index.js +152 -152
- package/dist/constants.d.ts +5 -5
- package/dist/constants.js +5 -5
- package/dist/extensions/NEEDLE_compression_texture/NEEDLE_compression_texture.d.ts +24 -24
- package/dist/extensions/NEEDLE_compression_texture/NEEDLE_compression_texture.js +54 -54
- package/dist/extensions/NEEDLE_compression_texture/index.d.ts +1 -1
- package/dist/extensions/NEEDLE_compression_texture/index.js +1 -1
- package/dist/extensions/NEEDLE_mesh_compression/NEEDLE_mesh_compression.d.ts +26 -26
- package/dist/extensions/NEEDLE_mesh_compression/NEEDLE_mesh_compression.js +85 -85
- package/dist/extensions/NEEDLE_mesh_compression/index.d.ts +1 -1
- package/dist/extensions/NEEDLE_mesh_compression/index.js +1 -1
- package/dist/extensions/NEEDLE_opaque/NEEDLE_opaque.d.ts +27 -27
- package/dist/extensions/NEEDLE_opaque/NEEDLE_opaque.js +897 -897
- package/dist/extensions/NEEDLE_opaque/index.d.ts +3 -3
- package/dist/extensions/NEEDLE_opaque/index.js +28 -28
- package/dist/extensions/NEEDLE_progressive/NEEDLE_progressive.d.ts +48 -48
- package/dist/extensions/NEEDLE_progressive/NEEDLE_progressive.js +69 -69
- package/dist/extensions/NEEDLE_progressive/index.d.ts +1 -1
- package/dist/extensions/NEEDLE_progressive/index.js +1 -1
- package/dist/extensions/NEEDLE_progressive_mesh_settings/index.d.ts +23 -23
- package/dist/extensions/NEEDLE_progressive_mesh_settings/index.js +41 -41
- package/dist/extensions/NEEDLE_progressive_texture_settings/index.d.ts +25 -25
- package/dist/extensions/NEEDLE_progressive_texture_settings/index.js +67 -67
- package/dist/extensions/index.d.ts +7 -7
- package/dist/extensions/index.js +17 -17
- package/dist/extensions/utils.d.ts +6 -6
- package/dist/extensions/utils.js +72 -72
- package/dist/functions/weld.3.d.ts +76 -76
- package/dist/functions/weld.3.js +414 -414
- package/dist/index.d.ts +8 -8
- package/dist/index.js +8 -8
- package/dist/scripts/index.d.ts +2 -2
- package/dist/scripts/index.js +2 -2
- package/dist/scripts/pack-gltf.d.ts +16 -16
- package/dist/scripts/pack-gltf.js +120 -120
- package/dist/scripts/stats.d.ts +2 -2
- package/dist/scripts/stats.js +18 -18
- package/dist/transforms/index.d.ts +9 -9
- package/dist/transforms/index.js +9 -9
- package/dist/transforms/needle_animation_transform.d.ts +2 -2
- package/dist/transforms/needle_animation_transform.js +5 -5
- package/dist/transforms/needle_asset.d.ts +3 -3
- package/dist/transforms/needle_asset.js +35 -35
- package/dist/transforms/needle_common.d.ts +16 -16
- package/dist/transforms/needle_common.js +1 -1
- package/dist/transforms/needle_mesh_transform.d.ts +10 -10
- package/dist/transforms/needle_mesh_transform.js +140 -140
- package/dist/transforms/needle_progressive.d.ts +42 -42
- package/dist/transforms/needle_progressive.js +933 -921
- package/dist/transforms/needle_texture_transform.d.ts +13 -13
- package/dist/transforms/needle_texture_transform.js +202 -196
- package/dist/transforms/needle_toktx.d.ts +25 -25
- package/dist/transforms/needle_toktx.js +230 -230
- package/dist/transforms/needle_webp.d.ts +13 -13
- package/dist/transforms/needle_webp.js +37 -37
- package/dist/transforms/toktx.d.ts +52 -52
- package/dist/transforms/toktx.js +221 -221
- package/dist/transforms/toktx_types.d.ts +78 -78
- package/dist/transforms/toktx_types.js +62 -62
- package/dist/transforms/util.d.ts +21 -21
- package/dist/transforms/util.js +90 -90
- package/dist/utils/compression_utils.d.ts +2 -2
- package/dist/utils/compression_utils.js +6 -6
- package/dist/utils/fileutils.d.ts +6 -6
- package/dist/utils/fileutils.js +141 -141
- package/dist/utils/guid.d.ts +1 -1
- package/dist/utils/guid.js +5 -5
- package/dist/utils/index.d.ts +10 -9
- package/dist/utils/index.js +10 -9
- package/dist/utils/merge.d.ts +12 -12
- package/dist/utils/merge.js +31 -31
- package/dist/utils/mesh.d.ts +13 -13
- package/dist/utils/mesh.js +107 -107
- package/dist/utils/nodeio.d.ts +5 -5
- package/dist/utils/nodeio.js +14 -14
- package/dist/utils/stats.d.ts +54 -53
- package/dist/utils/stats.js +191 -166
- package/dist/utils/texture.d.ts +2 -2
- package/dist/utils/texture.js +8 -8
- package/dist/utils/validate.d.ts +5 -0
- package/dist/utils/validate.js +16 -0
- package/dist/utils/version.d.ts +1 -1
- package/dist/utils/version.js +5 -5
- package/package.json +77 -76
- package/scripts/clear-caches.mjs +3 -3
- package/scripts/defaults.mjs +11 -11
- package/scripts/fileutils.mjs +5 -5
- package/scripts/index.mjs +41 -41
- package/scripts/limit-caches.mjs +3 -3
- package/scripts/make-progressive.mjs +16 -16
- package/scripts/pack-gltf.mjs +107 -107
- package/tsconfig.json +19 -19
|
@@ -1,230 +1,230 @@
|
|
|
1
|
-
import { BufferUtils, FileUtils, ImageUtils, uuid } from "@gltf-transform/core";
|
|
2
|
-
import { listTextureSlots, getTextureChannelMask, fitPowerOfTwo, TextureResizeFilter } from "@gltf-transform/functions";
|
|
3
|
-
import { KHRTextureBasisu } from "@gltf-transform/extensions";
|
|
4
|
-
import { checkKTXSoftware, createParams, KTX_SOFTWARE_VERSION_MIN, Mode } from "./toktx.js";
|
|
5
|
-
import { existsSync, mkdirSync } from "fs";
|
|
6
|
-
import { formatBytes, spawn, waitExit } from './util.js';
|
|
7
|
-
import { ETC1S_DEFAULTS, UASTC_DEFAULTS } from "./toktx_types.js";
|
|
8
|
-
import fs from 'fs/promises';
|
|
9
|
-
import os from 'os';
|
|
10
|
-
import tmp from 'tmp';
|
|
11
|
-
import pLimit from 'p-limit';
|
|
12
|
-
import { replaceExtension } from "../utils/fileutils.js";
|
|
13
|
-
import { ceilMultipleOfFour, isMultipleOfFour } from "../utils/index.js";
|
|
14
|
-
import sharp from "sharp";
|
|
15
|
-
const NUM_CPUS = os.cpus().length || 1; // microsoft/vscode#112122
|
|
16
|
-
let didLogEnvironmentInfo = false;
|
|
17
|
-
export class NeedleToktxContext {
|
|
18
|
-
limit;
|
|
19
|
-
basisuExtension;
|
|
20
|
-
version;
|
|
21
|
-
tmpPathBase;
|
|
22
|
-
document;
|
|
23
|
-
env;
|
|
24
|
-
numTextures;
|
|
25
|
-
_prepareFailed = false;
|
|
26
|
-
constructor(options) {
|
|
27
|
-
this.limit = pLimit(options.jobs || NUM_CPUS);
|
|
28
|
-
// allow overriding toktx installation location / adding additional search paths
|
|
29
|
-
const isUsingWindows = process.platform === 'win32';
|
|
30
|
-
this.env = {
|
|
31
|
-
PATH: process.env.PATH ?? ""
|
|
32
|
-
};
|
|
33
|
-
if (options?.toktxPath?.length && this.env) {
|
|
34
|
-
this.env.PATH = `${options.toktxPath};${this.env.PATH}`;
|
|
35
|
-
}
|
|
36
|
-
// Use global env variable
|
|
37
|
-
if (process.env.NEEDLE_TOKTX?.length) {
|
|
38
|
-
if (!existsSync(process.env.NEEDLE_TOKTX)) {
|
|
39
|
-
if (options.debug)
|
|
40
|
-
console.log("WARN: NEEDLE_TOKTX is set to a non-existing path: \"" + process.env.NEEDLE_TOKTX + "\"");
|
|
41
|
-
}
|
|
42
|
-
else {
|
|
43
|
-
if (options.debug)
|
|
44
|
-
console.log("INFO: Adding NEEDLE_TOKTX to PATH: \"" + process.env.NEEDLE_TOKTX + "\"");
|
|
45
|
-
this.env.PATH = `${process.env.NEEDLE_TOKTX};${this.env.PATH}`;
|
|
46
|
-
}
|
|
47
|
-
}
|
|
48
|
-
// fallback default installation directory:
|
|
49
|
-
if (isUsingWindows) {
|
|
50
|
-
const defaultPath = "C:\\Program Files\\KTX-Software\\bin";
|
|
51
|
-
if (existsSync(defaultPath))
|
|
52
|
-
this.env.PATH = `${this.env.PATH};${defaultPath}`;
|
|
53
|
-
}
|
|
54
|
-
// append default system paths
|
|
55
|
-
if (isUsingWindows && process.env.Path?.length)
|
|
56
|
-
this.env.PATH = `${this.env.PATH};${process.env.Path}`;
|
|
57
|
-
const ktxPaths = this.env.PATH.split(";").filter(p => p.toLocaleLowerCase().includes("ktx"));
|
|
58
|
-
if (options.debug) {
|
|
59
|
-
console.log("INFO: Found these paths including ktx:\n\t" + ktxPaths.join("\n\t") + "");
|
|
60
|
-
}
|
|
61
|
-
else if (ktxPaths?.length <= 0) {
|
|
62
|
-
if (didLogEnvironmentInfo === false)
|
|
63
|
-
console.log("INFO: Environment variable 'NEEDLE_TOKTX' not set. Use 'NEEDLE_TOKTX' to specify the directory of the toktx installation (path/to/bin)");
|
|
64
|
-
didLogEnvironmentInfo = true;
|
|
65
|
-
}
|
|
66
|
-
}
|
|
67
|
-
async prepare(document) {
|
|
68
|
-
try {
|
|
69
|
-
this.document = document;
|
|
70
|
-
const logger = document.getLogger();
|
|
71
|
-
this.version = await checkKTXSoftware(logger, this.env);
|
|
72
|
-
if (this.version === null) {
|
|
73
|
-
logger.warn(`WARN: toktx not found, skipping texture compression. Please install KTX-Software >= ${KTX_SOFTWARE_VERSION_MIN}`);
|
|
74
|
-
this._prepareFailed = true;
|
|
75
|
-
return false;
|
|
76
|
-
}
|
|
77
|
-
this.basisuExtension = document.createExtension(KHRTextureBasisu).setRequired(true);
|
|
78
|
-
const tmpDir = `${tmp.tmpdir}/gltf-transform`;
|
|
79
|
-
if (!(existsSync(tmpDir)))
|
|
80
|
-
mkdirSync(tmpDir);
|
|
81
|
-
this.tmpPathBase = tmpDir + "/" + uuid();
|
|
82
|
-
this.numTextures = document.getRoot().listTextures().length;
|
|
83
|
-
}
|
|
84
|
-
catch (e) {
|
|
85
|
-
console.error(e);
|
|
86
|
-
console.log("Error preparing toktx transform");
|
|
87
|
-
this._prepareFailed = true;
|
|
88
|
-
return false;
|
|
89
|
-
}
|
|
90
|
-
return true;
|
|
91
|
-
}
|
|
92
|
-
async process(index, texture, settings, context) {
|
|
93
|
-
const task = async () => {
|
|
94
|
-
if (this._prepareFailed)
|
|
95
|
-
return false;
|
|
96
|
-
const doc = this.document;
|
|
97
|
-
const logger = doc.getLogger();
|
|
98
|
-
const slots = listTextureSlots(texture);
|
|
99
|
-
let channels = getTextureChannelMask(texture);
|
|
100
|
-
// const textureLabel =
|
|
101
|
-
// texture.getURI() ||
|
|
102
|
-
// texture.getName() ||
|
|
103
|
-
// `${index + 1}/${doc.getRoot().listTextures().length}`;
|
|
104
|
-
const prefix = `toktx:texture[${index}] ${texture.getName()}`;
|
|
105
|
-
let srcMimeType = texture.getMimeType();
|
|
106
|
-
if (srcMimeType === 'image/ktx2') {
|
|
107
|
-
logger.debug(`${prefix}: Skipping, already KTX.`);
|
|
108
|
-
return;
|
|
109
|
-
}
|
|
110
|
-
else if (srcMimeType !== 'image/png' && srcMimeType !== 'image/jpeg') {
|
|
111
|
-
logger.warn(`${prefix}: Skipping, unsupported texture type "${texture.getMimeType()}".`);
|
|
112
|
-
return;
|
|
113
|
-
}
|
|
114
|
-
if (slots.length > 0)
|
|
115
|
-
logger.debug(`${prefix}: Slots → [${slots.join(', ')}]`);
|
|
116
|
-
let srcImage = texture.getImage();
|
|
117
|
-
let srcExtension = texture.getURI()
|
|
118
|
-
? FileUtils.extension(texture.getURI())
|
|
119
|
-
: ImageUtils.mimeTypeToExtension(texture.getMimeType());
|
|
120
|
-
const srcSize = texture.getSize();
|
|
121
|
-
const srcBytes = srcImage ? srcImage.byteLength : null;
|
|
122
|
-
if (!srcImage || !srcSize || !srcBytes) {
|
|
123
|
-
logger.warn(`${prefix}: Skipping, unreadable texture.`);
|
|
124
|
-
return;
|
|
125
|
-
}
|
|
126
|
-
if (!isMultipleOfFour(srcSize[0]) || !isMultipleOfFour(srcSize[1])) {
|
|
127
|
-
const limitInputPixels = true;
|
|
128
|
-
const instance = sharp(srcImage, { limitInputPixels }).toFormat('png');
|
|
129
|
-
const srcSize = ImageUtils.getSize(srcImage, srcMimeType);
|
|
130
|
-
const dstSize = fitPowerOfTwo(srcSize, 'nearest-pot');
|
|
131
|
-
dstSize[0] = ceilMultipleOfFour(dstSize[0]);
|
|
132
|
-
dstSize[1] = ceilMultipleOfFour(dstSize[1]);
|
|
133
|
-
logger.warn(`${prefix}: Resizing ${srcSize.join('x')} → ${dstSize.join('x')}px`);
|
|
134
|
-
instance.resize(dstSize[0], dstSize[1], { fit: 'fill', kernel: TextureResizeFilter.LANCZOS3 });
|
|
135
|
-
srcImage = BufferUtils.toView(await instance.toBuffer());
|
|
136
|
-
srcExtension = 'png';
|
|
137
|
-
srcMimeType = 'image/png';
|
|
138
|
-
}
|
|
139
|
-
const tmpName = this.tmpPathBase + "_" + index;
|
|
140
|
-
const inPath = tmpName + '.' + srcExtension;
|
|
141
|
-
const outPath = tmpName + '.ktx2';
|
|
142
|
-
// const inBytes = image.byteLength;
|
|
143
|
-
await fs.writeFile(inPath, Buffer.from(srcImage));
|
|
144
|
-
let mode = Mode.ETC1S;
|
|
145
|
-
let quality = 128;
|
|
146
|
-
let compression = 1;
|
|
147
|
-
if (settings) {
|
|
148
|
-
if (settings.mode === "auto") {
|
|
149
|
-
const useUASTC = slots.includes("normalTexture") || slots.includes("metallicRoughnessTexture");
|
|
150
|
-
mode = useUASTC ? Mode.UASTC : Mode.ETC1S;
|
|
151
|
-
// in auto-mode we use higher quality for normal and metallicRoughness textures
|
|
152
|
-
quality = useUASTC ? 255 : 128;
|
|
153
|
-
compression = useUASTC ? 4 : 1;
|
|
154
|
-
}
|
|
155
|
-
else {
|
|
156
|
-
mode = settings.mode === "ETC1S" ? Mode.ETC1S : Mode.UASTC;
|
|
157
|
-
}
|
|
158
|
-
}
|
|
159
|
-
else if (slots) {
|
|
160
|
-
const isNormal = slots.includes("normalTexture");
|
|
161
|
-
const isMetallicRoughness = slots.includes("metallicRoughnessTexture");
|
|
162
|
-
const betterQuality = isNormal || isMetallicRoughness;
|
|
163
|
-
mode = Mode.ETC1S;
|
|
164
|
-
// https://github.khronos.org/KTX-Software/ktxtools/toktx.html
|
|
165
|
-
quality = betterQuality ? 255 : 128;
|
|
166
|
-
// ETC1S / BasisLZ compression level, an encoding speed vs. quality tradeoff. Range is [0,5], default is 1. Higher values are slower but give higher quality.
|
|
167
|
-
compression = betterQuality ? 4 : 1;
|
|
168
|
-
// For the product usecase we use UASTC for meallicroughness and normal textures
|
|
169
|
-
if (context.config.usecase === "product") {
|
|
170
|
-
if (betterQuality) {
|
|
171
|
-
mode = Mode.UASTC;
|
|
172
|
-
}
|
|
173
|
-
quality = 255;
|
|
174
|
-
compression = 4;
|
|
175
|
-
}
|
|
176
|
-
}
|
|
177
|
-
const defaults = mode === Mode.ETC1S ? ETC1S_DEFAULTS : UASTC_DEFAULTS;
|
|
178
|
-
// TODO: should we ensure that the texture is using all channels? Unused channels are discarded anyways. The issue we have here is that textures that are solely used from pointers have unknown channels. We register them with RGBA when generating progressive assets as well as assign texture json pointers /textures/ to
|
|
179
|
-
// Alpha blended materials to make sure RGBA channels are used when progressive GLBs are compressed (in needle_progressive 377)
|
|
180
|
-
// https://linear.app/needle/issue/NE-4985
|
|
181
|
-
// channels = TextureChannel.R | TextureChannel.G | TextureChannel.B | TextureChannel.A;
|
|
182
|
-
//@ts-ignore
|
|
183
|
-
const toktxParams = createParams(texture, slots, channels, srcSize, logger, this.numTextures, { ...defaults, mode, quality, compression });
|
|
184
|
-
await fs.writeFile(inPath, Buffer.from(srcImage));
|
|
185
|
-
const params = [
|
|
186
|
-
'create',
|
|
187
|
-
...toktxParams,
|
|
188
|
-
inPath,
|
|
189
|
-
outPath,
|
|
190
|
-
];
|
|
191
|
-
logger.debug(`${prefix}: Spawning → toktx ${mode.toUpperCase()} ${params.join(' ')}`);
|
|
192
|
-
try {
|
|
193
|
-
// COMPRESS: Run `ktx create` CLI tool.
|
|
194
|
-
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
195
|
-
const [status, stdout, stderr] = await waitExit(spawn('ktx', params, { env: this.env }));
|
|
196
|
-
if (status !== 0) {
|
|
197
|
-
logger.error(`${prefix}: Failed → \n\n${stderr.toString()}`);
|
|
198
|
-
}
|
|
199
|
-
else {
|
|
200
|
-
const result = await fs.readFile(outPath);
|
|
201
|
-
texture.setImage(result).setMimeType('image/ktx2');
|
|
202
|
-
// TODO: we also need to remove the old file when the extension has changed - this should probably be done outside of this method to also support webp
|
|
203
|
-
// If this is a texture that was loaded from a file, we need to update the URI
|
|
204
|
-
if (texture.getURI()) {
|
|
205
|
-
const oldUri = texture.getURI();
|
|
206
|
-
const newUri = replaceExtension(oldUri, 'ktx2');
|
|
207
|
-
texture.setURI(newUri);
|
|
208
|
-
}
|
|
209
|
-
const dstBytes = texture.getImage().byteLength;
|
|
210
|
-
logger.debug(`${prefix}: ${formatBytes(srcBytes)} → ${formatBytes(dstBytes)}`);
|
|
211
|
-
return true;
|
|
212
|
-
}
|
|
213
|
-
}
|
|
214
|
-
catch (err) {
|
|
215
|
-
logger.error(`${prefix}: Toktx failed → ${err}`);
|
|
216
|
-
}
|
|
217
|
-
return false;
|
|
218
|
-
};
|
|
219
|
-
return this.limit(task);
|
|
220
|
-
}
|
|
221
|
-
async finalize(document) {
|
|
222
|
-
const usesKTX2 = document
|
|
223
|
-
.getRoot()
|
|
224
|
-
.listTextures()
|
|
225
|
-
.some((t) => t.getMimeType() === 'image/ktx2');
|
|
226
|
-
if (!usesKTX2) {
|
|
227
|
-
this.basisuExtension?.dispose();
|
|
228
|
-
}
|
|
229
|
-
}
|
|
230
|
-
}
|
|
1
|
+
import { BufferUtils, FileUtils, ImageUtils, uuid } from "@gltf-transform/core";
|
|
2
|
+
import { listTextureSlots, getTextureChannelMask, fitPowerOfTwo, TextureResizeFilter } from "@gltf-transform/functions";
|
|
3
|
+
import { KHRTextureBasisu } from "@gltf-transform/extensions";
|
|
4
|
+
import { checkKTXSoftware, createParams, KTX_SOFTWARE_VERSION_MIN, Mode } from "./toktx.js";
|
|
5
|
+
import { existsSync, mkdirSync } from "fs";
|
|
6
|
+
import { formatBytes, spawn, waitExit } from './util.js';
|
|
7
|
+
import { ETC1S_DEFAULTS, UASTC_DEFAULTS } from "./toktx_types.js";
|
|
8
|
+
import fs from 'fs/promises';
|
|
9
|
+
import os from 'os';
|
|
10
|
+
import tmp from 'tmp';
|
|
11
|
+
import pLimit from 'p-limit';
|
|
12
|
+
import { replaceExtension } from "../utils/fileutils.js";
|
|
13
|
+
import { ceilMultipleOfFour, isMultipleOfFour } from "../utils/index.js";
|
|
14
|
+
import sharp from "sharp";
|
|
15
|
+
const NUM_CPUS = os.cpus().length || 1; // microsoft/vscode#112122
|
|
16
|
+
let didLogEnvironmentInfo = false;
|
|
17
|
+
export class NeedleToktxContext {
|
|
18
|
+
limit;
|
|
19
|
+
basisuExtension;
|
|
20
|
+
version;
|
|
21
|
+
tmpPathBase;
|
|
22
|
+
document;
|
|
23
|
+
env;
|
|
24
|
+
numTextures;
|
|
25
|
+
_prepareFailed = false;
|
|
26
|
+
constructor(options) {
|
|
27
|
+
this.limit = pLimit(options.jobs || NUM_CPUS);
|
|
28
|
+
// allow overriding toktx installation location / adding additional search paths
|
|
29
|
+
const isUsingWindows = process.platform === 'win32';
|
|
30
|
+
this.env = {
|
|
31
|
+
PATH: process.env.PATH ?? ""
|
|
32
|
+
};
|
|
33
|
+
if (options?.toktxPath?.length && this.env) {
|
|
34
|
+
this.env.PATH = `${options.toktxPath};${this.env.PATH}`;
|
|
35
|
+
}
|
|
36
|
+
// Use global env variable
|
|
37
|
+
if (process.env.NEEDLE_TOKTX?.length) {
|
|
38
|
+
if (!existsSync(process.env.NEEDLE_TOKTX)) {
|
|
39
|
+
if (options.debug)
|
|
40
|
+
console.log("WARN: NEEDLE_TOKTX is set to a non-existing path: \"" + process.env.NEEDLE_TOKTX + "\"");
|
|
41
|
+
}
|
|
42
|
+
else {
|
|
43
|
+
if (options.debug)
|
|
44
|
+
console.log("INFO: Adding NEEDLE_TOKTX to PATH: \"" + process.env.NEEDLE_TOKTX + "\"");
|
|
45
|
+
this.env.PATH = `${process.env.NEEDLE_TOKTX};${this.env.PATH}`;
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
// fallback default installation directory:
|
|
49
|
+
if (isUsingWindows) {
|
|
50
|
+
const defaultPath = "C:\\Program Files\\KTX-Software\\bin";
|
|
51
|
+
if (existsSync(defaultPath))
|
|
52
|
+
this.env.PATH = `${this.env.PATH};${defaultPath}`;
|
|
53
|
+
}
|
|
54
|
+
// append default system paths
|
|
55
|
+
if (isUsingWindows && process.env.Path?.length)
|
|
56
|
+
this.env.PATH = `${this.env.PATH};${process.env.Path}`;
|
|
57
|
+
const ktxPaths = this.env.PATH.split(";").filter(p => p.toLocaleLowerCase().includes("ktx"));
|
|
58
|
+
if (options.debug) {
|
|
59
|
+
console.log("INFO: Found these paths including ktx:\n\t" + ktxPaths.join("\n\t") + "");
|
|
60
|
+
}
|
|
61
|
+
else if (ktxPaths?.length <= 0) {
|
|
62
|
+
if (didLogEnvironmentInfo === false)
|
|
63
|
+
console.log("INFO: Environment variable 'NEEDLE_TOKTX' not set. Use 'NEEDLE_TOKTX' to specify the directory of the toktx installation (path/to/bin)");
|
|
64
|
+
didLogEnvironmentInfo = true;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
async prepare(document) {
|
|
68
|
+
try {
|
|
69
|
+
this.document = document;
|
|
70
|
+
const logger = document.getLogger();
|
|
71
|
+
this.version = await checkKTXSoftware(logger, this.env);
|
|
72
|
+
if (this.version === null) {
|
|
73
|
+
logger.warn(`WARN: toktx not found, skipping texture compression. Please install KTX-Software >= ${KTX_SOFTWARE_VERSION_MIN}`);
|
|
74
|
+
this._prepareFailed = true;
|
|
75
|
+
return false;
|
|
76
|
+
}
|
|
77
|
+
this.basisuExtension = document.createExtension(KHRTextureBasisu).setRequired(true);
|
|
78
|
+
const tmpDir = `${tmp.tmpdir}/gltf-transform`;
|
|
79
|
+
if (!(existsSync(tmpDir)))
|
|
80
|
+
mkdirSync(tmpDir);
|
|
81
|
+
this.tmpPathBase = tmpDir + "/" + uuid();
|
|
82
|
+
this.numTextures = document.getRoot().listTextures().length;
|
|
83
|
+
}
|
|
84
|
+
catch (e) {
|
|
85
|
+
console.error(e);
|
|
86
|
+
console.log("Error preparing toktx transform");
|
|
87
|
+
this._prepareFailed = true;
|
|
88
|
+
return false;
|
|
89
|
+
}
|
|
90
|
+
return true;
|
|
91
|
+
}
|
|
92
|
+
async process(index, texture, settings, context) {
|
|
93
|
+
const task = async () => {
|
|
94
|
+
if (this._prepareFailed)
|
|
95
|
+
return false;
|
|
96
|
+
const doc = this.document;
|
|
97
|
+
const logger = doc.getLogger();
|
|
98
|
+
const slots = listTextureSlots(texture);
|
|
99
|
+
let channels = getTextureChannelMask(texture);
|
|
100
|
+
// const textureLabel =
|
|
101
|
+
// texture.getURI() ||
|
|
102
|
+
// texture.getName() ||
|
|
103
|
+
// `${index + 1}/${doc.getRoot().listTextures().length}`;
|
|
104
|
+
const prefix = `toktx:texture[${index}] ${texture.getName()}`;
|
|
105
|
+
let srcMimeType = texture.getMimeType();
|
|
106
|
+
if (srcMimeType === 'image/ktx2') {
|
|
107
|
+
logger.debug(`${prefix}: Skipping, already KTX.`);
|
|
108
|
+
return;
|
|
109
|
+
}
|
|
110
|
+
else if (srcMimeType !== 'image/png' && srcMimeType !== 'image/jpeg') {
|
|
111
|
+
logger.warn(`${prefix}: Skipping, unsupported texture type "${texture.getMimeType()}".`);
|
|
112
|
+
return;
|
|
113
|
+
}
|
|
114
|
+
if (slots.length > 0)
|
|
115
|
+
logger.debug(`${prefix}: Slots → [${slots.join(', ')}]`);
|
|
116
|
+
let srcImage = texture.getImage();
|
|
117
|
+
let srcExtension = texture.getURI()
|
|
118
|
+
? FileUtils.extension(texture.getURI())
|
|
119
|
+
: ImageUtils.mimeTypeToExtension(texture.getMimeType());
|
|
120
|
+
const srcSize = texture.getSize();
|
|
121
|
+
const srcBytes = srcImage ? srcImage.byteLength : null;
|
|
122
|
+
if (!srcImage || !srcSize || !srcBytes) {
|
|
123
|
+
logger.warn(`${prefix}: Skipping, unreadable texture.`);
|
|
124
|
+
return;
|
|
125
|
+
}
|
|
126
|
+
if (!isMultipleOfFour(srcSize[0]) || !isMultipleOfFour(srcSize[1])) {
|
|
127
|
+
const limitInputPixels = true;
|
|
128
|
+
const instance = sharp(srcImage, { limitInputPixels }).toFormat('png');
|
|
129
|
+
const srcSize = ImageUtils.getSize(srcImage, srcMimeType);
|
|
130
|
+
const dstSize = fitPowerOfTwo(srcSize, 'nearest-pot');
|
|
131
|
+
dstSize[0] = ceilMultipleOfFour(dstSize[0]);
|
|
132
|
+
dstSize[1] = ceilMultipleOfFour(dstSize[1]);
|
|
133
|
+
logger.warn(`${prefix}: Resizing ${srcSize.join('x')} → ${dstSize.join('x')}px`);
|
|
134
|
+
instance.resize(dstSize[0], dstSize[1], { fit: 'fill', kernel: TextureResizeFilter.LANCZOS3 });
|
|
135
|
+
srcImage = BufferUtils.toView(await instance.toBuffer());
|
|
136
|
+
srcExtension = 'png';
|
|
137
|
+
srcMimeType = 'image/png';
|
|
138
|
+
}
|
|
139
|
+
const tmpName = this.tmpPathBase + "_" + index;
|
|
140
|
+
const inPath = tmpName + '.' + srcExtension;
|
|
141
|
+
const outPath = tmpName + '.ktx2';
|
|
142
|
+
// const inBytes = image.byteLength;
|
|
143
|
+
await fs.writeFile(inPath, Buffer.from(srcImage));
|
|
144
|
+
let mode = Mode.ETC1S;
|
|
145
|
+
let quality = 128;
|
|
146
|
+
let compression = 1;
|
|
147
|
+
if (settings) {
|
|
148
|
+
if (settings.mode === "auto") {
|
|
149
|
+
const useUASTC = slots.includes("normalTexture") || slots.includes("metallicRoughnessTexture");
|
|
150
|
+
mode = useUASTC ? Mode.UASTC : Mode.ETC1S;
|
|
151
|
+
// in auto-mode we use higher quality for normal and metallicRoughness textures
|
|
152
|
+
quality = useUASTC ? 255 : 128;
|
|
153
|
+
compression = useUASTC ? 4 : 1;
|
|
154
|
+
}
|
|
155
|
+
else {
|
|
156
|
+
mode = settings.mode === "ETC1S" ? Mode.ETC1S : Mode.UASTC;
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
else if (slots) {
|
|
160
|
+
const isNormal = slots.includes("normalTexture");
|
|
161
|
+
const isMetallicRoughness = slots.includes("metallicRoughnessTexture");
|
|
162
|
+
const betterQuality = isNormal || isMetallicRoughness;
|
|
163
|
+
mode = Mode.ETC1S;
|
|
164
|
+
// https://github.khronos.org/KTX-Software/ktxtools/toktx.html
|
|
165
|
+
quality = betterQuality ? 255 : 128;
|
|
166
|
+
// ETC1S / BasisLZ compression level, an encoding speed vs. quality tradeoff. Range is [0,5], default is 1. Higher values are slower but give higher quality.
|
|
167
|
+
compression = betterQuality ? 4 : 1;
|
|
168
|
+
// For the product usecase we use UASTC for meallicroughness and normal textures
|
|
169
|
+
if (context.config.usecase === "product") {
|
|
170
|
+
if (betterQuality) {
|
|
171
|
+
mode = Mode.UASTC;
|
|
172
|
+
}
|
|
173
|
+
quality = 255;
|
|
174
|
+
compression = 4;
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
const defaults = mode === Mode.ETC1S ? ETC1S_DEFAULTS : UASTC_DEFAULTS;
|
|
178
|
+
// TODO: should we ensure that the texture is using all channels? Unused channels are discarded anyways. The issue we have here is that textures that are solely used from pointers have unknown channels. We register them with RGBA when generating progressive assets as well as assign texture json pointers /textures/ to
|
|
179
|
+
// Alpha blended materials to make sure RGBA channels are used when progressive GLBs are compressed (in needle_progressive 377)
|
|
180
|
+
// https://linear.app/needle/issue/NE-4985
|
|
181
|
+
// channels = TextureChannel.R | TextureChannel.G | TextureChannel.B | TextureChannel.A;
|
|
182
|
+
//@ts-ignore
|
|
183
|
+
const toktxParams = createParams(texture, slots, channels, srcSize, logger, this.numTextures, { ...defaults, mode, quality, compression });
|
|
184
|
+
await fs.writeFile(inPath, Buffer.from(srcImage));
|
|
185
|
+
const params = [
|
|
186
|
+
'create',
|
|
187
|
+
...toktxParams,
|
|
188
|
+
inPath,
|
|
189
|
+
outPath,
|
|
190
|
+
];
|
|
191
|
+
logger.debug(`${prefix}: Spawning → toktx ${mode.toUpperCase()} ${params.join(' ')}`);
|
|
192
|
+
try {
|
|
193
|
+
// COMPRESS: Run `ktx create` CLI tool.
|
|
194
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
195
|
+
const [status, stdout, stderr] = await waitExit(spawn('ktx', params, { env: this.env }));
|
|
196
|
+
if (status !== 0) {
|
|
197
|
+
logger.error(`${prefix}: Failed → \n\n${stderr.toString()}`);
|
|
198
|
+
}
|
|
199
|
+
else {
|
|
200
|
+
const result = await fs.readFile(outPath);
|
|
201
|
+
texture.setImage(result).setMimeType('image/ktx2');
|
|
202
|
+
// TODO: we also need to remove the old file when the extension has changed - this should probably be done outside of this method to also support webp
|
|
203
|
+
// If this is a texture that was loaded from a file, we need to update the URI
|
|
204
|
+
if (texture.getURI()) {
|
|
205
|
+
const oldUri = texture.getURI();
|
|
206
|
+
const newUri = replaceExtension(oldUri, 'ktx2');
|
|
207
|
+
texture.setURI(newUri);
|
|
208
|
+
}
|
|
209
|
+
const dstBytes = texture.getImage().byteLength;
|
|
210
|
+
logger.debug(`${prefix}: ${formatBytes(srcBytes)} → ${formatBytes(dstBytes)}`);
|
|
211
|
+
return true;
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
catch (err) {
|
|
215
|
+
logger.error(`${prefix}: Toktx failed → ${err}`);
|
|
216
|
+
}
|
|
217
|
+
return false;
|
|
218
|
+
};
|
|
219
|
+
return this.limit(task);
|
|
220
|
+
}
|
|
221
|
+
async finalize(document) {
|
|
222
|
+
const usesKTX2 = document
|
|
223
|
+
.getRoot()
|
|
224
|
+
.listTextures()
|
|
225
|
+
.some((t) => t.getMimeType() === 'image/ktx2');
|
|
226
|
+
if (!usesKTX2) {
|
|
227
|
+
this.basisuExtension?.dispose();
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
}
|
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
import { Document, ILogger, Texture } from '@gltf-transform/core';
|
|
2
|
-
import { NEEDLE_compression_texture_schema } from '../extensions/index.js';
|
|
3
|
-
import { INeedleTextureTransformContext, NeedleTransformContext } from './needle_common.js';
|
|
4
|
-
export interface NeedleWebPOptions {
|
|
5
|
-
}
|
|
6
|
-
export declare class NeedleWebPContext implements INeedleTextureTransformContext {
|
|
7
|
-
options: NeedleWebPOptions;
|
|
8
|
-
logger: ILogger;
|
|
9
|
-
constructor(options: NeedleWebPOptions);
|
|
10
|
-
prepare(document: Document): Promise<void>;
|
|
11
|
-
process(index: number, texture: Texture, settings: Pick<NEEDLE_compression_texture_schema, "quality">, context: NeedleTransformContext): Promise<boolean>;
|
|
12
|
-
finalize(document: Document): Promise<void>;
|
|
13
|
-
}
|
|
1
|
+
import { Document, ILogger, Texture } from '@gltf-transform/core';
|
|
2
|
+
import { NEEDLE_compression_texture_schema } from '../extensions/index.js';
|
|
3
|
+
import { INeedleTextureTransformContext, NeedleTransformContext } from './needle_common.js';
|
|
4
|
+
export interface NeedleWebPOptions {
|
|
5
|
+
}
|
|
6
|
+
export declare class NeedleWebPContext implements INeedleTextureTransformContext {
|
|
7
|
+
options: NeedleWebPOptions;
|
|
8
|
+
logger: ILogger;
|
|
9
|
+
constructor(options: NeedleWebPOptions);
|
|
10
|
+
prepare(document: Document): Promise<void>;
|
|
11
|
+
process(index: number, texture: Texture, settings: Pick<NEEDLE_compression_texture_schema, "quality">, context: NeedleTransformContext): Promise<boolean>;
|
|
12
|
+
finalize(document: Document): Promise<void>;
|
|
13
|
+
}
|
|
@@ -1,37 +1,37 @@
|
|
|
1
|
-
import sharp from 'sharp';
|
|
2
|
-
import { BufferUtils, HTTPUtils } from '@gltf-transform/core';
|
|
3
|
-
export class NeedleWebPContext {
|
|
4
|
-
options;
|
|
5
|
-
logger;
|
|
6
|
-
constructor(options) {
|
|
7
|
-
this.options = options;
|
|
8
|
-
}
|
|
9
|
-
async prepare(document) {
|
|
10
|
-
this.logger = document.getLogger();
|
|
11
|
-
}
|
|
12
|
-
async process(index, texture, settings, context) {
|
|
13
|
-
const options = {
|
|
14
|
-
quality: settings.quality,
|
|
15
|
-
// effort: remap(options.effort, 100, 6),
|
|
16
|
-
// lossless: options.lossless,
|
|
17
|
-
// nearLossless: options.nearLossless,
|
|
18
|
-
};
|
|
19
|
-
if (typeof options !== "string" && options.quality >= 0) {
|
|
20
|
-
// the quality range is 0-1 so we remap to 0-100
|
|
21
|
-
options.quality = Math.round(settings.quality * 100);
|
|
22
|
-
}
|
|
23
|
-
if (options.quality <= 0) {
|
|
24
|
-
options.quality = 1;
|
|
25
|
-
}
|
|
26
|
-
// console.log(options);
|
|
27
|
-
this.logger.info(`→ Will compress texture ${texture.getName()} to webp with quality ${options.quality}`);
|
|
28
|
-
const buffer = await sharp(texture.getImage()).toFormat('webp', options).toBuffer();
|
|
29
|
-
texture
|
|
30
|
-
.setMimeType('image/webp')
|
|
31
|
-
.setImage(BufferUtils.toView(buffer))
|
|
32
|
-
.setURI(HTTPUtils.basename(texture.getURI()) + '.webp');
|
|
33
|
-
return true;
|
|
34
|
-
}
|
|
35
|
-
async finalize(document) {
|
|
36
|
-
}
|
|
37
|
-
}
|
|
1
|
+
import sharp from 'sharp';
|
|
2
|
+
import { BufferUtils, HTTPUtils } from '@gltf-transform/core';
|
|
3
|
+
export class NeedleWebPContext {
|
|
4
|
+
options;
|
|
5
|
+
logger;
|
|
6
|
+
constructor(options) {
|
|
7
|
+
this.options = options;
|
|
8
|
+
}
|
|
9
|
+
async prepare(document) {
|
|
10
|
+
this.logger = document.getLogger();
|
|
11
|
+
}
|
|
12
|
+
async process(index, texture, settings, context) {
|
|
13
|
+
const options = {
|
|
14
|
+
quality: settings.quality,
|
|
15
|
+
// effort: remap(options.effort, 100, 6),
|
|
16
|
+
// lossless: options.lossless,
|
|
17
|
+
// nearLossless: options.nearLossless,
|
|
18
|
+
};
|
|
19
|
+
if (typeof options !== "string" && options.quality >= 0) {
|
|
20
|
+
// the quality range is 0-1 so we remap to 0-100
|
|
21
|
+
options.quality = Math.round(settings.quality * 100);
|
|
22
|
+
}
|
|
23
|
+
if (options.quality <= 0) {
|
|
24
|
+
options.quality = 1;
|
|
25
|
+
}
|
|
26
|
+
// console.log(options);
|
|
27
|
+
this.logger.info(`→ Will compress texture ${texture.getName()} to webp with quality ${options.quality}`);
|
|
28
|
+
const buffer = await sharp(texture.getImage()).toFormat('webp', options).toBuffer();
|
|
29
|
+
texture
|
|
30
|
+
.setMimeType('image/webp')
|
|
31
|
+
.setImage(BufferUtils.toView(buffer))
|
|
32
|
+
.setURI(HTTPUtils.basename(texture.getURI()) + '.webp');
|
|
33
|
+
return true;
|
|
34
|
+
}
|
|
35
|
+
async finalize(document) {
|
|
36
|
+
}
|
|
37
|
+
}
|