@needle-tools/gltf-build-pipeline 0.1.0

package/CHANGELOG.md

@@ -0,0 +1,140 @@
+# Changelog
+All notable changes to this package will be documented in this file.
+
+The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
+and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
+
+## [1.1.3-alpha] - 2023-04-04
+- change: file utils use needle.config.json to find files if no directory was specified
+
+## [1.1.2-alpha] - 2023-04-04
+- change: add generator info
+- change: try catch toktx exception
+- change: improve relative path handling
+
+## [1.1.1-alpha] - 2023-04-04
+- change: use env variable set by npm to resolve relative paths
+
+## [1.1.0-alpha.1] - 2023-04-04
+- add: option to disallow toktx using `no-toktx` commandline parameter
+
+## [1.0.0-alpha] - 2023-04-03
+- update to gltf-transform 3.1.4
+
+## [0.11.2-pre] - 2023-01-30
+- fix: clean up logic for finding toktx to only use one approach
+
+## [0.11.1-pre] - 2023-01-27
+- fix: getting correct gltf transform texture for texture definition
+- fix: opaque extension now also uses getTexture util method
+
+## [0.11.0-pre] - 2023-01-25
+- add: script `transform` does run progressive and gltf-pack
+- change: script `transform:progressive` can now process all files in passed in directory
+
+## [0.10.10-pre] - 2023-01-25
+- add: `NEEDLE_TOKTX` environment variable
+- fix: env variable handling
+
+## [0.10.9-pre] - 2023-01-20
+- fix: set mime-type to KTX2 when using cached texture data that was processed with default toktx settings
+
+## [0.10.8-pre] - 2023-01-18
+- fix: pack-gltf should run toktx with default settings when no compression extension was found in texture
+
+## [0.10.7-pre] - 2023-01-18
+- fix: pack-gltf when invoked with single glb path
+
+## [0.10.6-pre.1] - 2023-01-13
+- fix: support for glbs using old texture pointer format ``textures/`` (new format is ``/textures/``)
+
+## [0.10.5-pre] - 2023-01-12
+- fix: pack-gltf running in sequence again to avoid wasm memory error with many glbs
+
+## [0.10.4-pre] - 2023-01-11
+- change: simplify mesh logs `lockBorder` setting
+- change: pack-gltf now searches glb files in subdirectories
+
+## [0.10.3-pre] - 2023-01-09
+- fix: issue with pack-gltf running out of memory when invoked for directory with many glbs
+- change: await squoosh pool close
+- change: disable extension read-write logs by default
+
+## [0.10.2-pre] - 2023-01-07
+- add: mesh simplification now welds vertices with tiny threshold to be able to simplify disjointed meshes 
+- change: progressive loading handling of unknown slot, dont throw for unknown texture slot but instead log warning and proceed without making the texture progressively loaded
+- change: update gltf-transform dependency to 2.5.1
+
+## [0.10.1-pre] - 2023-01-04
+- fix: error in progressive texture resize where `maxSize` could be 0
+
+## [0.10.0-pre] - 2023-01-04
+- add: caching for texture transforms
+- change: use one needle_transform function to handle compressions of textures in one loop instead of mulitple toktx and squoosh transform methods
+
+## [0.9.1-pre] - 2023-01-03
+- add: opaque extension support for primitive extensions to support mesh compression settings passed through when using progressive transformations
+
+## [0.9.0-pre] - 2023-01-03
+- add: ``NEEDLE_mesh_compression`` support for running gltf-transform simplify per primitive
+
+## [0.8.1-pre.1] - 2022-12-14
+- add: quality option to ``NEEDLE_compression_texture`` for webp compression
+- fix: progressive glb should always use name from guid and index
+
+## [0.8.0-pre] - 2022-12-13
+- add: webp compression mode
+- change: progressive texture glbs use guid as filename
+- change: pack-gltf can now take directory as argument
+
+## [0.7.1-pre] - 2022-12-08
+- fix: processing gltf multiple times did cause texture json pointers being resolved wrongly including loosing previously set extensions
+
+## [0.7.0-pre] - 2022-12-05
+- add: make-progressive tool to transform an existing glbs and resize images with high res images being loaded from external glbs
+
+## [0.6.4-pre] - 2022-12-03
+- add: compression mode ``none`` to skip toktx compression
+- fix: issue with texture pointer not finding previously assigned sampler when writing
+- fix: if commandline parsing fails take path containing ``KTX-Software`` as additional toktx search path
+
+## [0.6.3-pre] - 2022-10-27
+- fix: options.file nullreference error
+
+## [0.6.2-pre] - 2022-10-20
+- change: toktx temp folder adjusted, use uuid for shorter names, generate uuid once per file [PR](https://github.com/donmccurdy/glTF-Transform/pull/708)
+
+## [0.6.1-pre] - 2022-10-20
+- fix: NEEDLE_compression_settings havent been passed to toktx anymore
+
+## [0.6.0-pre] - 2022-10-19
+- add: generate toktx textures in ``glTF-transform`` subdirectory, input and output textures having the same random name and are prefixed with the texture index
+
+## [0.5.1-pre] - 2022-10-12
+- fix texture extensions surviving multiple compressions (e.g. compression same glb multiple times)
+- fix cli args when invoked via npm run
+
+## [0.5.0-pre] - 2022-10-11
+- add draco support again
+
+## [0.4.1-pre] - 2022-10-11
+- fix texture extension pass-through
+
+## [0.4.0-pre] - 2022-10-11
+- update toktx to gltf-Transform v2.4.4
+
+## [0.3.0-pre] - 2022-10-06
+- add: handle NEEDLE_compression_texture extension and API to toktx to control cli arguments per texture 
+
+## [0.2.2-pre] - 2022-10-03
+- add: allow setting ``--toktxPath`` from commandline on windows
+
+## [0.2.1-pre] - 2022-10-03
+- fix bug where EXT_texture_exr extension would be inserted in all textures
+
+## [0.2.0-pre] - 2022-10-02
+- handle KHR_animation_pointer
+- handle texture json pointers
+
+## [0.1.3-pre] - 2022-09-26
+- initial release

package/clear-caches.mjs

@@ -0,0 +1,4 @@
+
+import { clearCache } from './dist/cache/index.js';
+
+clearCache();

package/defaults.mjs

@@ -0,0 +1,12 @@
+import { ALL_EXTENSIONS } from '@gltf-transform/extensions';
+import { ALL_EXTENSIONS as NEEDLE_EXTENSIONS } from "./dist/extensions/NEEDLE_opaque/index.js";
+import { NEEDLE_progressive_texture } from './dist/extensions/NEEDLE_progressive_texture/index.js';
+
+
+export function registerExtensions(node) {
+    node.registerExtensions(ALL_EXTENSIONS);
+    node.registerExtensions(NEEDLE_EXTENSIONS);
+    node.registerExtensions([
+        NEEDLE_progressive_texture
+    ])
+}

package/dist/cache/cache.d.ts

@@ -0,0 +1,5 @@
+import { ExtensibleProperty } from "@gltf-transform/core";
+export declare function clearCache(): void;
+export declare function addToCache(key: string, bytes: Uint8Array): void;
+export declare function tryGetFromCache(key: string): Uint8Array | null;
+export declare function getKey(property: ExtensibleProperty): string;

package/dist/cache/cache.js

@@ -0,0 +1,82 @@
+import { Texture, Primitive } from "@gltf-transform/core";
+import tmp from 'tmp';
+import fs from 'fs';
+import path from 'path';
+export function clearCache() {
+    const dir = getCacheDirectory();
+    fs.rmSync(dir, { recursive: true });
+    console.log(`Cleared cache at ${dir}`);
+}
+export function addToCache(key, bytes) {
+    const dir = getCacheDirectory();
+    const path = `${dir}/${key}.bin`;
+    console.log(`Added ${(bytes.byteLength / 1024).toFixed(1)} KB to cache at ${path}`);
+    fs.writeFileSync(path, bytes);
+}
+export function tryGetFromCache(key) {
+    const dir = getCacheDirectory();
+    const path = `${dir}/${key}.bin`;
+    if (fs.existsSync(path)) {
+        const bytes = fs.readFileSync(path);
+        return bytes;
+    }
+    return null;
+}
+export function getKey(property) {
+    let hash = 0;
+    let prefix = "prop";
+    const extensions = property.listExtensions();
+    const extras = property.getExtras();
+    // const slots = listTextureSlots(doc, texture);
+    // const channels = getTextureChannelMask(doc, texture);
+    hash += hashObject({
+        extensions,
+        extras,
+    });
+    if (property instanceof Texture) {
+        prefix = "image";
+        const image = property.getImage();
+        const bytes = image?.buffer;
+        if (bytes)
+            hash += hashBuffer(bytes, image.byteOffset, image.byteLength);
+    }
+    else if (property instanceof Primitive) {
+        prefix = "prim";
+        const indices = property.getIndices();
+        const attributes = property.listAttributes();
+        const targets = property.listTargets();
+        hash += hashObject({
+            indices,
+            attributes,
+            targets
+        });
+    }
+    let key = hash.toString();
+    key = `${prefix}-${key}`;
+    return key;
+}
+function getCacheDirectory() {
+    const tmpDir = path.join(tmp.tmpdir, "Needle", "build-pipeline-cache");
+    if (!fs.existsSync(tmpDir)) {
+        fs.mkdirSync(tmpDir, { recursive: true });
+    }
+    return tmpDir;
+}
+function hashObject(obj) {
+    const str = JSON.stringify(obj);
+    let hash = 0;
+    for (let i = 0; i < str.length; i++) {
+        hash = ((hash << 5) - hash) + str.charCodeAt(i);
+        hash |= 0;
+    }
+    return hash;
+}
+function hashBuffer(buffer, offset, length) {
+    let hash = 0;
+    const view = new Uint8Array(buffer, offset, length);
+    for (let i = 0; i < view.length; i++) {
+        hash = ((hash << 5) - hash) + view[i];
+        hash |= 0;
+    }
+    return hash;
+}

package/dist/cache/index.d.ts

@@ -0,0 +1 @@
+export * from "./cache.js";

package/dist/cache/index.js

@@ -0,0 +1 @@
+export * from "./cache.js";

package/dist/extensions/NEEDLE_compression_texture/NEEDLE_compression_texture.d.ts

@@ -0,0 +1,24 @@
+import { Extension, ExtensionProperty, PropertyType, ReaderContext, WriterContext } from "@gltf-transform/core";
+export declare const EXTENSION_NAME = "NEEDLE_compression_texture";
+export declare interface NEEDLE_compression_texture_schema {
+    mode: string;
+    slot: string;
+    /** quality in range of 0 - 1 */
+    quality: number;
+    maxSize: number;
+}
+export declare class NEEDLE_compression_texture extends Extension {
+    static EXTENSION_NAME: string;
+    extensionName: string;
+    create(toktxInfo: NEEDLE_compression_texture_schema): NEEDLE_compression_texture_property;
+    read(context: ReaderContext): this;
+    write(context: WriterContext): this;
+}
+export declare class NEEDLE_compression_texture_property extends ExtensionProperty {
+    static EXTENSION_NAME: string;
+    extensionName: string;
+    parentTypes: PropertyType[];
+    propertyType: any;
+    init(): void;
+    model: NEEDLE_compression_texture_schema;
+}

package/dist/extensions/NEEDLE_compression_texture/NEEDLE_compression_texture.js

@@ -0,0 +1,54 @@
+import { Extension, ExtensionProperty, PropertyType } from "@gltf-transform/core";
+import { getTextureDefinition } from "../utils.js";
+export const EXTENSION_NAME = "NEEDLE_compression_texture";
+const debugLog = false;
+export class NEEDLE_compression_texture extends Extension {
+    static EXTENSION_NAME = EXTENSION_NAME;
+    extensionName = EXTENSION_NAME;
+    create(toktxInfo) {
+        const ext = new NEEDLE_compression_texture_property(this.document.getGraph(), "");
+        ext.model = toktxInfo;
+        return ext;
+    }
+    read(context) {
+        if (!context.jsonDoc.json.textures)
+            return this;
+        if (debugLog)
+            console.log("READ " + EXTENSION_NAME);
+        context.textures?.forEach((tex, _) => {
+            const textureDefinition = getTextureDefinition(context, tex);
+            if (textureDefinition?.extensions && textureDefinition.extensions[EXTENSION_NAME]) {
+                const prop = new NEEDLE_compression_texture_property(this.document.getGraph(), EXTENSION_NAME);
+                prop.model = textureDefinition.extensions[EXTENSION_NAME];
+                tex.setExtension(EXTENSION_NAME, prop);
+            }
+        });
+        return this;
+    }
+    write(context) {
+        if (debugLog)
+            console.log("WRITE " + EXTENSION_NAME);
+        const root = this.document.getRoot();
+        const textures = root.listTextures();
+        for (const tex of textures) {
+            const ext = tex.getExtension(EXTENSION_NAME);
+            if (ext) {
+                const def = getTextureDefinition(context, tex);
+                if (def) {
+                    def.extensions = def.extensions || {};
+                    def.extensions[EXTENSION_NAME] = ext.model;
+                }
+            }
+        }
+        return this;
+    }
+}
+export class NEEDLE_compression_texture_property extends ExtensionProperty {
+    static EXTENSION_NAME = EXTENSION_NAME;
+    init() {
+        this.extensionName = EXTENSION_NAME;
+        this.parentTypes = [PropertyType.TEXTURE];
+        this.propertyType = EXTENSION_NAME;
+    }
+    model;
+}

package/dist/extensions/NEEDLE_compression_texture/index.d.ts

@@ -0,0 +1 @@
+export * from "./NEEDLE_compression_texture.js";

package/dist/extensions/NEEDLE_compression_texture/index.js

@@ -0,0 +1 @@
+export * from "./NEEDLE_compression_texture.js";

package/dist/extensions/NEEDLE_mesh_compression/NEEDLE_mesh_compression.d.ts

@@ -0,0 +1,23 @@
+import { Extension, ExtensionProperty, PropertyType, ReaderContext } from "@gltf-transform/core";
+declare interface NEEDLE_mesh_compression_model {
+    useSimplifier: boolean;
+    error: number;
+    ratio: number;
+    lockBorder: boolean;
+}
+export declare class NEEDLE_mesh_compression extends Extension {
+    static EXTENSION_NAME: string;
+    extensionName: string;
+    create(model: NEEDLE_mesh_compression_model): NEEDLE_mesh_compression_property;
+    read(context: ReaderContext): this;
+    write(ctx: any): this;
+}
+export declare class NEEDLE_mesh_compression_property extends ExtensionProperty {
+    static EXTENSION_NAME: string;
+    readonly extensionName: string;
+    parentTypes: PropertyType[];
+    propertyType: string;
+    init(): void;
+    model: NEEDLE_mesh_compression_model;
+}
+export {};

package/dist/extensions/NEEDLE_mesh_compression/NEEDLE_mesh_compression.js

@@ -0,0 +1,43 @@
+import { Extension, ExtensionProperty, PropertyType } from "@gltf-transform/core";
+const EXTENSION_NAME = "NEEDLE_mesh_compression";
+const debugLog = false;
+export class NEEDLE_mesh_compression extends Extension {
+    static EXTENSION_NAME = EXTENSION_NAME;
+    extensionName = EXTENSION_NAME;
+    create(model) {
+        const ext = new NEEDLE_mesh_compression_property(this.document.getGraph(), "");
+        ext.model = model;
+        return ext;
+    }
+    read(context) {
+        if (!context.jsonDoc.json.meshes)
+            return;
+        if (debugLog)
+            console.log("READ " + EXTENSION_NAME);
+        context.meshes?.forEach((mesh, index) => {
+            const meshDefinition = context.jsonDoc.json.meshes[index];
+            if (!meshDefinition)
+                return;
+            const primitives = mesh.listPrimitives();
+            for (let i = 0; i < primitives.length; i++) {
+                const primitive = meshDefinition.primitives[i];
+                if (primitive.extensions && primitive.extensions[EXTENSION_NAME]) {
+                    const model = primitive.extensions[EXTENSION_NAME];
+                    primitives[i].setExtension(EXTENSION_NAME, this.create(model));
+                }
+            }
+        });
+        return this;
+    }
+    write(ctx) {
+        return this;
+    }
+}
+export class NEEDLE_mesh_compression_property extends ExtensionProperty {
+    static EXTENSION_NAME = EXTENSION_NAME;
+    extensionName = EXTENSION_NAME;
+    parentTypes = [PropertyType.PRIMITIVE];
+    propertyType = EXTENSION_NAME;
+    init() { }
+    model;
+}

package/dist/extensions/NEEDLE_mesh_compression/index.d.ts

@@ -0,0 +1 @@
+export * from "./NEEDLE_mesh_compression.js";

package/dist/extensions/NEEDLE_mesh_compression/index.js

@@ -0,0 +1 @@
+export * from "./NEEDLE_mesh_compression.js";

package/dist/extensions/NEEDLE_opaque/NEEDLE_opaque.d.ts

@@ -0,0 +1,20 @@
+import { WriterContext, PropertyType } from '@gltf-transform/core';
+declare interface IExtensibleProperty {
+}
+export type Options = {
+    debug?: boolean;
+    write?: boolean;
+};
+export interface IOpaqueExtension {
+}
+export interface IOpaqueProperty {
+    /** used by progressive texture extension to pass an extension between documents */
+    getExtensionDefinition(): object;
+}
+export interface IExtensionWriter {
+    /** used by progressive texture extension to pass an extension between documents */
+    assignTo?(obj: object): any;
+    write(context: WriterContext, prop: IExtensibleProperty): void;
+}
+export declare function createOpaqueExtension(name: string, types: PropertyType | PropertyType[], opts?: Options): IOpaqueExtension;
+export {};