@needle-tools/engine 3.32.21-alpha → 3.32.22-alpha

package/plugins/vite/build-pipeline.js

@@ -0,0 +1,97 @@
+import { exec, spawn } from 'child_process';
+import { getOutputDirectory, loadConfig, tryLoadProjectConfig } from './config.js';
+import { existsSync, readdirSync } from 'fs';
+
+// see https://linear.app/needle/issue/NE-3798
+
+/**
+ * Runs the needle build pipeline as part of the vite build process
+ * @param {import('../types').userSettings} userSettings
+ * @returns {import('vite').Plugin}
+ */
+export const needleBuildPipeline = async (command, config, userSettings) => {
+    // we only want to run compression here if this is a distribution build
+    if (command !== "build") return;
+    if (userSettings.noBuildPipeline) return;
+
+    const meta = await loadConfig();
+    let shouldRun = false;
+    if (meta && meta.developmentBuild === false) {
+        shouldRun = true;
+    }
+    if (!shouldRun) {
+        log("Skipping build pipeline because this is a development build");
+        return;
+    }
+
+    /** @type {Promise<any>|null} */
+    let task = null;
+    let taskFinished = false;
+    let taskSucceeded = false;
+    return {
+        name: 'needle-buildpipeline',
+        enforce: "post",
+        apply: 'build',
+        buildEnd() {
+            // start the compression process once vite is done copying the files
+            task = invokeBuildPipeline().then((res) => {
+                taskFinished = true;
+                taskSucceeded = res;
+            });
+        },
+        closeBundle() {
+            // this is the last hook that is called, so we can wait for the task to finish here
+            if (taskFinished) return;
+            // delay the final log slightly to give other plugins a chance to log their stuff
+            wait(100).then(() => {
+                if (!taskFinished) log("Waiting for postprocessing to finish...")
+            });
+            return task.then(_ => {
+                log("finished", taskSucceeded ? "successfully" : "with errors");
+            });
+        },
+    }
+}
+
+function log(...args) {
+    console.log("[needle-buildpipeline]", ...args);
+}
+
+/**
+ * @param {*} opts 
+ * @returns {Promise<boolean>}
+ */
+async function invokeBuildPipeline(opts) {
+    await wait(1000);
+    const outputDirectory = getOutputDirectory() + "/assets";
+    if (!existsSync(outputDirectory)) {
+        log("No output directory found at ", outputDirectory);
+        return;
+    }
+    const files = readdirSync(outputDirectory).filter(f => f.endsWith(".glb") || f.endsWith(".gltf"));
+    log(files.length + " file(s) to process");
+
+    const cmd = "npm run transform --prefix node_modules/@needle-tools/gltf-build-pipeline";
+    log("Running command \"" + cmd + "\" at " + process.cwd() + "...");
+    const sub = exec(cmd);
+    sub.stdout.on('data', data => {
+        if (data.length <= 0) return;
+        // ensure that it doesnt end with a newline
+        if (data.endsWith("\n")) data = data.slice(0, -1);
+        console.log(data);
+    });
+    sub.stderr.on('data', console.error);
+    return new Promise((resolve, reject) => {
+        sub.on('exit', (code) => {
+            resolve(code === 0);
+        });
+    });
+}
+
+function wait(ms) {
+    return new Promise((resolve, reject) => {
+        setTimeout(() => {
+            resolve();
+        }, ms);
+    });
+}

package/plugins/vite/index.js

@@ -43,6 +43,7 @@ export { needleFacebookInstantGames } from "./facebook-instant-games.js";
 import { vite_4_4_hack } from "./vite-4.4-hack.js";
 import { needleImportsLogger } from "./imports-logger.js";
 import { needleBuildInfo } from "./buildinfo.js";
+import { needleBuildPipeline } from "./build-pipeline.js";
 
 
 export * from "./gzip.js";
@@ -79,6 +80,7 @@ export const needlePlugins = async (command, config, userSettings) => {
         vite_4_4_hack(command, config, userSettings),
         needleFacebookInstantGames(command, config, userSettings),
         needleImportsLogger(command, config, userSettings),
+        needleBuildPipeline(command, config, userSettings),
     ];
     array.push(await editorConnection(command, config, userSettings, array));
     return array;

package/src/engine/extensions/NEEDLE_progressive.ts

@@ -22,6 +22,7 @@ let show_lod0 = false;
 if (debug) {
     window.addEventListener("keyup", evt => {
         if (evt.key === "p") {
+            console.log("Toggle progressive textures", debug_toggle_maps);
             debug_toggle_maps.forEach((map, material) => {
                 Object.entries(map).forEach(([key, value]) => {
                     if (show_lod0) {
@@ -43,51 +44,68 @@ export class NEEDLE_progressive implements GLTFLoaderPlugin {
         return EXTENSION_NAME;
     }
 
-    static assignTextureLOD(context: Context, source: SourceIdentifier | undefined, material: Material, level: number = 0) : Promise<any> {
-        if (!material) return Promise.resolve(null);
-
-        const promises: Array<Promise<Texture | null>> = [];
-
-
-        if (material instanceof RawShaderMaterial) {
-            // iterate uniforms of custom shaders
-            for (const slot of Object.keys(material.uniforms)) {
-                const val = material.uniforms[slot].value;
-                if (val instanceof Texture) {
-                    const task = this.assignTextureLODForSlot(context, source, material, level, slot, val);
-                    promises.push(task);
+    /** Load a different resolution of a texture (if available)  
+     * @param context the context
+     * @param source the sourceid of the file from which the texture is loaded (this is usually the component's sourceId)
+     * @param materialOrTexture the material or texture to load the LOD for (if passing in a material all textures in the material will be loaded)
+     * @param level the level of detail to load (0 is the highest resolution) - currently only 0 is supported
+     */
+    static assignTextureLOD(context: Context, source: SourceIdentifier | undefined, materialOrTexture: Material | Texture, level: number = 0): Promise<any> {
+        if (!materialOrTexture) return Promise.resolve(null);
+
+        if (materialOrTexture instanceof Material) {
+            const material = materialOrTexture;
+            const promises: Array<Promise<Texture | null>> = [];
+
+            if (material instanceof RawShaderMaterial) {
+                // iterate uniforms of custom shaders
+                for (const slot of Object.keys(material.uniforms)) {
+                    const val = material.uniforms[slot].value;
+                    if (val instanceof Texture) {
+                        const task = this.assignTextureLODForSlot(context, source, val, level, material, slot);
+                        promises.push(task);
+                    }
                 }
             }
-        }
-        else {
-            for (const slot of Object.keys(material)) {
-                const val = material[slot];
-                if (val instanceof Texture) {
-                    const task = this.assignTextureLODForSlot(context, source, material, level, slot, val);
-                    promises.push(task);
+            else {
+                for (const slot of Object.keys(material)) {
+                    const val = material[slot];
+                    if (val instanceof Texture) {
+                        const task = this.assignTextureLODForSlot(context, source, val, level, material, slot);
+                        promises.push(task);
+                    }
                 }
             }
+            return PromiseAllWithErrors(promises);
+        }
+
+        if (materialOrTexture instanceof Texture) {
+            const texture = materialOrTexture;
+            return this.assignTextureLODForSlot(context, source, texture, level, null, null);
         }
 
-        return PromiseAllWithErrors(promises);
+        return Promise.resolve(null);
     }
 
-    private static assignTextureLODForSlot(context: Context, source: SourceIdentifier | undefined, material: Material, level: number, slot: string, val: any): Promise<Texture | null> {
+    private static assignTextureLODForSlot(context: Context, source: SourceIdentifier | undefined, val: any, level: number, material: Material | null, slot: string | null): Promise<Texture | null> {
         if (val?.isTexture !== true) return Promise.resolve(null);
 
-        if (debug) console.log("-----------\n", "FIND", material.name, slot, val?.name, val?.userData, val, material);
+        if (debug) console.log("-----------\n", "FIND", material?.name, slot, val?.name, val?.userData, val, material);
 
-        return NEEDLE_progressive.getOrLoadTexture(context, source, material, slot, val, level).then(t => {
+        return NEEDLE_progressive.getOrLoadTexture(context, source, val, level, material, slot).then(t => {
 
             if (t?.isTexture === true) {
 
-                if (debug) console.warn("Assign LOD", material.name, slot, t.name, t["guid"], material, "Prev:", val, "Now:", t, "\n--------------");
+                if (debug) console.warn("Assign LOD", material?.name, slot, t.name, t["guid"], material, "Prev:", val, "Now:", t, "\n--------------");
 
-                material[slot] = t;
                 t.needsUpdate = true;
-                material.needsUpdate = true;
 
-                if (debug) {
+                if (material && slot) {
+                    material[slot] = t;
+                    material.needsUpdate = true;
+                }
+
+                if (debug && material && slot) {
                     let debug_map = debug_toggle_maps.get(material);
                     if (!debug_map) {
                         debug_map = {};
@@ -146,16 +164,16 @@ export class NEEDLE_progressive implements GLTFLoaderPlugin {
     private static resolved: { [key: string]: Texture } = {};
     private static currentlyLoading: { [key: string]: Promise<Texture | null> } = {};
 
-    private static async getOrLoadTexture(context: Context, source: SourceIdentifier | undefined, material: Material, slot: string, current: Texture, _level: number): Promise<Texture | null> {
+    private static async getOrLoadTexture(context: Context, source: SourceIdentifier | undefined, current: Texture, _level: number, material: Material | null, slot: string | null): Promise<Texture | null> {
 
         const key = current.uuid;
 
         let progressiveInfo: ProgressiveTextureSchema | undefined;
 
         // See https://github.com/needle-tools/needle-engine-support/issues/133
-        if(current.source && current.source[$progressiveTextureExtension])
+        if (current.source && current.source[$progressiveTextureExtension])
             progressiveInfo = current.source[$progressiveTextureExtension];
-        if(!progressiveInfo) progressiveInfo = NEEDLE_progressive.cache.get(key);
+        if (!progressiveInfo) progressiveInfo = NEEDLE_progressive.cache.get(key);
 
         if (progressiveInfo) {
             if (debug)
@@ -171,12 +189,12 @@ export class NEEDLE_progressive implements GLTFLoaderPlugin {
                     let res = this.resolved[resolveKey];
                     // check if the texture has been disposed or not
                     if (res.image?.data || res.source?.data) {
-                        if (debug) console.log("Texture has already been loaded: " + resolveKey, material.name, slot, current.name, res);
+                        if (debug) console.log("Texture has already been loaded: " + resolveKey, material?.name, slot, current.name, res);
                         res = this.copySettings(current, res);
                         return res;
                     }
                     else if (res) {
-                        if(debug) console.warn("Texture has been disposed, will load again: " + resolveKey, material.name, slot, current.name, res.source.data);
+                        if (debug) console.warn("Texture has been disposed, will load again: " + resolveKey, material?.name, slot, current.name, res.source.data);
                     }
                 }
 
@@ -184,7 +202,7 @@ export class NEEDLE_progressive implements GLTFLoaderPlugin {
                 try {
                     if (this.currentlyLoading[resolveKey] !== undefined) {
                         if (debug)
-                            console.log("Already loading:", material.name + "." + slot, resolveKey);
+                            console.log("Already loading:", material?.name + "." + slot, resolveKey);
                         let res = await this.currentlyLoading[resolveKey];
                         if (res)
                             res = this.copySettings(current, res);
@@ -196,19 +214,19 @@ export class NEEDLE_progressive implements GLTFLoaderPlugin {
                         addDracoAndKTX2Loaders(loader, context);
 
 
-                        if (debug) console.warn("Start loading " + uri, material.name, slot, ext.guid);
+                        if (debug) console.warn("Start loading " + uri, material?.name, slot, ext.guid);
                         if (debug) {
                             await delay(Math.random() * 1000);
                         }
 
                         const gltf = await loader.loadAsync(uri);
                         const parser = gltf.parser;
-                        if (debug) console.log("Loading finished " + uri, material.name, slot, ext.guid);
+                        if (debug) console.log("Loading finished " + uri, material?.name, slot, ext.guid);
                         let index = -1;
                         let found = false;
-                        
+
                         if (!gltf.parser.json?.textures) {
-                            if (debug) console.warn("No textures in glTF " + uri + " - may be a bug", material.name, slot, ext.guid);
+                            if (debug) console.warn("No textures in glTF " + uri + " - may be a bug", material?.name, slot, ext.guid);
                             return resolve(null);
                         }
 
@@ -234,7 +252,7 @@ export class NEEDLE_progressive implements GLTFLoaderPlugin {
                         }
                         this.resolved[resolveKey] = tex as Texture;
                         if (debug)
-                            console.log(material.name, slot, "change \"" + current.name + "\" → \"" + tex.name + "\"", uri, index, tex, material, resolveKey);
+                            console.log(material?.name, slot, "change \"" + current.name + "\" → \"" + tex.name + "\"", uri, index, tex, material, resolveKey);
                         resolve(tex);
                     });
                     this.currentlyLoading[resolveKey] = request;
@@ -343,7 +361,7 @@ export class NEEDLE_progressive implements GLTFLoaderPlugin {
     private static _currentProgressiveLoadingInfo: Map<Context, ProgressiveLoadingInfo[]> = new Map();
 
     // called whenever a progressive loading event starts
-    private static onProgressiveLoadStart(context: Context, source: SourceIdentifier | undefined, uri: string, material: Material, slot: string): ProgressiveLoadingInfo {
+    private static onProgressiveLoadStart(context: Context, source: SourceIdentifier | undefined, uri: string, material: Material | null, slot: string | null): ProgressiveLoadingInfo {
         if (!this._currentProgressiveLoadingInfo.has(context)) {
             this._currentProgressiveLoadingInfo.set(context, []);
         }
@@ -379,11 +397,11 @@ export class ProgressiveLoadingInfo {
     readonly context: Context;
     readonly source: SourceIdentifier | undefined;
     readonly uri: string;
-    readonly material?: Material;
-    readonly slot?: string;
+    readonly material?: Material | null;
+    readonly slot?: string | null;
     // TODO: can contain information if the event is a background process / preloading or if the object is currently visible
 
-    constructor(context: Context, source: SourceIdentifier | undefined, uri: string, material?: Material, slot?: string) {
+    constructor(context: Context, source: SourceIdentifier | undefined, uri: string, material?: Material | null, slot?: string | null) {
         this.context = context;
         this.source = source;
         this.uri = uri;

package/src/engine-components/SpriteRenderer.ts

@@ -1,8 +1,10 @@
 import * as THREE from "three";
 import { Material, NearestFilter, Texture } from "three";
 
+import { Context } from "../engine/engine_context.js";
 import { serializable, serializeable } from "../engine/engine_serialization_decorator.js";
 import { getParam } from "../engine/engine_utils.js";
+import { NEEDLE_progressive } from "../engine/extensions/NEEDLE_progressive.js";
 import { Behaviour } from "./Component.js";
 import { RGBAColor } from "./js-extensions/RGBAColor.js";
 
@@ -70,7 +72,6 @@ class Vec2 {
 }
 
 export class Sprite {
-
     @serializable()
     guid?: string;
     @serializable(Texture)
@@ -83,6 +84,7 @@ export class Sprite {
     vertices!: Array<Vec2>;
 
     _geometry?: THREE.BufferGeometry;
+    _hasLoadedProgressive: boolean = false;
 }
 
 const $spriteTexOwner = Symbol("spriteOwner");
@@ -101,7 +103,7 @@ export class SpriteData {
     @serializable()
     index: number = 0;
 
-    update() {
+    update(context: Context, sourceId: string | undefined, material: Material | undefined) {
         if (!this.spriteSheet) return;
         const index = this.index;
         if (index < 0 || index >= this.spriteSheet.sprites.length)
@@ -113,6 +115,21 @@ export class SpriteData {
         if (tex.minFilter == NearestFilter && tex.magFilter == NearestFilter)
             tex.anisotropy = 1;
         tex.needsUpdate = true;
+
+        if (!slice._hasLoadedProgressive) {
+            slice._hasLoadedProgressive = true;
+            const previousTexture = tex;
+            NEEDLE_progressive.assignTextureLOD(context, sourceId, tex, 0).then(res => {
+                if (res instanceof Texture) {
+                    slice.texture = res;
+                    const shouldUpdateInMaterial = material?.["map"] === previousTexture;
+                    if (shouldUpdateInMaterial) {
+                        material["map"] = res;
+                        material.needsUpdate = true;
+                    }
+                }
+            });
+        }
     }
 }
 
@@ -137,8 +154,11 @@ export class SpriteRenderer extends Behaviour {
         if (value === this._spriteSheet) return;
         if (typeof value === "number") {
             const index = Math.floor(value);
-            if (index === value)
-                this.spriteIndex = index;
+            // if (value === index)
+            this.spriteIndex = index;
+            // else if (debug) {
+            //     console.log("Spritesheet framedrop", index, value);
+            // }
             return;
         }
         else {
@@ -172,7 +192,7 @@ export class SpriteRenderer extends Behaviour {
 
     awake(): void {
         this._currentSprite = undefined;
-        if(debug) {
+        if (debug) {
             console.log("Awake", this.name, this, this.sprite);
         }
     }
@@ -218,6 +238,7 @@ export class SpriteRenderer extends Behaviour {
             }
             this.sharedMaterial = mat;
             this._currentSprite = new THREE.Mesh(SpriteUtils.getOrCreateGeometry(sprite), mat);
+            NEEDLE_progressive.assignTextureLOD(this.context, this.sourceId, mat, 0);
         }
         else {
             this._currentSprite.geometry = SpriteUtils.getOrCreateGeometry(sprite);
@@ -231,7 +252,7 @@ export class SpriteRenderer extends Behaviour {
                 this.gameObject.add(this._currentSprite);
         }
 
-        if(this._currentSprite){
+        if (this._currentSprite) {
             this._currentSprite.layers.set(this.layer)
         }
 
@@ -240,6 +261,6 @@ export class SpriteRenderer extends Behaviour {
             this.sharedMaterial.transparent = this.transparent;
         }
         this._currentSprite.castShadow = this.castShadows;
-        this._spriteSheet?.update();
+        this._spriteSheet?.update(this.context, this.sourceId, this.sharedMaterial);
     }
 }

package/src/engine-components/ui/Graphic.ts

@@ -1,8 +1,10 @@
 import { Color, LinearSRGBColorSpace, Object3D, SRGBColorSpace, Texture } from 'three';
 import * as ThreeMeshUI from 'three-mesh-ui'
+import type { Options } from 'three-mesh-ui/build/types/core/elements/MeshUIBaseElement.js';
 import SimpleStateBehavior from "three-mesh-ui/examples/behaviors/states/SimpleStateBehavior.js"
 
 import { serializable } from '../../engine/engine_serialization_decorator.js';
+import { NEEDLE_progressive } from '../../engine/extensions/NEEDLE_progressive.js';
 import { GameObject } from '../Component.js';
 import { RGBAColor } from "../js-extensions/RGBAColor.js"
 import { BaseUIComponent } from "./BaseUIComponent.js";
@@ -117,7 +119,7 @@ export class Graphic extends BaseUIComponent implements IGraphic, IRectTransform
         }
     }
 
-    setOptions(opts: object) {
+    setOptions(opts: Options) {
         this.makePanel();
         if (this.uiObject) {
             //@ts-ignore
@@ -218,9 +220,14 @@ export class Graphic extends BaseUIComponent implements IGraphic, IRectTransform
                 }
             // }
             this.setOptions({ backgroundImage: tex, borderRadius: 0, backgroundOpacity: this.color.alpha, backgroundSize: "stretch" });
+            NEEDLE_progressive.assignTextureLOD(this.context, this.sourceId, tex, 0).then(res => {
+                if (res instanceof Texture) {
+                    this.setOptions({ backgroundImage: res });
+                }
+            });
         }
         else {
-            this.setOptions({ backgroundImage: null, borderRadius: 0, backgroundOpacity: this.color.alpha });
+            this.setOptions({ backgroundImage: undefined, borderRadius: 0, backgroundOpacity: this.color.alpha });
         }
         this.markDirty();
     }

package/src/engine-components/ui/Image.ts

@@ -40,7 +40,8 @@ export class Image extends MaskableGraphic {
     private pixelsPerUnitMultiplier: number = 1;
 
     private isBuiltinSprite() {
-        switch (this.sprite?.texture?.name) {
+        const sprite = this.sprite;
+        switch (sprite?.texture?.name) {
             case "InputFieldBackground":
             case "UISprite":
             case "Background":
@@ -49,7 +50,7 @@ export class Image extends MaskableGraphic {
         }
         // this is a hack/workaround for production builds where the name of the sprite is missing
         // need to remove this!!!!
-        if (!this.sprite?.texture?.name?.length && this.sprite?.texture?.image?.width === 32 && this.sprite?.texture?.image?.height === 32)
+        if (!sprite?.texture?.name?.length && sprite?.texture?.image?.width === 32 && sprite?.texture?.image?.height === 32)
             return true;
         return false;
     }

package/src/engine-components-experimental/networking/PlayerSync.ts

@@ -276,7 +276,7 @@ export class PlayerState extends Behaviour {
                     }
                     else if (debug) console.warn("PlayerState.start → owner is still undefined but dontDestroy is set to true", this.name);
                 }
-                else console.log("PlayerState.start → owner is assigned", this.owner);
+                else if(debug) console.log("PlayerState.start → owner is assigned", this.owner);
             }, 2000);
         }
     }