@needle-tools/engine 4.8.2 → 4.8.3-next.64a46ce

package/plugins/vite/peer.js

@@ -1,8 +1,8 @@
-
 const peerjsString = `/* needle: fix for peerjs */ window.global = window; var parcelRequire;`
 
 /**
  * @param {import('../types').userSettings} userSettings
+ * @returns {import("vite").Plugin | undefined}
  */
 export const needlePeerjs = (command, config, userSettings) => {
 
@@ -23,7 +23,65 @@ export const needlePeerjs = (command, config, userSettings) => {
                         },
                     ]
                 }
-            }
+            },
+        },
+        enforce: "pre",
+        transform: (code, id) => {
+            return patchWebRTCAdapterForGemini(code, id);
         }
     }
 }
+
+
+
+
+/**
+ * @param {string} code
+ * @param {string} id
+ * @returns {string | undefined}
+ */
+function patchWebRTCAdapterForGemini(code, id) {
+    // Match navigator.mediaDevices.getUserMedia assignments
+    const assignmentRegex = /(navigator\d{0,1})\.mediaDevices\.getUserMedia\s*=\s*function/gm;
+
+    let didTransform = false;
+    let transformedCode = code;
+    let match;
+
+    while ((match = assignmentRegex.exec(code)) !== null) {
+        const navigatorRef = match[1];
+        const matchStart = match.index;
+
+        // Find the end of the function assignment
+        let braceCount = 0;
+        let inFunction = false;
+        let endIndex = matchStart;
+
+        for (let i = matchStart; i < code.length; i++) {
+            const char = code[i];
+            if (char === '{') {
+                braceCount++;
+                inFunction = true;
+            } else if (char === '}') {
+                braceCount--;
+                if (inFunction && braceCount === 0) {
+                    endIndex = i + 1;
+                    break;
+                }
+            }
+        }
+
+        const originalAssignment = code.substring(matchStart, endIndex);
+        const wrappedAssignment = `if (Object.getOwnPropertyDescriptor(${navigatorRef}.mediaDevices, "getUserMedia")?.writable) {\n    ${originalAssignment}\n}`;
+        didTransform = true;
+        transformedCode = transformedCode.replace(originalAssignment, wrappedAssignment);
+        // console.log("-------------------------------\nTRANSFORMED\n", id, "\nOriginal:", originalAssignment, "\nWrapped:", wrappedAssignment, "\n\n");
+    }
+
+    if(!didTransform) return undefined;
+
+    console.log("[needle:peerjs] Fixed WebRTC assignment");
+
+    return transformedCode;
+
+}

package/plugins/vite/poster-client.js

@@ -1,40 +1,46 @@
 
 async function generatePoster() {
-    const { screenshot2 } = await import("@needle-tools/engine");
 
     try {
-        const needleEngine = document.querySelector("needle-engine");
-        if (!needleEngine) return null;
+        const { screenshot2, onStart } = await import("@needle-tools/engine");
 
         // Keep in sync with og:image:width meta tags
         // https://developers.facebook.com/docs/sharing/best-practices/
         const width = 1080;
         const height = 1080;
-        const context = await needleEngine.getContext();
 
-        // wait a fixed time for e.g. fonts to load
-        while(context.time.frameCount < 2)
-            await new Promise((resolve) => setTimeout(resolve, 200));
+        return new Promise(res => {
+            onStart(async context => {
 
-        const mimeType = "image/webp";
+                if (context.lodsManager.manager) {
+                    await context.lodsManager.manager.awaitLoading({ frames: 20, maxPromisesPerObject: 2 });
+                }
 
-        // We're reading back as a blob here because that's async, and doesn't seem
-        // to stress the GPU so much on memory-constrained devices.
-        const blob = await screenshot2({context, width, height, mimeType, type: "blob"});
-        
-        // We can only send a DataURL, so we need to convert it back here.
-        const dataUrl = await new Promise((resolve, reject) => {
-            const reader = new FileReader();
-            reader.onload = function() {
-                resolve(reader.result);
-            };
-            reader.onloadend = function() {
-                resolve(null);
-            };
-            reader.readAsDataURL(blob);
-        });
+                const mimeType = "image/webp";
+
+                // We're reading back as a blob here because that's async, and doesn't seem
+                // to stress the GPU so much on memory-constrained devices.
+                const blob = await screenshot2({ context, width, height, mimeType, type: "blob" });
+
+                // We can only send a DataURL, so we need to convert it back here.
+                const dataUrl = await new Promise((resolve, reject) => {
+                    const reader = new FileReader();
+                    reader.onload = function () {
+                        resolve(reader.result);
+                    };
+                    reader.onloadend = function () {
+                        resolve(null);
+                    };
+                    reader.readAsDataURL(blob);
+                });
+
+
+                res(dataUrl);
+
+
+            }, { once: true });
+        })
 
-        return dataUrl;
     }
     catch (e) {
         console.error(e);
@@ -42,33 +48,11 @@ async function generatePoster() {
     }
 }
 
-async function sendPoster() {
-    const blob = await generatePoster();
-    import.meta.hot.send("needle:screenshot", { data: blob });
-}
 
-// communicate via vite
 if (import.meta.hot) {
-    // wait for needle engine to be fully loaded
-    const needleEngine = document.querySelector("needle-engine");
-    needleEngine?.addEventListener("loadfinished", () => {
-        // wait a moment
-        setTimeout(() => {
-            sendPoster();
-        }, 200);
-    });
-
-    // for debugging: build extra button with dev-only options
-    /*
-    var button = document.createElement("button");
-    button.id = "send-msg";
-    button.innerHTML = "Generate Poster";
-    button.style.position = "fixed";
-    button.style.zIndex = "9999";
-    document.body.appendChild(button);
-
-    document.querySelector("#send-msg").addEventListener("click", async () => {
-        sendPoster();
-    });
-    */
+    async function run() {
+        const blob = await generatePoster();
+        import.meta.hot.send("needle:screenshot", { data: blob });
+    }
+    run();
 }

package/plugins/vite/poster.js

@@ -36,7 +36,7 @@ export const needlePoster = (command, config, userSettings) => {
                 }
                 try {
                     const targetPath = "./" + getPosterPath();
-                    console.debug(`Received poster, saving to ${targetPath}`);
+                    console.debug(`[needle:poster] Saving automatic poster to ${targetPath}`);
                     // remove data:image/png;base64, from the beginning of the string
                     if (targetPath.endsWith(".webp"))
                         data.data = data.data.replace(/^data:image\/webp;base64,/, "");
@@ -47,10 +47,9 @@ export const needlePoster = (command, config, userSettings) => {
                         fs.mkdirSync(dir, { recursive: true })
                     }
                     fs.writeFileSync(targetPath, Buffer.from(data.data, "base64"));
-                    console.debug("Saved poster to file");
                 }
                 catch (err) {
-                    console.error("Failed to save poster", err.message);
+                    console.error("[needle:poster] Failed to save poster", err.message);
                 }
             });
         },

package/src/engine/engine_context.ts

@@ -980,14 +980,14 @@ export class Context implements IContext {
 		if (this.isInXR) return true;
 		if (!this._isVisible) return false;
 		// Make sure not to call getComputedStyle multiple times per frame
-		if (this.time.frame === this._lastStyleComputedFrame) return this._lastStyleComputedResult;
-		this._lastStyleComputedFrame = this.time.frame;
+		if (!this._needsVisibleUpdate && this._lastStyleComputedResult !== undefined) return this._lastStyleComputedResult;
+		this._needsVisibleUpdate = false;
 		const style = getComputedStyle(this.domElement);
 		this._lastStyleComputedResult = style.visibility !== "hidden" && style.display !== "none" && style.opacity !== "0";
 		return this._lastStyleComputedResult;
 	}
-	private _lastStyleComputedFrame: number = -1;
-	private _lastStyleComputedResult: boolean = true;
+	private _needsVisibleUpdate: boolean = true;
+	private _lastStyleComputedResult: boolean | undefined = undefined;
 
 	private _createId: number = 0;
 	private async internalOnCreate(opts?: ContextCreateArgs): Promise<boolean> {
@@ -1391,6 +1391,7 @@ export class Context implements IContext {
 			this.renderer.info.reset();
 		}
 
+		this._needsVisibleUpdate = true;
 
 		const sessionStarted = frame !== null && this._xrFrame === null;
 		this._xrFrame = frame;
@@ -1420,8 +1421,8 @@ export class Context implements IContext {
 
 		Context.Current = this;
 		this.time.update();
-		if (debugframerate)
-			console.log("FPS", (this.time.smoothedFps).toFixed(0));
+		
+		if (debugframerate) console.log("FPS", (this.time.smoothedFps).toFixed(0));
 
 
 		looputils.processNewScripts(this);
@@ -1460,7 +1461,6 @@ export class Context implements IContext {
 		}
 		this.executeCoroutines(FrameEvent.EarlyUpdate);
 		invokeLifecycleFunctions(this, FrameEvent.EarlyUpdate);
-		if (this.onHandlePaused()) return false;
 
 		this._currentFrameEvent = FrameEvent.Update;
 
@@ -1474,8 +1474,6 @@ export class Context implements IContext {
 		}
 		this.executeCoroutines(FrameEvent.Update);
 		invokeLifecycleFunctions(this, FrameEvent.Update);
-		if (this.onHandlePaused()) return false;
-
 		this._currentFrameEvent = FrameEvent.LateUpdate;
 
 		for (let i = 0; i < this.scripts_lateUpdate.length; i++) {
@@ -1490,7 +1488,6 @@ export class Context implements IContext {
 		// this.mainLight = null;
 		this.executeCoroutines(FrameEvent.LateUpdate);
 		invokeLifecycleFunctions(this, FrameEvent.LateUpdate);
-		if (this.onHandlePaused()) return false;
 
 		if (this.physicsSteps === undefined) {
 			this.physicsSteps = 1;
@@ -1499,8 +1496,6 @@ export class Context implements IContext {
 			this.internalUpdatePhysics(this.physicsSteps);
 		}
 
-		if (this.onHandlePaused()) return false;
-
 		if (this.isVisibleToUser || this.runInBackground) {
 
 			this._currentFrameEvent = FrameEvent.OnBeforeRender;
@@ -1607,8 +1602,6 @@ export class Context implements IContext {
 		}
 		this.handleRendererContextLost();
 
-		if(this.time.frame % 10 !== 0) return;
-
 		this._isRendering = true;
 		this.renderRequiredTextures();
 

package/src/engine/engine_networking_streams.ts

@@ -8,7 +8,7 @@ import { showBalloonMessage } from "./debug/index.js";
 import { Application } from "./engine_application.js";
 import { Context } from "./engine_context.js";
 import type { IModel } from "./engine_networking_types.js";
-import { type IComponent,isComponent } from "./engine_types.js";
+import { type IComponent, isComponent } from "./engine_types.js";
 import { getParam } from "./engine_utils.js";
 
 
@@ -113,8 +113,8 @@ class CallHandle extends EventDispatcher<any> {
     }
 }
 
-function applySdpTransform(sdp){
-    sdp = sdp.replace("a=fmtp:111 minptime=10;useinbandfec=1","a=fmtp:111 ptime=5;useinbandfec=1;stereo=1;maxplaybackrate=48000;maxaveragebitrat=128000;sprop-stereo=1");
+function applySdpTransform(sdp) {
+    sdp = sdp.replace("a=fmtp:111 minptime=10;useinbandfec=1", "a=fmtp:111 ptime=5;useinbandfec=1;stereo=1;maxplaybackrate=48000;maxaveragebitrat=128000;sprop-stereo=1");
     return sdp;
 }
 
@@ -237,10 +237,19 @@ export class PeerHandle extends EventDispatcher<any> {
         this.context = context;
         this.id = id;
         this.setupPeer();
-        navigator["getUserMedia"] = (
-            navigator["getUserMedia"] || navigator["webkitGetUserMedia"] ||
-            navigator["mozGetUserMedia"] || navigator["msGetUserMedia"]
-        );
+        const isGetUserMediaWriteable = Object.getOwnPropertyDescriptor(navigator, "getUserMedia")?.writable;
+        try {
+            if (isGetUserMediaWriteable) {
+                navigator["getUserMedia"] = (
+                    navigator["getUserMedia"] || navigator["webkitGetUserMedia"] ||
+                    navigator["mozGetUserMedia"] || navigator["msGetUserMedia"]
+                );
+            }
+            else if (debug) console.warn("[PeerJs] getUserMedia is not writable");
+        }
+        catch (err) {
+            if (debug) console.error("[PeerJs] Error setting getUserMedia", err);
+        }
     }
 
     private _enabled: boolean = false;
@@ -479,7 +488,7 @@ export class NetworkedStreams extends EventDispatcher<any> {
         }
 
         if (!context) throw new Error("Failed to create NetworkedStreams because context is undefined");
-        else if(!(context instanceof Context)) throw new Error("Failed to create NetworkedStreams because context is not an instance of Context");
+        else if (!(context instanceof Context)) throw new Error("Failed to create NetworkedStreams because context is not an instance of Context");
         if (!peer) throw new Error("Failed to create NetworkedStreams because peer is undefined");
 
         this.context = context;

package/src/engine-components/OrbitControls.ts

@@ -565,6 +565,11 @@ export class OrbitControls extends Behaviour implements ICameraController {
         }
     }
 
+    onPausedChanged(isPaused: boolean): void {
+        if (!this._controls) return;
+        if (isPaused) this._controls.enabled = false;
+    }
+
 
     /** @internal */
     onBeforeRender() {