@needle-tools/engine 4.7.0-next.bbbc637 → 4.7.0-next.cac0716

package/plugins/common/logger.js

@@ -1,7 +1,9 @@
 import { createWriteStream, existsSync, mkdirSync, readdirSync, rmSync, statSync, write } from "fs";
 
 const filename_timestamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
+const debug = false;
 
+// #region public api
 
 /**
  * @typedef {"server" | "client" | "client-http"} ProcessType
@@ -21,23 +23,23 @@ export function patchConsoleLogs() {
 
     console.log = (...args) => {
         originalConsoleLog(...args);
-        captureLogMessage("server", 'log', args);
+        captureLogMessage("server", 'log', args, null);
     };
     console.error = (...args) => {
         originalConsoleError(...args);
-        captureLogMessage("server", 'error', args);
+        captureLogMessage("server", 'error', args, null);
     };
     console.warn = (...args) => {
         originalConsoleWarn(...args);
-        captureLogMessage("server", 'warn', args);
+        captureLogMessage("server", 'warn', args, null);
     };
     console.info = (...args) => {
         originalConsoleInfo(...args);
-        captureLogMessage("server", 'info', args);
+        captureLogMessage("server", 'info', args, null);
     };
     console.debug = (...args) => {
         originalConsoleDebug(...args);
-        captureLogMessage("server", 'debug', args);
+        captureLogMessage("server", 'debug', args, null);
     };
 
     // Restore original console methods
@@ -54,78 +56,72 @@ export function patchConsoleLogs() {
 
 
 let isCapturing = false;
+/** @type {Set<string>} */
+const isCapturingLogMessage = new Set();
 
-/** @type {Array<{ process: ProcessType, key: string, log:any, timestamp:number }>} */
+/** @type {Array<{ process: ProcessType, key: string, log:any, timestamp:number, connectionId: string | null }>} */
 const queue = new Array();
 
 /**
  * @param {ProcessType} process
  * @param {string} key
  * @param {any} log
+ * @param {string | null} connectionId - Optional connection ID for client logs.
+ * @param {number} [time] - Optional timestamp, defaults to current time.
  */
-export function captureLogMessage(process, key, log, time = Date.now()) {
+export function captureLogMessage(process, key, log, connectionId, time = Date.now()) {
+
+    if (isCapturingLogMessage.has(log)) {
+        return; // prevent circular logs
+    }
+
     if (isCapturing) {
-        queue.push({ process, key, log, timestamp: Date.now() });
+        queue.push({ process, key, log, timestamp: Date.now(), connectionId });
         return;
     }
     isCapturing = true;
+    isCapturingLogMessage.add(log);
 
     try {
-        const str = stringifyLog(log);
-        const prefix = `${getTimestamp(time, true)}, ${process}.${key}: `;
-        const separator = "";
-        writeToFile(process, indent(`${prefix}${separator}${str}`, prefix.length, separator));
+        let str = stringifyLog(log);
+        if (str.trim().length > 0) {
+            // if(process === "server") str = stripAnsiColors(str);
+            const prefix = `${getTimestamp(time, true)}, ${process}${connectionId ? (`[${connectionId}]`) : ""}.${key}: `;
+            const separator = "";
+            const finalLog = indent(`${prefix}${separator}${removeEmptyLinesAtStart(str)}`, prefix.length, separator)
+            writeToFile(process, finalLog, connectionId);
+        }
     } finally {
         isCapturing = false;
+        isCapturingLogMessage.delete(log);
     }
 
     let queued = queue.pop();
     if (queued) {
-        captureLogMessage(queued.process, queued.key, queued.log, queued.timestamp);
+        captureLogMessage(queued.process, queued.key, queued.log, queued.connectionId, queued.timestamp);
     }
 }
 
 
-// #region utility functions
+// #region stringify log
 
-/**
- * Returns the current timestamp in ISO format.
- * @param {number} [date] - Optional date to format, defaults to current date.
- */
-function getTimestamp(date, timeOnly = false) {
-    const now = date ? new Date(date) : new Date();
-    if (timeOnly) {
-        return now.toTimeString().split(' ')[0]; // HH:MM:SS
-    }
-    return now.toISOString();
-}
 
 /**
+ * Stringifies a log message, handling circular references and formatting.
  * @param {any} log
  * @param {Set<any>} [seen]
  */
-function stringifyLog(log, seen = new Set()) {
+function stringifyLog(log, seen = new Set(), depth = 0) {
 
-    if (seen.has(log)) {
-        return "[circular]";
-    }
-    seen.add(log);
-    if (Array.isArray(log)) {
-        let res = "";
-        for (let item of log) {
-            if (res) res += ", ";
-            res += stringifyLog(item, seen);
-        }
-        return res;
-    }
-    if (typeof log === "object") {
-        let entries = Object.entries(log).map(([key, value]) => {
-            return `"${key}": ${stringifyLog(value, seen)}`;
-        });
-        return `{ ${entries.join(", ")} }`;
+    const isServer = typeof window === "undefined";
+    const stringify_limits = {
+        string: isServer ? 100_000 : 2000,
+        object_keys: isServer ? 100 : 200,
+        array_items: isServer ? 2_000 : 100,
     }
 
     if (typeof log === "string") {
+        if (log.length > stringify_limits.string) log = `${log.slice(0, stringify_limits.string)}... <truncated ${log.length - stringify_limits.string} characters>`;
         return log;
     }
     if (typeof log === "number" || typeof log === "boolean") {
@@ -137,9 +133,93 @@ function stringifyLog(log, seen = new Set()) {
     if (log === undefined) {
         return "undefined";
     }
+    if (typeof log === "function") {
+        return "<function>";
+    }
+
+    if (seen.has(log)) return "<circular>";
+
+    if (Array.isArray(log)) {
+        seen.add(log);
+        return stringifyArray(log);
+    }
+    if (typeof log === "object") {
+        seen.add(log);
+        // const str = JSON.stringify(log, (key, value) => {
+        //     if (typeof value === "function") return "<function>";
+        //     if (typeof value === "string") return stringifyLog(value, seen, depth + 1);
+        //     if (typeof value === "object") {
+        //         if (seen.has(value)) return "<circular>";
+        //         seen.add(value);
+        //     }
+        //     return value;
+        // });
+        // return str;
+        const keys = Object.keys(log);
+        let res = "{";
+        for (let i = 0; i < keys.length; i++) {
+            const key = keys[i];
+            let value = log[key];
+
+            if (typeof value === "number") {
+                // clamp precision for numbers
+                value = Number(value.toFixed(6));
+            }
+            let str = stringifyLog(value, seen, depth + 1);
+            if (typeof value === "object") {
+                if (Array.isArray(value)) {
+                    str = `[${str}]`;
+                }
+            }
+            else if (typeof value === "string") {
+                str = `"${str}"`;
+            }
+            if (i > 0) res += ", ";
+            res += `"${key}":${str}`;
+        }
+        res += "}";
+        return res;
+        // let entries = Object.entries(log).map(([key, value], index) => {
+        //     if (index > stringify_limits.object_keys) return `"${key}": <truncated>`;
+        //     return `"${key}": ${stringifyLog(value, seen, depth + 1)}`;
+        // });
+        // return `{ ${entries.join(", ")} }`;
+    }
     return String(log);
+
+    function stringifyArray(arr) {
+        let res = "";
+        for (let i = 0; i < arr.length; i++) {
+            let entry = arr[i];
+            if (res && i > 0) res += ", ";
+            if (i > stringify_limits.array_items) {
+                res += "<truncated " + (arr.length - i) + ">";
+                break;
+            }
+            res += stringifyLog(entry, seen, depth + 1);
+        }
+        return res;
+    }
+}
+
+
+
+
+// #region utility functions
+
+/**
+ * Returns the current timestamp in ISO format.
+ * @param {number} [date] - Optional date to format, defaults to current date.
+ */
+function getTimestamp(date, timeOnly = false) {
+    const now = date ? new Date(date) : new Date();
+    if (timeOnly) {
+        return now.toTimeString().split(' ')[0]; // HH:MM:SS
+    }
+    return now.toISOString();
 }
 
+
 /**
  * Indents a string by a specified length.
  * @param {string} str - The string to indent.
@@ -158,10 +238,35 @@ function indent(str, length, separator = "") {
     return lines.join("\n");
 }
 
+/**
+ * Removes empty lines at the start of a string.
+ * @param {string} str - The string to process.
+ */
+function removeEmptyLinesAtStart(str) {
+    const lines = str.split("\n");
+    for (let i = 0; i < lines.length; i++) {
+        const line = lines[i].trim();
+        if (line.length > 0) {
+            lines[i] = line; // keep the first non-empty line
+            return lines.slice(i).join("\n");
+        }
+    }
+    return "";
+}
+
+/**
+ * Strips ANSI color codes from a string.
+ * @param {string} str - The string to process.
+ */
+function stripAnsiColors(str) {
+    // This pattern catches most ANSI escape sequences
+    return str.replace(/\x1b\[[0-9;]*[a-zA-Z]/g, '');
+}
+
 
 // #region log to file
 
-/** @type {Map<ProcessType, import("fs").WriteStream>} */
+/** @type {Map<string, import("fs").WriteStream>} */
 const filestreams = new Map();
 const fileLogDirectory = "node_modules/.needle/logs";
 // cleanup old log files
@@ -184,17 +289,20 @@ if (existsSync(fileLogDirectory)) {
  * Writes a log message to the file.
  * @param {ProcessType} process
  * @param {string} log
+ * @param {string | null} connectionId - Optional connection ID for client logs.
  */
-function writeToFile(process, log) {
-    if (!filestreams.has(process)) {
+function writeToFile(process, log, connectionId) {
+    const filename = `${process}.needle.log`; //connectionId && process === "client" ? `${process}-${connectionId}.needle.log` : `${process}.needle.log`;
+
+    if (!filestreams.has(filename)) {
         if (!existsSync(fileLogDirectory)) {
             mkdirSync(fileLogDirectory, { recursive: true });
         }
-        filestreams.set(process, createWriteStream(`${fileLogDirectory}/needle.${filename_timestamp}.${process}.log`, { flags: 'a' }));
+        filestreams.set(filename, createWriteStream(`${fileLogDirectory}/${filename_timestamp}.${filename}`, { flags: 'a' }));
     }
-    const writeStream = filestreams.get(process);
+    const writeStream = filestreams.get(filename);
     if (!writeStream) {
-        console.error(`No write stream for process: ${process}`);
+        if (debug) console.error(`No write stream for process: ${filename}`);
         return;
     }
     writeStream.write(log + '\n');

package/plugins/types/userconfig.d.ts

@@ -1,5 +1,4 @@
 import { License } from "./license";
-import { NeedlePWAOptions } from "./webmanifest.js";
 
 export type needleModules = {
     webpack: object | undefined
@@ -118,5 +117,8 @@ export type userSettings = {
      */
     openBrowser?: boolean;
 
+    /** Automatically import MaterialX for needle engine in 'main.ts' */
     loadMaterialX?: boolean;
+
+    disableLogging?: boolean;
 }

package/plugins/vite/logger.client.js

@@ -7,11 +7,17 @@
  */
 function sendLogToServer(level, ...message) {
     if ("hot" in import.meta) {
+        message = stringifyLog(message);
         // @ts-ignore
         import.meta.hot.send("needle:client-log", { level, message: message });
     }
 }
 
+// const obj = {
+//     hello: "world"
+// }
+// obj["test"] = obj;
+// sendLogToServer("internal", "Test circular reference", obj);
 
 if (import.meta && "hot" in import.meta) {
 
@@ -65,11 +71,14 @@ User Activation: ${"userActivation" in navigator ? JSON.stringify(navigator.user
                     if (device) {
                         const adapterInfo = device.adapterInfo;
                         if (adapterInfo) {
-                            sendLogToServer("internal", [`WebGPU adapter info`,
-                                [adapterInfo.vendor, adapterInfo.architecture, adapterInfo.device]
-                                    .filter(e => e.trim()?.length)
-                                    .join(", ")
-                            ]);
+                            sendLogToServer("internal", [`WebGPU adapter info`, {
+                                vendor: adapterInfo.vendor,
+                                architecture: adapterInfo.architecture,
+                                device: adapterInfo.device,
+                                description: adapterInfo.description,
+                                features: adapterInfo.features,
+                                limits: adapterInfo.limits
+                            }]);
                         }
                     }
                 });
@@ -146,4 +155,108 @@ User Activation: ${"userActivation" in navigator ? JSON.stringify(navigator.user
 
 
 
-}
+}
+
+
+
+
+
+// #region copied from common/logger.js
+
+
+/**
+ * Stringifies a log message, handling circular references and formatting.
+ * @param {any} log
+ * @param {Set<any>} [seen]
+ */
+function stringifyLog(log, seen = new Set(), depth = 0) {
+
+    const isServer = typeof window === "undefined";
+    const stringify_limits = {
+        string: isServer ? 100_000 : 2000,
+        object_keys: isServer ? 100 : 200,
+        array_items: isServer ? 2_000 : 100,
+    }
+
+    if (typeof log === "string") {
+        if (log.length > stringify_limits.string) log = `${log.slice(0, stringify_limits.string)}... <truncated ${log.length - stringify_limits.string} characters>`;
+        return log;
+    }
+    if (typeof log === "number" || typeof log === "boolean") {
+        return String(log);
+    }
+    if (log === null) {
+        return "null";
+    }
+    if (log === undefined) {
+        return "undefined";
+    }
+    if (typeof log === "function") {
+        return "<function>";
+    }
+
+    if (seen.has(log)) return "<circular>";
+
+    if (Array.isArray(log)) {
+        seen.add(log);
+        return stringifyArray(log);
+    }
+    if (typeof log === "object") {
+        seen.add(log);
+        // const str = JSON.stringify(log, (key, value) => {
+        //     if (typeof value === "function") return "<function>";
+        //     if (typeof value === "string") return stringifyLog(value, seen, depth + 1);
+        //     if (typeof value === "object") {
+        //         if (seen.has(value)) return "<circular>";
+        //         seen.add(value);
+        //     }
+        //     return value;
+        // });
+        // return str;
+        const keys = Object.keys(log);
+        let res = "{";
+        for (let i = 0; i < keys.length; i++) {
+            const key = keys[i];
+            let value = log[key];
+
+            if (typeof value === "number") {
+                // clamp precision for numbers
+                value = Number(value.toFixed(6));
+            }
+            let str = stringifyLog(value, seen, depth + 1);
+            if (typeof value === "object") {
+                if (Array.isArray(value)) {
+                    str = `[${str}]`;
+                }
+            }
+            else if (typeof value === "string") {
+                str = `"${str}"`;
+            }
+            if (i > 0) res += ", ";
+            res += `"${key}":${str}`;
+        }
+        res += "}";
+        return res;
+        // let entries = Object.entries(log).map(([key, value], index) => {
+        //     if (index > stringify_limits.object_keys) return `"${key}": <truncated>`;
+        //     return `"${key}": ${stringifyLog(value, seen, depth + 1)}`;
+        // });
+        // return `{ ${entries.join(", ")} }`;
+    }
+    return String(log);
+
+    function stringifyArray(arr) {
+        let res = "";
+        for (let i = 0; i < arr.length; i++) {
+            let entry = arr[i];
+            if (res && i > 0) res += ", ";
+            if (i > stringify_limits.array_items) {
+                res += "<truncated " + (arr.length - i) + ">";
+                break;
+            }
+            res += stringifyLog(entry, seen, depth + 1);
+        }
+        return res;
+    }
+}
+

package/plugins/vite/logger.js

@@ -10,12 +10,16 @@ const __dirname = path.dirname(__filename);
 /**
  * write logs to local file
  * @param {import('../types/userconfig.js').userSettings} userSettings
- * @returns {import('vite').Plugin}
+ * @returns {import('vite').Plugin | null}
  */
 export const needleLogger = (command, config, userSettings) => {
 
+    if (userSettings?.disableLogging === true) {
+        return null;
+    }
+
     patchConsoleLogs();
-    captureLogMessage("server", "info", "Vite started with command \"" + command + "\" in " + __dirname);
+    captureLogMessage("server", "info", "Vite started with command \"" + command + "\" in " + __dirname, null);
 
     return {
         name: "needle:logger",
@@ -58,12 +62,22 @@ export const needleLogger = (command, config, userSettings) => {
  * @param {import('vite').PreviewServer | import('vite').ViteDevServer} server
  */
 function logRequests(server, log_http_requests = false) {
+    /**
+     * Logs a message to the server console and captures it.
+     * @type {Map<import("vite").WebSocket, {id:string}>}
+     */
+    const connectedClients = new Map();
+    let index = 0;
+
     if ("ws" in server) {
         // Clent connections
         server.ws.on('connection', (socket, request) => {
-            captureLogMessage("server", "connection", "New websocket connection established");
+            const clientId = String(index++);
+            connectedClients.set(socket, { id: clientId });
+            const ip = request.socket.remoteAddress || 'unknown';
+            captureLogMessage("server", "connection", `New websocket connection established ${clientId} from ${ip}`, clientId);
             socket.on('close', () => {
-                captureLogMessage("server", "connection", "Websocket connection closed");
+                captureLogMessage("server", "connection", `Websocket connection closed ${clientId}`, clientId);
             });
         });
         // Client log messages via websocket
@@ -72,13 +86,15 @@ function logRequests(server, log_http_requests = false) {
                 console.warn("Received empty log data, ignoring");
                 return;
             }
-            captureLogMessage("client", data.level, data.message);
+            const socket = client.socket;
+            const info = connectedClients.get(socket);
+            captureLogMessage("client", data.level, data.message, info ? info.id : null);
         });
     }
     // Log HTTP requests
     if (log_http_requests) {
         server.middlewares.use((req, res, next) => {
-            captureLogMessage("client-http", "info", [req.method, req.url]);
+            captureLogMessage("client-http", "info", [req.method, req.url], null);
             next();
         });
     }

package/plugins/vite/materialx.js

@@ -1,8 +1,8 @@
 import { existsSync } from 'fs';
 
 const materialx_packagejson_path = "node_modules/@needle-tools/materialx/package.json";
-const materialx_import_chunk =
-    `import { useNeedleMaterialX } from "@needle-tools/materialx/needle";
+const materialx_import_chunk = `
+import { useNeedleMaterialX } from "@needle-tools/materialx/needle";
 useNeedleMaterialX();
 `
 
@@ -20,8 +20,10 @@ export const needleMaterialXLoader = (command, config, userSettings) => {
         transform: (code, id) => {
             if (id.endsWith("src/main.ts")) {
                 if (userSettings?.loadMaterialX !== false && existsSync(materialx_packagejson_path)) {
-                    console.log("[needle-materialx-loader] Adding MaterialX import to main.ts");
-                    code = materialx_import_chunk + "\n" + code;
+                    if (!code.includes("@needle-tools/materialx")) {
+                        console.log("[needle-materialx-loader] Adding MaterialX import to main.ts");
+                        code = materialx_import_chunk + "\n" + code;
+                    }
                 }
             }
             return code;

package/src/engine/engine_addressables.ts

@@ -188,14 +188,11 @@ export class AssetReference {
     private _urlName: string;
     private _progressListeners: ProgressCallback[] = [];
 
-    private _hash?: string;
-    private _hashedUri: string;
-
     private _isLoadingRawBinary: boolean = false;
     private _rawBinary?: ArrayBufferLike | null;
 
     /** @internal */
-    constructor(uri: string, hash?: string, asset: any = null) {
+    constructor(uri: string, _hash?: string, asset: any = null) {
         this._url = uri;
 
         const lastUriPart = uri.lastIndexOf("/");
@@ -210,14 +207,8 @@ export class AssetReference {
         else {
             this._urlName = uri;
         }
-
-        this._hash = hash;
-        if (uri.includes("?v="))
-            this._hashedUri = uri;
-        else
-            this._hashedUri = hash ? uri + "?v=" + hash : uri;
+        
         if (asset !== null) this.asset = asset;
-
         registerPrefabProvider(this._url, this.onResolvePrefab.bind(this));
     }
 
@@ -263,8 +254,8 @@ export class AssetReference {
         if (this._isLoadingRawBinary) return null;
         if (this._rawBinary !== undefined) return this._rawBinary;
         this._isLoadingRawBinary = true;
-        if (debug) console.log("Preload", this._hashedUri);
-        const res = await BlobStorage.download(this._hashedUri, p => {
+        if (debug) console.log("Preload", this.url);
+        const res = await BlobStorage.download(this.url, p => {
             this.raiseProgressEvent(p);
         });
         this._rawBinary = res?.buffer ?? null;
@@ -294,7 +285,7 @@ export class AssetReference {
         // console.log("START LOADING");
         if (this._rawBinary) {
             if (!(this._rawBinary instanceof ArrayBuffer)) {
-                console.error("Invalid raw binary data", this._rawBinary);
+                console.error("Failed loading: Invalid raw binary data. Must be of type ArrayBuffer. " + (typeof this._rawBinary));
                 return null;
             }
             this._loading = getLoader().parseSync(context, this._rawBinary, this.url, null);
@@ -302,7 +293,7 @@ export class AssetReference {
         }
         else {
             if (debug) console.log("Load async", this.url);
-            this._loading = getLoader().loadSync(context, this._hashedUri, this.url, null, prog => {
+            this._loading = getLoader().loadSync(context, this.url, this.url, null, prog => {
                 this.raiseProgressEvent(prog);
             });
         }

package/src/engine/engine_lightdata.ts

@@ -53,10 +53,12 @@ export class LightDataRegistry implements ILightDataRegistry {
     }
 
     tryGetSkybox(sourceId?: SourceIdentifier | null): Texture | null {
+        if (debugLightmap) console.log("[Get Skybox]", sourceId, this._lightmaps)
         return this.tryGet(sourceId, LightmapType.Skybox, 0);
     }
 
     tryGetReflection(sourceId?: SourceIdentifier | null): Texture | null {
+        if (debugLightmap) console.log("[Get Reflection]", sourceId, this._lightmaps)
         return this.tryGet(sourceId, LightmapType.Reflection, 0);
     }
 
@@ -66,9 +68,15 @@ export class LightDataRegistry implements ILightDataRegistry {
             return null;
         }
         const entry = this._lightmaps.get(sourceId);
-        if (!entry) return null;
+        if (!entry) {
+            if (debugLightmap) console.warn(`[Lighting] No ${LightmapType[type]} texture entry for`, sourceId);
+            return null;
+        }
         const arr = entry.get(type);
-        if (arr === undefined) return null;
+        if (arr === undefined) {
+            if (debugLightmap) console.warn(`[Lighting] No ${LightmapType[type]} texture for`, sourceId, "index", index);
+            return null;
+        }
         if (!arr?.length || arr.length <= index) return null;
         return arr[index];
     }