@needle-tools/engine 4.7.0-next.da47826 → 4.7.0-next.fe38209

package/plugins/common/logger.js

@@ -0,0 +1,328 @@
+import { createWriteStream, existsSync, mkdirSync, readdirSync, rmSync, statSync, write } from "fs";
+
+const filename_timestamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
+const debug = false;
+
+// #region public api
+
+/**
+ * @typedef {"server" | "client" | "client-http"} ProcessType
+ */
+
+let originalConsoleLog = console.log;
+let originalConsoleError = console.error;
+let originalConsoleWarn = console.warn;
+let originalConsoleInfo = console.info;
+let originalConsoleDebug = console.debug;
+let didPatch = false;
+let unpatchFunction = null;
+
+export function patchConsoleLogs() {
+    if (didPatch) return unpatchFunction;
+    didPatch = true;
+
+    console.log = (...args) => {
+        originalConsoleLog(...args);
+        captureLogMessage("server", 'log', args, null);
+    };
+    console.error = (...args) => {
+        originalConsoleError(...args);
+        captureLogMessage("server", 'error', args, null);
+    };
+    console.warn = (...args) => {
+        originalConsoleWarn(...args);
+        captureLogMessage("server", 'warn', args, null);
+    };
+    console.info = (...args) => {
+        originalConsoleInfo(...args);
+        captureLogMessage("server", 'info', args, null);
+    };
+    console.debug = (...args) => {
+        originalConsoleDebug(...args);
+        captureLogMessage("server", 'debug', args, null);
+    };
+
+    // Restore original console methods
+    unpatchFunction = () => {
+        didPatch = false;
+        console.log = originalConsoleLog;
+        console.error = originalConsoleError;
+        console.warn = originalConsoleWarn;
+        console.info = originalConsoleInfo;
+        console.debug = originalConsoleDebug;
+    }
+    return unpatchFunction;
+}
+
+
+let isCapturing = false;
+/** @type {Set<string>} */
+const isCapturingLogMessage = new Set();
+
+/** @type {Array<{ process: ProcessType, key: string, log:any, timestamp:number, connectionId: string | null }>} */
+const queue = new Array();
+
+/**
+ * @param {ProcessType} process
+ * @param {string} key
+ * @param {any} log
+ * @param {string | null} connectionId - Optional connection ID for client logs.
+ * @param {number} [time] - Optional timestamp, defaults to current time.
+ */
+export function captureLogMessage(process, key, log, connectionId, time = Date.now()) {
+
+    if (isCapturingLogMessage.has(log)) {
+        return; // prevent circular logs
+    }
+
+    if (isCapturing) {
+        queue.push({ process, key, log, timestamp: Date.now(), connectionId });
+        return;
+    }
+    isCapturing = true;
+    isCapturingLogMessage.add(log);
+
+    try {
+        let str = stringifyLog(log);
+        if (str.trim().length > 0) {
+            // if(process === "server") str = stripAnsiColors(str);
+            const prefix = `${getTimestamp(time, true)}, ${process}${connectionId ? (`[${connectionId}]`) : ""}.${key}: `;
+            const separator = "";
+            const finalLog = indent(`${prefix}${separator}${removeEmptyLinesAtStart(str)}`, prefix.length, separator)
+            writeToFile(process, finalLog, connectionId);
+        }
+    } finally {
+        isCapturing = false;
+        isCapturingLogMessage.delete(log);
+    }
+
+    let queued = queue.pop();
+    if (queued) {
+        captureLogMessage(queued.process, queued.key, queued.log, queued.connectionId, queued.timestamp);
+    }
+}
+
+
+// #region stringify log
+
+
+/**
+ * Stringifies a log message, handling circular references and formatting.
+ * @param {any} log
+ * @param {Set<any>} [seen]
+ */
+function stringifyLog(log, seen = new Set(), depth = 0) {
+
+    const isServer = typeof window === "undefined";
+    const stringify_limits = {
+        string: isServer ? 100_000 : 2000,
+        object_keys: isServer ? 300 : 100,
+        object_depth: isServer ? 10 : 3,
+        array_items: isServer ? 2_000 : 100,
+    }
+
+    if (typeof log === "string") {
+        if (log.length > stringify_limits.string) log = `${log.slice(0, stringify_limits.string)}... <truncated ${log.length - stringify_limits.string} characters>`;
+        return log;
+    }
+    if (typeof log === "number" || typeof log === "boolean") {
+        return String(log);
+    }
+    if (log === null) {
+        return "null";
+    }
+    if (log === undefined) {
+        return "undefined";
+    }
+    if (typeof log === "function") {
+        return "<function>";
+    }
+
+    if (seen.has(log)) return "<circular>";
+
+    if (Array.isArray(log)) {
+        seen.add(log);
+        return stringifyArray(log);
+    }
+    if (typeof log === "object") {
+
+        if (depth > stringify_limits.object_depth) {
+            return "<object too deep>";
+        }
+
+        seen.add(log);
+        // const str = JSON.stringify(log, (key, value) => {
+        //     if (typeof value === "function") return "<function>";
+        //     if (typeof value === "string") return stringifyLog(value, seen, depth + 1);
+        //     if (typeof value === "object") {
+        //         if (seen.has(value)) return "<circular>";
+        //         seen.add(value);
+        //     }
+        //     return value;
+        // });
+        // return str;
+        const keys = Object.keys(log);
+        let res = "{";
+        for (let i = 0; i < keys.length; i++) {
+            const key = keys[i];
+            let value = log[key];
+
+            if (typeof value === "number") {
+                // clamp precision for numbers
+                value = Number(value.toFixed(6));
+            }
+            let str = stringifyLog(value, seen, depth + 1);
+            if (typeof value === "object") {
+                if (Array.isArray(value)) {
+                    str = `[${str}]`;
+                }
+            }
+            else if (typeof value === "string") {
+                str = `"${str}"`;
+            }
+            if (i > 0) res += ", ";
+            res += `"${key}":${str}`;
+        }
+        res += "}";
+        return res;
+        // let entries = Object.entries(log).map(([key, value], index) => {
+        //     if (index > stringify_limits.object_keys) return `"${key}": <truncated>`;
+        //     return `"${key}": ${stringifyLog(value, seen, depth + 1)}`;
+        // });
+        // return `{ ${entries.join(", ")} }`;
+    }
+    return String(log);
+
+    function stringifyArray(arr) {
+        let res = "";
+        for (let i = 0; i < arr.length; i++) {
+            let entry = arr[i];
+            if (res && i > 0) res += ", ";
+            if (i > stringify_limits.array_items) {
+                res += "<truncated " + (arr.length - i) + ">";
+                break;
+            }
+            res += stringifyLog(entry, seen, depth + 1);
+        }
+        return res;
+    }
+}
+
+
+
+
+
+// #region utility functions
+
+/**
+ * Returns the current timestamp in ISO format.
+ * @param {number} [date] - Optional date to format, defaults to current date.
+ */
+function getTimestamp(date, timeOnly = false) {
+    const now = date ? new Date(date) : new Date();
+    if (timeOnly) {
+        return now.toTimeString().split(' ')[0]; // HH:MM:SS
+    }
+    return now.toISOString();
+}
+
+
+/**
+ * Indents a string by a specified length.
+ * @param {string} str - The string to indent.
+ * @param {number} length - The number of spaces to indent each line.
+ * @returns {string} The indented string.
+ */
+function indent(str, length, separator = "") {
+    const lines = str.split("\n");
+    const prefixStr = " ".repeat(length) + separator;
+    for (let i = 1; i < lines.length; i++) {
+        let entry = lines[i].trim();
+        if (entry.length === 0) continue; // skip empty lines
+        // indent the line
+        lines[i] = prefixStr + entry;
+    }
+    return lines.join("\n");
+}
+
+/**
+ * Removes empty lines at the start of a string.
+ * @param {string} str - The string to process.
+ */
+function removeEmptyLinesAtStart(str) {
+    const lines = str.split("\n");
+    for (let i = 0; i < lines.length; i++) {
+        const line = lines[i].trim();
+        if (line.length > 0) {
+            lines[i] = line; // keep the first non-empty line
+            return lines.slice(i).join("\n");
+        }
+    }
+    return "";
+}
+
+/**
+ * Strips ANSI color codes from a string.
+ * @param {string} str - The string to process.
+ */
+function stripAnsiColors(str) {
+    // This pattern catches most ANSI escape sequences
+    return str.replace(/\x1b\[[0-9;]*[a-zA-Z]/g, '');
+}
+
+
+// #region log to file
+
+/** @type {Map<string, import("fs").WriteStream>} */
+const filestreams = new Map();
+const fileLogDirectory = "node_modules/.needle/logs";
+// cleanup old log files
+if (existsSync(fileLogDirectory)) {
+    const files = readdirSync(fileLogDirectory);
+    // sort by age and keep the last 10 files
+    files.sort((a, b) => {
+        const aStat = statSync(`${fileLogDirectory}/${a}`);
+        const bStat = statSync(`${fileLogDirectory}/${b}`);
+        return aStat.mtimeMs - bStat.mtimeMs;
+    });
+    // remove all but the last 30 files
+    const filesToKeep = 30;
+    for (let i = 0; i < files.length - filesToKeep; i++) {
+        rmSync(`${fileLogDirectory}/${files[i]}`, { force: true });
+    }
+}
+
+/**
+ * Writes a log message to the file.
+ * @param {ProcessType} process
+ * @param {string} log
+ * @param {string | null} connectionId - Optional connection ID for client logs.
+ */
+function writeToFile(process, log, connectionId) {
+    const filename = `${process}.needle.log`; //connectionId && process === "client" ? `${process}-${connectionId}.needle.log` : `${process}.needle.log`;
+
+    if (!filestreams.has(filename)) {
+        if (!existsSync(fileLogDirectory)) {
+            mkdirSync(fileLogDirectory, { recursive: true });
+        }
+        filestreams.set(filename, createWriteStream(`${fileLogDirectory}/${filename_timestamp}.${filename}`, { flags: 'a' }));
+    }
+    const writeStream = filestreams.get(filename);
+    if (!writeStream) {
+        if (debug) console.error(`No write stream for process: ${filename}`);
+        return;
+    }
+    writeStream.write(log + '\n');
+}
+
+
+
+// #region process exit
+function onExit() {
+    filestreams.forEach((stream) => stream.end());
+    filestreams.clear();
+}
+const events = ['SIGTERM', 'SIGINT', 'beforeExit', 'rejectionHandled', 'uncaughtException', 'exit'];
+for (const event of events) {
+    process.on(event, onExit);
+}

package/plugins/types/userconfig.d.ts

@@ -1,5 +1,4 @@
 import { License } from "./license";
-import { NeedlePWAOptions } from "./webmanifest.js";
 
 export type needleModules = {
     webpack: object | undefined
@@ -49,7 +48,7 @@ export type userSettings = {
     /** Custom configuration for facebook instant games. */
     facebookInstantGames?: {}
     /** Set to true to create an imports.log file that shows all module imports. The file is generated when stopping the server. */
-    logModuleImportChains?: boolean;
+    debugImportChains?: boolean;
 
     /** Set to true to disable generating the buildinfo.json file in your output directory */
     noBuildInfo?: boolean;
@@ -117,4 +116,9 @@ export type userSettings = {
      * @default undefined
      */
     openBrowser?: boolean;
+
+    /** Automatically import MaterialX for needle engine in 'main.ts' */
+    loadMaterialX?: boolean;
+
+    disableLogging?: boolean;
 }

package/plugins/vite/alias.js

@@ -3,14 +3,33 @@ import path from 'path';
 
 const projectDir = process.cwd() + "/";
 
+/**
+ * @typedef {"auto-resolve" | ((res: string, packageName: string, index: number, path: string) => string | null | void)} PackageResolveValue
+ */
+
 /** these are alias callbacks as in the vite.alias dictionary
  * the first argument is the already resoled absolute path (it is only invoked if the path was found in node_modules)
  * the 2,3,4 args are the same as in vite.alias (packageName, index, path);
  */
 /**
- * @type {Record<string, null | ((res: string, packageName: string, index: number, path: string) => string | null | void)>}
+ * @type {Record<string, PackageResolveValue>}
  */
 const packages_to_resolve = {
+    // We are currently overriding "three" resolution to ensure that all dependencies resolve to the same three.js version.
+    // This is hacky, but the alternative is potentially having conflicting three.js versions since some packages are 
+    // stubborn with their peer dependencies or just slow (slower as we) with updating.
+    // NOT adding this allows node.js to correctly resolve `exports` specified in three.js package.json;
+    // since we're overriding resolution here we need to manually resolve the subset of exports that we use.
+    'three/addons': (res, packageName, index, path) => {
+        return "three/examples/jsm";
+    },
+    'three/nodes': (res, packageName, index, path) => {
+        return "three/examples/jsm/nodes/Nodes.js";
+    },
+    'three': (res, packageName, index, _path) => {
+        return path.resolve(projectDir, 'node_modules', 'three');
+    },
+
     // Handle all previous imports where users did import using @needle-engine/src
     '@needle-tools/engine/src': (res, packageName, index, path) => {
         // resolve old engine/src imports UNLESS it's the asap plugin (the asap plugin currently only exists in the src folder)
@@ -23,21 +42,25 @@ const packages_to_resolve = {
     because this is automatically done by vite according to whatever we define in our package.json exports
     This did previously prevent us from declaring proper exports in package.json
     */
-    '@needle-tools/engine': (res, packageName, index, path) => {
+    '@needle-tools/engine': (res, packageName, index, _path) => {
         // Check if the import is something like @needle-tools/engine/engine/engine_utils
         // in which case we want to resolve into the lib directory
-        if (path.startsWith("@needle-tools/engine/engine")) {
+        if (_path.startsWith("@needle-tools/engine/engine")) {
             return res + "/lib";
         }
+        const node_modules_path = path.resolve(projectDir, 'node_modules', '@needle-tools/engine');
+        if (existsSync(node_modules_path)) {
+            return node_modules_path;
+        }
     },
 
-    /* 
-    Removed. Three.js is manually resolved below to ensure all dependencies resolve to the same three.js version.
-    'three': null,
-    */
-    'peerjs': null,
-    'websocket-ts': null,
-    'md5': null,
+    // Auto resolve to 'node_modules/<name>' or '@needle-tools/engine/node_modules/<name>'
+    'peerjs': "auto-resolve",
+    'websocket-ts': "auto-resolve",
+    'md5': "auto-resolve",
+    'three-mesh-bvh': "auto-resolve",
+    'postprocessing': "auto-resolve",
+    '@dimforge/rapier3d-compat': "auto-resolve",
 }
 
 /**
@@ -67,6 +90,7 @@ export const needleViteAlias = (command, config, userSettings) => {
     }
 
 
+    /** @type {import("vite").Plugin} */
     const aliasPlugin = {
         name: "needle-alias",
         config(config) {
@@ -75,11 +99,9 @@ export const needleViteAlias = (command, config, userSettings) => {
             if (!config.resolve.alias) config.resolve.alias = {};
             const aliasDict = config.resolve.alias;
 
-            addThreeJSResolvers(aliasDict);
-
             for (const name in packages_to_resolve) {
                 if (!aliasDict[name]) {
-                    addPathResolver(name, aliasDict, packages_to_resolve[name]);
+                    addPathResolver(name, aliasDict, packages_to_resolve[name], debug ?? false);
                 }
             }
 
@@ -127,43 +149,36 @@ export const needleViteAlias = (command, config, userSettings) => {
     if (debug) return [debuggingPlugin, aliasPlugin];
     return [aliasPlugin];
 
-    function addThreeJSResolvers(aliasDict) {
-        // We are currently overriding "three" resolution to ensure that all dependencies resolve to the same three.js version.
-        // This is hacky, but the alternative is potentially having conflicting three.js versions since some packages are 
-        // stubborn with their peer dependencies or just slow (slower as we) with updating.
-        // NOT adding this allows node.js to correctly resolve `exports` specified in three.js package.json;
-        // since we're overriding resolution here we need to manually resolve the subset of exports that we use.
-        aliasDict['three/addons'] = (res, packageName, index, path) => {
-            return "three/examples/jsm";
-        };
-        aliasDict['three/nodes'] = (res, packageName, index, path) => {
-            return "three/examples/jsm/nodes/Nodes.js";
-        };
-        aliasDict['three'] = (res, packageName, index, _path) => {
-            return path.resolve(projectDir, 'node_modules', 'three');
-        };
-    }
-
-    function addPathResolver(name, aliasDict, cb) {
+    /**
+     * Adds a path resolver to the alias dictionary.
+     * @param {string} name - The name of the package to resolve.
+     * @param {import("vite").AliasOptions} aliasDict - The alias dictionary to add the resolver to.
+     * @param {PackageResolveValue | null} value - A callback function to override the default resolution behavior.
+     * @param {boolean} debug - Whether to log debug information.
+     * @returns {void}
+     */
+    function addPathResolver(name, aliasDict, value, debug) {
         // If a package at the node_modules path exist we resolve the request there
         // introduced in 89a50718c38940abb99ee16c5e029065e41d7d65
-        const fullpath = path.resolve(projectDir, 'node_modules', name);
-        if (typeof cb !== "function") cb = null;
-        const isDevEnvironment = process.env.NODE_ENV === "development";
+        const callback = typeof value === "function" ? value : null;
+
+        let fullpath = path.resolve(projectDir, 'node_modules', name);
+        if (!existsSync(fullpath)) {
+            fullpath = path.resolve(projectDir, 'node_modules', "@needle-tools/engine", "node_modules", name);
+        }
 
         if (existsSync(fullpath)) {
             aliasDict[name] = (packageName, index, path) => {
-                if (cb !== null && !isDevEnvironment) {
-                    const overrideResult = cb(fullpath, packageName, index, path);
-                    if (overrideResult !== undefined)
+                if (callback !== null) {
+                    const overrideResult = callback(fullpath, packageName, index, path);
+                    if (typeof overrideResult === "string")
                         if (existsSync(overrideResult)) {
-                            console.warn(`[needle-alias] \"${path}\" was requested and resolved to \"${overrideResult}\"`);
+                            if (debug) console.warn(`[needle-alias] \"${path}\" was requested and resolved to \"${overrideResult}\"`);
                             return overrideResult;
                         }
                 }
 
-                if (path != packageName) {
-                    // TODO: we might want to check if the package.json exports contains the path to see if it's valid
+                if (debug && path != packageName) {
                     console.warn(`[needle-alias] \"${path}\" was requested and resolved to \"${fullpath}\"`);
                 }
 

package/plugins/vite/imports-logger.js

@@ -6,7 +6,7 @@ import fs from 'fs';
  */
 export const needleImportsLogger = (command, config, userSettings) => {
 
-    if (!userSettings.logModuleImportChains) return;
+    if (!userSettings.debugImportChains) return;
 
     const graph =  {
         allNodes: new Map(),

package/plugins/vite/index.js

@@ -66,6 +66,8 @@ export { needleBuildInfo } from "./buildinfo.js";
 import { needleServer } from "./server.js";
 import { needleNPM } from "./npm.js";
 import { needleTransformCode } from "./transform.js";
+import { needleMaterialXLoader } from "./materialx.js";
+import { needleLogger } from "./logger.js";
 export { needleServer } from "./server.js";
 
 
@@ -110,6 +112,7 @@ export const needlePlugins = async (command, config = undefined, userSettings =
     userSettings = { ...defaultUserSettings, ...userSettings };
 
     const array = [
+        needleLogger(command, config, userSettings),
         needleDefines(command, config, userSettings),
         needleLicense(command, config, userSettings),
         needleViteAlias(command, config, userSettings),
@@ -133,6 +136,7 @@ export const needlePlugins = async (command, config = undefined, userSettings =
         needlePWA(command, config, userSettings),
         needleServer(command, config, userSettings),
         needleNPM(command, config, userSettings),
+        needleMaterialXLoader(command, config, userSettings),
     ];
 
     const asap = await needleAsap(command, config, userSettings);