@needle-tools/engine 2.63.2-pre → 2.63.2-pre.1

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@needle-tools/engine",
-    "version": "2.63.2-pre",
+    "version": "2.63.2-pre.1",
     "description": "Needle Engine is a web-based runtime for 3D apps. It runs on your machine for development, and can be deployed anywhere. It is flexible, extensible, and collaboration and XR come naturally.",
     "main": "dist/needle-engine.umd.cjs",
     "module": "dist/needle-engine.js",

package/src/plugins/vite/build.js

@@ -0,0 +1,20 @@
+
+export const needleBuild = (command, config, userSettings) => {
+
+    // TODO: need to set this when building a dist
+    const isDeployOnlyBuild = config?.deployOnly === true;
+
+    return {
+        name: 'build',
+        config(config) {
+            if (!config.build) {
+                config.build = {};
+            }
+            if (isDeployOnlyBuild)
+            {
+                console.log("Deploy only build - will not empty output directory")
+                config.build.emptyOutDir = false;
+            }
+        }
+    }
+}

package/src/plugins/vite/config.js

@@ -0,0 +1,48 @@
+import { existsSync, readFileSync } from 'fs';
+
+export async function loadConfig(path) {
+    try {
+        // First try to get the path from the config
+        if (!path) {
+            const configJson = tryLoadProjectConfig();
+            if (configJson?.codegenDirectory) {
+                path = configJson.codegenDirectory + "/meta.json";
+            }
+        }
+        // If that fails, try the default path
+        if (!path)
+            path = './src/generated/meta.json';
+
+        if (existsSync(path)) {
+            const text = readFileSync(path, 'utf8');
+            if (!text) return null;
+            return JSON.parse(text);
+        }
+        else console.error("Could not find config file at " + path);
+        return null;
+    }
+    catch (err) {
+        console.error("Error loading config file");
+        console.error(err);
+        return null;
+    }
+}
+
+export function tryLoadProjectConfig() {
+    try {
+        const root = process.cwd();
+        const path = root + '/needle.config.json';
+        if (existsSync(path)) {
+            const text = readFileSync(path);
+            if (!text) return null;
+            const json = JSON.parse(text);
+            return json;
+        }
+    }
+    catch (err) {
+        console.error("Error loading config file");
+        console.error(err);
+    }
+
+    return null;
+}

package/src/plugins/vite/gzip.js

@@ -0,0 +1,6 @@
+
+// import { existsSync } from 'fs';
+
+export function useGzip(config) {
+    return config?.gzip ? true : false;
+}

package/src/plugins/vite/index.js

@@ -0,0 +1,23 @@
+import { needleBuild } from "./build.js";
+import { needleMeta } from "./meta.js"
+import { needlePoster } from "./poster.js"
+import { needleReload } from "./reload.js"
+
+export * from "./gzip.js";
+export * from "./config.js";
+
+const defaultUserSettings = {
+    allowRemoveMetaTags: true,
+    allowHotReload: true,
+}
+
+export const needlePlugins = (command, config, userSettings) => {
+    // ensure we have user settings initialized with defaults
+    userSettings = { ...defaultUserSettings, ...userSettings }
+    return [
+        needleMeta(command, config, userSettings),
+        needlePoster(command),
+        needleReload(command, config, userSettings),
+        needleBuild(command, config, userSettings)
+    ]
+}

package/src/plugins/vite/meta.js

@@ -0,0 +1,128 @@
+import { loadConfig } from './config.js';
+import fs from 'fs';
+import { getPosterPath } from './poster.js';
+
+export const needleMeta = (command, config, userSettings) => {
+
+    // we can check if this is a build
+    // const isBuild = command === 'build';
+
+    async function updateConfig() {
+        config = await loadConfig();
+    }
+
+    if (!userSettings) userSettings = {};
+
+    return {
+        // replace meta tags
+        name: 'meta-tags',
+        transformIndexHtml: {
+            enforce: 'pre',
+            transform(html, _ctx) {
+
+                html = insertNeedleCredits(html);
+
+                if (userSettings.allowMetaPlugin === false) return [];
+
+                // this is useful to get the latest config exported from editor
+                // whenever vite wants to transform the html
+                updateConfig();
+
+                // early out of the config is invalid / doesn't contain meta information
+                // TODO might be better to handle these edge cases / special cases right from Unity/Blender and not here
+                if (!config) return [];
+
+                let meta = config.meta;
+
+                if (!meta) {
+                    meta = {};
+                    meta.title = config.sceneName;
+                }
+
+                if (!meta.image?.length) {
+                    const path = getPosterPath();
+                    if (fs.existsSync('./' + path))
+                        meta.image = path;
+                }
+
+                const tags = [];
+
+                let img = meta.image;
+                if (img?.length) {
+                    // for a regular build the absolutePath is url (since we dont know the deployment target)
+                    if (config.absolutePath?.length) {
+                        const baseUrl = config.absolutePath;
+                        let url = baseUrl + "/" + img;
+                        url = removeDuplicateSlashesInUrl(url);
+                        // url = appendVersion(url);
+                        tags.push({ tag: 'meta', attrs: { name: 'twitter:card', content: 'summary_large_image' } });
+                        tags.push({ tag: 'meta', attrs: { name: 'twitter:image', content: url } });
+                        tags.push({ tag: 'meta', attrs: { name: 'og:image', content: url } });
+                    }
+                }
+
+                if (config.absolutePath?.length) {
+                    html = updateUrlMetaTag(html, config.absolutePath);
+                }
+
+                if (meta.title?.length) {
+                    tags.push({ tag: 'meta', attrs: { name: 'og:title', content: meta.title } });
+
+                    html = html.replace(
+                        /<title>(.*?)<\/title>/,
+                        `<title>${meta.title}</title>`,
+                    );
+
+                    if (userSettings.allowRemoveMetaTags !== false)
+                        html = removeMetaTag(html, 'og:title');
+
+                }
+
+                if (meta.description?.length) {
+                    tags.push({ tag: 'meta', attrs: { name: 'description', content: meta.description } });
+                    tags.push({ tag: 'meta', attrs: { name: 'og:description', content: meta.description } });
+
+                    if (userSettings.allowRemoveMetaTags !== false) {
+                        html = removeMetaTag(html, 'description');
+                        html = removeMetaTag(html, 'og:description');
+                    }
+                }
+
+                return { html, tags }
+            },
+        }
+    }
+}
+
+function updateUrlMetaTag(html, url) {
+    html = html.replace(`<meta name="url" content="http://needle.tools">`, `<meta name="url" content="${url}">`);
+    return html;
+}
+
+function appendVersion(str) {
+    return str + "?v=" + Date.now();
+}
+
+function removeDuplicateSlashesInUrl(url) {
+    return url.replace(/([^:]\/)\/+/g, "$1");
+}
+
+function removeMetaTag(html, name) {
+    // TODO: maybe we could also just replace the content
+    const regex = new RegExp(`<meta (name|property)="${name}".+?\/?>`, 'gs');
+    const newHtml = html.replace(
+        regex,
+        '',
+    );
+    // console.log(newHtml);
+    return newHtml;
+}
+
+function insertNeedleCredits(html) {
+    const needleCredits = `<!-- 🌵 Made with Needle — https://needle.tools -->`;
+    html = html.replace(
+        /<head>/,
+        needleCredits + "\n<head>",
+    );
+    return html;
+}

package/src/plugins/vite/poster-client.js

@@ -0,0 +1,60 @@
+
+
+async function generatePoster() {
+    const { screenshot } = await import("@needle-tools/engine/engine/engine_utils_screenshot");
+
+    try {
+        const needleEngine = document.querySelector("needle-engine");
+        if (!needleEngine) return null;
+
+        const width = 1920;
+        const height = 1920;
+        const context = await needleEngine.getContext();
+
+        // wait until a few update loops have run
+        while (context.time.frameCount < 3) {
+            await new Promise((resolve) => setTimeout(resolve, 50));
+        }
+
+        const mimeType = "image/webp";
+        console.log("Generating poster...");
+        const dataUrl = screenshot(context, width, height, mimeType);
+
+        return dataUrl;
+    }
+    catch (e) {
+        console.error(e);
+        return null;
+    }
+}
+
+async function sendPoster() {
+    const blob = await generatePoster();
+    import.meta.hot.send("needle:screenshot", { data: blob });
+}
+
+// communicate via vite
+if (import.meta.hot) {
+    // wait for needle engine to be fully loaded
+    const needleEngine = document.querySelector("needle-engine");
+    needleEngine?.addEventListener("loadfinished", () => {
+        // wait a moment
+        setTimeout(() => {
+            sendPoster();
+        }, 200);
+    });
+
+    // for debugging: build extra button with dev-only options
+    /*
+    var button = document.createElement("button");
+    button.id = "send-msg";
+    button.innerHTML = "Generate Poster";
+    button.style.position = "fixed";
+    button.style.zIndex = "9999";
+    document.body.appendChild(button);
+
+    document.querySelector("#send-msg").addEventListener("click", async () => {
+        sendPoster();
+    });
+    */
+}

package/src/plugins/vite/poster.js

@@ -0,0 +1,55 @@
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+export function getPosterPath() {
+    return "include/poster.webp";
+}
+
+export const needlePoster = (command) => {
+    // only relevant for local development
+    if (command === 'build') return [];
+
+    return {
+        name: 'save-screenshot',
+        configureServer(server) {
+            server.ws.on('needle:screenshot', async (data, client) => {
+                if(!data?.data){
+                    console.warn("Received empty screenshot data, ignoring");
+                    return;
+                }
+                const targetPath = "./" + getPosterPath();
+                console.log("Received poster, saving to " + targetPath);
+                // remove data:image/png;base64, from the beginning of the string
+                if (targetPath.endsWith(".webp"))
+                    data.data = data.data.replace(/^data:image\/webp;base64,/, "");
+                else
+                    data.data = data.data.replace(/^data:image\/png;base64,/, "");
+                const dir = path.dirname(targetPath);
+                if (!fs.existsSync(dir)) {
+                    fs.mkdirSync(dir, { recursive: true })
+                }
+                fs.writeFileSync(targetPath, Buffer.from(data.data, "base64"));
+            });
+        },
+        transformIndexHtml: {
+            enforce: 'pre',
+            transform(html, ctx) {
+                const file = path.join(__dirname, 'poster-client.js');
+                return [
+                    {
+                        tag: 'script',
+                        attrs: {
+                            type: 'module',
+                        },
+                        children: fs.readFileSync(file, 'utf8'),
+                        injectTo: 'body',
+                    },
+                ];
+            },
+        },
+    }
+};

package/src/plugins/vite/reload-client.js

@@ -0,0 +1,16 @@
+import { showBalloonMessage } from "@needle-tools/engine";
+
+try {
+    // communicate via vite
+    if (import.meta.hot) {
+        // listen to needle-reload event
+        import.meta.hot.on('needle:reload', (evt) => {
+            console.log("Received reload event");
+            showBalloonMessage("Detected files changing\npage will reload in a moment");
+        });
+
+    }
+}
+catch {
+    // ignore
+}

package/src/plugins/vite/reload.js

@@ -0,0 +1,342 @@
+import path from 'path';
+import { loadConfig, tryLoadProjectConfig } from './config.js';
+import { getPosterPath } from './poster.js';
+import * as crypto from 'crypto';
+import { existsSync, readFileSync, statSync } from 'fs';
+import { fileURLToPath } from 'url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+const filesUsingHotReload = new Set();
+
+export const needleReload = (command, config, userSettings) => {
+    if (command === "build") return;
+
+
+    let isUpdatingConfig = false;
+    const updateConfig = async () => {
+        if (isUpdatingConfig) return;
+        isUpdatingConfig = true;
+        const res = await loadConfig();
+        isUpdatingConfig = false;
+        if (res) config = res;
+    }
+
+
+    const projectConfig = tryLoadProjectConfig();
+    const buildDirectory = projectConfig?.buildDirectory?.length ? process.cwd().replaceAll("\\", "/") + "/" + projectConfig?.buildDirectory : "";
+    if (buildDirectory?.length) {
+        setTimeout(() => console.log("Build directory: ", buildDirectory), 100);
+    }
+
+    // These ignore patterns will be injected into user config to better control vite reloading
+    const ignorePatterns = ["dist/**/*", "src/generated/*", "**/package~/**/codegen/**/*", "**/codegen/register_types.js"];
+    if (projectConfig?.buildDirectory?.length) ignorePatterns.push(`${projectConfig?.buildDirectory}/**/*`);
+    if (projectConfig?.codegenDirectory?.length) ignorePatterns.push(`${projectConfig?.codegenDirectory}/**/*`);
+
+    return {
+        name: 'reload',
+        config(config) {
+            if (!config.server) config.server = { watch: { ignored: [] } };
+            else if (!config.server.watch) config.server.watch = { ignored: [] };
+            else if (!config.server.watch.ignored) config.server.watch.ignored = [];
+            for (const pattern of ignorePatterns)
+                config.server.watch.ignored.push(pattern);
+            setTimeout(() => console.log("Updated server ignore patterns: ", config.server.watch.ignored), 100);
+        },
+        handleHotUpdate(args) {
+            args.buildDirectory = buildDirectory;
+            return handleReload(args);
+        },
+        transform(src, id) {
+            if (!id.includes(".ts")) return;
+            updateConfig();
+            if (config?.allowHotReload === false) return;
+            if (userSettings?.allowHotReload === false) return;
+            src = insertScriptRegisterHotReloadCode(src, id);
+            return insertScriptHotReloadCode(src, id);
+        },
+        transformIndexHtml: {
+            enforce: 'pre',
+            transform(html, _) {
+                if (config?.allowHotReload === false) return [html];
+                if (userSettings?.allowHotReload === false) return [html];
+                const file = path.join(__dirname, 'reload-client.js');
+                return [
+                    {
+                        tag: 'script',
+                        attrs: {
+                            type: 'module',
+                        },
+                        children: readFileSync(file, 'utf8'),
+                        injectTo: 'body',
+                    },
+                ];
+            },
+        },
+    };
+}
+
+
+const ignorePatterns = [];
+const ignoreRegex = new RegExp(ignorePatterns.join("|"));
+
+let lastReloadTime = 0;
+const posterPath = getPosterPath();
+let reloadIsScheduled = false;
+const lockFileName = "needle.lock";
+
+function notifyClientWillReload(server, file) {
+    console.log("Send reload notification");
+    server.ws.send('needle:reload', { type: 'will-reload', file: file });
+}
+
+async function handleReload({ file, server, modules, read, buildDirectory }) {
+
+    // dont reload the full page on css changes
+    const isCss = file.endsWith(".css") || file.endsWith(".scss") || file.endsWith(".sass")
+    if (isCss) return;
+
+    // Dont reload the whole server when a file that is using hot reload changes
+    if (filesUsingHotReload.has(file)) {
+        console.log("File is using hot reload: " + file);
+        return;
+    }
+
+    // the poster is generated after the server has started
+    // we dont want to specifically handle the png or webp that gets generated
+    if (file.includes(posterPath)) return;
+
+    if (file.endsWith("build.log")) return;
+
+    // if (file.endsWith("/codegen/register_types.js" || file.endsWith("/generated/register_types.js"))) {
+    //     console.log("Ignore change in codegen file: " + file);
+    //     return [];
+    // }
+
+    // This was a test for ignoring files via regex patterns
+    // instead of relying on the vite server watch ignore array
+    // we could here also match paths that we know we dont want to track
+    if (ignorePatterns.length > 0 && ignoreRegex.test(file)) {
+        console.log("Ignore change in file: " + getFileNameLog(file));
+        return [];
+    }
+
+    // Ignore files changing in output directory
+    // this happens during build time when e.g. dist is not ignored in vite config
+    // we still dont want to reload the local server in that case
+    if (buildDirectory?.length) {
+        const dir = path.dirname(file).replaceAll("\\", "/");
+        if (dir.startsWith(buildDirectory)) {
+            console.log("Ignore change in build directory: " + getFileNameLog(file));
+            return [];
+        }
+    }
+
+    // Check if codegen files actually changed their content
+    // this will return false if its the first update
+    // meaning if its the first export after the server starts those will not trigger a reload
+    const shouldCheckIfContentChanged = file.includes("/codegen/") || file.includes("/generated/") || file.endsWith("gen.js");// || file.endsWith(".glb") || file.endsWith(".gltf") || file.endsWith(".bin");
+    if (shouldCheckIfContentChanged) {
+        if (reloadIsScheduled) {
+            return [];
+        }
+        if (await testIfFileContentChanged(file, read) === false) {
+            console.log("File content didnt change: " + getFileNameLog(file));
+            return [];
+        }
+    }
+
+    if (file.endsWith(".vue") || file.endsWith(".ts") || file.endsWith(".js") || file.endsWith(".jsx") || file.endsWith(".tsx"))
+        return;
+
+    if (file.endsWith(lockFileName)) return;
+    let fileSize = "";
+    const isGlbOrGltfFile = file.endsWith(".glb") || file.endsWith(".bin");
+    if (isGlbOrGltfFile && existsSync(file)) {
+        fileSize = statSync(file).size;
+        // the file is about to be created/written to
+        if (fileSize <= 0) {
+            // console.log("> File is changing: " + getFileNameLog(file));
+            return;
+        }
+    }
+
+    console.log("> Detected file change: ", getFileNameLog(file) + " (" + ((fileSize / (1024 * 1024)).toFixed(1)) + " MB)");
+    notifyClientWillReload(server);
+    scheduleReload(server);
+    return [];
+}
+
+
+async function scheduleReload(server, level = 0) {
+    if (reloadIsScheduled && level === 0) return;
+    reloadIsScheduled = true;
+
+    const lockFile = path.join(process.cwd(), lockFileName);
+    if (existsSync(lockFile)) {
+        if (level === 0)
+            console.log("Lock file exists, waiting for export to finish...");
+        setTimeout(() => scheduleReload(server, level += 1), 300);
+        return;
+    }
+
+    reloadIsScheduled = false;
+
+    const timeDiff = Date.now() - lastReloadTime;
+    if (timeDiff < 1000) {
+        // Sometimes file changes happen immediately after triggering a reload 
+        // we dont want to reload again in that case
+        console.log("Ignoring reload, last reload was too recent", timeDiff);
+        return;
+    }
+
+    lastReloadTime = Date.now();
+    const readableTime = new Date(lastReloadTime).toLocaleTimeString();
+    console.log("< Reloading... " + readableTime)
+    server.ws.send({
+        type: 'full-reload',
+        path: '*'
+    });
+}
+
+const projectDirectory = process.cwd().replaceAll("\\", "/");
+
+function getFileNameLog(file) {
+    if (file.startsWith(projectDirectory)) {
+        return file.substring(projectDirectory.length);
+    }
+    return file;
+}
+
+const hashes = new Map();
+const hash256 = crypto.createHash('sha256');
+
+async function testIfFileContentChanged(file, read) {
+    let content = await read(file);
+    content = removeVersionQueryArgument(content);
+
+    const hash = hash256.copy();
+    hash.update(content);
+    // compare if hash string changed
+    const newHash = hash.digest('hex');
+    const oldHash = hashes.get(file);
+    if (oldHash !== newHash) {
+        // console.log("Update hash for file: " + getFileNameLog(file) + " to " + newHash);
+        hashes.set(file, newHash);
+        // if its the first update we dont want to trigger a reload
+        if (!oldHash) return false;
+        return true;
+    }
+    return false;
+}
+function removeVersionQueryArgument(content) {
+    if (typeof content === "string") {
+        // Some codegen files include hashes for loading glb files (e.g. ?v=213213124)
+        // Or context.hash = "54543453"
+        // We dont want to use those hashes for detecting if the file changed
+        let res = content.replaceAll(/(v=[0-9]+)/g, "");
+        res = res.replaceAll(/(hash = \"[0-9]+\")/g, "hash = \"\"");
+        return res;
+    }
+    return content;
+}
+
+
+
+function insertScriptRegisterHotReloadCode(src, filePath) {
+    if (!filePath.includes("needle-engine.ts")) {
+        return src;
+    }
+
+    // this code injects a register call into the component method
+    const code = `
+
+import { register, unregister } from "./engine/engine_hot_reload";
+import { Component as ComponentType } from "./engine-components/Component";
+
+const prototype = ComponentType.prototype;
+const created = prototype.__internalNewInstanceCreated;
+prototype.__internalNewInstanceCreated = function (...args) {
+    created.call(this, ...args);
+    register(this);
+}
+const destroy = prototype.__internalDestroy;
+prototype.__internalDestroy = function (...args) {
+    destroy.call(this, ...args);
+    unregister(this);
+}
+        `
+    return code + src;
+}
+
+
+const HOT_RELOAD_START_MARKER = "NEEDLE_HOT_RELOAD_BEGIN";
+const HOT_RELOAD_END_MARKER = "NEEDLE_HOT_RELOAD_END";
+
+function insertScriptHotReloadCode(src, filePath) {
+    if (filePath.includes("engine_hot_reload")) return;
+    if (filePath.includes(".vite")) return;
+
+    const originalFilePath = filePath;
+
+    // default import path when outside package
+    let importPath = "@needle-tools/engine/engine/engine_hot_reload";
+
+    if (filePath.includes("package~/engine")) {
+        // convert local dev path to project node_modules path 
+        const folderName = "package~";
+        const startIndex = filePath.indexOf(folderName);
+        const newPath = process.cwd() + "/node_modules/@needle-tools/engine" + filePath.substring(startIndex + folderName.length);
+        filePath = newPath;
+    }
+
+    if (filePath.includes("@needle-tools/engine")) {
+        // only make engine components hot reloadable
+        if (!filePath.includes("engine/engine-components"))
+            return;
+        // make import path from engine package
+        const fullPathToHotReload = process.cwd() + "/node_modules/@needle-tools/engine/engine/engine_hot_reload.ts";
+        // console.log(fullPathToHotReload);
+        const fileDirectory = path.dirname(filePath);
+        // console.log("DIR", fileDirectory)
+        const relativePath = path.relative(fileDirectory, fullPathToHotReload);
+        importPath = relativePath.replace(/\\/g, "/");
+        // console.log("importPath: ", importPath);
+    }
+
+    // console.log(importPath, ">", filePath);
+
+
+    const code = `
+
+// ${HOT_RELOAD_START_MARKER}
+// Inserted by needle reload plugin (vite)
+import { applyChanges } from "${importPath}";
+//@ts-ignore
+if (import.meta.hot) {
+    //@ts-ignore
+    import.meta.hot.accept((newModule) => {
+        if (newModule) {
+
+            const success = applyChanges(newModule);
+            if(success === false){
+                //@ts-ignore
+                import.meta.hot.invalidate()
+            }
+        }
+    })
+}
+// ${HOT_RELOAD_END_MARKER}
+`
+
+    if (!filesUsingHotReload.has(originalFilePath))
+        filesUsingHotReload.add(originalFilePath);
+
+    return {
+        code: code + src,
+        map: null
+    }
+
+}