@utoo/web 1.1.0 → 1.2.0-rc.2

package/esm/webpackLoaders/worker/cjs.js

@@ -1,9 +1,255 @@
 import "systemjs/dist/system.js";
-import nodePolyFills from "./nodePolyFills";
+import nodePolyFills from "./polyfills/nodePolyFills";
+const fs = nodePolyFills.fs;
+const path = nodePolyFills.path;
+const installedModules = {};
+const statCache = {};
+const pkgJsonCache = {};
+const resolutionCache = {};
+const searchPathsCache = {};
+const statSync = (p) => {
+    if (p.includes("node_modules") &&
+        Object.prototype.hasOwnProperty.call(statCache, p)) {
+        if (statCache[p] === false)
+            throw new Error("ENOENT");
+        return statCache[p];
+    }
+    try {
+        const res = fs.statSync(p);
+        if (p.includes("node_modules"))
+            statCache[p] = res;
+        return res;
+    }
+    catch (e) {
+        if (p.includes("node_modules"))
+            statCache[p] = false;
+        throw e;
+    }
+};
+const existsSync = (p) => {
+    try {
+        statSync(p);
+        return true;
+    }
+    catch (_a) {
+        return false;
+    }
+};
+const executeModule = (moduleCode, moduleId, id, importMaps, entrypoint) => {
+    if (installedModules[moduleId]) {
+        return installedModules[moduleId].exports;
+    }
+    const context = path.dirname(moduleId);
+    let finalExports = {};
+    const moduleRequire = (childId) => loadModule(childId, context, importMaps, entrypoint);
+    moduleRequire.resolve = (request) => request;
+    const module = { exports: finalExports, require: moduleRequire };
+    if (moduleId.includes("node_modules")) {
+        installedModules[moduleId] = module;
+    }
+    // Hack for entrypoint
+    if (moduleId === entrypoint) {
+        moduleCode = "self.Buffer = require('buffer').Buffer;" + moduleCode;
+    }
+    try {
+        if (moduleId.endsWith(".json")) {
+            finalExports = JSON.parse(moduleCode);
+            module.exports = finalExports;
+        }
+        else {
+            new Function("require", "exports", "module", "__filename", "__dirname", moduleCode)(moduleRequire, module.exports, module, moduleId, context);
+            finalExports = module.exports;
+        }
+    }
+    catch (e) {
+        console.error(`Worker: Error executing module ${moduleId}:`, e);
+        throw new Error(`Failed to load dependency ${moduleId}: ${e.message}`);
+    }
+    const originalWarn = console.warn;
+    console.warn = (...args) => {
+        var _a;
+        const msg = ((_a = args[0]) === null || _a === void 0 ? void 0 : _a.toString()) || "";
+        if (msg.includes("(SystemJS Error#W3")) {
+            return;
+        }
+        originalWarn.apply(console, args);
+    };
+    try {
+        System.set(moduleId, { default: finalExports });
+    }
+    catch (e) {
+        // ignore
+    }
+    finally {
+        console.warn = originalWarn;
+    }
+    return finalExports;
+};
+const loadModule = (id, context, importMaps, entrypoint) => {
+    var _a, _b, _c, _d, _e, _f;
+    const cacheKey = `${context}:${id}`;
+    if (resolutionCache[cacheKey]) {
+        const cachedId = resolutionCache[cacheKey];
+        if (installedModules[cachedId]) {
+            return installedModules[cachedId].exports;
+        }
+        // Fast path: if we know the resolved path, try to load it directly
+        try {
+            const moduleCode = fs.readFileSync(cachedId, "utf8");
+            return executeModule(moduleCode, cachedId, id, importMaps, entrypoint);
+        }
+        catch (e) {
+            // If read fails, fall back to full resolution
+        }
+    }
+    // 1. Resolve
+    let resolvedId = id;
+    if (id.startsWith(".")) {
+        resolvedId = path.resolve(context, id);
+    }
+    // 2. Check Cache (SystemJS)
+    let dependency = System.get(resolvedId);
+    if (dependency)
+        return dependency.default;
+    if (id !== resolvedId) {
+        dependency = System.get(id);
+        if (dependency)
+            return dependency.default;
+    }
+    // 3. Check Node Polyfills
+    if (id in nodePolyFills) {
+        // @ts-ignore
+        return nodePolyFills[id];
+    }
+    if (resolvedId in nodePolyFills) {
+        // @ts-ignore
+        return nodePolyFills[resolvedId];
+    }
+    // 4. Check importMaps & FS
+    let moduleCode = importMaps[resolvedId] || importMaps[id];
+    let moduleId = importMaps[resolvedId] ? resolvedId : id;
+    // Fallback: Try resolving from node_modules
+    if (!moduleCode && !id.startsWith(".") && !id.startsWith("/")) {
+        let searchPaths = searchPathsCache[context];
+        if (!searchPaths) {
+            searchPaths = [];
+            // @ts-ignore
+            const cwd = ((_b = (_a = self.process) === null || _a === void 0 ? void 0 : _a.cwd) === null || _b === void 0 ? void 0 : _b.call(_a)) || ((_c = self.workerData) === null || _c === void 0 ? void 0 : _c.cwd) || "/";
+            const isInsideNodeModules = context.includes("/node_modules/") ||
+                context.includes("\\node_modules\\");
+            if (isInsideNodeModules) {
+                searchPaths.push(path.join(cwd, "node_modules"));
+            }
+            let currentDir = context;
+            while (true) {
+                if (path.basename(currentDir) !== "node_modules") {
+                    const nodeModulesPath = path.join(currentDir, "node_modules");
+                    if (!searchPaths.includes(nodeModulesPath)) {
+                        searchPaths.push(nodeModulesPath);
+                    }
+                }
+                const parent = path.dirname(currentDir);
+                if (parent === currentDir)
+                    break;
+                currentDir = parent;
+            }
+            searchPathsCache[context] = searchPaths;
+        }
+        for (const nodeModulesDir of searchPaths) {
+            const nodeModulesPath = path.join(nodeModulesDir, id);
+            // Check package.json first
+            const pkgJsonPath = path.join(nodeModulesPath, "package.json");
+            if (existsSync(pkgJsonPath)) {
+                try {
+                    let pkg;
+                    if (pkgJsonCache[pkgJsonPath]) {
+                        pkg = pkgJsonCache[pkgJsonPath];
+                    }
+                    else {
+                        pkg = JSON.parse(fs.readFileSync(pkgJsonPath, "utf8"));
+                        pkgJsonCache[pkgJsonPath] = pkg;
+                    }
+                    const mainField = pkg.main;
+                    if (mainField) {
+                        const candidates = [
+                            path.resolve(nodeModulesPath, mainField),
+                            path.resolve(nodeModulesPath, mainField) + ".js",
+                            path.resolve(nodeModulesPath, mainField) + ".json",
+                            path.resolve(nodeModulesPath, mainField, "index.js"),
+                        ];
+                        for (const candidate of candidates) {
+                            if (existsSync(candidate) && !statSync(candidate).isDirectory()) {
+                                resolvedId = candidate;
+                                moduleCode = fs.readFileSync(candidate, "utf8");
+                                moduleId = candidate;
+                                break;
+                            }
+                        }
+                    }
+                }
+                catch (_g) { }
+            }
+            if (!moduleCode) {
+                const extensions = ["", ".js", ".json", "/index.js"];
+                for (const ext of extensions) {
+                    const p = nodeModulesPath + ext;
+                    if (existsSync(p) && !statSync(p).isDirectory()) {
+                        resolvedId = p;
+                        moduleCode = fs.readFileSync(p, "utf8");
+                        moduleId = p;
+                        break;
+                    }
+                }
+            }
+            if (moduleCode)
+                break;
+        }
+    }
+    // Fallback: Try resolving absolute path (handling CWD stripping)
+    if (!moduleCode && id.startsWith("/")) {
+        // @ts-ignore
+        const cwd = ((_e = (_d = self.process) === null || _d === void 0 ? void 0 : _d.cwd) === null || _e === void 0 ? void 0 : _e.call(_d)) || ((_f = self.workerData) === null || _f === void 0 ? void 0 : _f.cwd) || "/";
+        let relativeId = id;
+        if (id.startsWith(cwd)) {
+            relativeId = id.slice(cwd.length);
+            if (relativeId.startsWith("/"))
+                relativeId = relativeId.slice(1);
+        }
+        const extensions = ["", ".js", ".json", "/index.js"];
+        for (const ext of extensions) {
+            const p = relativeId + ext;
+            if (existsSync(p) && !statSync(p).isDirectory()) {
+                resolvedId = p; // Use relative path for FS ops
+                moduleCode = fs.readFileSync(p, "utf8");
+                moduleId = id; // Keep original absolute path as module ID
+                break;
+            }
+        }
+    }
+    if (!moduleCode) {
+        // Try extensions
+        const extensions = ["", ".js", ".json", "/index.js"];
+        for (const ext of extensions) {
+            const p = resolvedId + ext;
+            if (existsSync(p) && !statSync(p).isDirectory()) {
+                resolvedId = p;
+                moduleCode = fs.readFileSync(p, "utf8");
+                moduleId = p;
+                break;
+            }
+        }
+    }
+    if (moduleCode) {
+        const cacheKey = `${context}:${id}`;
+        resolutionCache[cacheKey] = moduleId;
+        return executeModule(moduleCode, moduleId, id, importMaps, entrypoint);
+    }
+    console.error(`Worker: Dependency ${id} (resolved: ${resolvedId}) not found.`);
+    return {};
+};
 export async function cjs(entrypoint, importMaps) {
-    debugger;
     await Promise.all(Object.entries(importMaps).map(async ([k, v]) => {
-        if (v.startsWith("http")) {
+        if (v.startsWith("https://")) {
             try {
                 const response = await fetch(v);
                 if (response.ok) {
@@ -20,61 +266,10 @@ export async function cjs(entrypoint, importMaps) {
             }
         }
     }));
-    Object.assign(importMaps, nodePolyFills);
-    const require = (id) => {
-        let dependency = System.get(id);
-        if (dependency) {
-            return dependency.default;
-        }
-        if (id in nodePolyFills) {
-            // @ts-ignore
-            return nodePolyFills[id];
-        }
-        const moduleCode = importMaps[id];
-        if (!moduleCode) {
-            console.error(`Worker: Dependency ${id} not found in import maps.`);
-            return {};
-        }
-        let finalExports = {};
-        const module = { exports: finalExports };
-        const exports = module.exports;
-        try {
-            new Function("require", "exports", "module", moduleCode)(require, exports, module);
-            finalExports = module.exports;
-        }
-        catch (e) {
-            console.error(`Worker: Error executing dependency module ${id}:`, e);
-            throw new Error(`Failed to load CJS dependency ${id}: ${e.message}`);
-        }
-        System.set(id, { default: finalExports });
-        return finalExports;
-    };
-    require.resolve = (request) => request;
     // @ts-ignore
     // a hack for loader-runner resolving
-    self.__systemjs_require__ = require;
-    let entryPointCode;
-    for (const path in importMaps) {
-        const moduleCode = importMaps[path];
-        if (path === entrypoint) {
-            entryPointCode = moduleCode;
-            // FIXME
-            entryPointCode = "self.Buffer = require('buffer').Buffer;" + moduleCode;
-            break;
-        }
-    }
-    if (entryPointCode) {
-        let finalExports = {};
-        const module = { exports: finalExports };
-        const exports = module.exports;
-        try {
-            new Function("require", "exports", "module", entryPointCode)(require, exports, module);
-        }
-        catch (e) {
-            console.error(`Worker: Error executing entry point ${entrypoint}:`, e);
-        }
-    }
-    else {
-        console.warn("Warning: Entry point not found in import maps. No final execution step taken.");
-    }
+    self.__systemjs_require__ = (id) => loadModule(id, path.dirname(entrypoint), importMaps, entrypoint);
+    // @ts-ignore
+    self.__systemjs_require__.resolve = (request) => request;
+    loadModule(entrypoint, path.dirname(entrypoint), importMaps, entrypoint);
 }

package/esm/webpackLoaders/worker/index.d.ts

@@ -1,2 +1,2 @@
-declare const _default: null;
-export default _default;
+export declare function startLoaderWorker(): void;
+export default startLoaderWorker;

package/esm/webpackLoaders/worker/index.js

@@ -1,31 +1,41 @@
-import initWasm, { notifyWorkerAck, recvMessageInWorker, recvWorkerRequest, sendTaskMessage, } from "../../utoo";
+import { SabComClient } from "../../sabcom";
+import initWasm, { recvTaskMessageInWorker, sendTaskMessage, workerCreated, } from "../../utoo";
 import { cjs } from "./cjs";
 const binding = {
-    recvWorkerRequest,
-    recvMessageInWorker,
-    notifyWorkerAck,
+    recvTaskMessageInWorker,
     sendTaskMessage,
+    workerCreated,
 };
-self.process = {
-    env: {},
-    cwd: () => self.workerData.cwd,
-};
-self.onmessage = async (event) => {
-    let [module, memory, meta] = event.data;
-    await initWasm(module, memory).catch((err) => {
-        console.log(err);
-        throw err;
-    });
-    self.workerData = {
-        poolId: meta.workerData.poolId,
-        workerId: meta.workerData.workerId,
-        cwd: "./",
-        binding,
-        readFile: async (path) => {
-            // TODO: if we want that, just connect to @utoo/web internalProject endpoint port with comlink
-            throw new Error("readFile in loader not supported on browser ");
-        },
+export function startLoaderWorker() {
+    self.onmessage = async (event) => {
+        let [module, memory, meta] = event.data;
+        await initWasm(module, memory).catch((err) => {
+            console.log(err);
+            throw err;
+        });
+        const sabClient = meta.sab
+            ? new SabComClient(meta.sab, () => {
+                self.postMessage("sab_request");
+            })
+            : undefined;
+        self.workerData = {
+            workerId: meta.workerData.workerId,
+            cwd: meta.workerData.cwd,
+            binding,
+            sabClient,
+        };
+        self.process = {
+            env: {},
+            cwd: () => self.workerData.cwd,
+        };
+        console.log("Worker CWD:", self.process.cwd());
+        cjs(meta.loaderAssets.entrypoint, meta.loaderAssets.importMaps);
     };
-    cjs(meta.loaderAssets.entrypoint, meta.loaderAssets.importMaps);
-};
-export default null;
+}
+// @ts-ignore
+if (typeof __webpack_require__ !== "undefined") {
+    // @ts-ignore
+    self.startLoaderWorker = startLoaderWorker;
+}
+startLoaderWorker();
+export default startLoaderWorker;

package/esm/webpackLoaders/worker/polyfills/fastGlobPolyfill.d.ts

@@ -0,0 +1,2 @@
+declare const fastGlob: any;
+export default fastGlob;

package/esm/webpackLoaders/worker/polyfills/fastGlobPolyfill.js

@@ -0,0 +1,48 @@
+import micromatch from "micromatch";
+import * as path from "path";
+import * as fs from "./fsPolyfill";
+const walkSync = (currentDir, rootDir, entries) => {
+    let list;
+    try {
+        list = fs.readdirSync(currentDir, { withFileTypes: true });
+    }
+    catch (e) {
+        return;
+    }
+    for (const entry of list) {
+        const fullPath = path.join(currentDir, entry.name);
+        const relativePath = path.relative(rootDir, fullPath);
+        if (entry.isDirectory()) {
+            walkSync(fullPath, rootDir, entries);
+        }
+        else if (entry.isFile()) {
+            entries.push(relativePath);
+        }
+    }
+};
+const fastGlob = (patterns, options = {}) => {
+    return Promise.resolve(fastGlob.sync(patterns, options));
+};
+fastGlob.sync = (patterns, options = {}) => {
+    const cwd = options.cwd || "/";
+    const ignore = options.ignore || [];
+    const allFiles = [];
+    walkSync(cwd, cwd, allFiles);
+    const matched = micromatch(allFiles, patterns, {
+        ignore: ignore,
+        dot: options.dot,
+        cwd: cwd,
+    });
+    if (options.absolute) {
+        return matched.map((p) => path.join(cwd, p));
+    }
+    return matched;
+};
+fastGlob.stream = (patterns, options = {}) => {
+    throw new Error("fastGlob.stream is not implemented in polyfill");
+};
+fastGlob.async = fastGlob;
+fastGlob.generateTasks = () => [];
+fastGlob.isDynamicPattern = (p) => micromatch.scan(p).isGlob;
+fastGlob.escapePath = (p) => p.replace(/([*?|(){}[\]])/g, "\\$1");
+export default fastGlob;

package/esm/webpackLoaders/worker/polyfills/fsPolyfill.d.ts

@@ -0,0 +1,124 @@
+import { Buffer } from "buffer";
+export declare function readFile(path: string, options: any, cb: Function): void;
+export declare function readFileSync(path: string, options: any): string | Buffer<any>;
+export declare function readdirSync(path: string, options?: any): any;
+export declare function readdir(path: string, options: any, cb: Function): void;
+export declare function writeFileSync(path: string, data: string | Uint8Array, options?: any): void;
+export declare function writeFile(path: string, data: string | Uint8Array, options: any, cb: Function): void;
+export declare function mkdirSync(path: string, options?: any): void;
+export declare function mkdir(path: string, options: any, cb: Function): void;
+export declare function rmSync(path: string, options?: any): void;
+export declare function rm(path: string, options: any, cb: Function): void;
+export declare function rmdirSync(path: string, options?: any): void;
+export declare function rmdir(path: string, options: any, cb: Function): void;
+export declare function copyFileSync(src: string, dst: string): void;
+export declare function copyFile(src: string, dst: string, cb: Function): void;
+export declare function statSync(p: string): {
+    isDirectory: () => any;
+    isFile: () => any;
+    isSymbolicLink: () => any;
+    size: number;
+    mtime: Date;
+};
+export declare function lstatSync(p: string): {
+    isDirectory: () => any;
+    isFile: () => any;
+    isSymbolicLink: () => any;
+    size: number;
+    mtime: Date;
+};
+export declare function stat(path: string, cb: Function): void;
+export declare function lstat(path: string, cb: Function): void;
+export declare function realpathSync(p: string): string;
+export declare function realpath(p: string, cb: Function): void;
+export declare function accessSync(path: string, mode?: number): void;
+export declare function access(path: string, mode: number | Function, cb?: Function): void;
+export declare function existsSync(path: string): boolean;
+export declare const promises: {
+    readFile: (path: string, options?: any) => Promise<string | Buffer<any>>;
+    writeFile: (path: string, data: string | Uint8Array, options?: any) => Promise<void>;
+    readdir: (path: string, options?: any) => Promise<any>;
+    mkdir: (path: string, options?: any) => Promise<void>;
+    rm: (path: string, options?: any) => Promise<void>;
+    rmdir: (path: string, options?: any) => Promise<void>;
+    copyFile: (src: string, dst: string) => Promise<void>;
+    stat: (p: string) => Promise<{
+        isDirectory: () => any;
+        isFile: () => any;
+        isSymbolicLink: () => any;
+        size: number;
+        mtime: Date;
+    }>;
+    lstat: (p: string) => Promise<{
+        isDirectory: () => any;
+        isFile: () => any;
+        isSymbolicLink: () => any;
+        size: number;
+        mtime: Date;
+    }>;
+    realpath: (p: string) => Promise<string>;
+    access: (path: string, mode?: number) => Promise<void>;
+};
+export declare const constants: {
+    F_OK: number;
+    R_OK: number;
+    W_OK: number;
+    X_OK: number;
+};
+declare const _default: {
+    readFile: typeof readFile;
+    readFileSync: typeof readFileSync;
+    readdir: typeof readdir;
+    readdirSync: typeof readdirSync;
+    writeFile: typeof writeFile;
+    writeFileSync: typeof writeFileSync;
+    mkdir: typeof mkdir;
+    mkdirSync: typeof mkdirSync;
+    rm: typeof rm;
+    rmSync: typeof rmSync;
+    rmdir: typeof rmdir;
+    rmdirSync: typeof rmdirSync;
+    copyFile: typeof copyFile;
+    copyFileSync: typeof copyFileSync;
+    stat: typeof stat;
+    statSync: typeof statSync;
+    lstat: typeof lstat;
+    lstatSync: typeof lstatSync;
+    realpath: typeof realpath;
+    realpathSync: typeof realpathSync;
+    access: typeof access;
+    accessSync: typeof accessSync;
+    existsSync: typeof existsSync;
+    promises: {
+        readFile: (path: string, options?: any) => Promise<string | Buffer<any>>;
+        writeFile: (path: string, data: string | Uint8Array, options?: any) => Promise<void>;
+        readdir: (path: string, options?: any) => Promise<any>;
+        mkdir: (path: string, options?: any) => Promise<void>;
+        rm: (path: string, options?: any) => Promise<void>;
+        rmdir: (path: string, options?: any) => Promise<void>;
+        copyFile: (src: string, dst: string) => Promise<void>;
+        stat: (p: string) => Promise<{
+            isDirectory: () => any;
+            isFile: () => any;
+            isSymbolicLink: () => any;
+            size: number;
+            mtime: Date;
+        }>;
+        lstat: (p: string) => Promise<{
+            isDirectory: () => any;
+            isFile: () => any;
+            isSymbolicLink: () => any;
+            size: number;
+            mtime: Date;
+        }>;
+        realpath: (p: string) => Promise<string>;
+        access: (path: string, mode?: number) => Promise<void>;
+    };
+    constants: {
+        F_OK: number;
+        R_OK: number;
+        W_OK: number;
+        X_OK: number;
+    };
+};
+export default _default;