@utoo/web 1.1.0 → 1.2.0-rc.10

package/esm/webpackLoaders/polyfills/nodePolyFills.js

@@ -0,0 +1,230 @@
+const buffer = require("buffer");
+self.Buffer = buffer.Buffer;
+const process = require("process");
+const originalCwd = process.cwd;
+process.cwd = () => {
+    var _a;
+    // @ts-ignore
+    return ((_a = self.workerData) === null || _a === void 0 ? void 0 : _a.cwd) || (originalCwd === null || originalCwd === void 0 ? void 0 : originalCwd()) || "/";
+};
+if (!process.versions)
+    process.versions = {};
+if (!process.versions.node)
+    process.versions.node = "24.0.0";
+self.process = process;
+self.global = self;
+const path = require("path");
+const originalResolve = path.resolve;
+path.resolve = (...args) => {
+    var _a;
+    // @ts-ignore
+    const cwd = ((_a = self.workerData) === null || _a === void 0 ? void 0 : _a.cwd) || "/";
+    return originalResolve(cwd, ...args);
+};
+import * as fs from "./fsPolyfill";
+import * as workerThreads from "./workerThreadsPolyfill";
+const workerThreadsWithLiveWorkerData = {
+    ...workerThreads,
+    get workerData() {
+        // @ts-ignore
+        return self.workerData;
+    },
+    get threadId() {
+        var _a;
+        // @ts-ignore
+        return ((_a = self.workerData) === null || _a === void 0 ? void 0 : _a.workerId) || 0;
+    },
+};
+// Used to directly inject polyfill instance into systemjs
+export default {
+    get assert() {
+        return require("assert");
+    },
+    get "node:assert"() {
+        return require("assert");
+    },
+    buffer,
+    "node:buffer": buffer,
+    get child_process() {
+        return require("child_process");
+    },
+    get "node:child_process"() {
+        return require("child_process");
+    },
+    get cluster() {
+        return require("cluster");
+    },
+    get "node:cluster"() {
+        return require("cluster");
+    },
+    get console() {
+        return require("console");
+    },
+    get "node:console"() {
+        return require("console");
+    },
+    get constants() {
+        return require("constants");
+    },
+    get "node:constants"() {
+        return require("constants");
+    },
+    get crypto() {
+        return require("crypto");
+    },
+    get "node:crypto"() {
+        return require("crypto");
+    },
+    get dgram() {
+        return require("dgram");
+    },
+    get "node:dgram"() {
+        return require("dgram");
+    },
+    get dns() {
+        return require("dns");
+    },
+    get "node:dns"() {
+        return require("dns");
+    },
+    get domain() {
+        return require("domain");
+    },
+    get "node:domain"() {
+        return require("domain");
+    },
+    get events() {
+        return require("events");
+    },
+    get "node:events"() {
+        return require("events");
+    },
+    get http() {
+        return require("http");
+    },
+    get "node:http"() {
+        return require("http");
+    },
+    get http2() {
+        return require("http2");
+    },
+    get "node:http2"() {
+        return require("http2");
+    },
+    get https() {
+        return require("https");
+    },
+    get "node:https"() {
+        return require("https");
+    },
+    get module() {
+        return require("module");
+    },
+    get "node:module"() {
+        return require("module");
+    },
+    get net() {
+        return require("net");
+    },
+    get "node:net"() {
+        return require("net");
+    },
+    get os() {
+        return require("os");
+    },
+    get "node:os"() {
+        return require("os");
+    },
+    get punycode() {
+        return require("punycode");
+    },
+    get "node:punycode"() {
+        return require("punycode");
+    },
+    get querystring() {
+        return require("querystring");
+    },
+    get "node:querystring"() {
+        return require("querystring");
+    },
+    get readline() {
+        return require("readline");
+    },
+    get "node:readline"() {
+        return require("readline");
+    },
+    get repl() {
+        return require("repl");
+    },
+    get "node:repl"() {
+        return require("repl");
+    },
+    get stream() {
+        return require("stream");
+    },
+    get "node:stream"() {
+        return require("stream");
+    },
+    get string_decoder() {
+        return require("string_decoder");
+    },
+    get "node:string_decoder"() {
+        return require("string_decoder");
+    },
+    get sys() {
+        return require("util");
+    },
+    get "node:sys"() {
+        return require("util");
+    },
+    get timers() {
+        return require("timers");
+    },
+    get "node:timers"() {
+        return require("timers");
+    },
+    get tls() {
+        return require("tls");
+    },
+    get "node:tls"() {
+        return require("tls");
+    },
+    get tty() {
+        return require("tty");
+    },
+    get "node:tty"() {
+        return require("tty");
+    },
+    get vm() {
+        return require("vm");
+    },
+    get "node:vm"() {
+        return require("vm");
+    },
+    get zlib() {
+        return require("zlib");
+    },
+    get "node:zlib"() {
+        return require("zlib");
+    },
+    fs,
+    "node:fs": fs,
+    path,
+    "node:path": path,
+    process,
+    "node:process": process,
+    get url() {
+        return require("url");
+    },
+    get "node:url"() {
+        return require("url");
+    },
+    get util() {
+        return require("util");
+    },
+    get "node:util"() {
+        return require("util");
+    },
+    worker_threads: workerThreadsWithLiveWorkerData,
+    "node:worker_threads": workerThreadsWithLiveWorkerData,
+};

package/esm/webpackLoaders/polyfills/workerThreadsPolyfill.d.ts

@@ -0,0 +1,7 @@
+export declare const workerData: any;
+export declare const isMainThread = false;
+export declare const parentPort: {
+    postMessage: (message: any) => void;
+    on: (event: string, listener: (...args: any[]) => void) => void;
+    off: (event: string, listener: (...args: any[]) => void) => void;
+};

package/esm/webpackLoaders/polyfills/workerThreadsPolyfill.js

@@ -0,0 +1,16 @@
+// @ts-ignore
+export const workerData = self.workerData;
+export const isMainThread = false;
+export const parentPort = {
+    postMessage: (message) => self.postMessage(message),
+    on: (event, listener) => {
+        if (event === "message") {
+            self.onmessage = (e) => listener(e.data);
+        }
+    },
+    off: (event, listener) => {
+        if (event === "message") {
+            self.onmessage = null;
+        }
+    },
+};

package/esm/webpackLoaders/{worker/type.d.ts → types.d.ts}

@@ -1,11 +1,12 @@
 export interface LoaderRunnerMeta {
     workerData: {
-        poolId: string;
         workerId: number;
         cwd: string;
+        projectRoot: string;
     };
     loaderAssets: {
         importMaps: Record<string, string>;
         entrypoint: string;
     };
+    sab?: SharedArrayBuffer;
 }

package/esm/webpackLoaders/worker.d.ts

@@ -0,0 +1,2 @@
+export declare function startLoaderWorker(): void;
+export default startLoaderWorker;

package/esm/webpackLoaders/worker.js

@@ -0,0 +1,46 @@
+import { SabComClient } from "../utils/sabcom";
+import initWasm, { Project, recvTaskMessageInWorker, sendTaskMessage, workerCreated, } from "../utoo";
+import { cjs } from "./cjs";
+const binding = {
+    recvTaskMessageInWorker,
+    sendTaskMessage,
+    workerCreated,
+};
+export function startLoaderWorker() {
+    self.onmessage = async (event) => {
+        let [module, memory, meta] = event.data;
+        await initWasm(module, memory).catch((err) => {
+            console.log(err);
+            throw err;
+        });
+        const sabClient = meta.sab
+            ? new SabComClient(meta.sab, () => {
+                self.postMessage("sab_request");
+            })
+            : undefined;
+        // Initialize the thread-local state (tokio runtime).
+        // We don't need to pass threadWorkerUrl here because it's already stored in a global static in Rust.
+        Project.init("");
+        Project.setCwd(meta.workerData.cwd);
+        self.workerData = {
+            workerId: meta.workerData.workerId,
+            cwd: meta.workerData.cwd,
+            projectRoot: meta.workerData.projectRoot,
+            binding,
+            sabClient,
+            fs: Project,
+        };
+        self.process = {
+            env: {},
+            cwd: () => self.workerData.cwd,
+        };
+        cjs(meta.loaderAssets.entrypoint, meta.loaderAssets.importMaps);
+    };
+}
+// @ts-ignore
+if (typeof __webpack_require__ !== "undefined") {
+    // @ts-ignore
+    self.startLoaderWorker = startLoaderWorker;
+}
+startLoaderWorker();
+export default startLoaderWorker;

package/esm/worker.js

@@ -1,6 +1,6 @@
 import * as comlink from "comlink";
-import { internalEndpoint } from "./internalProject";
-import { HandShake } from "./message";
+import { internalEndpoint } from "./project/InternalProject";
+import { HandShake } from "./utils/message";
 const ConnectedPorts = new Set();
 self.addEventListener("message", (e) => {
     const port = e.ports[0];

package/package.json

@@ -1,42 +1,43 @@
 {
   "name": "@utoo/web",
-  "version": "1.1.0",
+  "version": "1.2.0-rc.10",
   "module": "esm/index.js",
   "types": "esm/index.d.ts",
   "files": [
-    "esm/*"
+    "esm"
   ],
   "scripts": {
-    "install-toolchain": "cargo install wasm-bindgen-cli@0.2.104 && brew install binaryen",
+    "install-toolchain": "cargo install wasm-bindgen-cli@0.2.104 && brew install binaryen llvm",
     "build-wasm": "cargo build -p utoo-wasm --target wasm32-unknown-unknown -Z build-std=panic_abort,std",
     "build-wasm:pm": "cargo build -p utoo-wasm --target wasm32-unknown-unknown --no-default-features -Z build-std=panic_abort,std",
     "bindgen-dev": "wasm-bindgen ../../target/wasm32-unknown-unknown/wasm-dev/utoo_wasm.wasm --out-dir src/utoo --out-name index --target web --debug --keep-debug --no-demangle",
     "bindgen-build": "wasm-bindgen ../../target/wasm32-unknown-unknown/release/utoo_wasm.wasm --out-dir src/utoo --out-name index --target web",
     "bindgen-build:local": "wasm-bindgen ../../target/wasm32-unknown-unknown/release-local/utoo_wasm.wasm --out-dir src/utoo --out-name index --target web",
-    "pretsc": "npm run build --workspace @utoo/pack-shared",
-    "tsc": "rm -rf esm && tsc -p ./tsconfig.json",
-    "dev": "npm run build-wasm -- --profile wasm-dev && npm run bindgen-dev && npm run tsc && cp src/utoo/index_bg.wasm esm/utoo",
-    "dev:pm": "npm run build-wasm:pm -- --profile wasm-dev && npm run bindgen-dev && npm run tsc && cp src/utoo/index_bg.wasm esm/utoo",
+    "tsc": "rm -rf esm && tsc -p ./tsconfig.build.json",
+    "copy-wasm": "cp src/utoo/index_bg.wasm esm/utoo",
+    "build:shared": "npx turbo run build --filter=@utoo/pack-shared",
+    "dev": "npm run build-wasm -- --profile wasm-dev && npm run bindgen-dev && npx turbo run tsc --filter=@utoo/web  && npm run build-loaderWorker && npm run copy-wasm",
+    "dev:pm": "npm run build-wasm:pm -- --profile wasm-dev && npm run bindgen-dev && npm run tsc && npm run copy-wasm",
     "wasm-opt": "wasm-opt src/utoo/index_bg.wasm -o esm/utoo/index_bg.wasm --enable-threads --enable-bulk-memory --enable-nontrapping-float-to-int -Oz",
-    "build": "npm run build-wasm -- --release && npm run bindgen-build && npm run build-loaderWorker && npm run tsc && npm run wasm-opt",
-    "build:local": "npm run build-wasm -- --profile release-local && npm run bindgen-build:local && npm run build-loaderWorker && npm run tsc && cp src/utoo/index_bg.wasm esm/utoo",
-    "build-loaderWorker": "node cli/umd.js -e ./src/webpackLoaders/worker/index.ts -o ./src/webpackLoaders/workerContent.js -t webworker --se true",
-    "prepublishOnly": "npm run build"
+    "build": "npm run build-wasm -- --release && npm run bindgen-build && npm run tsc && npm run build-loaderWorker && npm run wasm-opt",
+    "build:local": "CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ npm run build-wasm -- --profile release-local && npm run bindgen-build:local && npm run build:shared && npm run tsc && npm run build-loaderWorker && npm run copy-wasm",
+    "build-loaderWorker": "node cli/umd.js -e ./src/webpackLoaders/worker.ts -o ./esm/loaderWorker.js -t webworker",
+    "clean": "rm -rf esm src/utoo",
+    "prepublishOnly": "turbo run build --filter=@utoo/web"
   },
   "dependencies": {
+    "@utoo/pack-shared": "^0.0.8",
     "comlink": "^4.4.2",
-    "systemjs": "^6.15.1",
-    "loader-runner": "^4.3.0",
-    "less": "^4.0.0",
-    "@utoo/pack-shared": "^0.0.7"
+    "systemjs": "^6.15.1"
   },
   "devDependencies": {
-    "typescript": "^5.8.3",
     "@types/systemjs": "^6.15.1",
-    "webpack": "^5.0.0",
-    "webpack-merge": "^6.0.1",
+    "binaryen": "^125.0.0",
     "node-stdlib-browser": "^1.3.1",
     "ts-loader": "^9.5.2",
+    "typescript": "^5.8.3",
+    "webpack": "^5.0.0",
+    "webpack-merge": "^6.0.1",
     "yargs": "^18.0.0"
   },
   "engines": {

package/esm/internalProject.js

@@ -1,185 +0,0 @@
-import initWasm, { init_log_filter, Project as ProjectInternal, recvPoolRequest, recvWorkerTermination, } from "./utoo";
-// @ts-ignore
-import webpackLoadersCode from "./webpackLoaders/workerContent";
-let nextWorkerId = 0;
-const loaderWorkers = {};
-async function getTurbopackLoaderAssets() {
-    const turbopackLoaderAssets = {};
-    const dirEntries = await internalEndpoint.projectInternal.readDir(".turbopack");
-    await Promise.all(dirEntries.map(async (entry) => {
-        // only collect .js files
-        if (entry.type === "file" && entry.name.endsWith(".js")) {
-            turbopackLoaderAssets[`./${entry.name}`] =
-                await internalEndpoint.projectInternal.readToString(`.turbopack/${entry.name}`);
-        }
-    }));
-    return turbopackLoaderAssets;
-}
-const createOrScalePool = async (binding, loadersImportMap) => {
-    while (true) {
-        try {
-            let poolOptions = await recvPoolRequest();
-            const { filename: entrypoint, maxConcurrency } = poolOptions;
-            const concurrency = Math.max(maxConcurrency, navigator.hardwareConcurrency);
-            const workers = loaderWorkers[entrypoint] || (loaderWorkers[entrypoint] = []);
-            if (workers.length < concurrency) {
-                for (let i = workers.length; i < concurrency; i++) {
-                    nextWorkerId += 1;
-                    const turbopackLoaderAssets = await getTurbopackLoaderAssets();
-                    const blob = new Blob([webpackLoadersCode], {
-                        type: "text/javascript",
-                    });
-                    const workerUrl = URL.createObjectURL(blob);
-                    const worker = new Worker(workerUrl);
-                    worker.postMessage([
-                        // @ts-ignore
-                        initWasm.__wbindgen_wasm_module,
-                        binding.memory,
-                        {
-                            workerData: {
-                                poolId: entrypoint,
-                                workerId: nextWorkerId,
-                            },
-                            loaderAssets: {
-                                importMaps: { ...turbopackLoaderAssets, ...loadersImportMap },
-                                entrypoint: entrypoint.replace(".turbopack/", ""),
-                            },
-                        },
-                    ]);
-                    // @ts-ignore
-                    worker.workerId = nextWorkerId;
-                    // @ts-ignore
-                    workers.push(worker);
-                }
-            }
-            else if (workers.length > concurrency) {
-                const workersToStop = workers.splice(0, workers.length - concurrency);
-                workersToStop.forEach((worker) => worker.terminate());
-            }
-        }
-        catch (_e) {
-            // rust channel closed. do nothing
-            return;
-        }
-    }
-};
-const waitingForWorkerTermination = async () => {
-    while (true) {
-        try {
-            const { filename, workerId } = await recvWorkerTermination();
-            const workers = loaderWorkers[filename];
-            const workerIdx = workers.findIndex((worker) => worker.workerId === workerId);
-            if (workerIdx > -1) {
-                const worker = workers.splice(workerIdx, 1);
-                worker[0].terminate();
-            }
-        }
-        catch (_e) {
-            // rust channel closed. do nothing
-            return;
-        }
-    }
-};
-class InternalEndpoint {
-    // This should be called only once
-    async mount(opt) {
-        var _a;
-        this.options = opt;
-        const { cwd, wasmUrl, threadWorkerUrl, logFilter } = opt;
-        (_a = this.wasmInit) !== null && _a !== void 0 ? _a : (this.wasmInit = initWasm(wasmUrl));
-        await this.wasmInit;
-        // Initialize log filter after wasm init
-        const filter = logFilter || "pack_core=info,pack_api=info,utoo_wasm=info";
-        init_log_filter(filter);
-        this.projectInternal = new ProjectInternal(cwd, threadWorkerUrl);
-        return;
-    }
-    async install(packageLock, maxConcurrentDownloads) {
-        await this.wasmInit;
-        await this.projectInternal.install(packageLock, maxConcurrentDownloads);
-        return;
-    }
-    async build() {
-        var _a;
-        const binding = await this.wasmInit;
-        createOrScalePool(binding, (_a = this.options) === null || _a === void 0 ? void 0 : _a.loadersImportMap);
-        waitingForWorkerTermination();
-        return await this.projectInternal.build();
-    }
-    async readFile(path, encoding) {
-        await this.wasmInit;
-        let ret;
-        if (encoding === "utf8") {
-            ret = await this.projectInternal.readToString(path);
-        }
-        else {
-            ret = await this.projectInternal.read(path);
-        }
-        return ret;
-    }
-    async writeFile(path, content, _encoding) {
-        await this.wasmInit;
-        if (typeof content === "string") {
-            return await this.projectInternal.writeString(path, content);
-        }
-        else {
-            return await this.projectInternal.write(path, content);
-        }
-    }
-    async copyFile(src, dst) {
-        await this.wasmInit;
-        return await this.projectInternal.copyFile(src, dst);
-    }
-    async readdir(path, options) {
-        await this.wasmInit;
-        const dirEntries = (options === null || options === void 0 ? void 0 : options.recursive)
-            ? await this.projectInternal.readDir(path)
-            : // TODO: support recursive readDirAll
-                await this.projectInternal.readDir(path);
-        const rawDirents = dirEntries.map((e) => {
-            const dir = e.toJSON();
-            return {
-                name: dir.name,
-                type: dir.type,
-            };
-        });
-        // WARN: This is a hack, functions can not be structurally cloned
-        return rawDirents;
-    }
-    async mkdir(path, options) {
-        await this.wasmInit;
-        if (options === null || options === void 0 ? void 0 : options.recursive) {
-            return await this.projectInternal.createDirAll(path);
-        }
-        else {
-            return await this.projectInternal.createDir(path);
-        }
-    }
-    async rm(path, options) {
-        await this.wasmInit;
-        let metadata = (await this.projectInternal.metadata(path)).toJSON();
-        switch (metadata.type) {
-            case "file":
-                return await this.projectInternal.removeFile(path);
-            case "directory":
-                return await this.projectInternal.removeDir(path, !!(options === null || options === void 0 ? void 0 : options.recursive));
-            default:
-                // nothing to remove now
-                break;
-        }
-    }
-    async rmdir(path, options) {
-        await this.wasmInit;
-        return await this.projectInternal.removeDir(path, !!(options === null || options === void 0 ? void 0 : options.recursive));
-    }
-    async gzip(files) {
-        await this.wasmInit;
-        return await this.projectInternal.gzip(files);
-    }
-    async sigMd5(content) {
-        await this.wasmInit;
-        return await this.projectInternal.sigMd5(content);
-    }
-}
-const internalEndpoint = new InternalEndpoint();
-export { internalEndpoint };

package/esm/type.js

@@ -1,12 +0,0 @@
-export class Dirent {
-    constructor(rawDirent) {
-        this.rawDirent = rawDirent;
-        this.name = this.rawDirent.name;
-    }
-    isDirectory() {
-        return this.rawDirent.type === "directory";
-    }
-    isFile() {
-        return this.rawDirent.type === "file";
-    }
-}

package/esm/webpackLoaders/loaders/less-loader/index.d.ts

@@ -1,3 +0,0 @@
-export const __esModule: boolean;
-export default lessLoader;
-declare function lessLoader(source: any): Promise<void>;