@utoo/web 1.0.7 → 1.2.0-rc.1

package/esm/loaderWorker.js.LICENSE.txt

@@ -0,0 +1,44 @@
+/*!
+ * SystemJS 6.15.1
+ */
+
+/*!
+ * The buffer module from node.js, for the browser.
+ *
+ * @author   Feross Aboukhadijeh <feross@feross.org> <http://feross.org>
+ * @license  MIT
+ */
+
+/*!
+ * The buffer module from node.js, for the browser.
+ *
+ * @author   Feross Aboukhadijeh <https://feross.org>
+ * @license  MIT
+ */
+
+/*!
+ * fill-range <https://github.com/jonschlinkert/fill-range>
+ *
+ * Copyright (c) 2014-present, Jon Schlinkert.
+ * Licensed under the MIT License.
+ */
+
+/*!
+ * is-number <https://github.com/jonschlinkert/is-number>
+ *
+ * Copyright (c) 2014-present, Jon Schlinkert.
+ * Released under the MIT License.
+ */
+
+/*!
+ * to-regex-range <https://github.com/micromatch/to-regex-range>
+ *
+ * Copyright (c) 2015-present, Jon Schlinkert.
+ * Released under the MIT License.
+ */
+
+/*! https://mths.be/punycode v1.4.1 by @mathias */
+
+/*! ieee754. BSD-3-Clause License. Feross Aboukhadijeh <https://feross.org/opensource> */
+
+/*! safe-buffer. MIT License. Feross Aboukhadijeh <https://feross.org/opensource> */

package/esm/loaderWorkerPool.d.ts

@@ -0,0 +1,3 @@
+import { Binding } from "./type";
+import { Project as ProjectInternal } from "./utoo";
+export declare const runLoaderWorkerPool: (binding: Binding, projectCwd: string, projectInternal: ProjectInternal, workerUrl: string, loadersImportMap?: Record<string, string>) => Promise<void>;

package/esm/loaderWorkerPool.js

@@ -0,0 +1,125 @@
+import { SAB_OP_COPY_FILE, SAB_OP_MKDIR, SAB_OP_READ_DIR, SAB_OP_READ_FILE, SAB_OP_RM, SAB_OP_RMDIR, SAB_OP_WRITE_FILE, SabComHost, } from "./sabcom";
+import initWasm, { registerWorkerScheduler, workerCreated, } from "./utoo";
+let nextWorkerId = 0;
+const loaderWorkers = {};
+export const runLoaderWorkerPool = async (binding, projectCwd, projectInternal, workerUrl, loadersImportMap) => {
+    registerWorkerScheduler(async (creation) => {
+        const { options: { filename, cwd }, } = creation;
+        nextWorkerId += 1;
+        const workerId = nextWorkerId;
+        const sab = new SharedArrayBuffer(1024 * 1024 * 10); // 10MB
+        const sabHost = new SabComHost(sab);
+        const worker = new Worker(workerUrl, { name: filename });
+        worker.onmessage = async (event) => {
+            if (event.data === "sab_request") {
+                const { op, data: path } = sabHost.readRequest();
+                try {
+                    if (op === SAB_OP_READ_FILE) {
+                        const bytes = await projectInternal.read(path);
+                        sabHost.writeResponse(bytes);
+                    }
+                    else if (op === SAB_OP_READ_DIR) {
+                        const entries = await projectInternal.readDir(path);
+                        sabHost.writeResponse(JSON.stringify(entries.map((e) => e.toJSON())));
+                    }
+                    else if (op === SAB_OP_WRITE_FILE) {
+                        const { content, encoding } = JSON.parse(path);
+                        const filePath = content.path;
+                        const fileContent = content.data;
+                        // TODO: handle binary content (base64?)
+                        await projectInternal.writeString(filePath, fileContent);
+                        sabHost.writeResponse("ok");
+                    }
+                    else if (op === SAB_OP_MKDIR) {
+                        const { path: dirPath, recursive } = JSON.parse(path);
+                        if (recursive) {
+                            await projectInternal.createDirAll(dirPath);
+                        }
+                        else {
+                            await projectInternal.createDir(dirPath);
+                        }
+                        sabHost.writeResponse("ok");
+                    }
+                    else if (op === SAB_OP_RM) {
+                        const { path: rmPath, recursive } = JSON.parse(path);
+                        // Mimic internalProject.rm logic
+                        const metadata = await projectInternal.metadata(rmPath);
+                        const type = metadata.toJSON().type;
+                        if (type === "file") {
+                            await projectInternal.removeFile(rmPath);
+                        }
+                        else if (type === "directory") {
+                            await projectInternal.removeDir(rmPath, !!recursive);
+                        }
+                        sabHost.writeResponse("ok");
+                    }
+                    else if (op === SAB_OP_RMDIR) {
+                        const { path: rmPath, recursive } = JSON.parse(path);
+                        await projectInternal.removeDir(rmPath, !!recursive);
+                        sabHost.writeResponse("ok");
+                    }
+                    else if (op === SAB_OP_COPY_FILE) {
+                        const { src, dst } = JSON.parse(path);
+                        await projectInternal.copyFile(src, dst);
+                        sabHost.writeResponse("ok");
+                    }
+                    else {
+                        sabHost.writeError("Unknown op");
+                    }
+                }
+                catch (e) {
+                    sabHost.writeError(e.message);
+                }
+            }
+        };
+        let finalCwd = cwd;
+        let finalFilename = filename;
+        if (projectCwd) {
+            const sep = "/";
+            const pCwd = projectCwd.endsWith(sep)
+                ? projectCwd.slice(0, -1)
+                : projectCwd;
+            let cCwd = cwd.startsWith(sep) ? cwd.slice(1) : cwd;
+            if (cCwd === "." || cCwd === "./") {
+                cCwd = "";
+            }
+            finalCwd = cCwd ? `${pCwd}${sep}${cCwd}` : pCwd;
+            if (!filename.startsWith("/")) {
+                let fName = filename;
+                if (fName.startsWith("./"))
+                    fName = fName.slice(2);
+                finalFilename = `${pCwd}${sep}${fName}`;
+            }
+        }
+        worker.postMessage([
+            // @ts-ignore
+            initWasm.__wbindgen_wasm_module,
+            binding.memory,
+            {
+                workerData: {
+                    cwd: finalCwd,
+                    workerId: workerId,
+                },
+                loaderAssets: {
+                    importMaps: { ...loadersImportMap },
+                    entrypoint: finalFilename,
+                },
+                sab,
+            },
+        ]);
+        const workers = loaderWorkers[filename] || (loaderWorkers[filename] = new Map());
+        workers.set(workerId, worker);
+        workerCreated(workerId);
+    }, (termination) => {
+        const { workerId, options } = termination;
+        const entrypoint = options.filename;
+        const workers = loaderWorkers[entrypoint];
+        if (workers) {
+            const worker = workers.get(workerId);
+            if (worker) {
+                worker.terminate();
+                workers.delete(workerId);
+            }
+        }
+    });
+};

package/esm/project.d.ts

@@ -1,6 +1,7 @@
 import { BuildOutput, Dirent, PackFile, ProjectEndpoint, ProjectOptions, ServiceWorkerOptions } from "./type";
 export declare class Project implements ProjectEndpoint {
     #private;
+    private options;
     readonly cwd: string;
     readonly serviceWorkerOptions?: ServiceWorkerOptions;
     private remote;

package/esm/project.js

@@ -21,6 +21,7 @@ const ConnectedPorts = new Set();
 export class Project {
     constructor(options) {
         var _a, _b;
+        this.options = options;
         _Project_mount.set(this, void 0);
         const { cwd, workerUrl, wasmUrl, threadWorkerUrl, serviceWorker, logFilter, loadersImportMap, } = options;
         this.cwd = cwd;
@@ -43,6 +44,7 @@ export class Project {
             cwd,
             wasmUrl,
             threadWorkerUrl,
+            loaderWorkerUrl: this.options.loaderWorkerUrl,
             loadersImportMap,
             logFilter,
         }), "f");

package/esm/sabcom.d.ts

@@ -0,0 +1,31 @@
+export declare const SAB_STATE_IDLE = 0;
+export declare const SAB_STATE_REQUEST = 1;
+export declare const SAB_STATE_RESPONSE = 2;
+export declare const SAB_STATE_ERROR = 3;
+export declare const SAB_OP_READ_FILE = 1;
+export declare const SAB_OP_READ_DIR = 2;
+export declare const SAB_OP_WRITE_FILE = 3;
+export declare const SAB_OP_MKDIR = 4;
+export declare const SAB_OP_RM = 5;
+export declare const SAB_OP_RMDIR = 6;
+export declare const SAB_OP_COPY_FILE = 7;
+export declare class SabComHost {
+    private sab;
+    private int32;
+    private uint8;
+    constructor(sab: SharedArrayBuffer);
+    readRequest(): {
+        op: number;
+        data: string;
+    };
+    writeResponse(data: Uint8Array | string): void;
+    writeError(message: string): void;
+}
+export declare class SabComClient {
+    private sab;
+    private notifyHost;
+    private int32;
+    private uint8;
+    constructor(sab: SharedArrayBuffer, notifyHost: () => void);
+    call(op: number, data: string): Uint8Array<ArrayBuffer>;
+}

package/esm/sabcom.js

@@ -0,0 +1,71 @@
+export const SAB_STATE_IDLE = 0;
+export const SAB_STATE_REQUEST = 1;
+export const SAB_STATE_RESPONSE = 2;
+export const SAB_STATE_ERROR = 3;
+export const SAB_OP_READ_FILE = 1;
+export const SAB_OP_READ_DIR = 2;
+export const SAB_OP_WRITE_FILE = 3;
+export const SAB_OP_MKDIR = 4;
+export const SAB_OP_RM = 5;
+export const SAB_OP_RMDIR = 6;
+export const SAB_OP_COPY_FILE = 7;
+// Layout:
+// 0: State (Int32)
+// 1: Op (Int32)
+// 2: Data Length (Int32)
+// 12...: Data (Uint8) - Start at byte 12 (3 * 4 bytes)
+export class SabComHost {
+    constructor(sab) {
+        this.sab = sab;
+        this.int32 = new Int32Array(sab);
+        this.uint8 = new Uint8Array(sab);
+    }
+    readRequest() {
+        const op = this.int32[1];
+        const len = this.int32[2];
+        const data = new TextDecoder().decode(this.uint8.slice(12, 12 + len));
+        return { op, data };
+    }
+    writeResponse(data) {
+        if (typeof data === "string") {
+            data = new TextEncoder().encode(data);
+        }
+        // TODO: Check size overflow
+        this.int32[2] = data.length;
+        this.uint8.set(data, 12);
+        Atomics.store(this.int32, 0, SAB_STATE_RESPONSE);
+        Atomics.notify(this.int32, 0);
+    }
+    writeError(message) {
+        const data = new TextEncoder().encode(message);
+        this.int32[2] = data.length;
+        this.uint8.set(data, 12);
+        Atomics.store(this.int32, 0, SAB_STATE_ERROR);
+        Atomics.notify(this.int32, 0);
+    }
+}
+export class SabComClient {
+    constructor(sab, notifyHost) {
+        this.sab = sab;
+        this.notifyHost = notifyHost;
+        this.int32 = new Int32Array(sab);
+        this.uint8 = new Uint8Array(sab);
+    }
+    call(op, data) {
+        const encoded = new TextEncoder().encode(data);
+        this.int32[1] = op;
+        this.int32[2] = encoded.length;
+        this.uint8.set(encoded, 12);
+        Atomics.store(this.int32, 0, SAB_STATE_REQUEST);
+        this.notifyHost();
+        Atomics.wait(this.int32, 0, SAB_STATE_REQUEST);
+        const state = Atomics.load(this.int32, 0);
+        if (state === SAB_STATE_ERROR) {
+            const len = this.int32[2];
+            const msg = new TextDecoder().decode(this.uint8.slice(12, 12 + len));
+            throw new Error(msg);
+        }
+        const len = this.int32[2];
+        return this.uint8.slice(12, 12 + len);
+    }
+}

package/esm/type.d.ts

@@ -44,6 +44,7 @@ export interface ProjectOptions {
     cwd: string;
     workerUrl: string;
     threadWorkerUrl: string;
+    loaderWorkerUrl?: string;
     wasmUrl?: string;
     serviceWorker?: ServiceWorkerOptions;
     logFilter?: string;

package/esm/utoo/index.d.ts

@@ -1,38 +1,27 @@
 /**
- * @returns {Promise<WorkerTermination>}
+ * @param {any} message
+ * @returns {Promise<void>}
  */
-export function recvWorkerTermination(): Promise<WorkerTermination>;
+export function sendTaskMessage(message: any): Promise<void>;
 /**
- * @param {number} task_id
  * @param {number} worker_id
- * @returns {Promise<void>}
+ * @returns {Promise<WasmTaskMessage>}
  */
-export function notifyWorkerAck(task_id: number, worker_id: number): Promise<void>;
+export function recvTaskMessageInWorker(worker_id: number): Promise<WasmTaskMessage>;
+export function init_pack(): void;
 /**
- * @param {number} task_id
- * @param {string} message
- * @returns {Promise<void>}
+ * @param {string} filter
  */
-export function sendTaskMessage(task_id: number, message: string): Promise<void>;
+export function init_log_filter(filter: string): void;
 /**
  * @param {number} worker_id
- * @returns {Promise<string>}
- */
-export function recvMessageInWorker(worker_id: number): Promise<string>;
-/**
- * @param {string} pool_id
- * @returns {Promise<number>}
  */
-export function recvWorkerRequest(pool_id: string): Promise<number>;
+export function workerCreated(worker_id: number): void;
 /**
- * @returns {Promise<PoolOptions>}
+ * @param {Function} creator
+ * @param {Function} terminator
  */
-export function recvPoolRequest(): Promise<PoolOptions>;
-/**
- * @param {string} filter
- */
-export function init_log_filter(filter: string): void;
-export function init_pack(): void;
+export function registerWorkerScheduler(creator: Function, terminator: Function): void;
 /**
  * Entry point for web workers
  * @param {number} ptr
@@ -79,28 +68,6 @@ export class Metadata {
     __wbg_ptr: number | undefined;
     free(): void;
 }
-export class PoolOptions {
-    static __wrap(ptr: any): any;
-    __destroy_into_raw(): number | undefined;
-    __wbg_ptr: number | undefined;
-    free(): void;
-    /**
-     * @param {string} arg0
-     */
-    set filename(arg0: string);
-    /**
-     * @returns {string}
-     */
-    get filename(): string;
-    /**
-     * @param {number} arg0
-     */
-    set maxConcurrency(arg0: number);
-    /**
-     * @returns {number}
-     */
-    get maxConcurrency(): number;
-}
 export class Project {
     /**
      * @param {string} cwd
@@ -197,7 +164,43 @@ export class Project {
      */
     copyFile(src: string, dst: string): Promise<void>;
 }
-export class WorkerTermination {
+export class WasmTaskMessage {
+    static __wrap(ptr: any): any;
+    __destroy_into_raw(): number | undefined;
+    __wbg_ptr: number | undefined;
+    free(): void;
+    /**
+     * @param {number} arg0
+     */
+    set taskId(arg0: number);
+    /**
+     * @returns {number}
+     */
+    get taskId(): number;
+    /**
+     * @param {string} arg0
+     */
+    set data(arg0: string);
+    /**
+     * @returns {string}
+     */
+    get data(): string;
+}
+export class WebWorkerCreation {
+    static __wrap(ptr: any): any;
+    __destroy_into_raw(): number | undefined;
+    __wbg_ptr: number | undefined;
+    free(): void;
+    /**
+     * @param {WebWorkerOptions} arg0
+     */
+    set options(arg0: WebWorkerOptions);
+    /**
+     * @returns {WebWorkerOptions}
+     */
+    get options(): WebWorkerOptions;
+}
+export class WebWorkerOptions {
     static __wrap(ptr: any): any;
     __destroy_into_raw(): number | undefined;
     __wbg_ptr: number | undefined;
@@ -210,6 +213,28 @@ export class WorkerTermination {
      * @returns {string}
      */
     get filename(): string;
+    /**
+     * @param {string} arg0
+     */
+    set cwd(arg0: string);
+    /**
+     * @returns {string}
+     */
+    get cwd(): string;
+}
+export class WebWorkerTermination {
+    static __wrap(ptr: any): any;
+    __destroy_into_raw(): number | undefined;
+    __wbg_ptr: number | undefined;
+    free(): void;
+    /**
+     * @param {WebWorkerOptions} arg0
+     */
+    set options(arg0: WebWorkerOptions);
+    /**
+     * @returns {WebWorkerOptions}
+     */
+    get options(): WebWorkerOptions;
     /**
      * @param {number} arg0
      */