@utoo/web 1.0.0 → 1.0.1

package/esm/internalProject.d.ts

@@ -0,0 +1,29 @@
+import { PackFile, ProjectEndpoint, ProjectOptions } from "./type";
+import initWasm, { Project as ProjectInternal } from "./utoo";
+declare class InternalEndpoint implements ProjectEndpoint {
+    projectInternal?: ProjectInternal;
+    wasmInit?: ReturnType<typeof initWasm>;
+    options?: Omit<ProjectOptions, "workerUrl" | "serviceWorker">;
+    mount(opt: Omit<ProjectOptions, "workerUrl" | "serviceWorker">): Promise<void>;
+    install(packageLock: string, maxConcurrentDownloads?: number): Promise<void>;
+    build(): Promise<any>;
+    readFile(path: string, encoding?: "utf8"): Promise<any>;
+    writeFile(path: string, content: string | Uint8Array, _encoding?: "utf8"): Promise<void>;
+    copyFile(src: string, dst: string): Promise<void>;
+    readdir(path: string, options?: {
+        recursive?: boolean;
+    }): Promise<any>;
+    mkdir(path: string, options?: {
+        recursive?: boolean;
+    }): Promise<void>;
+    rm(path: string, options?: {
+        recursive?: boolean;
+    }): Promise<void>;
+    rmdir(path: string, options?: {
+        recursive?: boolean;
+    }): Promise<void>;
+    gzip(files: PackFile[]): Promise<Uint8Array<ArrayBufferLike>>;
+    sigMd5(content: Uint8Array): Promise<string>;
+}
+declare const internalEndpoint: InternalEndpoint;
+export { internalEndpoint };

package/esm/internalProject.js

@@ -0,0 +1,184 @@
+import initWasm, { init_log_filter, Project as ProjectInternal, recvPoolRequest, recvWorkerTermination, } from "./utoo";
+// @ts-ignore
+import webpackLoadersCode from "./webpackLoaders/workerContent";
+let nextWorkerId = 0;
+const loaderWorkers = {};
+async function getTurbopackLoaderAssets() {
+    const turbopackLoaderAssets = {};
+    const dirEntries = await internalEndpoint.projectInternal.readDir(".turbopack");
+    await Promise.all(dirEntries.map(async (entry) => {
+        // only collect .js files
+        if (entry.type === "file" && entry.name.endsWith(".js")) {
+            turbopackLoaderAssets[`./${entry.name}`] =
+                await internalEndpoint.projectInternal.readToString(`.turbopack/${entry.name}`);
+        }
+    }));
+    return turbopackLoaderAssets;
+}
+const createOrScalePool = async (binding, loadersImportMap) => {
+    while (true) {
+        try {
+            let poolOptions = await recvPoolRequest();
+            const { filename: entrypoint, maxConcurrency } = poolOptions;
+            const workers = loaderWorkers[entrypoint] || (loaderWorkers[entrypoint] = []);
+            if (workers.length < maxConcurrency) {
+                for (let i = workers.length; i < maxConcurrency; i++) {
+                    nextWorkerId += 1;
+                    const turbopackLoaderAssets = await getTurbopackLoaderAssets();
+                    const blob = new Blob([webpackLoadersCode], {
+                        type: "text/javascript",
+                    });
+                    const workerUrl = URL.createObjectURL(blob);
+                    const worker = new Worker(workerUrl);
+                    worker.postMessage([
+                        // @ts-ignore
+                        initWasm.__wbindgen_wasm_module,
+                        binding.memory,
+                        {
+                            workerData: {
+                                poolId: entrypoint,
+                                workerId: nextWorkerId,
+                            },
+                            loaderAssets: {
+                                importMaps: { ...turbopackLoaderAssets, ...loadersImportMap },
+                                entrypoint: entrypoint.replace(".turbopack/", ""),
+                            },
+                        },
+                    ]);
+                    // @ts-ignore
+                    worker.workerId = nextWorkerId;
+                    // @ts-ignore
+                    workers.push(worker);
+                }
+            }
+            else if (workers.length > maxConcurrency) {
+                const workersToStop = workers.splice(0, workers.length - maxConcurrency);
+                workersToStop.forEach((worker) => worker.terminate());
+            }
+        }
+        catch (_e) {
+            // rust channel closed. do nothing
+            return;
+        }
+    }
+};
+const waitingForWorkerTermination = async () => {
+    while (true) {
+        try {
+            const { filename, workerId } = await recvWorkerTermination();
+            const workers = loaderWorkers[filename];
+            const workerIdx = workers.findIndex((worker) => worker.workerId === workerId);
+            if (workerIdx > -1) {
+                const worker = workers.splice(workerIdx, 1);
+                worker[0].terminate();
+            }
+        }
+        catch (_e) {
+            // rust channel closed. do nothing
+            return;
+        }
+    }
+};
+class InternalEndpoint {
+    // This should be called only once
+    async mount(opt) {
+        var _a;
+        this.options = opt;
+        const { cwd, wasmUrl, threadWorkerUrl, logFilter } = opt;
+        (_a = this.wasmInit) !== null && _a !== void 0 ? _a : (this.wasmInit = initWasm(wasmUrl));
+        await this.wasmInit;
+        // Initialize log filter after wasm init
+        const filter = logFilter || "pack_core=info,pack_api=info,utoo_wasm=info";
+        init_log_filter(filter);
+        this.projectInternal = new ProjectInternal(cwd, threadWorkerUrl);
+        return;
+    }
+    async install(packageLock, maxConcurrentDownloads) {
+        await this.wasmInit;
+        await this.projectInternal.install(packageLock, maxConcurrentDownloads);
+        return;
+    }
+    async build() {
+        var _a;
+        const binding = await this.wasmInit;
+        createOrScalePool(binding, (_a = this.options) === null || _a === void 0 ? void 0 : _a.loadersImportMap);
+        waitingForWorkerTermination();
+        return await this.projectInternal.build();
+    }
+    async readFile(path, encoding) {
+        await this.wasmInit;
+        let ret;
+        if (encoding === "utf8") {
+            ret = await this.projectInternal.readToString(path);
+        }
+        else {
+            ret = await this.projectInternal.read(path);
+        }
+        return ret;
+    }
+    async writeFile(path, content, _encoding) {
+        await this.wasmInit;
+        if (typeof content === "string") {
+            return await this.projectInternal.writeString(path, content);
+        }
+        else {
+            return await this.projectInternal.write(path, content);
+        }
+    }
+    async copyFile(src, dst) {
+        await this.wasmInit;
+        return await this.projectInternal.copyFile(src, dst);
+    }
+    async readdir(path, options) {
+        await this.wasmInit;
+        const dirEntries = (options === null || options === void 0 ? void 0 : options.recursive)
+            ? await this.projectInternal.readDir(path)
+            : // TODO: support recursive readDirAll
+                await this.projectInternal.readDir(path);
+        const rawDirents = dirEntries.map((e) => {
+            const dir = e.toJSON();
+            return {
+                name: dir.name,
+                type: dir.type,
+            };
+        });
+        // WARN: This is a hack, functions can not be structurally cloned
+        return rawDirents;
+    }
+    async mkdir(path, options) {
+        await this.wasmInit;
+        if (options === null || options === void 0 ? void 0 : options.recursive) {
+            return await this.projectInternal.createDirAll(path);
+        }
+        else {
+            return await this.projectInternal.createDir(path);
+        }
+    }
+    async rm(path, options) {
+        await this.wasmInit;
+        let metadata = (await this.projectInternal.metadata(path)).toJSON();
+        switch (metadata.type) {
+            case "file":
+                return await this.projectInternal.removeFile(path);
+            case "directory":
+                return await this.projectInternal.removeDir(path, !!(options === null || options === void 0 ? void 0 : options.recursive));
+            default:
+                // nothing to remove now
+                break;
+        }
+    }
+    async rmdir(path, options) {
+        await this.wasmInit;
+        return await this.projectInternal.removeDir(path, !!(options === null || options === void 0 ? void 0 : options.recursive));
+    }
+    async gzip(files) {
+        await this.wasmInit;
+        return await this.projectInternal.gzip(files);
+    }
+    async sigMd5(content) {
+        await this.wasmInit;
+        return await this.projectInternal.sigMd5(content);
+    }
+}
+const internalEndpoint = new InternalEndpoint();
+export { internalEndpoint };

package/esm/project.d.ts

@@ -1,7 +1,8 @@
-import { BuildOutput, Dirent, PackFile, ProjectEndpoint, ProjectOptions } from "./type";
+import { BuildOutput, Dirent, PackFile, ProjectEndpoint, ProjectOptions, ServiceWorkerOptions } from "./type";
 export declare class Project implements ProjectEndpoint {
     #private;
-    private serviceWorkerOptions?;
+    readonly cwd: string;
+    readonly serviceWorkerOptions?: ServiceWorkerOptions;
     private remote;
     constructor(options: ProjectOptions);
     private connectWorker;

package/esm/project.js

@@ -22,14 +22,13 @@ export class Project {
     constructor(options) {
         var _a, _b;
         _Project_mount.set(this, void 0);
-        const { cwd, workerUrl, wasmUrl, threadWorkerUrl, serviceWorker, logFilter, } = options;
+        const { cwd, workerUrl, wasmUrl, threadWorkerUrl, serviceWorker, logFilter, loadersImportMap, } = options;
+        this.cwd = cwd;
         this.serviceWorkerOptions = serviceWorker;
         const { port1, port2 } = new MessageChannel();
         (_a = this.remote) !== null && _a !== void 0 ? _a : (this.remote = comlink.wrap(port1));
         if (!ProjectWorker) {
-            ProjectWorker = workerUrl
-                ? new Worker(workerUrl)
-                : new Worker(new URL("./worker", import.meta.url));
+            ProjectWorker = new Worker(workerUrl);
             window.addEventListener("message", (e) => {
                 this.connectWorker(e);
             });
@@ -44,6 +43,7 @@ export class Project {
             cwd,
             wasmUrl,
             threadWorkerUrl,
+            loadersImportMap,
             logFilter,
         }), "f");
     }

package/esm/threadWorker.js

@@ -1,10 +1,4 @@
 import initWasm from "./utoo";
-// Read logFilter from URL query string and set to globalThis
-const urlParams = new URLSearchParams(self.location.search);
-const logFilter = urlParams.get("logFilter");
-if (logFilter) {
-    globalThis.__UTOO_LOG_FILTER__ = logFilter;
-}
 // this is for wasm_thread to spawn new worker thread.
 // see: https://github.com/utooland/wasm_thread/blob/94438ff771ee0a6a55d79e49a655707970acb615/src/wasm32/js/web_worker.js#L10
 self.wasm_bindgen = initWasm;

package/esm/type.d.ts

@@ -1,5 +1,5 @@
 import { Issue } from "@utoo/pack-shared";
-import { DirEntryType } from "./utoo";
+import initWasm, { DirEntryType } from "./utoo";
 export interface RawDirent {
     name: string;
     type: DirEntryType;
@@ -42,13 +42,15 @@ export interface ProjectEndpoint {
 }
 export interface ProjectOptions {
     cwd: string;
-    workerUrl?: string;
+    workerUrl: string;
     threadWorkerUrl: string;
     wasmUrl?: string;
     serviceWorker?: ServiceWorkerOptions;
     logFilter?: string;
+    loadersImportMap?: Record<string, string>;
 }
 export interface ServiceWorkerOptions {
     url: string;
     scope: string;
 }
+export type Binding = Awaited<ReturnType<typeof initWasm>>;

package/esm/utoo/index.d.ts

@@ -1,3 +1,37 @@
+/**
+ * @returns {Promise<WorkerTermination>}
+ */
+export function recvWorkerTermination(): Promise<WorkerTermination>;
+/**
+ * @param {number} task_id
+ * @param {number} worker_id
+ * @returns {Promise<void>}
+ */
+export function notifyWorkerAck(task_id: number, worker_id: number): Promise<void>;
+/**
+ * @param {number} task_id
+ * @param {string} message
+ * @returns {Promise<void>}
+ */
+export function sendTaskMessage(task_id: number, message: string): Promise<void>;
+/**
+ * @param {number} worker_id
+ * @returns {Promise<string>}
+ */
+export function recvMessageInWorker(worker_id: number): Promise<string>;
+/**
+ * @param {string} pool_id
+ * @returns {Promise<number>}
+ */
+export function recvWorkerRequest(pool_id: string): Promise<number>;
+/**
+ * @returns {Promise<PoolOptions>}
+ */
+export function recvPoolRequest(): Promise<PoolOptions>;
+/**
+ * @param {string} filter
+ */
+export function init_log_filter(filter: string): void;
 export function init_pack(): void;
 /**
  * Entry point for web workers
@@ -45,6 +79,28 @@ export class Metadata {
     __wbg_ptr: number | undefined;
     free(): void;
 }
+export class PoolOptions {
+    static __wrap(ptr: any): any;
+    __destroy_into_raw(): number | undefined;
+    __wbg_ptr: number | undefined;
+    free(): void;
+    /**
+     * @param {string} arg0
+     */
+    set filename(arg0: string);
+    /**
+     * @returns {string}
+     */
+    get filename(): string;
+    /**
+     * @param {number} arg0
+     */
+    set maxConcurrency(arg0: number);
+    /**
+     * @returns {number}
+     */
+    get maxConcurrency(): number;
+}
 export class Project {
     /**
      * @param {string} cwd
@@ -141,6 +197,28 @@ export class Project {
      */
     copyFile(src: string, dst: string): Promise<void>;
 }
+export class WorkerTermination {
+    static __wrap(ptr: any): any;
+    __destroy_into_raw(): number | undefined;
+    __wbg_ptr: number | undefined;
+    free(): void;
+    /**
+     * @param {string} arg0
+     */
+    set filename(arg0: string);
+    /**
+     * @returns {string}
+     */
+    get filename(): string;
+    /**
+     * @param {number} arg0
+     */
+    set workerId(arg0: number);
+    /**
+     * @returns {number}
+     */
+    get workerId(): number;
+}
 export default __wbg_init;
 export function initSync(module: any, memory: any): any;
 declare function __wbg_init(module_or_path: any, memory: any): Promise<any>;