@utoo/pack 1.0.0 → 1.0.1-alpha.0

package/cjs/binding.d.ts

@@ -9,6 +9,20 @@ export declare class ExternalObject<T> {
     [K: symbol]: T
   }
 }
+export interface PoolOptions {
+  filename: string
+  maxConcurrency: number
+}
+export interface WorkerTermination {
+  filename: string
+  workerId: number
+}
+export declare function recvPoolRequest(): Promise<PoolOptions>
+export declare function recvWorkerTermination(): Promise<WorkerTermination>
+export declare function recvWorkerRequest(filename: string): Promise<number>
+export declare function recvMessageInWorker(workerId: number): Promise<string>
+export declare function notifyWorkerAck(taskId: number, workerId: number): Promise<void>
+export declare function sendTaskMessage(taskId: number, message: string): Promise<void>
 export interface NapiEndpointConfig {
   
 }

package/cjs/binding.js

@@ -319,7 +319,13 @@ if (!nativeBinding) {
     }
     throw new Error(`Failed to load native binding`);
 }
-const { endpointWriteToDisk, endpointServerChangedSubscribe, endpointClientChangedSubscribe, projectNew, projectUpdate, projectOnExit, projectShutdown, projectWriteAllEntrypointsToDisk, projectEntrypointsSubscribe, projectHmrEvents, projectHmrIdentifiersSubscribe, projectUpdateInfoSubscribe, projectTraceSource, projectGetSourceForAsset, projectGetSourceMap, projectGetSourceMapSync, rootTaskDispose, initCustomTraceSubscriber, teardownTraceSubscriber } = nativeBinding;
+const { recvPoolRequest, recvWorkerTermination, recvWorkerRequest, recvMessageInWorker, notifyWorkerAck, sendTaskMessage, endpointWriteToDisk, endpointServerChangedSubscribe, endpointClientChangedSubscribe, projectNew, projectUpdate, projectOnExit, projectShutdown, projectWriteAllEntrypointsToDisk, projectEntrypointsSubscribe, projectHmrEvents, projectHmrIdentifiersSubscribe, projectUpdateInfoSubscribe, projectTraceSource, projectGetSourceForAsset, projectGetSourceMap, projectGetSourceMapSync, rootTaskDispose, initCustomTraceSubscriber, teardownTraceSubscriber } = nativeBinding;
+module.exports.recvPoolRequest = recvPoolRequest;
+module.exports.recvWorkerTermination = recvWorkerTermination;
+module.exports.recvWorkerRequest = recvWorkerRequest;
+module.exports.recvMessageInWorker = recvMessageInWorker;
+module.exports.notifyWorkerAck = notifyWorkerAck;
+module.exports.sendTaskMessage = sendTaskMessage;
 module.exports.endpointWriteToDisk = endpointWriteToDisk;
 module.exports.endpointServerChangedSubscribe = endpointServerChangedSubscribe;
 module.exports.endpointClientChangedSubscribe = endpointClientChangedSubscribe;

package/cjs/project.js

@@ -36,6 +36,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.TurbopackInternalError = void 0;
 exports.projectFactory = projectFactory;
 const util_1 = require("util");
+const worker_threads_1 = require("worker_threads");
 const binding = __importStar(require("./binding"));
 const util_2 = require("./util");
 class TurbopackInternalError extends Error {
@@ -200,9 +201,60 @@ function projectFactory() {
         };
         return iterator;
     }
+    const loaderWorkers = {};
+    const createOrScalePool = async () => {
+        while (true) {
+            try {
+                let poolOptions = await binding.recvPoolRequest();
+                const { filename, maxConcurrency } = poolOptions;
+                const workers = loaderWorkers[filename] || (loaderWorkers[filename] = []);
+                if (workers.length < maxConcurrency) {
+                    for (let i = workers.length; i < maxConcurrency; i++) {
+                        const worker = new worker_threads_1.Worker(filename, {
+                            workerData: {
+                                poolId: filename,
+                                bindingPath: require.resolve("./binding.js"),
+                            },
+                        });
+                        worker.unref();
+                        workers.push(worker);
+                    }
+                }
+                else if (workers.length > maxConcurrency) {
+                    const workersToStop = workers.splice(0, workers.length - maxConcurrency);
+                    workersToStop.forEach((worker) => worker.terminate());
+                }
+            }
+            catch (e) {
+                return;
+            }
+        }
+    };
+    const waitingForWorkerTermination = async () => {
+        while (true) {
+            try {
+                const { filename, workerId } = await binding.recvWorkerTermination();
+                const workers = loaderWorkers[filename];
+                const workerIdx = workers.findIndex((worker) => worker.threadId === workerId);
+                if (workerIdx > -1) {
+                    const worker = workers.splice(workerIdx, 1);
+                    worker[0].terminate();
+                }
+            }
+            catch (e) {
+                return;
+            }
+        }
+    };
     class ProjectImpl {
         constructor(nativeProject) {
             this._nativeProject = nativeProject;
+            if (typeof binding.recvPoolRequest === "function") {
+                createOrScalePool();
+            }
+            if (typeof binding.recvWorkerTermination === "function") {
+                waitingForWorkerTermination();
+            }
         }
         async update(options) {
             await withErrorCause(async () => binding.projectUpdate(this._nativeProject, await rustifyPartialProjectOptions(options)));

package/cjs/types.d.ts

@@ -112,8 +112,8 @@ export interface ConfigComplete {
         clean?: boolean;
         copy?: Array<{
             from: string;
-            to: string;
-        }>;
+            to?: string;
+        } | string>;
         publicPath?: string;
     };
     target?: string;

package/esm/binding.d.ts

@@ -9,6 +9,20 @@ export declare class ExternalObject<T> {
     [K: symbol]: T
   }
 }
+export interface PoolOptions {
+  filename: string
+  maxConcurrency: number
+}
+export interface WorkerTermination {
+  filename: string
+  workerId: number
+}
+export declare function recvPoolRequest(): Promise<PoolOptions>
+export declare function recvWorkerTermination(): Promise<WorkerTermination>
+export declare function recvWorkerRequest(filename: string): Promise<number>
+export declare function recvMessageInWorker(workerId: number): Promise<string>
+export declare function notifyWorkerAck(taskId: number, workerId: number): Promise<void>
+export declare function sendTaskMessage(taskId: number, message: string): Promise<void>
 export interface NapiEndpointConfig {
   
 }

package/esm/binding.js

@@ -319,7 +319,13 @@ if (!nativeBinding) {
     }
     throw new Error(`Failed to load native binding`);
 }
-const { endpointWriteToDisk, endpointServerChangedSubscribe, endpointClientChangedSubscribe, projectNew, projectUpdate, projectOnExit, projectShutdown, projectWriteAllEntrypointsToDisk, projectEntrypointsSubscribe, projectHmrEvents, projectHmrIdentifiersSubscribe, projectUpdateInfoSubscribe, projectTraceSource, projectGetSourceForAsset, projectGetSourceMap, projectGetSourceMapSync, rootTaskDispose, initCustomTraceSubscriber, teardownTraceSubscriber } = nativeBinding;
+const { recvPoolRequest, recvWorkerTermination, recvWorkerRequest, recvMessageInWorker, notifyWorkerAck, sendTaskMessage, endpointWriteToDisk, endpointServerChangedSubscribe, endpointClientChangedSubscribe, projectNew, projectUpdate, projectOnExit, projectShutdown, projectWriteAllEntrypointsToDisk, projectEntrypointsSubscribe, projectHmrEvents, projectHmrIdentifiersSubscribe, projectUpdateInfoSubscribe, projectTraceSource, projectGetSourceForAsset, projectGetSourceMap, projectGetSourceMapSync, rootTaskDispose, initCustomTraceSubscriber, teardownTraceSubscriber } = nativeBinding;
+module.exports.recvPoolRequest = recvPoolRequest;
+module.exports.recvWorkerTermination = recvWorkerTermination;
+module.exports.recvWorkerRequest = recvWorkerRequest;
+module.exports.recvMessageInWorker = recvMessageInWorker;
+module.exports.notifyWorkerAck = notifyWorkerAck;
+module.exports.sendTaskMessage = sendTaskMessage;
 module.exports.endpointWriteToDisk = endpointWriteToDisk;
 module.exports.endpointServerChangedSubscribe = endpointServerChangedSubscribe;
 module.exports.endpointClientChangedSubscribe = endpointClientChangedSubscribe;

package/esm/project.js

@@ -1,4 +1,5 @@
 import { isDeepStrictEqual } from "util";
+import { Worker } from "worker_threads";
 import * as binding from "./binding";
 import { rustifyEnv } from "./util";
 export class TurbopackInternalError extends Error {
@@ -162,9 +163,60 @@ export function projectFactory() {
         };
         return iterator;
     }
+    const loaderWorkers = {};
+    const createOrScalePool = async () => {
+        while (true) {
+            try {
+                let poolOptions = await binding.recvPoolRequest();
+                const { filename, maxConcurrency } = poolOptions;
+                const workers = loaderWorkers[filename] || (loaderWorkers[filename] = []);
+                if (workers.length < maxConcurrency) {
+                    for (let i = workers.length; i < maxConcurrency; i++) {
+                        const worker = new Worker(filename, {
+                            workerData: {
+                                poolId: filename,
+                                bindingPath: require.resolve("./binding.js"),
+                            },
+                        });
+                        worker.unref();
+                        workers.push(worker);
+                    }
+                }
+                else if (workers.length > maxConcurrency) {
+                    const workersToStop = workers.splice(0, workers.length - maxConcurrency);
+                    workersToStop.forEach((worker) => worker.terminate());
+                }
+            }
+            catch (e) {
+                return;
+            }
+        }
+    };
+    const waitingForWorkerTermination = async () => {
+        while (true) {
+            try {
+                const { filename, workerId } = await binding.recvWorkerTermination();
+                const workers = loaderWorkers[filename];
+                const workerIdx = workers.findIndex((worker) => worker.threadId === workerId);
+                if (workerIdx > -1) {
+                    const worker = workers.splice(workerIdx, 1);
+                    worker[0].terminate();
+                }
+            }
+            catch (e) {
+                return;
+            }
+        }
+    };
     class ProjectImpl {
         constructor(nativeProject) {
             this._nativeProject = nativeProject;
+            if (typeof binding.recvPoolRequest === "function") {
+                createOrScalePool();
+            }
+            if (typeof binding.recvWorkerTermination === "function") {
+                waitingForWorkerTermination();
+            }
         }
         async update(options) {
             await withErrorCause(async () => binding.projectUpdate(this._nativeProject, await rustifyPartialProjectOptions(options)));

package/esm/types.d.ts

@@ -112,8 +112,8 @@ export interface ConfigComplete {
         clean?: boolean;
         copy?: Array<{
             from: string;
-            to: string;
-        }>;
+            to?: string;
+        } | string>;
         publicPath?: string;
     };
     target?: string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@utoo/pack",
-  "version": "1.0.0",
+  "version": "1.0.1-alpha.0",
   "main": "cjs/index.js",
   "module": "esm/index.js",
   "exports": {
@@ -85,12 +85,12 @@
   },
   "repository": "git@github.com:utooland/utoo.git",
   "optionalDependencies": {
-    "@utoo/pack-darwin-arm64": "1.0.0",
-    "@utoo/pack-darwin-x64": "1.0.0",
-    "@utoo/pack-linux-arm64-gnu": "1.0.0",
-    "@utoo/pack-linux-arm64-musl": "1.0.0",
-    "@utoo/pack-linux-x64-gnu": "1.0.0",
-    "@utoo/pack-linux-x64-musl": "1.0.0",
-    "@utoo/pack-win32-x64-msvc": "1.0.0"
+    "@utoo/pack-darwin-arm64": "1.0.1-alpha.0",
+    "@utoo/pack-darwin-x64": "1.0.1-alpha.0",
+    "@utoo/pack-linux-arm64-gnu": "1.0.1-alpha.0",
+    "@utoo/pack-linux-arm64-musl": "1.0.1-alpha.0",
+    "@utoo/pack-linux-x64-gnu": "1.0.1-alpha.0",
+    "@utoo/pack-linux-x64-musl": "1.0.1-alpha.0",
+    "@utoo/pack-win32-x64-msvc": "1.0.1-alpha.0"
   }
 }