@xylabs/threads 4.6.4 → 4.7.0

package/dist/master/invocation-proxy.js

@@ -1,121 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.createProxyFunction = createProxyFunction;
-exports.createProxyModule = createProxyModule;
-const debug_1 = __importDefault(require("debug"));
-const observable_fns_1 = require("observable-fns");
-const common_1 = require("../common");
-const observable_promise_1 = require("../observable-promise");
-const transferable_1 = require("../transferable");
-const messages_1 = require("../types/messages");
-const debugMessages = (0, debug_1.default)('threads:master:messages');
-let nextJobUID = 1;
-const dedupe = (array) => [...new Set(array)];
-const isJobErrorMessage = (data) => data && data.type === messages_1.WorkerMessageType.error;
-const isJobResultMessage = (data) => data && data.type === messages_1.WorkerMessageType.result;
-const isJobStartMessage = (data) => data && data.type === messages_1.WorkerMessageType.running;
-function createObservableForJob(worker, jobUID) {
-    return new observable_fns_1.Observable((observer) => {
-        let asyncType;
-        const messageHandler = ((event) => {
-            debugMessages('Message from worker:', event.data);
-            if (!event.data || event.data.uid !== jobUID)
-                return;
-            if (isJobStartMessage(event.data)) {
-                asyncType = event.data.resultType;
-            }
-            else if (isJobResultMessage(event.data)) {
-                if (asyncType === 'promise') {
-                    if (event.data.payload !== undefined) {
-                        observer.next((0, common_1.deserialize)(event.data.payload));
-                    }
-                    observer.complete();
-                    worker.removeEventListener('message', messageHandler);
-                }
-                else {
-                    if (event.data.payload) {
-                        observer.next((0, common_1.deserialize)(event.data.payload));
-                    }
-                    if (event.data.complete) {
-                        observer.complete();
-                        worker.removeEventListener('message', messageHandler);
-                    }
-                }
-            }
-            else if (isJobErrorMessage(event.data)) {
-                const error = (0, common_1.deserialize)(event.data.error);
-                if (asyncType === 'promise' || !asyncType) {
-                    observer.error(error);
-                }
-                else {
-                    observer.error(error);
-                }
-                worker.removeEventListener('message', messageHandler);
-            }
-        });
-        worker.addEventListener('message', messageHandler);
-        return () => {
-            if (asyncType === 'observable' || !asyncType) {
-                const cancelMessage = {
-                    type: messages_1.MasterMessageType.cancel,
-                    uid: jobUID,
-                };
-                worker.postMessage(cancelMessage);
-            }
-            worker.removeEventListener('message', messageHandler);
-        };
-    });
-}
-function prepareArguments(rawArgs) {
-    if (rawArgs.length === 0) {
-        return {
-            args: [],
-            transferables: [],
-        };
-    }
-    const args = [];
-    const transferables = [];
-    for (const arg of rawArgs) {
-        if ((0, transferable_1.isTransferDescriptor)(arg)) {
-            args.push((0, common_1.serialize)(arg.send));
-            transferables.push(...arg.transferables);
-        }
-        else {
-            args.push((0, common_1.serialize)(arg));
-        }
-    }
-    return {
-        args,
-        transferables: transferables.length === 0 ? transferables : dedupe(transferables),
-    };
-}
-function createProxyFunction(worker, method) {
-    return ((...rawArgs) => {
-        const uid = nextJobUID++;
-        const { args, transferables } = prepareArguments(rawArgs);
-        const runMessage = {
-            args,
-            method,
-            type: messages_1.MasterMessageType.run,
-            uid,
-        };
-        debugMessages('Sending command to run function to worker:', runMessage);
-        try {
-            worker.postMessage(runMessage, transferables);
-        }
-        catch (error) {
-            return observable_promise_1.ObservablePromise.from(Promise.reject(error));
-        }
-        return observable_promise_1.ObservablePromise.from((0, observable_fns_1.multicast)(createObservableForJob(worker, uid)));
-    });
-}
-function createProxyModule(worker, methodNames) {
-    const proxy = {};
-    for (const methodName of methodNames) {
-        proxy[methodName] = createProxyFunction(worker, methodName);
-    }
-    return proxy;
-}

package/dist/master/pool-types.js

@@ -1,14 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.PoolEventType = void 0;
-var PoolEventType;
-(function (PoolEventType) {
-    PoolEventType["initialized"] = "initialized";
-    PoolEventType["taskCanceled"] = "taskCanceled";
-    PoolEventType["taskCompleted"] = "taskCompleted";
-    PoolEventType["taskFailed"] = "taskFailed";
-    PoolEventType["taskQueued"] = "taskQueued";
-    PoolEventType["taskQueueDrained"] = "taskQueueDrained";
-    PoolEventType["taskStart"] = "taskStart";
-    PoolEventType["terminated"] = "terminated";
-})(PoolEventType || (exports.PoolEventType = PoolEventType = {}));

package/dist/master/pool.d.ts

@@ -1,50 +0,0 @@
-import { Observable } from 'observable-fns';
-import type { PoolEvent, QueuedTask, TaskRunFunction } from './pool-types';
-import { PoolEventType } from './pool-types';
-import { Thread } from './thread';
-export declare namespace Pool {
-    type Event<ThreadType extends Thread = any> = PoolEvent<ThreadType>;
-    type EventType = PoolEventType;
-}
-export interface Pool<ThreadType extends Thread> {
-    completed(allowResolvingImmediately?: boolean): Promise<any>;
-    settled(allowResolvingImmediately?: boolean): Promise<Error[]>;
-    events(): Observable<PoolEvent<ThreadType>>;
-    queue<Return>(task: TaskRunFunction<ThreadType, Return>): QueuedTask<ThreadType, Return>;
-    terminate(force?: boolean): Promise<void>;
-}
-interface PoolOptions {
-    concurrency?: number;
-    maxQueuedJobs?: number;
-    name?: string;
-    size?: number;
-}
-declare class WorkerPool<ThreadType extends Thread> implements Pool<ThreadType> {
-    static EventType: typeof PoolEventType;
-    private readonly debug;
-    private readonly eventObservable;
-    private readonly options;
-    private readonly workers;
-    private readonly eventSubject;
-    private initErrors;
-    private isClosing;
-    private nextTaskID;
-    private taskQueue;
-    constructor(spawnWorker: () => Promise<ThreadType>, optionsOrSize?: number | PoolOptions);
-    private findIdlingWorker;
-    private runPoolTask;
-    private run;
-    private scheduleWork;
-    private taskCompletion;
-    settled(allowResolvingImmediately?: boolean): Promise<Error[]>;
-    completed(allowResolvingImmediately?: boolean): Promise<void>;
-    events(): Observable<PoolEvent<ThreadType>>;
-    queue(taskFunction: TaskRunFunction<ThreadType, any>): QueuedTask<ThreadType, any>;
-    terminate(force?: boolean): Promise<void>;
-}
-declare function PoolConstructor<ThreadType extends Thread>(spawnWorker: () => Promise<ThreadType>, optionsOrSize?: number | PoolOptions): WorkerPool<ThreadType>;
-export declare const Pool: typeof PoolConstructor & {
-    EventType: typeof PoolEventType;
-};
-export { PoolEvent, PoolEventType, QueuedTask, } from './pool-types';
-export { Thread } from './thread';

package/dist/master/pool.js

@@ -1,262 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.Thread = exports.PoolEventType = exports.Pool = void 0;
-const debug_1 = __importDefault(require("debug"));
-const observable_fns_1 = require("observable-fns");
-const ponyfills_1 = require("../ponyfills");
-const implementation_1 = require("./implementation");
-const pool_types_1 = require("./pool-types");
-const thread_1 = require("./thread");
-let nextPoolID = 1;
-function createArray(size) {
-    const array = [];
-    for (let index = 0; index < size; index++) {
-        array.push(index);
-    }
-    return array;
-}
-function delay(ms) {
-    return new Promise(resolve => setTimeout(resolve, ms));
-}
-function flatMap(array, mapper) {
-    return array.reduce((flattened, element) => [...flattened, ...mapper(element)], []);
-}
-function slugify(text) {
-    return text.replaceAll(/\W/g, ' ').trim().replaceAll(/\s+/g, '-');
-}
-function spawnWorkers(spawnWorker, count) {
-    return createArray(count).map(() => ({
-        init: spawnWorker(),
-        runningTasks: [],
-    }));
-}
-class WorkerPool {
-    static EventType = pool_types_1.PoolEventType;
-    debug;
-    eventObservable;
-    options;
-    workers;
-    eventSubject = new observable_fns_1.Subject();
-    initErrors = [];
-    isClosing = false;
-    nextTaskID = 1;
-    taskQueue = [];
-    constructor(spawnWorker, optionsOrSize) {
-        const options = typeof optionsOrSize === 'number' ? { size: optionsOrSize } : optionsOrSize || {};
-        const { size = implementation_1.defaultPoolSize } = options;
-        this.debug = (0, debug_1.default)(`threads:pool:${slugify(options.name || String(nextPoolID++))}`);
-        this.options = options;
-        this.workers = spawnWorkers(spawnWorker, size);
-        this.eventObservable = (0, observable_fns_1.multicast)(observable_fns_1.Observable.from(this.eventSubject));
-        Promise.all(this.workers.map(worker => worker.init)).then(() => this.eventSubject.next({
-            size: this.workers.length,
-            type: pool_types_1.PoolEventType.initialized,
-        }), (error) => {
-            this.debug('Error while initializing pool worker:', error);
-            this.eventSubject.error(error);
-            this.initErrors.push(error);
-        });
-    }
-    findIdlingWorker() {
-        const { concurrency = 1 } = this.options;
-        return this.workers.find(worker => worker.runningTasks.length < concurrency);
-    }
-    async runPoolTask(worker, task) {
-        const workerID = this.workers.indexOf(worker) + 1;
-        this.debug(`Running task #${task.id} on worker #${workerID}...`);
-        this.eventSubject.next({
-            taskID: task.id,
-            type: pool_types_1.PoolEventType.taskStart,
-            workerID,
-        });
-        try {
-            const returnValue = await task.run(await worker.init);
-            this.debug(`Task #${task.id} completed successfully`);
-            this.eventSubject.next({
-                returnValue,
-                taskID: task.id,
-                type: pool_types_1.PoolEventType.taskCompleted,
-                workerID,
-            });
-        }
-        catch (error) {
-            this.debug(`Task #${task.id} failed`);
-            this.eventSubject.next({
-                error,
-                taskID: task.id,
-                type: pool_types_1.PoolEventType.taskFailed,
-                workerID,
-            });
-        }
-    }
-    async run(worker, task) {
-        const runPromise = (async () => {
-            const removeTaskFromWorkersRunningTasks = () => {
-                worker.runningTasks = worker.runningTasks.filter(someRunPromise => someRunPromise !== runPromise);
-            };
-            await delay(0);
-            try {
-                await this.runPoolTask(worker, task);
-            }
-            finally {
-                removeTaskFromWorkersRunningTasks();
-                if (!this.isClosing) {
-                    this.scheduleWork();
-                }
-            }
-        })();
-        worker.runningTasks.push(runPromise);
-    }
-    scheduleWork() {
-        this.debug('Attempt de-queueing a task in order to run it...');
-        const availableWorker = this.findIdlingWorker();
-        if (!availableWorker)
-            return;
-        const nextTask = this.taskQueue.shift();
-        if (!nextTask) {
-            this.debug('Task queue is empty');
-            this.eventSubject.next({ type: pool_types_1.PoolEventType.taskQueueDrained });
-            return;
-        }
-        this.run(availableWorker, nextTask);
-    }
-    taskCompletion(taskID) {
-        return new Promise((resolve, reject) => {
-            const eventSubscription = this.events().subscribe((event) => {
-                if (event.type === pool_types_1.PoolEventType.taskCompleted && event.taskID === taskID) {
-                    eventSubscription.unsubscribe();
-                    resolve(event.returnValue);
-                }
-                else if (event.type === pool_types_1.PoolEventType.taskFailed && event.taskID === taskID) {
-                    eventSubscription.unsubscribe();
-                    reject(event.error);
-                }
-                else if (event.type === pool_types_1.PoolEventType.terminated) {
-                    eventSubscription.unsubscribe();
-                    reject(new Error('Pool has been terminated before task was run.'));
-                }
-            });
-        });
-    }
-    async settled(allowResolvingImmediately = false) {
-        const getCurrentlyRunningTasks = () => flatMap(this.workers, worker => worker.runningTasks);
-        const taskFailures = [];
-        const failureSubscription = this.eventObservable.subscribe((event) => {
-            if (event.type === pool_types_1.PoolEventType.taskFailed) {
-                taskFailures.push(event.error);
-            }
-        });
-        if (this.initErrors.length > 0) {
-            throw this.initErrors[0];
-        }
-        if (allowResolvingImmediately && this.taskQueue.length === 0) {
-            await (0, ponyfills_1.allSettled)(getCurrentlyRunningTasks());
-            return taskFailures;
-        }
-        await new Promise((resolve, reject) => {
-            const subscription = this.eventObservable.subscribe({
-                error: reject,
-                next(event) {
-                    if (event.type === pool_types_1.PoolEventType.taskQueueDrained) {
-                        subscription.unsubscribe();
-                        resolve(void 0);
-                    }
-                },
-            });
-        });
-        await (0, ponyfills_1.allSettled)(getCurrentlyRunningTasks());
-        failureSubscription.unsubscribe();
-        return taskFailures;
-    }
-    async completed(allowResolvingImmediately = false) {
-        const settlementPromise = this.settled(allowResolvingImmediately);
-        const earlyExitPromise = new Promise((resolve, reject) => {
-            const subscription = this.eventObservable.subscribe({
-                error: reject,
-                next(event) {
-                    if (event.type === pool_types_1.PoolEventType.taskQueueDrained) {
-                        subscription.unsubscribe();
-                        resolve(settlementPromise);
-                    }
-                    else if (event.type === pool_types_1.PoolEventType.taskFailed) {
-                        subscription.unsubscribe();
-                        reject(event.error);
-                    }
-                },
-            });
-        });
-        const errors = await Promise.race([settlementPromise, earlyExitPromise]);
-        if (errors.length > 0) {
-            throw errors[0];
-        }
-    }
-    events() {
-        return this.eventObservable;
-    }
-    queue(taskFunction) {
-        const { maxQueuedJobs = Number.POSITIVE_INFINITY } = this.options;
-        if (this.isClosing) {
-            throw new Error('Cannot schedule pool tasks after terminate() has been called.');
-        }
-        if (this.initErrors.length > 0) {
-            throw this.initErrors[0];
-        }
-        const taskID = this.nextTaskID++;
-        const taskCompletion = this.taskCompletion(taskID);
-        taskCompletion.catch((error) => {
-            this.debug(`Task #${taskID} errored:`, error);
-        });
-        const task = {
-            cancel: () => {
-                if (!this.taskQueue.includes(task))
-                    return;
-                this.taskQueue = this.taskQueue.filter(someTask => someTask !== task);
-                this.eventSubject.next({
-                    taskID: task.id,
-                    type: pool_types_1.PoolEventType.taskCanceled,
-                });
-            },
-            id: taskID,
-            run: taskFunction,
-            then: taskCompletion.then.bind(taskCompletion),
-        };
-        if (this.taskQueue.length >= maxQueuedJobs) {
-            throw new Error('Maximum number of pool tasks queued. Refusing to queue another one.\n'
-                + 'This usually happens for one of two reasons: We are either at peak '
-                + "workload right now or some tasks just won't finish, thus blocking the pool.");
-        }
-        this.debug(`Queueing task #${task.id}...`);
-        this.taskQueue.push(task);
-        this.eventSubject.next({
-            taskID: task.id,
-            type: pool_types_1.PoolEventType.taskQueued,
-        });
-        this.scheduleWork();
-        return task;
-    }
-    async terminate(force) {
-        this.isClosing = true;
-        if (!force) {
-            await this.completed(true);
-        }
-        this.eventSubject.next({
-            remainingQueue: [...this.taskQueue],
-            type: pool_types_1.PoolEventType.terminated,
-        });
-        this.eventSubject.complete();
-        await Promise.all(this.workers.map(async (worker) => thread_1.Thread.terminate(await worker.init)));
-    }
-}
-function PoolConstructor(spawnWorker, optionsOrSize) {
-    return new WorkerPool(spawnWorker, optionsOrSize);
-}
-;
-PoolConstructor.EventType = pool_types_1.PoolEventType;
-exports.Pool = PoolConstructor;
-var pool_types_2 = require("./pool-types");
-Object.defineProperty(exports, "PoolEventType", { enumerable: true, get: function () { return pool_types_2.PoolEventType; } });
-var thread_2 = require("./thread");
-Object.defineProperty(exports, "Thread", { enumerable: true, get: function () { return thread_2.Thread; } });

package/dist/master/register.d.ts

@@ -1 +0,0 @@
-export {};

package/dist/master/register.js

@@ -1,11 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const index_1 = require("./index");
-if (typeof globalThis !== 'undefined') {
-    ;
-    globalThis.Worker = index_1.Worker;
-}
-else if (window !== undefined) {
-    ;
-    window.Worker = index_1.Worker;
-}

package/dist/master/spawn.js

@@ -1,114 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.spawn = spawn;
-const debug_1 = __importDefault(require("debug"));
-const observable_fns_1 = require("observable-fns");
-const common_1 = require("../common");
-const promise_1 = require("../promise");
-const symbols_1 = require("../symbols");
-const master_1 = require("../types/master");
-const invocation_proxy_1 = require("./invocation-proxy");
-const debugMessages = (0, debug_1.default)('threads:master:messages');
-const debugSpawn = (0, debug_1.default)('threads:master:spawn');
-const debugThreadUtils = (0, debug_1.default)('threads:master:thread-utils');
-const isInitMessage = (data) => data && data.type === 'init';
-const isUncaughtErrorMessage = (data) => data && data.type === 'uncaughtError';
-const initMessageTimeout = typeof process !== 'undefined' && process.env !== undefined && process.env.THREADS_WORKER_INIT_TIMEOUT
-    ? Number.parseInt(process.env.THREADS_WORKER_INIT_TIMEOUT, 10)
-    : 10_000;
-async function withTimeout(promise, timeoutInMs, errorMessage) {
-    let timeoutHandle;
-    const timeout = new Promise((resolve, reject) => {
-        timeoutHandle = setTimeout(() => reject(new Error(errorMessage)), timeoutInMs);
-    });
-    const result = await Promise.race([promise, timeout]);
-    clearTimeout(timeoutHandle);
-    return result;
-}
-function receiveInitMessage(worker) {
-    return new Promise((resolve, reject) => {
-        const messageHandler = ((event) => {
-            debugMessages('Message from worker before finishing initialization:', event.data);
-            if (isInitMessage(event.data)) {
-                worker.removeEventListener('message', messageHandler);
-                resolve(event.data);
-            }
-            else if (isUncaughtErrorMessage(event.data)) {
-                worker.removeEventListener('message', messageHandler);
-                reject((0, common_1.deserialize)(event.data.error));
-            }
-        });
-        worker.addEventListener('message', messageHandler);
-    });
-}
-function createEventObservable(worker, workerTermination) {
-    return new observable_fns_1.Observable((observer) => {
-        const messageHandler = ((messageEvent) => {
-            const workerEvent = {
-                data: messageEvent.data,
-                type: master_1.WorkerEventType.message,
-            };
-            observer.next(workerEvent);
-        });
-        const rejectionHandler = ((errorEvent) => {
-            debugThreadUtils('Unhandled promise rejection event in thread:', errorEvent);
-            const workerEvent = {
-                error: new Error(errorEvent.reason),
-                type: master_1.WorkerEventType.internalError,
-            };
-            observer.next(workerEvent);
-        });
-        worker.addEventListener('message', messageHandler);
-        worker.addEventListener('unhandledrejection', rejectionHandler);
-        workerTermination.then(() => {
-            const terminationEvent = { type: master_1.WorkerEventType.termination };
-            worker.removeEventListener('message', messageHandler);
-            worker.removeEventListener('unhandledrejection', rejectionHandler);
-            observer.next(terminationEvent);
-            observer.complete();
-        });
-    });
-}
-function createTerminator(worker) {
-    const [termination, resolver] = (0, promise_1.createPromiseWithResolver)();
-    const terminate = async () => {
-        debugThreadUtils('Terminating worker');
-        await worker.terminate();
-        resolver();
-    };
-    return { terminate, termination };
-}
-function setPrivateThreadProps(raw, worker, workerEvents, terminate) {
-    const workerErrors = workerEvents
-        .filter(event => event.type === master_1.WorkerEventType.internalError)
-        .map(errorEvent => errorEvent.error);
-    return Object.assign(raw, {
-        [symbols_1.$errors]: workerErrors,
-        [symbols_1.$events]: workerEvents,
-        [symbols_1.$terminate]: terminate,
-        [symbols_1.$worker]: worker,
-    });
-}
-async function spawn(worker, options) {
-    debugSpawn('Initializing new thread');
-    const timeout = options && options.timeout ? options.timeout : initMessageTimeout;
-    const initMessage = await withTimeout(receiveInitMessage(worker), timeout, `Timeout: Did not receive an init message from worker after ${timeout}ms. Make sure the worker calls expose().`);
-    const exposed = initMessage.exposed;
-    const { termination, terminate } = createTerminator(worker);
-    const events = createEventObservable(worker, termination);
-    if (exposed.type === 'function') {
-        const proxy = (0, invocation_proxy_1.createProxyFunction)(worker);
-        return setPrivateThreadProps(proxy, worker, events, terminate);
-    }
-    else if (exposed.type === 'module') {
-        const proxy = (0, invocation_proxy_1.createProxyModule)(worker, exposed.methods);
-        return setPrivateThreadProps(proxy, worker, events, terminate);
-    }
-    else {
-        const type = exposed.type;
-        throw new Error(`Worker init message states unexpected type of expose(): ${type}`);
-    }
-}

package/dist/master/thread.d.ts

@@ -1,8 +0,0 @@
-import type { Observable } from 'observable-fns';
-import type { Thread as ThreadType, WorkerEvent } from '../types/master';
-export type Thread = ThreadType;
-export declare const Thread: {
-    errors<ThreadT extends ThreadType>(thread: ThreadT): Observable<Error>;
-    events<ThreadT extends ThreadType>(thread: ThreadT): Observable<WorkerEvent>;
-    terminate<ThreadT extends ThreadType>(thread: ThreadT): Promise<void>;
-};

package/dist/master/thread.js

@@ -1,18 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.Thread = void 0;
-const symbols_1 = require("../symbols");
-function fail(message) {
-    throw new Error(message);
-}
-exports.Thread = {
-    errors(thread) {
-        return thread[symbols_1.$errors] || fail('Error observable not found. Make sure to pass a thread instance as returned by the spawn() promise.');
-    },
-    events(thread) {
-        return thread[symbols_1.$events] || fail('Events observable not found. Make sure to pass a thread instance as returned by the spawn() promise.');
-    },
-    terminate(thread) {
-        return thread[symbols_1.$terminate]();
-    },
-};