@travetto/worker 5.0.0-rc.9 → 5.0.1

package/README.md

@@ -16,7 +16,7 @@ yarn add @travetto/worker
 This module provides the necessary primitives for handling dependent workers.  A worker can be an individual actor or could be a pool of workers. Node provides ipc (inter-process communication) functionality out of the box. This module builds upon that by providing enhanced event management, richer process management, as well as constructs for orchestrating a conversation between two processes.
 
 ## Execution Pools
-With respect to managing multiple executions, [WorkPool](https://github.com/travetto/travetto/tree/main/module/worker/src/pool.ts#L34) is provided to allow for concurrent operation, and processing of jobs concurrently.  To manage the flow of jobs, [WorkQueue](https://github.com/travetto/travetto/tree/main/module/worker/src/queue.ts#L6) is provided to support a wide range of use cases. [WorkQueue](https://github.com/travetto/travetto/tree/main/module/worker/src/queue.ts#L6) allows for manual control of iteration, which is useful for event driven work loads. 
+With respect to managing multiple executions, [WorkPool](https://github.com/travetto/travetto/tree/main/module/worker/src/pool.ts#L32) is provided to allow for concurrent operation, and processing of jobs concurrently.  To manage the flow of jobs, [AsyncQueue](https://github.com/travetto/travetto/tree/main/module/runtime/src/queue.ts#L6) is used to support a wide range of use cases. [AsyncQueue](https://github.com/travetto/travetto/tree/main/module/runtime/src/queue.ts#L6) allows for manual control of iteration, which is useful for event driven work loads. 
 
 ## IPC Support
-Within the `comm` package, there is support for two primary communication elements: [ChildCommChannel](https://github.com/travetto/travetto/tree/main/module/worker/src/comm/child.ts#L6) and [ParentCommChannel](https://github.com/travetto/travetto/tree/main/module/worker/src/comm/parent.ts#L10).  Usually [ParentCommChannel](https://github.com/travetto/travetto/tree/main/module/worker/src/comm/parent.ts#L10) indicates it is the owner of the sub process. [ChildCommChannel](https://github.com/travetto/travetto/tree/main/module/worker/src/comm/child.ts#L6) indicates that it has been created/spawned/forked by the parent and will communicate back to it's parent. This generally means that a [ParentCommChannel](https://github.com/travetto/travetto/tree/main/module/worker/src/comm/parent.ts#L10) can be destroyed (i.e. killing the subprocess) where a [ChildCommChannel](https://github.com/travetto/travetto/tree/main/module/worker/src/comm/child.ts#L6) can only exit the process, but the channel cannot be destroyed.
+Within the `comm` package, there is support for two primary communication elements: [ChildCommChannel](https://github.com/travetto/travetto/tree/main/module/worker/src/ipc/child.ts#L6) and [ParentCommChannel](https://github.com/travetto/travetto/tree/main/module/worker/src/ipc/parent.ts#L10).  Usually [ParentCommChannel](https://github.com/travetto/travetto/tree/main/module/worker/src/ipc/parent.ts#L10) indicates it is the owner of the sub process. [ChildCommChannel](https://github.com/travetto/travetto/tree/main/module/worker/src/ipc/child.ts#L6) indicates that it has been created/spawned/forked by the parent and will communicate back to it's parent. This generally means that a [ParentCommChannel](https://github.com/travetto/travetto/tree/main/module/worker/src/ipc/parent.ts#L10) can be destroyed (i.e. killing the subprocess) where a [ChildCommChannel](https://github.com/travetto/travetto/tree/main/module/worker/src/ipc/child.ts#L6) can only exit the process, but the channel cannot be destroyed.

package/__index__.ts

@@ -1,9 +1,5 @@
-export * from './src/comm/channel';
-export * from './src/comm/child';
-export * from './src/comm/parent';
-export * from './src/comm/types';
-export * from './src/support/barrier';
-export * from './src/support/timeout';
-export * from './src/support/error';
-export * from './src/queue';
+export * from './src/ipc/channel';
+export * from './src/ipc/child';
+export * from './src/ipc/parent';
+export * from './src/ipc/types';
 export * from './src/pool';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@travetto/worker",
-  "version": "5.0.0-rc.9",
+  "version": "5.0.1",
   "description": "Process management utilities, with a focus on inter-process communication",
   "keywords": [
     "exec",
@@ -25,7 +25,7 @@
     "directory": "module/worker"
   },
   "dependencies": {
-    "@travetto/runtime": "^5.0.0-rc.9",
+    "@travetto/runtime": "^5.0.1",
     "generic-pool": "^3.9.0"
   },
   "travetto": {

package/src/pool.ts

@@ -1,9 +1,7 @@
 import os from 'node:os';
 import { Options, Pool, createPool } from 'generic-pool';
 
-import { Env, Util } from '@travetto/runtime';
-
-import { WorkQueue } from './queue';
+import { Env, Util, AsyncQueue } from '@travetto/runtime';
 
 type ItrSource<I> = Iterable<I> | AsyncIterable<I>;
 
@@ -67,10 +65,10 @@ export class WorkPool {
       },
       validate: async (x: Worker<I, O>) => x.active ?? true
     }, {
-      max: WorkPool.DEFAULT_SIZE,
-      min: 1,
       evictionRunIntervalMillis: 5000,
       ...(opts ?? {}),
+      max: opts?.max ?? WorkPool.DEFAULT_SIZE,
+      min: opts?.min ?? 1,
     });
 
 
@@ -143,7 +141,7 @@ export class WorkPool {
    * Process a given input source as an async iterable
    */
   static runStream<I, O>(worker: WorkerInput<I, O>, input: ItrSource<I>, opts?: WorkPoolConfig<I, O>): AsyncIterable<O> {
-    const itr = new WorkQueue<O>();
+    const itr = new AsyncQueue<O>();
     const res = this.run(worker, input, {
       ...opts,
       onComplete: (ev, inp, finishIdx) => {
@@ -163,7 +161,7 @@ export class WorkPool {
     value: O;
     total: number;
   }> {
-    const itr = new WorkQueue<{ idx: number, value: O, total: number }>();
+    const itr = new AsyncQueue<{ idx: number, value: O, total: number }>();
     const res = this.run(worker, input, {
       ...opts,
       onComplete: (ev, inp, finishIdx) => {

package/src/queue.ts

@@ -1,85 +0,0 @@
-import { Util } from '@travetto/runtime';
-
-/**
- * WorkQueue, a manual async iterator.  Items are added manually, and consumed asynchronously
- */
-export class WorkQueue<X> implements AsyncIterator<X>, AsyncIterable<X> {
-
-  #queue: X[] = [];
-  #done = false;
-  #ready = Util.resolvablePromise();
-  #size: number;
-
-  /**
-   * Initial set of items
-   */
-  constructor(initial: Iterable<X> = [], signal?: AbortSignal) {
-    this.#queue.push(...initial);
-    this.#size = this.#queue.length;
-    signal?.addEventListener('abort', () => this.close());
-    if (signal?.aborted) {
-      this.close();
-    }
-  }
-
-  // Allow for iteration
-  [Symbol.asyncIterator](): AsyncIterator<X> {
-    return this;
-  }
-
-  /**
-   * Wait for next event to fire
-   */
-  async next(): Promise<IteratorResult<X>> {
-    while (!this.#done && !this.#queue.length) {
-      await this.#ready.promise;
-      this.#ready = Util.resolvablePromise();
-    }
-    return { value: (this.#queue.length ? this.#queue.shift() : undefined)!, done: this.#done };
-  }
-
-  /**
-   * Queue next event to fire
-   * @param {boolean} immediate Determines if item(s) should be append or prepended to the queue
-   */
-  add(item: X, immediate = false): void {
-    this.#queue[immediate ? 'unshift' : 'push'](item);
-    this.#size += 1;
-    this.#ready.resolve();
-  }
-
-  /**
-   * Queue a list of data to stream
-   * @param {boolean} immediate Determines if item(s) should be append or prepended to the queue
-   */
-  addAll(items: Iterable<X>): void {
-    const copy = [...items];
-    this.#queue.push(...copy);
-    this.#size += copy.length;
-    this.#ready.resolve();
-  }
-
-  /**
-   * Close the iterator
-   */
-  close(): void {
-    this.#done = true;
-    this.#ready.resolve();
-  }
-
-  /**
-   * Throw an error from the queue, rejecting and terminating immediately
-   */
-  async throw(e?: Error): Promise<IteratorResult<X>> {
-    this.#done = true;
-    this.#ready.reject(e);
-    return { value: undefined, done: this.#done };
-  }
-
-  /**
-   * Get size, will change as items are added
-   */
-  get size(): number {
-    return this.#size;
-  }
-}

package/src/support/barrier.ts

@@ -1,89 +0,0 @@
-import { TimeSpan, Util } from '@travetto/runtime';
-
-import { Timeout } from './timeout';
-
-function canCancel(o: unknown): o is { cancel(): unknown } {
-  return !!o && (typeof o === 'object') && 'cancel' in o && typeof o.cancel === 'function';
-}
-
-/**
- * Build an execution barrier to handle various limitations
- */
-export class Barrier {
-  /**
-   * Listen for an unhandled event, as a promise
-   */
-  static listenForUnhandled(): Promise<unknown> & { cancel?: () => void } {
-    const uncaught = Util.resolvablePromise<Promise<unknown> & { cancel?: () => void }>();
-    const onError = (err: Error): void => { Util.queueMacroTask().then(() => uncaught.reject(err)); };
-    process.on('unhandledRejection', onError).on('uncaughtException', onError);
-    const cancel = (): void => {
-      process.off('unhandledRejection', onError).off('unhandledException', onError);
-      uncaught.resolve(undefined!); // Close the promise
-    };
-    Object.defineProperty(uncaught.promise, 'cancel', { value: cancel });
-    return uncaught.promise;
-  }
-
-  #support: string[] = [];
-  #barriers = new Map<string, Promise<unknown>>([]);
-
-  constructor(
-    timeout?: number | TimeSpan,
-    unhandled?: boolean
-  ) {
-    if (timeout !== undefined) {
-      this.add(new Timeout(timeout).wait(), true);
-    }
-    if (unhandled) {
-      this.add(Barrier.listenForUnhandled());
-    }
-  }
-
-  /**
-   * Add a new barrier
-   */
-  add(p: (() => Promise<unknown>) | Promise<unknown>, support = false): this {
-    if (!('then' in p)) {
-      p = p();
-    }
-    const k = Util.uuid();
-    p = p
-      .finally(() => this.#barriers.delete(k))
-      .catch(err => { this.cleanup(); throw err; });
-
-    if (!support) {
-      p = p.then(() => this.cleanup());
-    } else {
-      this.#support.push(k);
-    }
-
-    this.#barriers.set(k, p);
-    return this;
-  }
-
-  /**
-   * Clean up, and cancel all cancellable barriers
-   */
-  cleanup(): void {
-    for (const k of this.#support) {
-      const el = this.#barriers.get(k);
-      if (canCancel(el)) {
-        el.cancel();
-      }
-    }
-    this.#barriers.clear();
-  }
-
-  /**
-   * Wait for all barriers to clear out
-   */
-  async wait(): Promise<Error | undefined> {
-    let capturedError: Error | undefined;
-    // Wait for all barriers to be satisfied
-    while (this.#barriers.size) {
-      await Promise.race(this.#barriers.values()).catch(err => capturedError ??= err);
-    }
-    return capturedError;
-  }
-}

package/src/support/error.ts

@@ -1,6 +0,0 @@
-import { AppError } from '@travetto/runtime';
-
-/**
- * Represents an execution error
- */
-export class ExecutionError extends AppError { }

package/src/support/timeout.ts

@@ -1,42 +0,0 @@
-import timers from 'node:timers/promises';
-import { TimeSpan, TimeUtil, Util } from '@travetto/runtime';
-import { ExecutionError } from './error';
-
-/**
- * Timeout support, throws self on timeout
- */
-export class Timeout extends ExecutionError {
-
-  #timeout?: AbortController;
-  #resolver = Util.resolvablePromise();
-  #duration: number;
-
-  constructor(duration: number | TimeSpan, op: string = 'Operation') {
-    super(`${op} timed out after ${duration}${typeof duration === 'number' ? 'ms' : ''}`);
-    this.#duration = TimeUtil.asMillis(duration);
-  }
-
-  /**
-   * Stop timeout from firing
-   */
-  cancel(): void {
-    if (this.#timeout) {
-      this.#resolver.resolve();
-      this.#timeout.abort();
-      this.#timeout = undefined;
-    }
-  }
-
-  /**
-   * Wait for timeout as a promise
-   */
-  wait(): Promise<void> {
-    if (!this.#timeout) {
-      this.#timeout = new AbortController();
-      timers.setTimeout(this.#duration, undefined, { ref: false, signal: this.#timeout.signal })
-        .then(() => this.#resolver.reject(this))
-        .catch(() => false);
-    }
-    return this.#resolver.promise;
-  }
-}