@travetto/worker 8.0.0-alpha.8 → 8.0.0

package/README.md

@@ -13,10 +13,10 @@ npm install @travetto/worker
 yarn add @travetto/worker
 ```
 
-This module provides the necessary primitives for handling dependent workers.  A worker can be an individual actor or could be a pool of workers. Node provides ipc (inter-process communication) functionality out of the box. This module builds upon that by providing enhanced event management, richer process management, as well as constructs for orchestrating a conversation between two processes.
+This module provides the necessary primitives for handling dependent workers. A worker can be an individual actor or could be a pool of workers. Node provides ipc (inter-process communication) functionality out of the box. This module builds upon that by providing enhanced event management, richer process management, as well as constructs for orchestrating a conversation between two processes.
 
 ## Execution Pools
-With respect to managing multiple executions, [WorkPool](https://github.com/travetto/travetto/tree/main/module/worker/src/pool.ts#L34) is provided to allow for concurrent operation, and processing of jobs concurrently.  To manage the flow of jobs, [AsyncQueue](https://github.com/travetto/travetto/tree/main/module/runtime/src/queue.ts#L4) is used to support a wide range of use cases. [AsyncQueue](https://github.com/travetto/travetto/tree/main/module/runtime/src/queue.ts#L4) allows for manual control of iteration, which is useful for event driven work loads. 
+With respect to managing multiple executions, [WorkPool](https://github.com/travetto/travetto/tree/main/module/worker/src/pool.ts#L21) is provided to allow for concurrent operation, and processing of jobs concurrently. To manage the flow of jobs, [AsyncQueue](https://github.com/travetto/travetto/tree/main/module/runtime/src/queue.ts#L4) is used to support a wide range of use cases. [AsyncQueue](https://github.com/travetto/travetto/tree/main/module/runtime/src/queue.ts#L4) allows for manual control of iteration, which is useful for event driven work loads. 
 
 ## IPC Support
 To handle communication between processes, [IpcChannel](https://github.com/travetto/travetto/tree/main/module/worker/src/ipc.ts#L9) is provided. This class abstracts the underlying IPC mechanism and provides a simple interface for sending and receiving messages. It also includes event management capabilities, allowing for easy handling of different message types. By default the class assumes it is running in a child process, but it can also be used in a parent process (by passing in the child process) to communicate with child processes.

package/__index__.ts

@@ -1,2 +1,3 @@
 export * from './src/ipc.ts';
-export * from './src/pool.ts';
+export * from './src/pool.ts';
+export * from './src/types.ts';

package/package.json

@@ -1,11 +1,10 @@
 {
   "name": "@travetto/worker",
-  "version": "8.0.0-alpha.8",
-  "type": "module",
+  "version": "8.0.0",
   "description": "Process management utilities, with a focus on inter-process communication",
   "keywords": [
-    "exec",
     "child-process",
+    "exec",
     "ipc",
     "travetto",
     "typescript"
@@ -13,26 +12,27 @@
   "homepage": "https://travetto.io",
   "license": "MIT",
   "author": {
-    "email": "travetto.framework@gmail.com",
-    "name": "Travetto Framework"
+    "name": "Travetto Framework",
+    "email": "travetto.framework@gmail.com"
+  },
+  "repository": {
+    "url": "git+https://github.com/travetto/travetto.git",
+    "directory": "module/worker"
   },
   "files": [
     "__index__.ts",
     "src"
   ],
+  "type": "module",
   "main": "__index__.ts",
-  "repository": {
-    "url": "git+https://github.com/travetto/travetto.git",
-    "directory": "module/worker"
+  "publishConfig": {
+    "access": "public"
   },
   "dependencies": {
-    "@travetto/runtime": "^8.0.0-alpha.8",
+    "@travetto/runtime": "^8.0.0",
     "generic-pool": "^3.9.0"
   },
   "travetto": {
     "displayName": "Worker"
-  },
-  "publishConfig": {
-    "access": "public"
   }
 }

package/src/ipc.ts

@@ -7,7 +7,6 @@ import { ShutdownManager, Util } from '@travetto/runtime';
  * Channel that represents ipc communication
  */
 export class IpcChannel<V = unknown> {
-
   #emitter = new EventEmitter();
   subProcess: NodeJS.Process | ChildProcess;
   parentId: number;
@@ -37,7 +36,7 @@ export class IpcChannel<V = unknown> {
    * Determines if channel is active
    */
   get active(): boolean {
-    return (this.subProcess instanceof ChildProcess) ? !this.subProcess.killed : !!this.subProcess.connected;
+    return this.subProcess instanceof ChildProcess ? !this.subProcess.killed : !!this.subProcess.connected;
   }
 
   /**
@@ -48,7 +47,7 @@ export class IpcChannel<V = unknown> {
     if (!this.active) {
       throw new Error('Cannot send message to inactive process');
     } else if (this.subProcess.send && this.subProcess.connected) {
-      this.subProcess.send({ ...(data ?? {}), type: eventType }, undefined, undefined, (error) => error && console.error(error));
+      this.subProcess.send({ ...(data ?? {}), type: eventType }, undefined, undefined, error => error && console.error(error));
     } else {
       throw new Error('this.subProcess.send was not defined');
     }
@@ -88,7 +87,7 @@ export class IpcChannel<V = unknown> {
           this.subProcess.kill();
           await Promise.race([complete, Util.nonBlockingTimeout(1000)]);
         }
-      } catch { }
+      } catch {}
     }
     this.release();
   }
@@ -101,4 +100,4 @@ export class IpcChannel<V = unknown> {
     this.subProcess.removeAllListeners();
     this.#emitter.removeAllListeners();
   }
-}
+}

package/src/pool.ts

@@ -1,76 +1,70 @@
 import os from 'node:os';
-import { type Options, type Pool, createPool } from 'generic-pool';
 
-import { Env, Util, AsyncQueue } from '@travetto/runtime';
+import { createPool, type Pool } from 'generic-pool';
 
-type IterableSource<I> = Iterable<I> | AsyncIterable<I>;
-type WorkerExecutor<I, O> = (input: I, idx: number) => Promise<O>;
+import { AsyncQueue, Env, Util } from '@travetto/runtime';
 
-/**
- * Worker definition
- */
-export interface Worker<I, O = unknown> {
-  active: boolean;
-  id: unknown;
-  init?(): Promise<unknown>;
-  execute: WorkerExecutor<I, O>;
-  destroy?(): Promise<void>;
-  release?(): unknown;
-}
-
-type WorkerFactoryInput<I, O = unknown> = Partial<Worker<I, O>> & { execute: WorkerExecutor<I, O> };
-type WorkerInput<I, O> = (() => WorkerFactoryInput<I, O>) | WorkerExecutor<I, O>;
-type WorkPoolConfig<I, O> = Options & {
-  onComplete?: (output: O, input: I, finishIdx: number) => void;
-  onError?(event: Error, input: I, finishIdx: number): (unknown | Promise<unknown>);
-  shutdown?: AbortSignal;
-};
-
-const isWorkerFactory = <I, O>(value: WorkerInput<I, O>): value is (() => WorkerFactoryInput<I, O>) => value.length === 0;
+import {
+  type IterableSource,
+  isWorkerFactory,
+  type Worker,
+  type WorkerInput,
+  type WorkPoolCompleteEvent,
+  type WorkPoolConfig,
+  type WorkPoolProgress,
+  WorkPoolResultError
+} from './types.ts';
 
 /**
  * Work pool support
  */
 export class WorkPool {
-
   static MAX_SIZE = os.availableParallelism();
-  static DEFAULT_SIZE = Math.max(Math.trunc(WorkPool.MAX_SIZE * .75), 4);
+  static DEFAULT_SIZE = Math.max(Math.trunc(WorkPool.MAX_SIZE * 0.75), 4);
+
+  static #shouldTrace(): boolean {
+    return (Env.DEBUG.value ?? '').includes('@travetto/worker');
+  }
 
   /** Build worker pool */
   static #buildPool<I, O>(input: WorkerInput<I, O>, options?: WorkPoolConfig<I, O>): Pool<Worker<I, O>> {
     let pendingAcquires = 0;
 
-    const trace = /@travetto\/worker/.test(Env.DEBUG.value ?? '');
+    const trace = this.#shouldTrace();
 
     // Create the pool
-    const pool = createPool({
-      async create() {
-        try {
-          pendingAcquires += 1;
-          const worker: Worker<I, O> = {
-            id: Util.uuid(),
-            active: true,
-            ...isWorkerFactory(input) ? input() : { execute: input }
-          };
-          await worker.init?.();
-          return worker;
-        } finally {
-          pendingAcquires -= 1;
-        }
-      },
-      async destroy(worker) {
-        if (trace) {
-          console.debug('Destroying', { pid: process.pid, worker: worker.id });
-        }
-        return worker.destroy?.();
+    const pool = createPool(
+      {
+        async create() {
+          try {
+            pendingAcquires += 1;
+            const factoryInput = isWorkerFactory(input) ? await input() : { execute: input };
+            const worker: Worker<I, O> = {
+              id: Util.uuid(),
+              active: true,
+              ...factoryInput
+            };
+            await worker.init?.();
+            return worker;
+          } finally {
+            pendingAcquires -= 1;
+          }
+        },
+        async destroy(worker) {
+          if (trace) {
+            console.debug('Destroying', { pid: process.pid, worker: worker.id });
+          }
+          return worker.destroy?.();
+        },
+        validate: async (worker: Worker<I, O>) => worker.active
       },
-      validate: async (worker: Worker<I, O>) => worker.active
-    }, {
-      evictionRunIntervalMillis: 5000,
-      ...(options ?? {}),
-      max: options?.max ?? WorkPool.DEFAULT_SIZE,
-      min: options?.min ?? 1,
-    });
+      {
+        evictionRunIntervalMillis: 5000,
+        ...(options ?? {}),
+        max: options?.max ?? WorkPool.DEFAULT_SIZE,
+        min: options?.min ?? 1
+      }
+    );
 
     // Listen for shutdown
     options?.shutdown?.addEventListener('abort', async () => {
@@ -88,27 +82,42 @@ export class WorkPool {
    * Process a given input source and worker, and fire on completion
    */
   static async run<I, O>(workerFactory: WorkerInput<I, O>, source: IterableSource<I>, options: WorkPoolConfig<I, O> = {}): Promise<void> {
-
-    const trace = /@travetto\/worker/.test(Env.DEBUG.value ?? '');
+    const trace = this.#shouldTrace();
     const pending = new Set<Promise<unknown>>();
     const errors: Error[] = [];
     let inputIdx = 0;
-    let finishIdx = 0;
 
     const pool = this.#buildPool(workerFactory, options);
 
+    const progress: WorkPoolProgress = {
+      completed: 0,
+      total: options.total ?? 0,
+      failed: 0
+    };
+
     for await (const nextInput of source) {
       const worker = await pool.acquire()!;
 
       if (trace) {
         console.debug('Acquired', { pid: process.pid, worker: worker.id });
       }
+      if (!options.total) {
+        progress.total = inputIdx + 1;
+      }
 
-      const completion = worker.execute(nextInput, inputIdx += 1)
-        .then(output => options.onComplete?.(output, nextInput, finishIdx += 1))
+      const completion = worker
+        .execute(nextInput, (inputIdx += 1))
+        .then(output => {
+          const success = options.isSuccess?.(output) ?? true;
+          progress.failed += +!success;
+          progress.completed += 1;
+          return options.onComplete?.({ output, input: nextInput, success, progress });
+        })
         .catch(error => {
           errors.push(error);
-          options?.onError?.(error, nextInput, finishIdx += 1);
+          progress.failed += 1;
+          progress.completed += 1;
+          options?.onError?.({ error, input: nextInput, progress });
         }) // Catch error
         .finally(async () => {
           if (trace) {
@@ -118,12 +127,12 @@ export class WorkPool {
             if (worker.active) {
               try {
                 await worker.release?.();
-              } catch { }
+              } catch {}
               await pool.release(worker);
             } else {
               await pool.destroy(worker);
             }
-          } catch { }
+          } catch {}
         });
 
       completion.finally(() => pending.delete(completion));
@@ -133,43 +142,28 @@ export class WorkPool {
     await Promise.all(Array.from(pending));
 
     if (errors.length) {
-      throw errors[0];
+      throw new WorkPoolResultError(errors);
     }
   }
 
-  /**
-   * Process a given input source as an async iterable
-   */
-  static runStream<I, O>(worker: WorkerInput<I, O>, input: IterableSource<I>, options?: WorkPoolConfig<I, O>): AsyncIterable<O> {
-    const queue = new AsyncQueue<O>();
-    const result = this.run(worker, input, {
-      ...options,
-      onComplete: (event, value, finishIdx) => {
-        queue.add(event);
-        options?.onComplete?.(event, value, finishIdx);
-      }
-    });
-    result.finally(() => queue.close());
-    return queue;
-  }
-
   /**
    * Process a given input source as an async iterable with progress information
    */
-  static runStreamProgress<I, O>(worker: WorkerInput<I, O>, input: IterableSource<I>, total: number, options?: WorkPoolConfig<I, O>): AsyncIterable<{
-    idx: number;
-    value: O;
-    total: number;
-  }> {
-    const queue = new AsyncQueue<{ idx: number, value: O, total: number }>();
-    const result = this.run(worker, input, {
+  static runStream<I, O>(
+    worker: WorkerInput<I, O>,
+    source: IterableSource<I>,
+    options?: WorkPoolConfig<I, O>
+  ): AsyncIterable<WorkPoolCompleteEvent<I, O>> {
+    const queue = new AsyncQueue<WorkPoolCompleteEvent<I, O>>();
+    const result = this.run(worker, source, {
       ...options,
-      onComplete: (event, value, finishIdx) => {
-        queue.add({ value: event, idx: finishIdx, total });
-        options?.onComplete?.(event, value, finishIdx);
+      onComplete: async event => {
+        await options?.onComplete?.(event);
+        queue.add(event);
+        return;
       }
     });
     result.finally(() => queue.close());
     return queue;
   }
-}
+}

package/src/types.ts

@@ -0,0 +1,62 @@
+import type { Options } from 'generic-pool';
+
+import { RuntimeError } from '@travetto/runtime';
+
+export type IterableSource<I> = Iterable<I> | AsyncIterable<I>;
+export type WorkerExecutor<I, O> = (input: I, idx: number) => Promise<O>;
+
+/**
+ * WorkPool results error.
+ *
+ * Hold all the errors for a given work pool execution
+ */
+export class WorkPoolResultError extends RuntimeError<{ errors: Error[] }> {
+  constructor(errors: Error[]) {
+    super('WorkPool errors have occurred', { category: 'data', details: { errors } });
+  }
+}
+
+/**
+ * Worker definition
+ */
+export interface Worker<I, O = unknown> {
+  active: boolean;
+  id: unknown;
+  init?(): Promise<unknown>;
+  execute: WorkerExecutor<I, O>;
+  destroy?(): Promise<void>;
+  release?(): unknown;
+}
+
+export type WorkPoolProgress = {
+  total: number;
+  completed: number;
+  failed: number;
+};
+
+export type WorkPoolCompleteEvent<I, O> = {
+  output: O;
+  input: I;
+  success: boolean;
+  progress: WorkPoolProgress;
+};
+
+export type WorkPoolErrorEvent<I> = {
+  error: Error;
+  input: I;
+  progress: WorkPoolProgress;
+};
+
+export type WorkerFactoryInput<I, O = unknown> = Partial<Worker<I, O>> & { execute: WorkerExecutor<I, O> };
+export type WorkerInput<I, O> = (() => WorkerFactoryInput<I, O> | Promise<WorkerFactoryInput<I, O>>) | WorkerExecutor<I, O>;
+export type WorkPoolConfig<I, O> = Options & {
+  isSuccess?: (output: O) => boolean;
+  onComplete?: (event: WorkPoolCompleteEvent<I, O>) => void | Promise<void>;
+  onError?<R = unknown>(event: WorkPoolErrorEvent<I>): R | Promise<R>;
+  shutdown?: AbortSignal;
+  total?: number;
+};
+
+export const isWorkerFactory = <I, O>(
+  value: WorkerInput<I, O>
+): value is () => WorkerFactoryInput<I, O> | Promise<WorkerFactoryInput<I, O>> => value.length === 0;