@travetto/worker 4.0.0-rc.0 → 4.0.0-rc.1

package/README.md

@@ -18,153 +18,5 @@ This module provides the necessary primitives for handling dependent workers.  A
 ## Execution Pools
 With respect to managing multiple executions, [WorkPool](https://github.com/travetto/travetto/tree/main/module/worker/src/pool.ts#L35) is provided to allow for concurrent operation, and processing of jobs concurrently.  To manage the flow of jobs, [WorkQueue](https://github.com/travetto/travetto/tree/main/module/worker/src/queue.ts#L6) is provided to support a wide range of use cases. [WorkQueue](https://github.com/travetto/travetto/tree/main/module/worker/src/queue.ts#L6) allows for manual control of iteration, which is useful for event driven work loads. 
 
-Below is a pool that will convert images on demand, while queuing as needed.
-
-**Code: Image processing queue, with a fixed batch/pool size**
-```typescript
-import { ExecUtil, ExecutionState } from '@travetto/base';
-import { Worker, WorkPool, WorkQueue } from '@travetto/worker';
-
-class ImageProcessor implements Worker<string> {
-  active = false;
-  proc: ExecutionState;
-
-  get id(): number | undefined {
-    return this.proc.process.pid;
-  }
-
-  async destroy(): Promise<void> {
-    this.proc.process.kill();
-  }
-
-  async execute(path: string): Promise<void> {
-    this.active = true;
-    try {
-      this.proc = ExecUtil.spawn('convert images', [path]);
-      await this.proc;
-    } catch {
-      // Do nothing
-    }
-    this.active = false;
-  }
-}
-
-export class ImageCompressor {
-
-  changes: AsyncIterable<unknown>;
-  pendingImages = new WorkQueue<string>();
-
-  begin(): void {
-    this.changes ??= WorkPool.runStream(() => new ImageProcessor(), this.pendingImages);
-  }
-
-  convert(...images: string[]): void {
-    this.pendingImages.addAll(images);
-  }
-}
-```
-
-Once a pool is constructed, it can be shutdown by calling the `.shutdown()` method, and awaiting the result.
-
 ## IPC Support
 Within the `comm` package, there is support for two primary communication elements: [ChildCommChannel](https://github.com/travetto/travetto/tree/main/module/worker/src/comm/child.ts#L6) and [ParentCommChannel](https://github.com/travetto/travetto/tree/main/module/worker/src/comm/parent.ts#L10).  Usually [ParentCommChannel](https://github.com/travetto/travetto/tree/main/module/worker/src/comm/parent.ts#L10) indicates it is the owner of the sub process. [ChildCommChannel](https://github.com/travetto/travetto/tree/main/module/worker/src/comm/child.ts#L6) indicates that it has been created/spawned/forked by the parent and will communicate back to it's parent. This generally means that a [ParentCommChannel](https://github.com/travetto/travetto/tree/main/module/worker/src/comm/parent.ts#L10) can be destroyed (i.e. killing the subprocess) where a [ChildCommChannel](https://github.com/travetto/travetto/tree/main/module/worker/src/comm/child.ts#L6) can only exit the process, but the channel cannot be destroyed.
-
-### IPC as a Worker
-A common pattern is to want to model a sub process as a worker, to be a valid candidate in a [WorkPool](https://github.com/travetto/travetto/tree/main/module/worker/src/pool.ts#L35).  The [WorkUtil](https://github.com/travetto/travetto/tree/main/module/worker/src/util.ts#L14) class provides a utility to facilitate this desire.
-
-**Code: Spawned Worker**
-```typescript
-import { ExecutionState } from '@travetto/base';
-
-import { ParentCommChannel } from './comm/parent';
-import { Worker } from './pool';
-
-type Simple<V> = (ch: ParentCommChannel<V>) => Promise<unknown | void>;
-type Param<V, X> = (ch: ParentCommChannel<V>, input: X) => Promise<unknown | void>;
-
-const empty = async (): Promise<void> => { };
-
-/**
- * Spawned worker
- */
-export class WorkUtil {
-  /**
-   * Create a process channel worker from a given spawn config
-   */
-  static spawnedWorker<V, X>(
-    worker: () => ExecutionState,
-    init: Simple<V>,
-    execute: Param<V, X>,
-    destroy: Simple<V> = empty): Worker<X> {
-    const channel = new ParentCommChannel<V>(worker());
-    return {
-      get id(): number | undefined { return channel.id; },
-      get active(): boolean { return channel.active; },
-      init: () => init(channel),
-      execute: inp => execute(channel, inp),
-      async destroy(): Promise<void> {
-        await destroy(channel);
-        await channel.destroy();
-      },
-    };
-  }
-}
-```
-
-When creating your work, via process spawning, you will need to provide the script (and any other features you would like in `SpawnConfig`).   Additionally you must, at a minimum, provide functionality to run whenever an input element is up for grabs in the input source.  This method will be provided the communication channel ([ParentCommChannel](https://github.com/travetto/travetto/tree/main/module/worker/src/comm/parent.ts#L10)) and the input value.  A simple example could look like:
-
-**Code: Spawning Pool**
-```typescript
-import { ExecUtil } from '@travetto/base';
-import { WorkPool, WorkUtil } from '@travetto/worker';
-
-export async function main(): Promise<void> {
-  await WorkPool.run(
-    () => WorkUtil.spawnedWorker<{ data: number }, number>(
-      () => ExecUtil.spawn('trv', ['main', '@travetto/worker/doc/spawned.ts']),
-      ch => ch.once('ready'), // Wait for child to indicate it is ready
-      async (channel, inp) => {
-        const res = channel.once('response'); //  Register response listener
-        channel.send('request', { data: inp }); // Send request
-
-        const { data } = await res; // Get answer
-        console.log('Request complete', { input: inp, output: data });
-
-        if (!(inp + inp === data)) {
-          // Ensure the answer is double the input
-          throw new Error(`Did not get the double: inp=${inp}, data=${data}`);
-        }
-      }
-    ), [1, 2, 3, 4, 5]);
-}
-```
-
-**Code: Spawned Worker**
-```typescript
-import timers from 'node:timers/promises';
-import { ChildCommChannel } from '@travetto/worker';
-
-export async function main(): Promise<void> {
-  const exec = new ChildCommChannel<{ data: string }>();
-
-  exec.on('request', data =>
-    exec.send('response', { data: (data.data + data.data) })); // When data is received, return double
-
-  exec.send('ready'); // Indicate the child is ready to receive requests
-
-  for await (const _ of timers.setInterval(5000)) {
-    // Keep-alive
-  }
-}
-```
-
-**Terminal: Output**
-```bash
-$ trv main doc/spawner.ts
-
-Request complete { input: 1, output: 2 }
-Request complete { input: 2, output: 4 }
-Request complete { input: 3, output: 6 }
-Request complete { input: 4, output: 8 }
-Request complete { input: 5, output: 10 }
-```

package/__index__.ts

@@ -7,4 +7,3 @@ export * from './src/support/timeout';
 export * from './src/support/error';
 export * from './src/queue';
 export * from './src/pool';
-export * from './src/util';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@travetto/worker",
-  "version": "4.0.0-rc.0",
+  "version": "4.0.0-rc.1",
   "description": "Process management utilities, with a focus on inter-process communication",
   "keywords": [
     "exec",
@@ -25,7 +25,7 @@
     "directory": "module/worker"
   },
   "dependencies": {
-    "@travetto/base": "^4.0.0-rc.0",
+    "@travetto/base": "^4.0.0-rc.1",
     "generic-pool": "^3.9.0"
   },
   "travetto": {

package/src/comm/channel.ts

@@ -1,8 +1,6 @@
 import { ChildProcess } from 'node:child_process';
 import { EventEmitter } from 'node:events';
 
-import { ExecUtil } from '@travetto/base';
-
 /**
  * Channel that represents communication between parent/child
  */
@@ -56,8 +54,8 @@ export class ProcessCommChannel<T extends NodeJS.Process | ChildProcess, V = unk
     console.debug('Sending', { pid: this.#parentId, id: this.id, eventType });
     if (!this.#proc) {
       throw new Error('this.proc was not defined');
-    } else if (this.#proc.send) {
-      this.#proc.send({ ...(data ?? {}), type: eventType });
+    } else if (this.#proc.send && this.#proc.connected) {
+      this.#proc.send({ ...(data ?? {}), type: eventType }, (err) => err && console.error(err));
     } else {
       throw new Error('this.proc.send was not defined');
     }
@@ -92,7 +90,8 @@ export class ProcessCommChannel<T extends NodeJS.Process | ChildProcess, V = unk
     if (this.#proc) {
       console.debug('Killing', { pid: this.#parentId, id: this.id });
       if (this.#proc !== process) {
-        ExecUtil.kill(this.#proc);
+        // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
+        (this.#proc as ChildProcess).kill();
       }
       this.#proc = undefined;
     }

package/src/comm/parent.ts

@@ -1,6 +1,6 @@
 import { ChildProcess } from 'node:child_process';
 
-import { ShutdownManager, ExecutionState } from '@travetto/base';
+import { ShutdownManager } from '@travetto/base';
 
 import { ProcessCommChannel } from './channel';
 
@@ -9,13 +9,12 @@ import { ProcessCommChannel } from './channel';
  */
 export class ParentCommChannel<U = unknown> extends ProcessCommChannel<ChildProcess, U> {
 
-  #complete: ExecutionState['result'];
+  #complete: Promise<void>;
 
-  constructor(state: ExecutionState) {
-    super(state.process);
+  constructor(proc: ChildProcess) {
+    super(proc);
     ShutdownManager.onGracefulShutdown(() => this.destroy(), this);
-    this.#complete = state.result
-      .finally(() => { this.proc = undefined; });
+    this.#complete = new Promise<void>(r => proc.on('close', r)).finally(() => { this.proc = undefined; });
   }
 
   /**

package/src/pool.ts

@@ -34,7 +34,7 @@ const isWorkerFactory = <I, O>(o: WorkerInput<I, O>): o is (() => Worker<I, O>)
  */
 export class WorkPool {
 
-  static MAX_SIZE = os.cpus().length - 1;
+  static MAX_SIZE = os.availableParallelism();
   static DEFAULT_SIZE = Math.min(WorkPool.MAX_SIZE, 4);
 
   /** Build worker pool */

package/src/queue.ts

@@ -67,6 +67,15 @@ export class WorkQueue<X> implements AsyncIterator<X>, AsyncIterable<X> {
     this.#ready.resolve();
   }
 
+  /**
+   * Throw an error from the queue, rejecting and terminating immediately
+   */
+  async throw(e?: Error): Promise<IteratorResult<X>> {
+    this.#done = true;
+    this.#ready.reject(e);
+    return { value: undefined, done: this.#done };
+  }
+
   /**
    * Get size, will change as items are added
    */

package/src/util.ts

@@ -1,35 +0,0 @@
-import { ExecutionState } from '@travetto/base';
-
-import { ParentCommChannel } from './comm/parent';
-import { Worker } from './pool';
-
-type Simple<V> = (ch: ParentCommChannel<V>) => Promise<unknown | void>;
-type Param<V, X> = (ch: ParentCommChannel<V>, input: X) => Promise<unknown | void>;
-
-const empty = async (): Promise<void> => { };
-
-/**
- * Spawned worker
- */
-export class WorkUtil {
-  /**
-   * Create a process channel worker from a given spawn config
-   */
-  static spawnedWorker<V, X>(
-    worker: () => ExecutionState,
-    init: Simple<V>,
-    execute: Param<V, X>,
-    destroy: Simple<V> = empty): Worker<X> {
-    const channel = new ParentCommChannel<V>(worker());
-    return {
-      get id(): number | undefined { return channel.id; },
-      get active(): boolean { return channel.active; },
-      init: () => init(channel),
-      execute: inp => execute(channel, inp),
-      async destroy(): Promise<void> {
-        await destroy(channel);
-        await channel.destroy();
-      },
-    };
-  }
-}