@lsdsoftware/utils 2.1.0 → 2.2.0

package/README.md

@@ -29,49 +29,33 @@ const result = await semaphore.runTask(async () => {
 Observable wrapper for net.connect (see connect-socket.test.ts for usage)
 
 
-### Worker Rotator
-Rotate worker instances over a request stream and emit worker lifecycle events.
-
-```typescript
-import { makeWorkerRotator } from "@lsdsoftware/utils"
-
-const events$ = makeWorkerRotator({
-  makeWorker,
-  workerTtlMs: 60_000,
-  request$,
-  maxPendingRequests: 100
-})
-```
-
-Contract:
-
-- The returned observable is cold. Each subscription creates its own rotator engine and subscribes to `request$`; share the returned observable if you want one engine with multiple observers.
-- Requests emitted before the first worker is ready, or between workers, are buffered and handed to the next worker.
-- `maxPendingRequests` caps the no-worker buffer and errors the rotator when exceeded. The default is `Infinity`.
-- Pending requests are considered handed off once they are passed to `worker.process`. Delivery retries, acknowledgements, and exactly-once guarantees belong in the worker or an upstream queue.
-- Completing `request$` means no more input, but it does not define the rotator lifecycle. The engine runs until the returned observable is unsubscribed or errors.
-- `worker.relieve()` is called during worker teardown. Implementations should make it safe to call more than once.
-
-
 ### CLI Worker Rotator
-Rotate child processes that communicate over stdin/stdout using a line-oriented request/response protocol, such as JSONL.
+Rotate child processes that communicate over stdin/stdout.
 
 ```typescript
 import { spawn } from "node:child_process"
 import { makeCLIWorkerRotator } from "@lsdsoftware/utils"
 
-const events$ = makeCLIWorkerRotator({
-  spawnWorkerProcess: () => spawn("my-jsonl-worker", {
+const subscription = makeCLIWorkerRotator({
+  spawnWorkerProcess: signal => spawn("my-cli-worker", {
+    signal,
     stdio: ["pipe", "pipe", "inherit"]
   }),
-  workerTtlMs: 60_000,
-  request$,
-  maxPendingRequests: 100
+  workerTTL: 60_000,
+  onEvent: event => console.debug('[cli-worker-rotator]', event)
+}).subscribe(worker => {
+  if (worker) {
+    worker.stdin.write("hello\n")
+  }
 })
 ```
 
-Each request writes exactly one stdin line. Each stdout line is paired with the next pending request in order. Child processes should write logs to stderr.
-
-For custom framing or multiplexed protocols, implement `Worker<R>` directly and use `makeWorkerRotator`.
+Contract:
 
-If a worker exits before producing a matching stdout line for a written request, that request's `output$` is not resolved by this adapter. Apply timeout or cancellation around each request if the caller needs bounded waits.
+- The returned observable is cold. Each subscription owns its own child process lifecycle.
+- The observable emits the child process when a worker starts, emits `null` when that worker closes, then repeats with a new worker.
+- `spawnWorkerProcess` receives an `AbortSignal`; pass it to `spawn` so unsubscribing can cancel an in-flight spawn.
+- `workerTTL` rotates a worker after the given number of milliseconds. It must be greater than zero when provided.
+- `retryConfig` controls retries after spawn failures. `repeatConfig` controls rotation after normal close or TTL expiry; the default waits 1000 ms before spawning the next worker.
+- `onEvent` receives `spawn`, `fail-spawn`, and `close` events for logging, tracing, or diagnostics.
+- Request framing, response matching, backpressure, and protocol-specific timeouts belong in the caller that uses the emitted stdin/stdout streams.

package/dist/cli-worker-rotator.d.ts

@@ -1,22 +1,26 @@
-import type { ChildProcessByStdio } from "child_process";
+import type { ChildProcess, ChildProcessByStdio } from "child_process";
 import * as rxjs from "rxjs";
 import type { Readable, Writable } from "stream";
-import { WorkerRotatorEvent } from "./worker-rotator.js";
-export type CLIWorker = ChildProcessByStdio<Writable, Readable, null>;
-export type CLIWorkerRotatorEvent = WorkerRotatorEvent<CLIWorker>;
-export interface CLIRequest {
-    input: string;
-    output$: rxjs.SubjectLike<string>;
-}
+export type CLIWorkerRotatorEvent = {
+    type: 'spawn';
+    worker: ChildProcess;
+} | {
+    type: 'fail-spawn';
+    reason: unknown;
+} | {
+    type: 'close';
+    worker: ChildProcess;
+    reason: unknown;
+};
 export interface CLIWorkerRotatorOptions {
-    spawnWorkerProcess: () => CLIWorker;
-    workerTtlMs: number;
-    request$: rxjs.Observable<CLIRequest>;
-    maxPendingRequests?: number;
+    spawnWorkerProcess: (signal: AbortSignal) => ChildProcessByStdio<Writable, Readable, null>;
+    workerTTL?: number;
+    retryConfig?: rxjs.RetryConfig;
+    repeatConfig?: rxjs.RepeatConfig;
+    onEvent?: (event: CLIWorkerRotatorEvent) => void;
 }
 /**
- * Rotates child processes that communicate over stdin/stdout using a
- * line-oriented request/response protocol, such as JSONL.
- * See the README for the protocol and lifecycle contract.
+ * Rotates child processes that communicate over stdin/stdout.
+ * See the README for the lifecycle contract.
  */
-export declare function makeCLIWorkerRotator({ spawnWorkerProcess, workerTtlMs, request$, maxPendingRequests }: CLIWorkerRotatorOptions): rxjs.Observable<CLIWorkerRotatorEvent>;
+export declare function makeCLIWorkerRotator({ spawnWorkerProcess, workerTTL, retryConfig, repeatConfig, onEvent }: CLIWorkerRotatorOptions): rxjs.Observable<ChildProcessByStdio<Writable, Readable, null> | null>;

package/dist/cli-worker-rotator.js

@@ -1,69 +1,28 @@
 import * as rxjs from "rxjs";
-import { makeLineReader } from "./line-reader.js";
-import { makeWorkerRotator } from "./worker-rotator.js";
 /**
- * Rotates child processes that communicate over stdin/stdout using a
- * line-oriented request/response protocol, such as JSONL.
- * See the README for the protocol and lifecycle contract.
+ * Rotates child processes that communicate over stdin/stdout.
+ * See the README for the lifecycle contract.
  */
-export function makeCLIWorkerRotator({ spawnWorkerProcess, workerTtlMs, request$, maxPendingRequests }) {
-    return makeWorkerRotator({
-        makeWorker: () => makeWorker(spawnWorkerProcess),
-        workerTtlMs,
-        request$,
-        maxPendingRequests
-    }).pipe(rxjs.map(event => ({ ...event, worker: event.worker.child })));
-}
-async function makeWorker(spawn) {
-    const child = spawn();
-    await new Promise((f, r) => child.once('spawn', f).once('error', r));
-    return {
-        child,
-        process(request$) {
-            return request$.pipe(rxjs.mergeMap(request => writeLn(child.stdin, request.input).pipe(rxjs.map(() => request), rxjs.catchError(err => {
-                request.output$.error(err);
-                return rxjs.EMPTY;
-            }))), rxjs.zipWith(readLines(child.stdout)), rxjs.tap(([request, line]) => request.output$.next(line)), rxjs.ignoreElements());
-        },
-        relieve() {
-            child.stdin.end();
-        },
-        quit$: rxjs.race(rxjs.fromEvent(child, 'close', (code, signal) => `Worker exit ${signal || code}`), rxjs.fromEvent(child.stdin, 'error', err => new Error('Worker stdin error', { cause: err })))
-    };
-}
-function readLines(stream) {
-    return new rxjs.Observable(subscriber => {
-        const lineReader = makeLineReader(line => subscriber.next(line));
-        const onError = (err) => subscriber.error(err);
-        const onFinish = () => subscriber.complete();
-        lineReader.once('error', onError);
-        lineReader.once('finish', onFinish);
-        stream.once('error', onError);
-        stream.pipe(lineReader);
-        return () => {
-            lineReader.off('error', onError);
-            lineReader.off('finish', onFinish);
-            stream.off('error', onError);
-            stream.unpipe(lineReader);
-            lineReader.destroy();
-        };
-    });
-}
-function writeLn(stream, line) {
-    return new rxjs.Observable(subscriber => {
-        try {
-            stream.write(line + "\n", err => {
-                if (err) {
-                    subscriber.error(err);
-                }
-                else {
-                    subscriber.next();
-                    subscriber.complete();
-                }
-            });
-        }
-        catch (err) {
-            subscriber.error(err);
-        }
+export function makeCLIWorkerRotator({ spawnWorkerProcess, workerTTL, retryConfig, repeatConfig = { delay: 1000 }, onEvent }) {
+    if (workerTTL != undefined && workerTTL <= 0) {
+        throw new Error('Invalid workerTTL');
+    }
+    return rxjs.defer(() => {
+        const abortCtl = new AbortController();
+        return rxjs.defer(() => {
+            const child = spawnWorkerProcess(abortCtl.signal);
+            return rxjs.race(rxjs.fromEvent(child, 'spawn').pipe(rxjs.take(1), rxjs.map(() => child)), rxjs.fromEvent(child, 'error').pipe(rxjs.map(err => { throw err; })));
+        }).pipe(rxjs.tap({
+            next: worker => onEvent?.({ type: 'spawn', worker }),
+            error: err => onEvent?.({ type: 'fail-spawn', reason: err })
+        }), retryConfig ? rxjs.retry(retryConfig) : rxjs.identity, rxjs.exhaustMap(worker => {
+            return rxjs.NEVER.pipe(rxjs.startWith(worker), rxjs.takeUntil(rxjs.race(rxjs.fromEvent(worker, 'close', (code, signal) => `Process exit (${signal || code})`), rxjs.fromEvent(worker.stdin, 'error'), workerTTL
+                ? rxjs.timer(workerTTL).pipe(rxjs.map(() => 'TTL expire'))
+                : rxjs.NEVER).pipe(rxjs.tap(reason => onEvent?.({ type: 'close', worker, reason })))), rxjs.endWith(null), rxjs.finalize(() => {
+                worker.stdin.end();
+            }));
+        }), repeatConfig ? rxjs.repeat(repeatConfig) : rxjs.identity, rxjs.finalize(() => {
+            abortCtl.abort();
+        }));
     });
 }

package/dist/cli-worker-rotator.test.js

@@ -2,50 +2,52 @@ import { describe } from "@service-broker/test-utils";
 import assert from "assert";
 import { EventEmitter } from "events";
 import * as rxjs from "rxjs";
-import { PassThrough, Writable } from "stream";
+import { PassThrough } from "stream";
 import { makeCLIWorkerRotator } from "./cli-worker-rotator.js";
 describe('cli-worker-rotator', ({ test }) => {
-    test('pairs stdout lines with requests in order', async () => {
-        const request$ = new rxjs.Subject;
-        const child = makeEchoChild();
-        const firstOutput$ = new rxjs.Subject;
-        const secondOutput$ = new rxjs.Subject;
+    test('emits each spawned worker and null when it closes', async () => {
+        const children = [makeChild(), makeChild()];
+        const spawned = [...children];
+        const emissions = await rxjs.firstValueFrom(makeCLIWorkerRotator({
+            spawnWorkerProcess: () => children.shift(),
+            onEvent: event => {
+                if (event.type === 'spawn') {
+                    setTimeout(() => event.worker.emit('close', 0, null), 0);
+                }
+            },
+            repeatConfig: { count: 2, delay: 0 }
+        }).pipe(rxjs.take(4), rxjs.toArray()));
+        assert.strictEqual(emissions.length, 4);
+        assert.strictEqual(emissions[0], spawned[0]);
+        assert.strictEqual(emissions[1], null);
+        assert.strictEqual(emissions[2], spawned[1]);
+        assert.strictEqual(emissions[3], null);
+    });
+    test('aborts an in-flight spawn when unsubscribed before the worker starts', async () => {
+        let aborted = false;
         const subscription = makeCLIWorkerRotator({
-            spawnWorkerProcess: () => child,
-            workerTtlMs: 1000,
-            request$
+            spawnWorkerProcess: signal => {
+                signal.addEventListener('abort', () => {
+                    aborted = true;
+                });
+                return makeChild({ autoSpawn: false });
+            }
         }).subscribe();
-        try {
-            request$.next({ input: 'one', output$: firstOutput$ });
-            request$.next({ input: 'two', output$: secondOutput$ });
-            const outputs = await Promise.all([
-                rxjs.firstValueFrom(firstOutput$),
-                rxjs.firstValueFrom(secondOutput$)
-            ]);
-            assert.deepStrictEqual(outputs, ['ONE', 'TWO']);
-        }
-        finally {
-            subscription.unsubscribe();
-        }
+        subscription.unsubscribe();
+        assert.strictEqual(aborted, true);
     });
 });
-function makeEchoChild() {
+function makeChild({ autoSpawn = true } = {}) {
     const child = new EventEmitter();
-    const stdout = new PassThrough();
-    let remainder = '';
-    child.stdin = new Writable({
-        write(chunk, _encoding, callback) {
-            remainder += chunk.toString();
-            const lines = remainder.split(/\r?\n/);
-            remainder = lines.pop();
-            for (const line of lines) {
-                stdout.write(line.toUpperCase() + '\n');
-            }
-            callback();
-        }
-    });
-    child.stdout = stdout;
+    const stdin = new PassThrough();
+    child.stdin = stdin;
+    child.stdout = new PassThrough();
     child.stderr = null;
-    setTimeout(() => child.emit('spawn'), 0);
+    stdin.on('close', () => {
+        child.emit('close', 0, null);
+    });
+    if (autoSpawn) {
+        setTimeout(() => child.emit('spawn'), 0);
+    }
     return child;
 }

package/dist/index.d.ts

@@ -2,4 +2,3 @@ export * from './connect-socket.js';
 export * from './cli-worker-rotator.js';
 export * from './line-reader.js';
 export * from './semaphore.js';
-export * from './worker-rotator.js';

package/dist/index.js

@@ -2,4 +2,3 @@ export * from './connect-socket.js';
 export * from './cli-worker-rotator.js';
 export * from './line-reader.js';
 export * from './semaphore.js';
-export * from './worker-rotator.js';

package/dist/index.test.d.ts

@@ -2,4 +2,3 @@ import './cli-worker-rotator.test.js';
 import './connect-socket.test.js';
 import './line-reader.test.js';
 import './semaphore.test.js';
-import './worker-rotator.test.js';

package/dist/index.test.js

@@ -2,4 +2,3 @@ import './cli-worker-rotator.test.js';
 import './connect-socket.test.js';
 import './line-reader.test.js';
 import './semaphore.test.js';
-import './worker-rotator.test.js';

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@lsdsoftware/utils",
   "type": "module",
-  "version": "2.1.0",
+  "version": "2.2.0",
   "description": "Useful JavaScript utilities",
   "main": "dist/index.js",
   "files": [

package/dist/worker-rotator.d.ts

@@ -1,25 +0,0 @@
-import * as rxjs from "rxjs";
-export interface Worker<R> {
-    process(request$: rxjs.Observable<R>): rxjs.Observable<never>;
-    relieve(): void;
-    quit$: rxjs.Observable<unknown>;
-}
-export type WorkerRotatorEvent<W> = {
-    type: 'hired' | 'relieved';
-    worker: W;
-} | {
-    type: 'quit';
-    worker: W;
-    reason: unknown;
-};
-export interface WorkerRotatorOptions<R, W extends Worker<R>> {
-    makeWorker: () => Promise<W>;
-    workerTtlMs: number;
-    request$: rxjs.Observable<R>;
-    maxPendingRequests?: number;
-}
-/**
- * Rotates workers over a request stream and emits worker lifecycle events.
- * See the README for the lifecycle and buffering contract.
- */
-export declare function makeWorkerRotator<R, W extends Worker<R>>({ makeWorker, workerTtlMs, request$, maxPendingRequests }: WorkerRotatorOptions<R, W>): rxjs.Observable<WorkerRotatorEvent<W>>;

package/dist/worker-rotator.js

@@ -1,28 +0,0 @@
-import * as rxjs from "rxjs";
-/**
- * Rotates workers over a request stream and emits worker lifecycle events.
- * See the README for the lifecycle and buffering contract.
- */
-export function makeWorkerRotator({ makeWorker, workerTtlMs, request$, maxPendingRequests = Infinity }) {
-    return rxjs.defer(() => {
-        if (maxPendingRequests !== Infinity && (!Number.isInteger(maxPendingRequests) || maxPendingRequests < 0)) {
-            throw new RangeError('maxPendingRequests must be a non-negative integer or Infinity');
-        }
-        const eventSubject = new rxjs.ReplaySubject(1);
-        return rxjs.defer(() => makeWorker()).pipe(rxjs.tap(worker => eventSubject?.next({ type: 'hired', worker })), rxjs.exhaustMap(worker => rxjs.NEVER.pipe(rxjs.startWith(worker), rxjs.takeUntil(rxjs.race(worker.quit$, rxjs.timer(workerTtlMs).pipe(rxjs.map(() => 'Worker TTL expired'))).pipe(rxjs.tap(reason => eventSubject?.next({ type: 'quit', worker, reason })))), rxjs.endWith(null), rxjs.finalize(() => {
-            worker.relieve();
-            eventSubject?.next({ type: 'relieved', worker });
-        }))), rxjs.repeat(), rxjs.share(), worker$ => request$.pipe(rxjs.window(worker$), rxjs.zipWith(worker$.pipe(rxjs.startWith(null))), rxjs.mergeScan((pending, [window$, worker]) => rxjs.concat(pending, window$).pipe(worker
-            ? request$ => worker.process(request$).pipe(rxjs.startWith([]))
-            : request$ => request$.pipe(bufferRequests(maxPendingRequests))), []), rxjs.ignoreElements()), rxjs.mergeWith(eventSubject));
-    });
-}
-function bufferRequests(maxPendingRequests) {
-    return request$ => request$.pipe(rxjs.reduce((pending, request) => {
-        if (pending.length >= maxPendingRequests) {
-            throw new Error(`Worker rotator exceeded max pending requests (${maxPendingRequests})`);
-        }
-        pending.push(request);
-        return pending;
-    }, []));
-}

package/dist/worker-rotator.test.d.ts

@@ -1 +0,0 @@
-export {};

package/dist/worker-rotator.test.js

@@ -1,129 +0,0 @@
-import { describe } from "@service-broker/test-utils";
-import assert from "assert";
-import * as rxjs from "rxjs";
-import { makeWorkerRotator } from "./worker-rotator.js";
-describe('worker-rotator', ({ test }) => {
-    test('buffers requests before the first worker is ready', async () => {
-        const request$ = new rxjs.Subject;
-        const worker = makeTestWorker();
-        const workerReady = defer();
-        const processed = [];
-        worker.processRequests$.subscribe(request => processed.push(request));
-        const subscription = makeWorkerRotator({
-            makeWorker: () => workerReady.promise,
-            workerTtlMs: 1000,
-            request$
-        }).subscribe();
-        try {
-            request$.next(1);
-            request$.next(2);
-            workerReady.resolve(worker);
-            await waitFor(() => processed.length == 2);
-            assert.deepStrictEqual(processed, [1, 2]);
-        }
-        finally {
-            subscription.unsubscribe();
-        }
-    });
-    test('buffers requests between workers', async () => {
-        const request$ = new rxjs.Subject;
-        const firstWorker = makeTestWorker();
-        const secondWorker = makeTestWorker();
-        const firstWorkerReady = defer();
-        const secondWorkerReady = defer();
-        const workersReady = [firstWorkerReady, secondWorkerReady];
-        const processed = [];
-        firstWorker.processRequests$.subscribe(request => processed.push(request));
-        secondWorker.processRequests$.subscribe(request => processed.push(request));
-        const subscription = makeWorkerRotator({
-            makeWorker: () => workersReady.shift().promise,
-            workerTtlMs: 1000,
-            request$
-        }).subscribe();
-        try {
-            firstWorkerReady.resolve(firstWorker);
-            await waitFor(() => firstWorker.processCalls == 1);
-            request$.next(1);
-            firstWorker.quit$.next('done');
-            request$.next(2);
-            secondWorkerReady.resolve(secondWorker);
-            await waitFor(() => processed.length == 2);
-            assert.deepStrictEqual(processed, [1, 2]);
-            assert(firstWorker.relieved);
-        }
-        finally {
-            subscription.unsubscribe();
-        }
-    });
-    test('errors when pending request cap is exceeded', async () => {
-        const request$ = new rxjs.Subject;
-        const workerReady = defer();
-        const error = defer();
-        makeWorkerRotator({
-            makeWorker: () => workerReady.promise,
-            workerTtlMs: 1000,
-            request$,
-            maxPendingRequests: 1
-        }).subscribe({
-            error: err => error.resolve(err)
-        });
-        request$.next(1);
-        request$.next(2);
-        const err = await error.promise;
-        assert(err instanceof Error);
-        assert.equal(err.message, 'Worker rotator exceeded max pending requests (1)');
-    });
-    test('emits lifecycle events', async () => {
-        const request$ = new rxjs.Subject;
-        const worker = makeTestWorker();
-        const events = [];
-        const subscription = makeWorkerRotator({
-            makeWorker: async () => worker,
-            workerTtlMs: 1000,
-            request$
-        }).subscribe(event => events.push(event));
-        try {
-            await waitFor(() => events.length >= 1);
-            worker.quit$.next('done');
-            await waitFor(() => events.length >= 3);
-            assert.deepStrictEqual(events.slice(0, 3).map(event => event.type), ['hired', 'quit', 'relieved']);
-            assert.equal(events[1].type == 'quit' && events[1].reason, 'done');
-        }
-        finally {
-            subscription.unsubscribe();
-        }
-    });
-});
-function makeTestWorker() {
-    const processRequests$ = new rxjs.Subject;
-    return {
-        processCalls: 0,
-        processRequests$,
-        quit$: new rxjs.Subject(),
-        relieved: false,
-        process(request$) {
-            this.processCalls++;
-            return request$.pipe(rxjs.tap(request => processRequests$.next(request)), rxjs.ignoreElements());
-        },
-        relieve() {
-            this.relieved = true;
-        }
-    };
-}
-function defer() {
-    let resolve;
-    let reject;
-    const promise = new Promise((resolvePromise, rejectPromise) => {
-        resolve = resolvePromise;
-        reject = rejectPromise;
-    });
-    return { promise, resolve, reject };
-}
-async function waitFor(condition) {
-    for (let i = 0; i < 100; i++) {
-        if (condition())
-            return;
-        await new Promise(resolve => setTimeout(resolve, 1));
-    }
-    assert(condition());
-}