@ls-stack/utils 3.65.0 → 3.66.0

package/dist/asyncQueue.mjs

@@ -0,0 +1,757 @@
+import { defer } from "./promiseUtils.mjs";
+import { p as durationObjToMs } from "./time-sr2lhQRw.mjs";
+import { evtmitter } from "evtmitter";
+import { Result, isResult, resultify, unknownToError } from "t-result";
+
+//#region src/asyncQueue.ts
+/**
+* @file AsyncQueue - A powerful, type-safe async task queue with advanced error
+*   handling
+*
+*   Features:
+*
+*   - Concurrency control with configurable limits
+*   - Error handling with stop-on-error and reject-pending options
+*   - Lazy start capability for batch preparation
+*   - Pause/resume functionality for flow control
+*   - Abort signal support for cancellation
+*   - Timeout support per task or globally
+*   - Event emission for progress tracking
+*   - Metadata support for task context
+*   - Reset functionality for error recovery
+*
+* @example
+*   Basic Usage
+*   ```typescript
+*   const queue = createAsyncQueue<string>({ concurrency: 3 });
+*
+*   queue.resultifyAdd(async () => {
+*   const response = await fetch('/api/data');
+*   return response.json();
+*   }).then(result => {
+*   if (result.ok) console.log('Success:', result.value);
+*   else console.error('Error:', result.error);
+*   });
+*
+*   await queue.onIdle(); // Wait for all tasks to complete
+*   ```
+*
+* @example
+*   Error Handling
+*   ```typescript
+*   const queue = createAsyncQueue<string>({
+*   stopOnError: true,
+*   rejectPendingOnError: true
+*   });
+*
+*   // Process batch with automatic error recovery
+*   const items = ['item1', 'item2', 'bad-item', 'item3'];
+*   for (const item of items) {
+*   queue.resultifyAdd(async () => processItem(item));
+*   }
+*
+*   await queue.onIdle();
+*
+*   if (queue.isStopped) {
+*   console.log(`Queue stopped after processing ${queue.completed} items`);
+*   queue.reset(); // Resume processing remaining items
+*   }
+*   ```
+*/
+/**
+* A powerful async task queue with advanced error handling and flow control
+*
+* @example
+*   Basic Usage
+*   ```typescript
+*   const queue = createAsyncQueue<string>({ concurrency: 2 });
+*
+*   const processedItems: string[] = [];
+*
+*   queue.resultifyAdd(async () => {
+*   await delay(100);
+*   return 'task completed';
+*   }).then(result => {
+*   if (result.ok) processedItems.push(result.value);
+*   });
+*
+*   await queue.onIdle();
+*   console.log('Processed:', processedItems);
+*   ```
+*
+* @example
+*   Error Recovery
+*   ```typescript
+*   const queue = createAsyncQueue<string>({
+*   stopOnError: true,
+*   rejectPendingOnError: false
+*   });
+*
+*   // Add batch of tasks
+*   const items = ['item1', 'item2', 'bad-item', 'item3'];
+*   items.forEach(item => {
+*   queue.resultifyAdd(async () => {
+*   if (item === 'bad-item') throw new Error('Processing failed');
+*   return item.toUpperCase();
+*   });
+*   });
+*
+*   await queue.onIdle();
+*
+*   if (queue.isStopped) {
+*   console.log(`Stopped at ${queue.failed} failures, ${queue.size} remaining`);
+*   // Reset and continue with remaining tasks
+*   queue.reset();
+*   await queue.onIdle();
+*   }
+*   ```
+*
+* @example
+*   Lazy Start
+*   ```typescript
+*   const queue = createAsyncQueue<string>({ autoStart: false });
+*
+*   // Prepare all tasks without starting
+*   queue.resultifyAdd(() => processTask1());
+*   queue.resultifyAdd(() => processTask2());
+*   queue.resultifyAdd(() => processTask3());
+*
+*   // Start processing when ready
+*   queue.start();
+*   await queue.onIdle();
+*   ```
+*
+* @template T - The type of value returned by successful tasks
+* @template E - The type of errors that tasks can produce (defaults to Error)
+* @template I - The type of metadata associated with tasks (defaults to
+*   unknown)
+*/
+var AsyncQueue = class AsyncQueue {
+	#queue = [];
+	#pending = 0;
+	#size = 0;
+	#concurrency;
+	#completed = 0;
+	#failed = 0;
+	#idleResolvers = [];
+	#sizeLessThanWaiters = [];
+	/**
+	* Event emitter for tracking task lifecycle
+	*
+	* @example
+	*   Listening to Events
+	*   ```typescript
+	*   const queue = createAsyncQueue<string>();
+	*
+	*   queue.events.on('start', (event) => {
+	*   console.log('Task started:', event.payload.meta);
+	*   });
+	*
+	*   queue.events.on('complete', (event) => {
+	*   console.log('Task completed:', event.payload.value);
+	*   });
+	*
+	*   queue.events.on('error', (event) => {
+	*   console.error('Task failed:', event.payload.error);
+	*   });
+	*   ```
+	*/
+	events = evtmitter();
+	#signal;
+	#taskTimeout;
+	#stopped = false;
+	#paused = false;
+	#started = false;
+	#stopOnError = false;
+	#rejectPendingOnError = false;
+	#autoStart = true;
+	#stoppedReason;
+	#rateLimit;
+	#taskExecutionTimes = [];
+	#rateLimitTimeouts = /* @__PURE__ */ new Set();
+	/** Array of all task failures with metadata for debugging and analysis */
+	failures = [];
+	/** Array of all task completions with metadata for debugging and analysis */
+	completions = [];
+	constructor({ concurrency = 1, signal, timeout: taskTimeout, stopOnError = false, rejectPendingOnError = false, autoStart = true, rateLimit } = {}) {
+		this.#concurrency = concurrency;
+		this.#signal = signal;
+		this.#taskTimeout = taskTimeout;
+		this.#stopOnError = stopOnError;
+		this.#rejectPendingOnError = rejectPendingOnError;
+		this.#autoStart = autoStart;
+		this.#started = autoStart;
+		this.#rateLimit = rateLimit;
+		this.events.on("error", (e) => {
+			this.failures.push(e.payload);
+		});
+		this.events.on("complete", (e) => {
+			this.completions.push(e.payload);
+		});
+	}
+	#getRateLimitIntervalMs() {
+		if (!this.#rateLimit) return 0;
+		return typeof this.#rateLimit.interval === "number" ? this.#rateLimit.interval : durationObjToMs(this.#rateLimit.interval);
+	}
+	#cleanupExpiredExecutionTimes(now) {
+		if (!this.#rateLimit) return;
+		const cutoff = now - this.#getRateLimitIntervalMs();
+		this.#taskExecutionTimes = this.#taskExecutionTimes.filter((time) => time > cutoff);
+	}
+	#isRateLimited() {
+		if (!this.#rateLimit) return false;
+		const now = Date.now();
+		this.#cleanupExpiredExecutionTimes(now);
+		return this.#taskExecutionTimes.length >= this.#rateLimit.maxTasks;
+	}
+	#getRateLimitDelay() {
+		if (!this.#rateLimit || this.#taskExecutionTimes.length === 0) return 0;
+		const oldestExecution = this.#taskExecutionTimes[0];
+		if (oldestExecution === void 0) return 0;
+		const timeUntilSlotOpens = oldestExecution + this.#getRateLimitIntervalMs() - Date.now();
+		return Math.max(0, timeUntilSlotOpens);
+	}
+	#recordTaskExecution() {
+		if (!this.#rateLimit) return;
+		const now = Date.now();
+		this.#taskExecutionTimes.push(now);
+		this.#cleanupExpiredExecutionTimes(now);
+	}
+	#enqueue(task) {
+		this.#queue.push(task);
+		this.#size++;
+	}
+	static #createTimeoutSignal(ms) {
+		const controller = new AbortController();
+		const id = setTimeout(() => {
+			controller.abort(new DOMException("The operation was aborted due to timeout", "TimeoutError"));
+		}, ms);
+		controller.signal.addEventListener("abort", () => {
+			clearTimeout(id);
+		}, { once: true });
+		return controller.signal;
+	}
+	#resolveSizeLessThanWaiters() {
+		if (this.#sizeLessThanWaiters.length === 0) return;
+		const remaining = [];
+		for (const waiter of this.#sizeLessThanWaiters) if (this.#size < waiter.limit) waiter.resolve();
+		else remaining.push(waiter);
+		this.#sizeLessThanWaiters = remaining;
+	}
+	/**
+	* Add a task that returns a Result to the queue
+	*
+	* Use this method when your task function already returns a Result type. For
+	* functions that throw errors or return plain values, use `resultifyAdd`
+	* instead.
+	*
+	* @example
+	*   ```typescript
+	*   const queue = createAsyncQueue<string>();
+	*
+	*   const result = await queue.add(async () => {
+	*     try {
+	*       const data = await fetchData();
+	*       return Result.ok(data);
+	*     } catch (error) {
+	*       return Result.err(error);
+	*     }
+	*   });
+	*
+	*   if (result.ok) {
+	*     console.log('Success:', result.value);
+	*   } else {
+	*     console.log('Error:', result.error);
+	*   }
+	*   ```;
+	*
+	* @param fn - Task function that returns a Result
+	* @param options - Optional configuration for this task
+	* @returns Promise that resolves with the task result
+	*/
+	async add(fn, options) {
+		if (this.#signal?.aborted) return Result.err(this.#signal.reason instanceof Error ? this.#signal.reason : new DOMException("This operation was aborted", "AbortError"));
+		if (this.#stopped) return Result.err(this.#stoppedReason ?? /* @__PURE__ */ new Error("Queue has been stopped"));
+		const deferred = defer();
+		const taskTimeout = this.#taskTimeout ?? options?.timeout;
+		const task = {
+			run: async (ctx) => {
+				return fn(ctx);
+			},
+			resolve: deferred.resolve,
+			reject: deferred.reject,
+			signal: options?.signal,
+			meta: options?.meta,
+			timeout: taskTimeout
+		};
+		this.#enqueue(task);
+		if (this.#autoStart && this.#started) this.#processQueue();
+		const r = await deferred.promise;
+		if (options?.onComplete) r.onOk(options.onComplete);
+		if (options?.onError) r.onErr(options.onError);
+		return r;
+	}
+	/**
+	* Add a task that returns a plain value or throws errors to the queue
+	*
+	* This is the most commonly used method. It automatically wraps your function
+	* to handle errors and convert them to Result types.
+	*
+	* @example
+	*   Basic Usage
+	*   ```typescript
+	*   const queue = createAsyncQueue<string>();
+	*
+	*   queue.resultifyAdd(async () => {
+	*   const response = await fetch('/api/data');
+	*   return response.json();
+	*   }).then(result => {
+	*   if (result.ok) {
+	*   console.log('Data:', result.value);
+	*   } else {
+	*   console.error('Failed:', result.error);
+	*   }
+	*   });
+	*   ```
+	*
+	* @example
+	*   With Callbacks
+	*   ```typescript
+	*   queue.resultifyAdd(
+	*   async () => processData(),
+	*   {
+	*   onComplete: (data) => console.log('Processed:', data),
+	*   onError: (error) => console.error('Failed:', error),
+	*   timeout: 5000
+	*   }
+	*   );
+	*   ```
+	*
+	* @param fn - Task function that returns a value or throws
+	* @param options - Optional configuration for this task
+	* @returns Promise that resolves with the task result wrapped in Result
+	*/
+	resultifyAdd(fn, options) {
+		return this.add((ctx) => resultify(async () => {
+			return fn(ctx);
+		}), options);
+	}
+	async #processQueue() {
+		if (this.#signal?.aborted) {
+			this.clear();
+			return;
+		}
+		if (this.#stopped || this.#paused || !this.#started) return;
+		if (this.#pending >= this.#concurrency || this.#queue.length === 0) return;
+		if (this.#isRateLimited()) {
+			const delay = this.#getRateLimitDelay();
+			if (delay > 0) {
+				const timeoutId = setTimeout(() => {
+					this.#rateLimitTimeouts.delete(timeoutId);
+					this.#processQueue();
+				}, delay);
+				this.#rateLimitTimeouts.add(timeoutId);
+				return;
+			}
+		}
+		const task = this.#queue.shift();
+		if (!task) return;
+		this.#pending++;
+		this.#size--;
+		this.#resolveSizeLessThanWaiters();
+		this.#recordTaskExecution();
+		const signals = [];
+		if (task.signal) signals.push(task.signal);
+		if (this.#signal) signals.push(this.#signal);
+		if (task.timeout !== void 0) signals.push(AsyncQueue.#createTimeoutSignal(task.timeout));
+		const signal = signals.length > 1 ? AbortSignal.any(signals) : signals[0];
+		let abortListener;
+		try {
+			if (signal?.aborted) throw signal.reason instanceof Error ? signal.reason : new DOMException("This operation was aborted", "AbortError");
+			const signalAbortPromise = new Promise((_, reject) => {
+				if (signal) {
+					abortListener = () => {
+						const reason = signal.reason;
+						const err$1 = reason instanceof Error ? reason : new DOMException("This operation was aborted", "AbortError");
+						setTimeout(() => {
+							reject(err$1);
+						}, 0);
+					};
+					signal.addEventListener("abort", abortListener, { once: true });
+				}
+			});
+			const taskRunPromise = task.run({
+				signal,
+				meta: task.meta
+			});
+			this.events.emit("start", { meta: task.meta });
+			const result = await Promise.race([taskRunPromise, signalAbortPromise]);
+			if (isResult(result)) {
+				task.resolve(result);
+				if (!result.ok) {
+					this.#failed++;
+					this.events.emit("error", {
+						meta: task.meta,
+						error: result.error
+					});
+					this.#stopOnErrorAndRejectPending(unknownToError(result.error));
+				} else {
+					this.#completed++;
+					this.events.emit("complete", {
+						meta: task.meta,
+						value: result.value
+					});
+				}
+			} else {
+				const error = /* @__PURE__ */ new Error("Response not a Result");
+				task.resolve(Result.err(error));
+				this.#failed++;
+				this.events.emit("error", {
+					meta: task.meta,
+					error
+				});
+				this.#stopOnErrorAndRejectPending(error);
+			}
+		} catch (error) {
+			const processedError = unknownToError(error);
+			task.resolve(Result.err(processedError));
+			this.#failed++;
+			this.events.emit("error", {
+				meta: task.meta,
+				error: processedError
+			});
+			this.#stopOnErrorAndRejectPending(processedError);
+		} finally {
+			if (signal && abortListener) signal.removeEventListener("abort", abortListener);
+			this.#pending--;
+			this.#processQueue();
+			if (this.#pending === 0 && this.#size === 0 && this.#rateLimitTimeouts.size === 0) this.#resolveIdleWaiters();
+		}
+	}
+	#resolveIdleWaiters() {
+		while (this.#idleResolvers.length > 0) {
+			const resolve = this.#idleResolvers.shift();
+			if (resolve) resolve();
+		}
+	}
+	#stopOnErrorAndRejectPending(error) {
+		if (!this.#stopOnError) return;
+		this.#stopped = true;
+		this.#stoppedReason = error;
+		if (this.#rejectPendingOnError) {
+			while (this.#queue.length > 0) {
+				const task = this.#queue.shift();
+				if (task) task.resolve(Result.err(error));
+			}
+			this.#size = 0;
+			this.#resolveSizeLessThanWaiters();
+		}
+		this.#resolveIdleWaiters();
+	}
+	/**
+	* Wait for the queue to become idle (no pending tasks, no queued tasks, and
+	* no rate-limit timers)
+	*
+	* This method resolves when:
+	*
+	* - All tasks have completed (success or failure)
+	* - The queue is stopped due to error (stopOnError), even with remaining tasks
+	* - There are no queued tasks, no running tasks, and no pending rate-limit
+	*   timers
+	*
+	* @example
+	*   ```typescript
+	*   const queue = createAsyncQueue<string>();
+	*
+	*   // Add multiple tasks
+	*   for (let i = 0; i < 10; i++) {
+	*   queue.resultifyAdd(async () => `task ${i}`);
+	*   }
+	*
+	*   // Wait for all tasks to complete
+	*   await queue.onIdle();
+	*
+	*   console.log(`Completed: ${queue.completed}, Failed: ${queue.failed}`);
+	*   ```
+	*
+	* @returns Promise that resolves when the queue is idle
+	*/
+	async onIdle() {
+		if (this.#stopped || this.#pending === 0 && this.#size === 0 && this.#rateLimitTimeouts.size === 0) return Promise.resolve();
+		return new Promise((resolve) => {
+			this.#idleResolvers.push(resolve);
+		});
+	}
+	/**
+	* Wait until the queued task count is below a limit
+	*
+	* Resolves immediately if `size < limit` at the moment of calling. This only
+	* considers queued (not yet started) tasks; running tasks are tracked by
+	* `pending`.
+	*
+	* @param limit Threshold that `size` must be below to resolve
+	*/
+	onSizeLessThan(limit) {
+		if (this.#size < limit) return Promise.resolve();
+		return new Promise((resolve) => {
+			this.#sizeLessThanWaiters.push({
+				limit,
+				resolve
+			});
+		});
+	}
+	/**
+	* Clear all queued tasks (does not affect currently running tasks)
+	*
+	* This removes all tasks waiting in the queue but allows currently executing
+	* tasks to complete normally.
+	*
+	* @example
+	*   ```typescript
+	*   const queue = createAsyncQueue({ concurrency: 1 });
+	*
+	*   // Add multiple tasks
+	*   queue.resultifyAdd(async () => longRunningTask()); // Will start immediately
+	*   queue.resultifyAdd(async () => task2()); // Queued
+	*   queue.resultifyAdd(async () => task3()); // Queued
+	*
+	*   // Clear remaining queued tasks
+	*   queue.clear();
+	*
+	*   // Only the first task will complete
+	*   await queue.onIdle();
+	*   ```;
+	*/
+	clear() {
+		this.#queue = [];
+		this.#size = 0;
+		for (const timeoutId of this.#rateLimitTimeouts) clearTimeout(timeoutId);
+		this.#rateLimitTimeouts.clear();
+		if (this.#pending === 0) this.#resolveIdleWaiters();
+		this.#resolveSizeLessThanWaiters();
+	}
+	/** Number of tasks that have completed successfully */
+	get completed() {
+		return this.#completed;
+	}
+	/** Number of tasks that have failed */
+	get failed() {
+		return this.#failed;
+	}
+	/** Number of tasks currently being processed */
+	get pending() {
+		return this.#pending;
+	}
+	/** Number of tasks waiting in the queue to be processed */
+	get size() {
+		return this.#size;
+	}
+	/**
+	* Manually start processing tasks (only needed if autoStart: false)
+	*
+	* @example
+	*   ```typescript
+	*   const queue = createAsyncQueue({ autoStart: false });
+	*
+	*   // Add tasks without starting processing
+	*   queue.resultifyAdd(async () => 'task1');
+	*   queue.resultifyAdd(async () => 'task2');
+	*
+	*   // Start processing when ready
+	*   queue.start();
+	*   await queue.onIdle();
+	*   ```;
+	*/
+	start() {
+		if (this.#stopped) return;
+		this.#started = true;
+		this.#processQueue();
+	}
+	/**
+	* Pause processing new tasks (currently running tasks continue)
+	*
+	* @example
+	*   ```typescript
+	*   const queue = createAsyncQueue();
+	*
+	*   // Start some tasks
+	*   queue.resultifyAdd(async () => longRunningTask1());
+	*   queue.resultifyAdd(async () => longRunningTask2());
+	*
+	*   // Pause before more tasks are picked up
+	*   queue.pause();
+	*
+	*   // Later, resume processing
+	*   queue.resume();
+	*   ```;
+	*/
+	pause() {
+		this.#paused = true;
+	}
+	/** Resume processing tasks after pause */
+	resume() {
+		this.#paused = false;
+		if (this.#started && !this.#stopped) this.#processQueue();
+	}
+	/**
+	* Reset the queue after being stopped, allowing new tasks to be processed
+	*
+	* This clears the stopped state and error reason, and resumes processing any
+	* remaining queued tasks if autoStart was enabled.
+	*
+	* @example
+	*   ```typescript
+	*   const queue = createAsyncQueue({ stopOnError: true });
+	*
+	*   // Add tasks that will cause the queue to stop
+	*   queue.resultifyAdd(async () => { throw new Error('fail'); });
+	*   queue.resultifyAdd(async () => 'remaining task');
+	*
+	*   await queue.onIdle();
+	*
+	*   if (queue.isStopped) {
+	*   console.log(`Queue stopped, ${queue.size} tasks remaining`);
+	*
+	*   // Reset and process remaining tasks
+	*   queue.reset();
+	*   await queue.onIdle();
+	*   }
+	*   ```
+	*/
+	reset() {
+		this.#stopped = false;
+		this.#stoppedReason = void 0;
+		if (this.#autoStart) {
+			this.#started = true;
+			this.#processQueue();
+		}
+	}
+	/** Whether the queue is stopped due to an error */
+	get isStopped() {
+		return this.#stopped;
+	}
+	/** Whether the queue is currently paused */
+	get isPaused() {
+		return this.#paused;
+	}
+	/** Whether the queue has been started (relevant for autoStart: false) */
+	get isStarted() {
+		return this.#started;
+	}
+	/** The error that caused the queue to stop (if any) */
+	get stoppedReason() {
+		return this.#stoppedReason;
+	}
+};
+/**
+* AsyncQueue variant that requires metadata for all tasks
+*
+* This class enforces that every task must include metadata, which is useful
+* when you need to track or identify tasks consistently.
+*
+* @example
+*   ```typescript
+*   interface TaskMeta {
+*   id: string;
+*   priority: number;
+*   }
+*
+*   const queue = createAsyncQueueWithMeta<string, TaskMeta>({ concurrency: 2 });
+*
+*   queue.resultifyAdd(
+*   async () => processImportantTask(),
+*   { meta: { id: 'task-1', priority: 1 } }
+*   );
+*
+*   // Listen to events with metadata
+*   queue.events.on('complete', (event) => {
+*   console.log(`Task ${event.payload.meta.id} completed`);
+*   });
+*   ```
+*
+* @template T - The type of value returned by successful tasks
+* @template I - The type of metadata (required for all tasks)
+* @template E - The type of errors that tasks can produce
+*/
+var AsyncQueueWithMeta = class extends AsyncQueue {
+	constructor(options) {
+		super(options);
+	}
+	add(fn, options) {
+		return super.add(fn, options);
+	}
+	resultifyAdd(fn, options) {
+		return super.resultifyAdd(fn, options);
+	}
+};
+/**
+* Create a new AsyncQueue instance
+*
+* @example
+*   Basic Queue
+*   ```typescript
+*   const queue = createAsyncQueue<string>({ concurrency: 3 });
+*   ```
+*
+* @example
+*   Error Handling Queue
+*   ```typescript
+*   const queue = createAsyncQueue<string>({
+*   concurrency: 2,
+*   stopOnError: true,
+*   rejectPendingOnError: true
+*   });
+*   ```
+*
+* @example
+*   Lazy Start Queue
+*   ```typescript
+*   const queue = createAsyncQueue<string>({
+*   autoStart: false,
+*   concurrency: 1
+*   });
+*   ```
+*
+* @template T - The type of value returned by successful tasks
+* @template E - The type of errors that tasks can produce (defaults to Error)
+* @param options - Configuration options for the queue
+* @returns A new AsyncQueue instance
+*/
+function createAsyncQueue(options) {
+	return new AsyncQueue(options);
+}
+/**
+* Create a new AsyncQueueWithMeta instance that requires metadata for all tasks
+*
+* @example
+*   ```typescript
+*   interface TaskInfo {
+*   taskId: string;
+*   userId: string;
+*   }
+*
+*   const queue = createAsyncQueueWithMeta<ProcessResult, TaskInfo>({
+*   concurrency: 5
+*   });
+*
+*   queue.resultifyAdd(
+*   async (ctx) => {
+*   console.log(`Processing task ${ctx.meta.taskId} for user ${ctx.meta.userId}`);
+*   return await processUserTask(ctx.meta.userId);
+*   },
+*   { meta: { taskId: '123', userId: 'user456' } }
+*   );
+*   ```
+*
+* @template T - The type of value returned by successful tasks
+* @template I - The type of metadata (required for all tasks)
+* @template E - The type of errors that tasks can produce (defaults to Error)
+* @param options - Configuration options for the queue
+* @returns A new AsyncQueueWithMeta instance
+*/
+function createAsyncQueueWithMeta(options) {
+	return new AsyncQueueWithMeta(options);
+}
+
+//#endregion
+export { createAsyncQueue, createAsyncQueueWithMeta };