@nitpicker/core 0.24.0 → 0.25.1

package/lib/worker/worker-pool.js

@@ -1,278 +0,0 @@
-import { Worker } from 'node:worker_threads';
-/**
- * Long-lived pool of Worker threads that share execution across many tasks.
- *
- * Replaces the previous "one worker per page" model where every analyzed page
- * paid the cost of spawning a new V8 isolate, importing JSDOM, importing the
- * plugin module, and tearing it all down. With the pool, each worker is
- * created once at the start of an analyze run and processes tasks via
- * message passing until the pool is terminated.
- *
- * ## Why this matters
- *
- * On a 750-page archive with `CONCURRENCY_LIMIT = 50`, the old model spawned
- * 50 workers nearly simultaneously and repeated this every time a worker
- * finished, producing recurring "boot waves" that drove peak memory above
- * 20GB. The pool model performs N initial boots and then reuses those V8
- * isolates for the rest of the run, eliminating the recurring cost entirely.
- *
- * ## Message protocol (Pool ↔ Worker)
- *
- * | Direction        | `type`        | Payload                          |
- * |------------------|---------------|----------------------------------|
- * | Pool → Worker    | `'task'`      | `{ taskId, data }`               |
- * | Pool → Worker    | `'shutdown'`  | (none)                           |
- * | Worker → Pool    | `'url'`       | `{ taskId, url }` (forwarded)    |
- * | Worker → Pool    | `'result'`    | `{ taskId, result, error? }`     |
- *
- * Each task carries a unique numeric `taskId` so that responses and URL
- * notifications from a worker can be routed back to the correct caller.
- */
-export class WorkerPool {
-    /** FIFO queue of slots ready to accept a task. */
-    #idle = [];
-    /** Tasks awaiting their result, keyed by taskId. */
-    #inFlight = new Map();
-    /** Monotonic task identifier source. */
-    #nextTaskId = 0;
-    /** FIFO queue of pending tasks waiting for an idle slot. */
-    #queue = [];
-    /** Once `terminate()` has been called, no new tasks may be submitted. */
-    #shuttingDown = false;
-    /** All worker slots, idle or busy. */
-    #slots = [];
-    /** Per-task data payload to send when a queued task is dispatched. */
-    #taskData = new Map();
-    /** Worker entry script path; reused when replacing crashed workers. */
-    #workerPath;
-    /**
-     * Constructs a pool with the given size and worker entry script.
-     * @param options - Pool size and worker entry script path.
-     */
-    constructor(options) {
-        const size = Math.max(1, Math.floor(options.size));
-        this.#workerPath = options.workerPath;
-        for (let i = 0; i < size; i++) {
-            this.#spawn();
-        }
-    }
-    /**
-     * Submits a task to the pool. Resolves with the worker's `result` payload
-     * once a worker has finished processing it. The task may be queued if all
-     * workers are currently busy.
-     * @template I - Shape of the per-task `initialData` payload.
-     * @template R - Expected result type returned by the worker module.
-     * @param params - Task payload (file path, batch indices, emitter, data).
-     * @returns The worker's result.
-     */
-    run(params) {
-        if (this.#shuttingDown) {
-            return Promise.reject(new Error('WorkerPool is shutting down'));
-        }
-        const taskId = this.#nextTaskId++;
-        const data = {
-            filePath: params.filePath,
-            num: params.num,
-            total: params.total,
-            ...params.initialData,
-        };
-        return new Promise((resolve, reject) => {
-            const task = {
-                taskId,
-                emitter: params.emitter,
-                resolve: resolve,
-                reject,
-            };
-            this.#taskData.set(taskId, data);
-            const slot = this.#idle.shift();
-            if (slot) {
-                this.#dispatch(slot, task);
-                return;
-            }
-            this.#queue.push(task);
-        });
-    }
-    /**
-     * Rejects every queued or in-flight task, sends each worker a `shutdown`
-     * message, and awaits its exit. After this call returns, no further
-     * submissions are accepted.
-     *
-     * Pending tasks are rejected synchronously so callers waiting on
-     * `pool.run()` Promises do not hang. Workers handle the shutdown message
-     * by exiting cleanly; if that fails, {@link #shutdownSlot} forces
-     * termination after a 5 second timeout.
-     */
-    async terminate() {
-        this.#shuttingDown = true;
-        const pendingError = new Error('WorkerPool terminated before task completed');
-        for (const task of this.#queue) {
-            task.reject(pendingError);
-            this.#taskData.delete(task.taskId);
-        }
-        this.#queue = [];
-        for (const task of this.#inFlight.values()) {
-            task.reject(pendingError);
-        }
-        this.#inFlight.clear();
-        await Promise.all(this.#slots.map((slot) => this.#shutdownSlot(slot)));
-        this.#slots = [];
-        this.#idle = [];
-    }
-    /**
-     * Completes the current task on `slot` and dispatches the next queued task.
-     * @param slot
-     * @param taskId
-     * @param result
-     * @param error
-     */
-    #completeTask(slot, taskId, result, error) {
-        const task = this.#inFlight.get(taskId);
-        this.#inFlight.delete(taskId);
-        slot.current = null;
-        if (task) {
-            if (error) {
-                task.reject(new Error(error));
-            }
-            else {
-                task.resolve(result);
-            }
-        }
-        const next = this.#queue.shift();
-        if (next) {
-            this.#dispatch(slot, next);
-        }
-        else {
-            this.#idle.push(slot);
-        }
-    }
-    /**
-     * Sends a queued task to an idle slot and registers it as in-flight.
-     *
-     * If `postMessage` throws (e.g. the worker died between handoff and
-     * dispatch), the task is rejected immediately and the slot is left out
-     * of the idle pool so the worker's `error` event can replace it via
-     * {@link #onWorkerCrash}.
-     * @param slot - Pool slot that owns the worker.
-     * @param task - Pending task to dispatch.
-     */
-    #dispatch(slot, task) {
-        const data = this.#taskData.get(task.taskId);
-        try {
-            slot.worker.postMessage({ type: 'task', taskId: task.taskId, data });
-        }
-        catch (error) {
-            this.#taskData.delete(task.taskId);
-            task.reject(error instanceof Error ? error : new Error(String(error)));
-            return;
-        }
-        slot.current = task;
-        this.#inFlight.set(task.taskId, task);
-        this.#taskData.delete(task.taskId);
-    }
-    /**
-     * Routes an incoming worker message to the matching pending task.
-     * @param slot
-     * @param message
-     */
-    #onMessage(slot, message) {
-        if (!message || typeof message !== 'object') {
-            return;
-        }
-        const msg = message;
-        if (msg.type === 'url' &&
-            typeof msg.taskId === 'number' &&
-            typeof msg.url === 'string') {
-            const task = this.#inFlight.get(msg.taskId);
-            if (task) {
-                void task.emitter.emit('url', msg.url);
-            }
-            return;
-        }
-        if (msg.type === 'result' && typeof msg.taskId === 'number') {
-            this.#completeTask(slot, msg.taskId, msg.result, msg.error);
-            return;
-        }
-    }
-    /**
-     * Handles an unrecoverable worker failure: rejects the current task,
-     * removes the dead slot, and spawns a replacement so the pool keeps its
-     * configured size. Queued tasks are picked up by the new slot when ready.
-     * @param slot
-     * @param error
-     */
-    #onWorkerCrash(slot, error) {
-        const task = slot.current;
-        slot.current = null;
-        if (task) {
-            this.#inFlight.delete(task.taskId);
-            task.reject(error instanceof Error ? error : new Error(String(error)));
-        }
-        const slotIndex = this.#slots.indexOf(slot);
-        if (slotIndex !== -1) {
-            this.#slots.splice(slotIndex, 1);
-        }
-        const idleIndex = this.#idle.indexOf(slot);
-        if (idleIndex !== -1) {
-            this.#idle.splice(idleIndex, 1);
-        }
-        slot.worker.removeAllListeners();
-        void slot.worker.terminate();
-        if (this.#shuttingDown) {
-            return;
-        }
-        const replacement = this.#spawn();
-        const next = this.#queue.shift();
-        if (next) {
-            const idleIdx = this.#idle.indexOf(replacement);
-            if (idleIdx !== -1) {
-                this.#idle.splice(idleIdx, 1);
-            }
-            this.#dispatch(replacement, next);
-        }
-    }
-    /**
-     * Sends a shutdown message and waits for the worker to exit cleanly.
-     * @param slot
-     */
-    #shutdownSlot(slot) {
-        return new Promise((resolve) => {
-            const finish = () => {
-                slot.worker.removeAllListeners();
-                resolve();
-            };
-            slot.worker.once('exit', finish);
-            slot.worker.once('error', finish);
-            try {
-                slot.worker.postMessage({ type: 'shutdown' });
-            }
-            catch {
-                void slot.worker.terminate().finally(finish);
-                return;
-            }
-            // Safety net in case the worker fails to acknowledge the shutdown.
-            setTimeout(() => {
-                void slot.worker.terminate().finally(finish);
-            }, 5000).unref();
-        });
-    }
-    /** Spawns a fresh worker, wires up its event handlers, and adds it to the pool. */
-    #spawn() {
-        const worker = new Worker(this.#workerPath);
-        const slot = { worker, current: null };
-        this.#slots.push(slot);
-        worker.on('message', (message) => {
-            this.#onMessage(slot, message);
-        });
-        worker.on('error', (error) => {
-            this.#onWorkerCrash(slot, error);
-        });
-        worker.on('messageerror', (error) => {
-            this.#onWorkerCrash(slot, error);
-        });
-        this.#idle.push(slot);
-        // Workers should not keep the event loop alive on their own;
-        // the pool's terminate() controls their lifetime.
-        worker.unref();
-        return slot;
-    }
-}

package/lib/worker/worker.d.ts

@@ -1,34 +0,0 @@
-/**
- * Worker thread entry point for plugin execution.
- *
- * This module runs inside a `new Worker(...)` created by the
- * {@link ./worker-pool.ts!WorkerPool}. Unlike the previous one-shot model,
- * each worker is long-lived and processes many tasks via a message loop until
- * it receives a `shutdown` message.
- *
- * ## Message protocol (Worker ↔ Pool)
- *
- * | Direction       | `type`        | Payload                          |
- * |-----------------|---------------|----------------------------------|
- * | Pool → Worker   | `'task'`      | `{ taskId, data }`               |
- * | Pool → Worker   | `'shutdown'`  | (none)                           |
- * | Worker → Pool   | `'url'`       | `{ taskId, url }`                |
- * | Worker → Pool   | `'result'`    | `{ taskId, result, error? }`     |
- *
- * ## Lifecycle
- *
- * 1. Listens on `parentPort` for `task` messages.
- * 2. For each task, delegates to {@link ./runner.ts!runner} which dynamically
- *    `import()`s the module pointed to by `data.filePath`. Subsequent tasks
- *    that target the same module reuse the import cache, so the heavy boot
- *    cost is paid once per module per worker.
- * 3. Posts a `result` message back to the pool and waits for the next task.
- * 4. On `shutdown`, exits the process so the V8 isolate is fully released.
- *
- * The `'url'` events emitted by plugins are forwarded with the active
- * `taskId` so the pool can route them back to the correct caller's emitter.
- * @see {@link ./worker-pool.ts!WorkerPool} for the main-thread counterpart
- * @see {@link ./runner.ts!runner} for the actual module loading logic
- * @module
- */
-export {};

package/lib/worker/worker.js

@@ -1,108 +0,0 @@
-/**
- * Worker thread entry point for plugin execution.
- *
- * This module runs inside a `new Worker(...)` created by the
- * {@link ./worker-pool.ts!WorkerPool}. Unlike the previous one-shot model,
- * each worker is long-lived and processes many tasks via a message loop until
- * it receives a `shutdown` message.
- *
- * ## Message protocol (Worker ↔ Pool)
- *
- * | Direction       | `type`        | Payload                          |
- * |-----------------|---------------|----------------------------------|
- * | Pool → Worker   | `'task'`      | `{ taskId, data }`               |
- * | Pool → Worker   | `'shutdown'`  | (none)                           |
- * | Worker → Pool   | `'url'`       | `{ taskId, url }`                |
- * | Worker → Pool   | `'result'`    | `{ taskId, result, error? }`     |
- *
- * ## Lifecycle
- *
- * 1. Listens on `parentPort` for `task` messages.
- * 2. For each task, delegates to {@link ./runner.ts!runner} which dynamically
- *    `import()`s the module pointed to by `data.filePath`. Subsequent tasks
- *    that target the same module reuse the import cache, so the heavy boot
- *    cost is paid once per module per worker.
- * 3. Posts a `result` message back to the pool and waits for the next task.
- * 4. On `shutdown`, exits the process so the V8 isolate is fully released.
- *
- * The `'url'` events emitted by plugins are forwarded with the active
- * `taskId` so the pool can route them back to the correct caller's emitter.
- * @see {@link ./worker-pool.ts!WorkerPool} for the main-thread counterpart
- * @see {@link ./runner.ts!runner} for the actual module loading logic
- * @module
- */
-import { parentPort } from 'node:worker_threads';
-import { UrlEventBus } from '../url-event-bus.js';
-import { runner } from './runner.js';
-/** Identifier for the task currently being processed; used to tag forwarded `'url'` events. */
-let currentTaskId = null;
-/** Long-lived event bus that forwards `'url'` events from plugins to the pool. */
-const emitter = new UrlEventBus();
-emitter.on('url', (url) => {
-    if (!parentPort) {
-        throw new Error('Use in worker thread');
-    }
-    if (currentTaskId === null) {
-        return;
-    }
-    parentPort.postMessage({
-        type: 'url',
-        taskId: currentTaskId,
-        url,
-    });
-});
-if (!parentPort) {
-    throw new Error('Use in worker thread');
-}
-parentPort.on('message', (message) => {
-    if (!message || typeof message !== 'object') {
-        return;
-    }
-    const msg = message;
-    if (msg.type === 'shutdown') {
-        // Defer to the next tick so any in-flight `postMessage` from the most
-        // recent task has a chance to flush before the worker exits.
-        setImmediate(() => process.exit(0));
-        return;
-    }
-    if (msg.type === 'task' && typeof msg.taskId === 'number' && msg.data) {
-        void runTask(msg.taskId, msg.data);
-    }
-});
-/**
- * Executes a single task and posts the result back to the pool.
- *
- * Errors are caught and reported via the `error` field on the `result` message
- * so the pool can reject the originating Promise without the worker crashing.
- * @param taskId - Identifier echoed back so the pool can match the response.
- * @param data - Payload from the pool, including `filePath` consumed by `runner()`.
- */
-async function runTask(taskId, data) {
-    currentTaskId = taskId;
-    try {
-        const result = await runner(data, emitter);
-        if (!parentPort) {
-            throw new Error('Use in worker thread');
-        }
-        parentPort.postMessage({
-            type: 'result',
-            taskId,
-            result,
-        });
-    }
-    catch (error) {
-        if (!parentPort) {
-            throw new Error('Use in worker thread');
-        }
-        const message = error instanceof Error ? error.message : String(error);
-        parentPort.postMessage({
-            type: 'result',
-            taskId,
-            result: null,
-            error: message,
-        });
-    }
-    finally {
-        currentTaskId = null;
-    }
-}