@nitpicker/core 0.6.4 → 0.7.0

package/lib/worker/worker-pool.js

@@ -0,0 +1,278 @@
+import { Worker } from 'node:worker_threads';
+/**
+ * Long-lived pool of Worker threads that share execution across many tasks.
+ *
+ * Replaces the previous "one worker per page" model where every analyzed page
+ * paid the cost of spawning a new V8 isolate, importing JSDOM, importing the
+ * plugin module, and tearing it all down. With the pool, each worker is
+ * created once at the start of an analyze run and processes tasks via
+ * message passing until the pool is terminated.
+ *
+ * ## Why this matters
+ *
+ * On a 750-page archive with `CONCURRENCY_LIMIT = 50`, the old model spawned
+ * 50 workers nearly simultaneously and repeated this every time a worker
+ * finished, producing recurring "boot waves" that drove peak memory above
+ * 20GB. The pool model performs N initial boots and then reuses those V8
+ * isolates for the rest of the run, eliminating the recurring cost entirely.
+ *
+ * ## Message protocol (Pool ↔ Worker)
+ *
+ * | Direction        | `type`        | Payload                          |
+ * |------------------|---------------|----------------------------------|
+ * | Pool → Worker    | `'task'`      | `{ taskId, data }`               |
+ * | Pool → Worker    | `'shutdown'`  | (none)                           |
+ * | Worker → Pool    | `'url'`       | `{ taskId, url }` (forwarded)    |
+ * | Worker → Pool    | `'result'`    | `{ taskId, result, error? }`     |
+ *
+ * Each task carries a unique numeric `taskId` so that responses and URL
+ * notifications from a worker can be routed back to the correct caller.
+ */
+export class WorkerPool {
+    /** FIFO queue of slots ready to accept a task. */
+    #idle = [];
+    /** Tasks awaiting their result, keyed by taskId. */
+    #inFlight = new Map();
+    /** Monotonic task identifier source. */
+    #nextTaskId = 0;
+    /** FIFO queue of pending tasks waiting for an idle slot. */
+    #queue = [];
+    /** Once `terminate()` has been called, no new tasks may be submitted. */
+    #shuttingDown = false;
+    /** All worker slots, idle or busy. */
+    #slots = [];
+    /** Per-task data payload to send when a queued task is dispatched. */
+    #taskData = new Map();
+    /** Worker entry script path; reused when replacing crashed workers. */
+    #workerPath;
+    /**
+     * Constructs a pool with the given size and worker entry script.
+     * @param options - Pool size and worker entry script path.
+     */
+    constructor(options) {
+        const size = Math.max(1, Math.floor(options.size));
+        this.#workerPath = options.workerPath;
+        for (let i = 0; i < size; i++) {
+            this.#spawn();
+        }
+    }
+    /**
+     * Submits a task to the pool. Resolves with the worker's `result` payload
+     * once a worker has finished processing it. The task may be queued if all
+     * workers are currently busy.
+     * @template I - Shape of the per-task `initialData` payload.
+     * @template R - Expected result type returned by the worker module.
+     * @param params - Task payload (file path, batch indices, emitter, data).
+     * @returns The worker's result.
+     */
+    run(params) {
+        if (this.#shuttingDown) {
+            return Promise.reject(new Error('WorkerPool is shutting down'));
+        }
+        const taskId = this.#nextTaskId++;
+        const data = {
+            filePath: params.filePath,
+            num: params.num,
+            total: params.total,
+            ...params.initialData,
+        };
+        return new Promise((resolve, reject) => {
+            const task = {
+                taskId,
+                emitter: params.emitter,
+                resolve: resolve,
+                reject,
+            };
+            this.#taskData.set(taskId, data);
+            const slot = this.#idle.shift();
+            if (slot) {
+                this.#dispatch(slot, task);
+                return;
+            }
+            this.#queue.push(task);
+        });
+    }
+    /**
+     * Rejects every queued or in-flight task, sends each worker a `shutdown`
+     * message, and awaits its exit. After this call returns, no further
+     * submissions are accepted.
+     *
+     * Pending tasks are rejected synchronously so callers waiting on
+     * `pool.run()` Promises do not hang. Workers handle the shutdown message
+     * by exiting cleanly; if that fails, {@link #shutdownSlot} forces
+     * termination after a 5 second timeout.
+     */
+    async terminate() {
+        this.#shuttingDown = true;
+        const pendingError = new Error('WorkerPool terminated before task completed');
+        for (const task of this.#queue) {
+            task.reject(pendingError);
+            this.#taskData.delete(task.taskId);
+        }
+        this.#queue = [];
+        for (const task of this.#inFlight.values()) {
+            task.reject(pendingError);
+        }
+        this.#inFlight.clear();
+        await Promise.all(this.#slots.map((slot) => this.#shutdownSlot(slot)));
+        this.#slots = [];
+        this.#idle = [];
+    }
+    /**
+     * Completes the current task on `slot` and dispatches the next queued task.
+     * @param slot
+     * @param taskId
+     * @param result
+     * @param error
+     */
+    #completeTask(slot, taskId, result, error) {
+        const task = this.#inFlight.get(taskId);
+        this.#inFlight.delete(taskId);
+        slot.current = null;
+        if (task) {
+            if (error) {
+                task.reject(new Error(error));
+            }
+            else {
+                task.resolve(result);
+            }
+        }
+        const next = this.#queue.shift();
+        if (next) {
+            this.#dispatch(slot, next);
+        }
+        else {
+            this.#idle.push(slot);
+        }
+    }
+    /**
+     * Sends a queued task to an idle slot and registers it as in-flight.
+     *
+     * If `postMessage` throws (e.g. the worker died between handoff and
+     * dispatch), the task is rejected immediately and the slot is left out
+     * of the idle pool so the worker's `error` event can replace it via
+     * {@link #onWorkerCrash}.
+     * @param slot - Pool slot that owns the worker.
+     * @param task - Pending task to dispatch.
+     */
+    #dispatch(slot, task) {
+        const data = this.#taskData.get(task.taskId);
+        try {
+            slot.worker.postMessage({ type: 'task', taskId: task.taskId, data });
+        }
+        catch (error) {
+            this.#taskData.delete(task.taskId);
+            task.reject(error instanceof Error ? error : new Error(String(error)));
+            return;
+        }
+        slot.current = task;
+        this.#inFlight.set(task.taskId, task);
+        this.#taskData.delete(task.taskId);
+    }
+    /**
+     * Routes an incoming worker message to the matching pending task.
+     * @param slot
+     * @param message
+     */
+    #onMessage(slot, message) {
+        if (!message || typeof message !== 'object') {
+            return;
+        }
+        const msg = message;
+        if (msg.type === 'url' &&
+            typeof msg.taskId === 'number' &&
+            typeof msg.url === 'string') {
+            const task = this.#inFlight.get(msg.taskId);
+            if (task) {
+                void task.emitter.emit('url', msg.url);
+            }
+            return;
+        }
+        if (msg.type === 'result' && typeof msg.taskId === 'number') {
+            this.#completeTask(slot, msg.taskId, msg.result, msg.error);
+            return;
+        }
+    }
+    /**
+     * Handles an unrecoverable worker failure: rejects the current task,
+     * removes the dead slot, and spawns a replacement so the pool keeps its
+     * configured size. Queued tasks are picked up by the new slot when ready.
+     * @param slot
+     * @param error
+     */
+    #onWorkerCrash(slot, error) {
+        const task = slot.current;
+        slot.current = null;
+        if (task) {
+            this.#inFlight.delete(task.taskId);
+            task.reject(error instanceof Error ? error : new Error(String(error)));
+        }
+        const slotIndex = this.#slots.indexOf(slot);
+        if (slotIndex !== -1) {
+            this.#slots.splice(slotIndex, 1);
+        }
+        const idleIndex = this.#idle.indexOf(slot);
+        if (idleIndex !== -1) {
+            this.#idle.splice(idleIndex, 1);
+        }
+        slot.worker.removeAllListeners();
+        void slot.worker.terminate();
+        if (this.#shuttingDown) {
+            return;
+        }
+        const replacement = this.#spawn();
+        const next = this.#queue.shift();
+        if (next) {
+            const idleIdx = this.#idle.indexOf(replacement);
+            if (idleIdx !== -1) {
+                this.#idle.splice(idleIdx, 1);
+            }
+            this.#dispatch(replacement, next);
+        }
+    }
+    /**
+     * Sends a shutdown message and waits for the worker to exit cleanly.
+     * @param slot
+     */
+    #shutdownSlot(slot) {
+        return new Promise((resolve) => {
+            const finish = () => {
+                slot.worker.removeAllListeners();
+                resolve();
+            };
+            slot.worker.once('exit', finish);
+            slot.worker.once('error', finish);
+            try {
+                slot.worker.postMessage({ type: 'shutdown' });
+            }
+            catch {
+                void slot.worker.terminate().finally(finish);
+                return;
+            }
+            // Safety net in case the worker fails to acknowledge the shutdown.
+            setTimeout(() => {
+                void slot.worker.terminate().finally(finish);
+            }, 5000).unref();
+        });
+    }
+    /** Spawns a fresh worker, wires up its event handlers, and adds it to the pool. */
+    #spawn() {
+        const worker = new Worker(this.#workerPath);
+        const slot = { worker, current: null };
+        this.#slots.push(slot);
+        worker.on('message', (message) => {
+            this.#onMessage(slot, message);
+        });
+        worker.on('error', (error) => {
+            this.#onWorkerCrash(slot, error);
+        });
+        worker.on('messageerror', (error) => {
+            this.#onWorkerCrash(slot, error);
+        });
+        this.#idle.push(slot);
+        // Workers should not keep the event loop alive on their own;
+        // the pool's terminate() controls their lifetime.
+        worker.unref();
+        return slot;
+    }
+}

package/lib/worker/worker.d.ts

@@ -1,26 +1,33 @@
 /**
  * Worker thread entry point for plugin execution.
  *
- * This module runs inside a `new Worker(...)` created by {@link ./run-in-worker.ts!runInWorker}.
- * It uses a message-based protocol to communicate with the main thread:
+ * This module runs inside a `new Worker(...)` created by the
+ * {@link ./worker-pool.ts!WorkerPool}. Unlike the previous one-shot model,
+ * each worker is long-lived and processes many tasks via a message loop until
+ * it receives a `shutdown` message.
  *
- * ## Message protocol (Worker -> Main)
+ * ## Message protocol (Worker ↔ Pool)
  *
- * | `type`     | Payload          | Description                                    |
- * |------------|------------------|------------------------------------------------|
- * | `'url'`    | `{ url: string }` | A URL was discovered during analysis           |
- * | `'finish'` | `{ result: R }`  | Analysis complete, carries the plugin result   |
+ * | Direction       | `type`        | Payload                          |
+ * |-----------------|---------------|----------------------------------|
+ * | Pool → Worker   | `'task'`      | `{ taskId, data }`               |
+ * | Pool → Worker   | `'shutdown'`  | (none)                           |
+ * | Worker → Pool   | `'url'`       | `{ taskId, url }`                |
+ * | Worker → Pool   | `'result'`    | `{ taskId, result, error? }`     |
  *
  * ## Lifecycle
  *
- * 1. Reads `workerData` containing the module path + plugin data
- * 2. Creates a local {@link ../url-event-bus.ts!UrlEventBus} that forwards `'url'`
- *    events to the main thread via `parentPort.postMessage`
- * 3. Delegates to {@link ./runner.ts!runner} for dynamic import and execution
- * 4. Posts the `'finish'` message with the result
+ * 1. Listens on `parentPort` for `task` messages.
+ * 2. For each task, delegates to {@link ./runner.ts!runner} which dynamically
+ *    `import()`s the module pointed to by `data.filePath`. Subsequent tasks
+ *    that target the same module reuse the import cache, so the heavy boot
+ *    cost is paid once per module per worker.
+ * 3. Posts a `result` message back to the pool and waits for the next task.
+ * 4. On `shutdown`, exits the process so the V8 isolate is fully released.
  *
- * The main thread terminates this Worker after receiving `'finish'`.
- * @see {@link ./run-in-worker.ts!runInWorker} for the main-thread counterpart
+ * The `'url'` events emitted by plugins are forwarded with the active
+ * `taskId` so the pool can route them back to the correct caller's emitter.
+ * @see {@link ./worker-pool.ts!WorkerPool} for the main-thread counterpart
  * @see {@link ./runner.ts!runner} for the actual module loading logic
  * @module
  */

package/lib/worker/worker.js

@@ -1,68 +1,108 @@
 /**
  * Worker thread entry point for plugin execution.
  *
- * This module runs inside a `new Worker(...)` created by {@link ./run-in-worker.ts!runInWorker}.
- * It uses a message-based protocol to communicate with the main thread:
+ * This module runs inside a `new Worker(...)` created by the
+ * {@link ./worker-pool.ts!WorkerPool}. Unlike the previous one-shot model,
+ * each worker is long-lived and processes many tasks via a message loop until
+ * it receives a `shutdown` message.
  *
- * ## Message protocol (Worker -> Main)
+ * ## Message protocol (Worker ↔ Pool)
  *
- * | `type`     | Payload          | Description                                    |
- * |------------|------------------|------------------------------------------------|
- * | `'url'`    | `{ url: string }` | A URL was discovered during analysis           |
- * | `'finish'` | `{ result: R }`  | Analysis complete, carries the plugin result   |
+ * | Direction       | `type`        | Payload                          |
+ * |-----------------|---------------|----------------------------------|
+ * | Pool → Worker   | `'task'`      | `{ taskId, data }`               |
+ * | Pool → Worker   | `'shutdown'`  | (none)                           |
+ * | Worker → Pool   | `'url'`       | `{ taskId, url }`                |
+ * | Worker → Pool   | `'result'`    | `{ taskId, result, error? }`     |
  *
  * ## Lifecycle
  *
- * 1. Reads `workerData` containing the module path + plugin data
- * 2. Creates a local {@link ../url-event-bus.ts!UrlEventBus} that forwards `'url'`
- *    events to the main thread via `parentPort.postMessage`
- * 3. Delegates to {@link ./runner.ts!runner} for dynamic import and execution
- * 4. Posts the `'finish'` message with the result
+ * 1. Listens on `parentPort` for `task` messages.
+ * 2. For each task, delegates to {@link ./runner.ts!runner} which dynamically
+ *    `import()`s the module pointed to by `data.filePath`. Subsequent tasks
+ *    that target the same module reuse the import cache, so the heavy boot
+ *    cost is paid once per module per worker.
+ * 3. Posts a `result` message back to the pool and waits for the next task.
+ * 4. On `shutdown`, exits the process so the V8 isolate is fully released.
  *
- * The main thread terminates this Worker after receiving `'finish'`.
- * @see {@link ./run-in-worker.ts!runInWorker} for the main-thread counterpart
+ * The `'url'` events emitted by plugins are forwarded with the active
+ * `taskId` so the pool can route them back to the correct caller's emitter.
+ * @see {@link ./worker-pool.ts!WorkerPool} for the main-thread counterpart
  * @see {@link ./runner.ts!runner} for the actual module loading logic
  * @module
  */
-import { parentPort, workerData } from 'node:worker_threads';
+import { parentPort } from 'node:worker_threads';
 import { UrlEventBus } from '../url-event-bus.js';
 import { runner } from './runner.js';
-const data = workerData;
+/** Identifier for the task currently being processed; used to tag forwarded `'url'` events. */
+let currentTaskId = null;
+/** Long-lived event bus that forwards `'url'` events from plugins to the pool. */
 const emitter = new UrlEventBus();
-/**
- * Forward URL discovery events from the plugin to the main thread.
- * The main thread's {@link ../url-event-bus.ts!UrlEventBus} re-emits these
- * so that the orchestrator can track discovered URLs.
- */
 emitter.on('url', (url) => {
     if (!parentPort) {
         throw new Error('Use in worker thread');
     }
+    if (currentTaskId === null) {
+        return;
+    }
     parentPort.postMessage({
         type: 'url',
+        taskId: currentTaskId,
         url,
     });
 });
-try {
-    const result = await runner(data, emitter);
-    if (!parentPort) {
-        throw new Error('Use in worker thread');
-    }
-    parentPort.postMessage({
-        type: 'finish',
-        result,
-    });
+if (!parentPort) {
+    throw new Error('Use in worker thread');
 }
-catch (error) {
-    if (!parentPort) {
-        throw new Error('Use in worker thread');
+parentPort.on('message', (message) => {
+    if (!message || typeof message !== 'object') {
+        return;
+    }
+    const msg = message;
+    if (msg.type === 'shutdown') {
+        // Defer to the next tick so any in-flight `postMessage` from the most
+        // recent task has a chance to flush before the worker exits.
+        setImmediate(() => process.exit(0));
+        return;
+    }
+    if (msg.type === 'task' && typeof msg.taskId === 'number' && msg.data) {
+        void runTask(msg.taskId, msg.data);
+    }
+});
+/**
+ * Executes a single task and posts the result back to the pool.
+ *
+ * Errors are caught and reported via the `error` field on the `result` message
+ * so the pool can reject the originating Promise without the worker crashing.
+ * @param taskId - Identifier echoed back so the pool can match the response.
+ * @param data - Payload from the pool, including `filePath` consumed by `runner()`.
+ */
+async function runTask(taskId, data) {
+    currentTaskId = taskId;
+    try {
+        const result = await runner(data, emitter);
+        if (!parentPort) {
+            throw new Error('Use in worker thread');
+        }
+        parentPort.postMessage({
+            type: 'result',
+            taskId,
+            result,
+        });
+    }
+    catch (error) {
+        if (!parentPort) {
+            throw new Error('Use in worker thread');
+        }
+        const message = error instanceof Error ? error.message : String(error);
+        parentPort.postMessage({
+            type: 'result',
+            taskId,
+            result: null,
+            error: message,
+        });
+    }
+    finally {
+        currentTaskId = null;
     }
-    const message = error instanceof Error ? error.message : String(error);
-    // eslint-disable-next-line no-console
-    console.error(`[worker] Fatal error: ${message}`);
-    parentPort.postMessage({
-        type: 'finish',
-        result: null,
-        error: message,
-    });
 }

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@nitpicker/core",
-	"version": "0.6.4",
+	"version": "0.7.0",
 	"description": "Plugin-based page analysis engine for Nitpicker",
 	"author": "D-ZERO",
 	"license": "Apache-2.0",
@@ -29,11 +29,11 @@
 	"dependencies": {
 		"@d-zero/dealer": "1.7.0",
 		"@d-zero/shared": "0.20.1",
-		"@nitpicker/crawler": "0.6.3",
-		"@nitpicker/types": "0.6.3",
+		"@nitpicker/crawler": "0.7.0",
+		"@nitpicker/types": "0.7.0",
 		"ansi-colors": "4.1.3",
 		"cosmiconfig": "9.0.0",
 		"jsdom": "28.1.0"
 	},
-	"gitHead": "840d4c33a2d0ea9469d511b4f74a69f7fb38ee2b"
+	"gitHead": "3b4c33adf8af69a46a85faf8c04eddb5ab563972"
 }

package/lib/find-nitpicker-modules-dir.d.ts

@@ -1,12 +0,0 @@
-/**
- * Walks up from the current module's directory to locate the parent
- * `@nitpicker` directory in the file system.
- *
- * This is used by plugin discovery and label resolution to find
- * where `@nitpicker/*` packages are installed on disk. The traversal
- * starts from `import.meta.url` of the calling module (i.e. this file,
- * which lives inside `@nitpicker/core`) and walks up until it finds
- * a directory named `@nitpicker`.
- * @returns Absolute path to the `@nitpicker` directory, or `null` if not found.
- */
-export declare function findNitpickerModulesDir(): string | null;

package/lib/find-nitpicker-modules-dir.js

@@ -1,23 +0,0 @@
-import path from 'node:path';
-/**
- * Walks up from the current module's directory to locate the parent
- * `@nitpicker` directory in the file system.
- *
- * This is used by plugin discovery and label resolution to find
- * where `@nitpicker/*` packages are installed on disk. The traversal
- * starts from `import.meta.url` of the calling module (i.e. this file,
- * which lives inside `@nitpicker/core`) and walks up until it finds
- * a directory named `@nitpicker`.
- * @returns Absolute path to the `@nitpicker` directory, or `null` if not found.
- */
-export function findNitpickerModulesDir() {
-    const __filename = new URL(import.meta.url).pathname;
-    let dir = path.dirname(__filename);
-    while (dir !== path.dirname(dir)) {
-        if (path.basename(dir) === '@nitpicker') {
-            return dir;
-        }
-        dir = path.dirname(dir);
-    }
-    return null;
-}

package/lib/hooks/actions.d.ts

@@ -1,9 +0,0 @@
-import type { StartAction } from '../types.js';
-declare const _default: {
-    audit: {
-        start: import("typescript-fsa").ActionCreator<StartAction>;
-        log: import("typescript-fsa").ActionCreator<string>;
-        finish: import("typescript-fsa").ActionCreator<void>;
-    };
-};
-export default _default;

package/lib/hooks/actions.js

@@ -1,9 +0,0 @@
-import { actionCreatorFactory } from 'typescript-fsa';
-const auditActionCreator = actionCreatorFactory('@@hook/audit');
-export default {
-    audit: {
-        start: auditActionCreator('START'),
-        log: auditActionCreator('LOG'),
-        finish: auditActionCreator('FINISH'),
-    },
-};

package/lib/hooks/child-process.d.ts

@@ -1 +0,0 @@
-export {};

package/lib/hooks/child-process.js

@@ -1,34 +0,0 @@
-// Run on child_process
-import { Archive } from '@nitpicker/crawler';
-import { isType } from 'typescript-fsa';
-import actions from './actions.js';
-const crateTable = (headers, data, options) => ({
-    headers,
-    data,
-    options,
-});
-process.on('message', async (action) => {
-    if (isType(action, actions.audit.start)) {
-        const mod = await import(action.payload.hookModulePath);
-        const pluginHook = mod.default;
-        const hook = await pluginHook(action.payload.settings || {}, action.payload.configFilePath);
-        if (hook.audit) {
-            const archive = await Archive.connect(action.payload.archiveTempDir, action.payload.hookModuleName);
-            const pluginReport = await hook.audit(archive, (log) => {
-                if (process.send) {
-                    process.send(actions.audit.log(log));
-                }
-            }, action.payload.configFilePath, crateTable);
-            if (pluginReport) {
-                const data = {
-                    name: action.payload.hookModuleName,
-                    ...pluginReport,
-                };
-                await archive.setData('reports', data);
-            }
-        }
-        if (process.send) {
-            process.send(actions.audit.finish());
-        }
-    }
-});

package/lib/hooks/index.d.ts

@@ -1 +0,0 @@
-export { definePlugin } from './define-plugin.js';

package/lib/hooks/index.js

@@ -1 +0,0 @@
-export { definePlugin } from './define-plugin.js';

package/lib/hooks/runner.d.ts

@@ -1,10 +0,0 @@
-import type { StartAction } from '../types.js';
-import type { ProcessLogger } from '@nitpicker/types';
-import actions from './actions.js';
-/**
- *
- * @param phase
- * @param startActionParam
- * @param log
- */
-export declare function runner(phase: keyof typeof actions, startActionParam: StartAction, log: ProcessLogger): Promise<void>;

package/lib/hooks/runner.js

@@ -1,32 +0,0 @@
-import childProcess from 'node:child_process';
-import path from 'node:path';
-import { isType } from 'typescript-fsa';
-import actions from './actions.js';
-const __filename = new URL(import.meta.url).pathname;
-const __dirname = path.dirname(__filename);
-/**
- *
- * @param phase
- * @param startActionParam
- * @param log
- */
-export function runner(phase, startActionParam, log) {
-    log('Fork process');
-    const child = childProcess.fork(path.join(__dirname, 'child-process'));
-    const actionOnPhase = actions[phase];
-    const session = new Promise((resolve) => {
-        child.on('message', (action) => {
-            if (isType(action, actionOnPhase.log)) {
-                log(action.payload);
-            }
-            if (isType(action, actionOnPhase.finish)) {
-                child.disconnect();
-                log('Finished');
-                resolve();
-            }
-        });
-    });
-    child.send(actionOnPhase.start(startActionParam));
-    log('Started');
-    return session;
-}

package/lib/index.d.ts

@@ -1,5 +0,0 @@
-export { Nitpicker } from './nitpicker.js';
-export { definePlugin } from './hooks/index.js';
-export { readPluginLabels } from './read-plugin-labels.js';
-export { runInWorker } from './worker/run-in-worker.js';
-export * from './types.js';

package/lib/index.js

@@ -1,5 +0,0 @@
-export { Nitpicker } from './nitpicker.js';
-export { definePlugin } from './hooks/index.js';
-export { readPluginLabels } from './read-plugin-labels.js';
-export { runInWorker } from './worker/run-in-worker.js';
-export * from './types.js';