@nitpicker/crawler 0.4.2 → 0.4.3

package/src/types.ts

@@ -1,30 +0,0 @@
-import type { CrawlerError } from './utils/index.js';
-
-/**
- * Event map for the `CrawlerOrchestrator` class.
- *
- * Each key represents an event name and its value is the payload type
- * passed to listeners subscribed via `on()` or `once()`.
- */
-export interface CrawlEvent {
-	/**
-	 * Emitted when the archive file write operation begins.
-	 */
-	writeFileStart: {
-		/** Absolute path of the archive file being written. */
-		filePath: string;
-	};
-
-	/**
-	 * Emitted when the archive file write operation completes.
-	 */
-	writeFileEnd: {
-		/** Absolute path of the archive file that was written. */
-		filePath: string;
-	};
-
-	/**
-	 * Emitted when an error occurs during crawling or archiving.
-	 */
-	error: CrawlerError;
-}

package/src/utils/array/each-splitted.spec.ts

@@ -1,38 +0,0 @@
-import { describe, it, expect } from 'vitest';
-
-import { eachSplitted } from './each-splitted.js';
-
-describe('eachSplitted', () => {
-	it('splits array into chunks and calls callback on each', async () => {
-		const chunks: number[][] = [];
-		await eachSplitted([1, 2, 3, 4, 5], 2, (items) => {
-			chunks.push(items);
-		});
-		expect(chunks).toEqual([[1, 2], [3, 4], [5]]);
-	});
-
-	it('handles empty array', async () => {
-		const chunks: number[][] = [];
-		await eachSplitted([], 2, (items) => {
-			chunks.push(items);
-		});
-		expect(chunks).toEqual([]);
-	});
-
-	it('handles chunk size larger than array', async () => {
-		const chunks: number[][] = [];
-		await eachSplitted([1, 2], 10, (items) => {
-			chunks.push(items);
-		});
-		expect(chunks).toEqual([[1, 2]]);
-	});
-
-	it('supports async callback', async () => {
-		const results: string[] = [];
-		await eachSplitted(['a', 'b', 'c'], 1, async (items) => {
-			await new Promise((resolve) => setTimeout(resolve, 1));
-			results.push(...items);
-		});
-		expect(results.toSorted()).toEqual(['a', 'b', 'c']);
-	});
-});

package/src/utils/array/each-splitted.ts

@@ -1,19 +0,0 @@
-import { splitArray } from '@d-zero/shared/split-array';
-
-/**
- * Splits an array into chunks of the specified size and executes a callback
- * on each chunk in parallel using `Promise.all`.
- * @template T - The element type of the array.
- * @param a - The array to split into chunks.
- * @param count - The maximum number of elements per chunk.
- * @param callback - A function to invoke on each chunk. May be synchronous or asynchronous.
- * @returns A promise that resolves when all chunk callbacks have completed.
- */
-export async function eachSplitted<T>(
-	a: T[],
-	count: number,
-	callback: (items: T[]) => void | Promise<void>,
-) {
-	const splitted = splitArray(a, count);
-	await Promise.all(splitted.map((items) => callback(items)));
-}

package/src/utils/array/index.ts

@@ -1 +0,0 @@
-export { eachSplitted } from './each-splitted.js';

package/src/utils/debug.ts

@@ -1,6 +0,0 @@
-import debug from 'debug';
-
-/** Root debug logger for the Nitpicker application. Namespace: `Nitpicker`. */
-export const globalLog = debug('Nitpicker');
-/** Debug logger for the utils package. Namespace: `Nitpicker:Utils`. */
-export const log = globalLog.extend('Utils');

package/src/utils/error/dom-evaluation-error.spec.ts

@@ -1,20 +0,0 @@
-import { describe, it, expect } from 'vitest';
-
-import { DOMEvaluationError } from './dom-evaluation-error.js';
-
-describe('DOMEvaluationError', () => {
-	it('is an instance of Error', () => {
-		const error = new DOMEvaluationError('test');
-		expect(error).toBeInstanceOf(Error);
-	});
-
-	it('sets the message', () => {
-		const error = new DOMEvaluationError('test message');
-		expect(error.message).toBe('test message');
-	});
-
-	it('has the correct name', () => {
-		const error = new DOMEvaluationError('test');
-		expect(error.name).toBe('Error');
-	});
-});

package/src/utils/error/dom-evaluation-error.ts

@@ -1,6 +0,0 @@
-/**
- * Error thrown when DOM evaluation (e.g., running scripts within a browser page context)
- * fails. This typically occurs during page scraping when JavaScript execution
- * in the browser context encounters an error.
- */
-export class DOMEvaluationError extends Error {}

package/src/utils/error/error-emitter.spec.ts

@@ -1,78 +0,0 @@
-import type { ErrorEvent } from './error-emitter.js';
-
-import { TypedAwaitEventEmitter } from '@d-zero/shared/typed-await-event-emitter';
-import { describe, it, expect, vi } from 'vitest';
-
-import { ErrorEmitter } from './error-emitter.js';
-
-/**
- * Test class that uses the ErrorEmitter decorator.
- */
-class TestEmitter extends TypedAwaitEventEmitter<ErrorEvent> {
-	/**
-	 * A method that throws an Error.
-	 * @param message - Error message.
-	 */
-	@ErrorEmitter()
-	failWithError(message: string): Promise<never> {
-		throw new Error(message);
-	}
-	/**
-	 * A method that throws a non-Error value.
-	 */
-	@ErrorEmitter()
-	failWithNonError(): Promise<never> {
-		throw 'string-error';
-	}
-	/**
-	 * A method that succeeds.
-	 * @param value - A value to return.
-	 * @returns The input value.
-	 */
-	@ErrorEmitter()
-	succeed(value: string): Promise<string> {
-		return Promise.resolve(value);
-	}
-}
-
-describe('ErrorEmitter', () => {
-	it('returns the method result on success', async () => {
-		const emitter = new TestEmitter();
-		const result = await emitter.succeed('hello');
-		expect(result).toBe('hello');
-	});
-
-	it('emits an error event when the method throws an Error', async () => {
-		const emitter = new TestEmitter();
-		const listener = vi.fn();
-		emitter.on('error', listener);
-
-		await expect(emitter.failWithError('test failure')).rejects.toThrow('test failure');
-		expect(listener).toHaveBeenCalledOnce();
-		expect(listener).toHaveBeenCalledWith(expect.any(Error));
-		expect(listener.mock.calls[0][0].message).toBe('test failure');
-	});
-
-	it('re-throws the error after emitting', async () => {
-		const emitter = new TestEmitter();
-		await expect(emitter.failWithError('boom')).rejects.toThrow('boom');
-	});
-
-	it('does not emit error event for non-Error throws', async () => {
-		const emitter = new TestEmitter();
-		const listener = vi.fn();
-		emitter.on('error', listener);
-
-		await expect(emitter.failWithNonError()).rejects.toBe('string-error');
-		expect(listener).not.toHaveBeenCalled();
-	});
-
-	it('does not emit error event on success', async () => {
-		const emitter = new TestEmitter();
-		const listener = vi.fn();
-		emitter.on('error', listener);
-
-		await emitter.succeed('ok');
-		expect(listener).not.toHaveBeenCalled();
-	});
-});

package/src/utils/error/error-emitter.ts

@@ -1,44 +0,0 @@
-import type { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-await-event-emitter';
-
-import { log } from '../debug.js';
-
-const errorLog = log.extend('ErrorEmitter');
-
-/**
- * Event payload type for error events emitted by classes using the {@link ErrorEmitter} decorator.
- * @template E - The specific error type, defaults to `Error`.
- */
-export type ErrorEvent<E extends Error = Error> = {
-	/** The error instance that was caught. */
-	error: E;
-};
-
-/**
- * A class method decorator factory that wraps the decorated method with error handling.
- * When the method throws an `Error`, it emits an `'error'` event on the class instance
- * (which must extend {@link EventEmitter}) with the caught error, then re-throws the error.
- * @template C - The class type, which must be an EventEmitter capable of emitting error events.
- * @template E - The error event type, defaults to {@link ErrorEvent}.
- * @returns A decorator function that wraps the target method with error-emitting behavior.
- */
-export function ErrorEmitter<
-	C extends EventEmitter<E>,
-	E extends ErrorEvent = ErrorEvent,
->() {
-	// eslint-disable-next-line @typescript-eslint/no-unsafe-function-type
-	return (method: Function, context: ClassMethodDecoratorContext) => {
-		return async function (this: C, ...args: unknown[]) {
-			const constructorName = String(this.constructor?.name || this.constructor || this);
-			const methodName = `${constructorName}.${String(context.name)}`;
-			try {
-				return await method.apply(this, args);
-			} catch (error: unknown) {
-				if (error instanceof Error) {
-					errorLog('%s: %O', methodName, error);
-					void this.emit('error', error);
-				}
-				throw error;
-			}
-		};
-	};
-}

package/src/utils/error/index.ts

@@ -1,3 +0,0 @@
-export { DOMEvaluationError } from './dom-evaluation-error.js';
-export { ErrorEmitter } from './error-emitter.js';
-export type { ErrorEvent } from './error-emitter.js';

package/src/utils/index.ts

@@ -1,5 +0,0 @@
-export * from './types/index.js';
-export * from './array/index.js';
-export * from './error/index.js';
-export * from './object/index.js';
-export { globalLog as log } from './debug.js';

package/src/utils/object/clean-object.spec.ts

@@ -1,24 +0,0 @@
-import { describe, it, expect } from 'vitest';
-
-import { cleanObject } from './clean-object.js';
-
-describe('cleanObject', () => {
-	it('removes undefined properties', () => {
-		const result = cleanObject({ a: 1, b: undefined, c: 'hello' });
-		expect(result).toEqual({ a: 1, c: 'hello' });
-	});
-
-	it('keeps null properties', () => {
-		const result = cleanObject({ a: null, b: 0, c: '' });
-		expect(result).toEqual({ a: null, b: 0, c: '' });
-	});
-
-	it('returns empty object for falsy input', () => {
-		expect(cleanObject()).toEqual({});
-	});
-
-	it('returns same properties when no undefined values', () => {
-		const obj = { x: 1, y: 2 };
-		expect(cleanObject(obj)).toEqual({ x: 1, y: 2 });
-	});
-});

package/src/utils/object/clean-object.ts

@@ -1,13 +0,0 @@
-/**
- * Creates a shallow copy of an object with all `undefined`-valued properties removed.
- * If the input is falsy (e.g., `undefined` or `null`), returns an empty object.
- * @template T - The type of the input object.
- * @param obj - The object to clean. If falsy, an empty `Partial<T>` is returned.
- * @returns A new object containing only the properties whose values are not `undefined`.
- */
-export function cleanObject<T extends Record<string, unknown>>(obj?: T) {
-	if (!obj) {
-		return {} as Partial<T>;
-	}
-	return Object.fromEntries(Object.entries(obj).filter(([, v]) => v !== undefined)) as T;
-}

package/src/utils/object/index.ts

@@ -1 +0,0 @@
-export * from './clean-object.js';

package/src/utils/types/index.ts

@@ -1 +0,0 @@
-export * from './types.js';

package/src/utils/types/types.ts

@@ -1,65 +0,0 @@
-import type { ExURL } from '@d-zero/shared/parse-url';
-
-// beholder is the canonical owner of scraping types — re-export
-export type {
-	PageData,
-	ImageElement,
-	SkippedPageData,
-	Resource,
-	AnchorData,
-	Meta,
-	NetworkLog,
-} from '@d-zero/beholder';
-
-export type { ExURL } from '@d-zero/shared/parse-url';
-export type { CompressType } from '@d-zero/shared/detect-compress';
-export type { CDNType } from '@d-zero/shared/detect-cdn';
-
-/**
- * Represents a discovered link during crawling, with its metadata from the HEAD request.
- */
-export interface Link {
-	/** The parsed URL of the link. */
-	url: ExURL;
-
-	/** Whether this link points to an external domain. */
-	isExternal: boolean;
-
-	/** Whether this link is in a lower layer (subdirectory) of a scope URL. */
-	isLowerLayer: boolean;
-
-	/** Destination data from the HEAD request, present only if the link was fetched. */
-	dest?: {
-		/** Chain of redirect URLs traversed. */
-		redirectPaths: string[];
-		/** HTTP status code of the final response. */
-		status: number;
-		/** HTTP status text of the final response. */
-		statusText: string;
-		/** The Content-Type header value, or `null` if unavailable. */
-		contentType: string | null;
-		/** The Content-Length header value in bytes, or `null` if unavailable. */
-		contentLength: number | null;
-		/** Raw HTTP response headers, or `null` if unavailable. */
-		responseHeaders: Record<string, string | string[] | undefined> | null;
-		/** The page title, if available from a title-only scrape. */
-		title?: string;
-	};
-}
-
-/**
- * An error event emitted during crawling or scraping.
- */
-export interface CrawlerError {
-	/** The process ID where the error occurred. */
-	pid: number;
-
-	/** Whether the error occurred in the main process (as opposed to a sub-process). */
-	isMainProcess: boolean;
-
-	/** The URL being processed when the error occurred, or `null` if not applicable. */
-	url: string | null;
-
-	/** The error object. */
-	error: Error;
-}

package/tsconfig.json

@@ -1,11 +0,0 @@
-{
-	"extends": "../../../tsconfig.json",
-	"compilerOptions": {
-		"composite": true,
-		"outDir": "./lib",
-		"rootDir": "./src",
-		"resolveJsonModule": true
-	},
-	"include": ["./src/**/*"],
-	"exclude": ["node_modules", "lib", "./src/**/*.spec.ts"]
-}