@nitpicker/crawler 0.4.2 → 0.4.3

package/src/crawler/handle-scrape-end.ts

@@ -1,115 +0,0 @@
-import type LinkList from './link-list.js';
-import type { CrawlerOptions } from './types.js';
-import type { AnchorData, Link, PageData } from '../utils/index.js';
-import type { ExURL } from '@d-zero/shared/parse-url';
-
-import { crawlerLog } from '../debug.js';
-
-import { injectScopeAuth } from './inject-scope-auth.js';
-import { isExternalUrl } from './is-external-url.js';
-import { isInAnyLowerLayer } from './is-in-any-lower-layer.js';
-
-/**
- * Process the result of a successful page scrape.
- *
- * Extracts anchors from the page (unless in metadata-only mode), enqueues
- * newly discovered URLs via the `addUrl` callback, and marks the URL
- * as done in the link list.
- * @param result - The scraped page data.
- * @param linkList - The link list managing the crawl queue.
- * @param scope - Map of hostnames to their scope URLs.
- * @param options - Crawler configuration options.
- * @param addUrl - Callback to enqueue a newly discovered URL. Accepts optional
- *   `{ metadataOnly: true }` to request metadata-only scraping.
- * @returns An object containing the constructed link and whether the page is external.
- */
-export function handleScrapeEnd(
-	result: PageData,
-	linkList: LinkList,
-	scope: ReadonlyMap<string, readonly ExURL[]>,
-	options: CrawlerOptions,
-	addUrl: (url: ExURL, opts?: { metadataOnly?: true }) => void,
-): { link: Link | null; isExternal: boolean } {
-	const isMetadataOnly = linkList.isMetadataOnly(result.url.withoutHash);
-	if (!isMetadataOnly) {
-		processAnchors(result.anchorList, scope, options, addUrl);
-	}
-
-	const link = linkList.done(
-		result.url,
-		scope,
-		{
-			page: result,
-		},
-		options,
-	);
-
-	crawlerLog('Scrape end URL: %s', result.url.href);
-	crawlerLog('Scrape end Status: %d', result.status);
-	crawlerLog('Scrape end Type: %s', result.contentType);
-	if (!result.isExternal) {
-		crawlerLog('Scrape end Anchors: %d URLs', result.anchorList.length);
-	}
-
-	return { link, isExternal: result.isExternal };
-}
-
-/**
- * Process anchor elements extracted from a scraped page and enqueue new URLs.
- *
- * For each anchor:
- * 1. Determines if it is external (outside the crawl scope)
- * 2. Injects authentication credentials from matching scope URLs
- * 3. Reconstructs the `withoutHash` URL with injected auth
- * 4. In recursive mode: enqueues internal lower-layer URLs for full scraping,
- *    and external URLs for metadata-only scraping (if `fetchExternal` is enabled)
- * 5. In non-recursive mode: enqueues all URLs for metadata-only scraping
- * @param anchors - The list of anchor data extracted from the page.
- * @param scope - Map of hostnames to their scope URLs.
- * @param options - Crawler configuration options.
- * @param addUrl - Callback to enqueue a newly discovered URL. Accepts optional
- *   `{ metadataOnly: true }` to request metadata-only scraping.
- */
-function processAnchors(
-	anchors: AnchorData[],
-	scope: ReadonlyMap<string, readonly ExURL[]>,
-	options: CrawlerOptions,
-	addUrl: (url: ExURL, opts?: { metadataOnly?: true }) => void,
-): void {
-	for (const anchor of anchors) {
-		const isExternal = isExternalUrl(anchor.href, scope);
-		anchor.isExternal = isExternal;
-
-		if (!isExternal && (!anchor.href.username || !anchor.href.password)) {
-			injectScopeAuth(anchor.href, scope);
-
-			const auth =
-				anchor.href.username && anchor.href.password
-					? `${anchor.href.username}:${anchor.href.password}@`
-					: '';
-			const host =
-				anchor.href.hostname + (anchor.href.port ? `:${anchor.href.port}` : '');
-			const newSearch = anchor.href.query ? `?${anchor.href.query}` : '';
-			const body = anchor.href.dirname
-				? `${anchor.href.paths.join('/')}${newSearch}`
-				: newSearch
-					? `${newSearch}`
-					: '';
-			const withoutHash = `${anchor.href.protocol}//${auth}${host}${body ? `/${body}` : ''}`;
-
-			anchor.href.withoutHash = withoutHash;
-		}
-
-		if (options.recursive) {
-			const scopes = scope.get(anchor.href.hostname);
-			if (scopes && isInAnyLowerLayer(anchor.href, scopes, options)) {
-				addUrl(anchor.href);
-			} else if (isExternal && options.fetchExternal) {
-				addUrl(anchor.href, { metadataOnly: true });
-			}
-			continue;
-		}
-
-		addUrl(anchor.href, { metadataOnly: true });
-	}
-}

package/src/crawler/handle-scrape-error.spec.ts

@@ -1,105 +0,0 @@
-import type { CrawlerOptions } from './types.js';
-
-import { tryParseUrl as parseUrl } from '@d-zero/shared/parse-url';
-import { describe, it, expect } from 'vitest';
-
-import { handleScrapeError } from './handle-scrape-error.js';
-import LinkList from './link-list.js';
-
-const defaultOptions: CrawlerOptions = {
-	interval: 0,
-	parallels: 1,
-	recursive: true,
-	fromList: false,
-	captureImages: false,
-	executablePath: null,
-	fetchExternal: false,
-	scope: ['https://example.com/'],
-	excludes: [],
-	excludeKeywords: [],
-	excludeUrls: [],
-	maxExcludedDepth: 10,
-	retry: 3,
-	verbose: false,
-	disableQueries: false,
-};
-
-/**
- * Create a scope map for testing.
- * @returns A scope map with example.com.
- */
-function createScope() {
-	return new Map([['example.com', [parseUrl('https://example.com/')!]]]);
-}
-
-describe('handleScrapeError', () => {
-	it('marks the URL as done when shutdown is true', () => {
-		const linkList = new LinkList();
-		const url = parseUrl('https://example.com/page')!;
-		const scope = createScope();
-		linkList.add(url);
-		linkList.progress(url);
-
-		const { link, result } = handleScrapeError(
-			{
-				url,
-				error: { name: 'Error', message: 'Browser crashed' },
-				shutdown: true,
-				pid: 1234,
-			},
-			linkList,
-			scope,
-			defaultOptions,
-		);
-
-		expect(link).not.toBeNull();
-		expect(link!.url.href).toBe(url.href);
-		expect(result).toBeDefined();
-		expect(result!.status).toBe(-1);
-	});
-
-	it('marks the URL as done when shutdown is false', () => {
-		const linkList = new LinkList();
-		const url = parseUrl('https://example.com/page')!;
-		const scope = createScope();
-		linkList.add(url);
-		linkList.progress(url);
-
-		const { link, result } = handleScrapeError(
-			{
-				url,
-				error: { name: 'Error', message: 'ERR_NAME_NOT_RESOLVED' },
-				shutdown: false,
-				pid: 5678,
-			},
-			linkList,
-			scope,
-			defaultOptions,
-		);
-
-		expect(link).not.toBeNull();
-		expect(link!.url.href).toBe(url.href);
-		expect(result).toBeDefined();
-		expect(result!.status).toBe(-1);
-	});
-
-	it('returns null link when url is null', () => {
-		const linkList = new LinkList();
-		const scope = createScope();
-
-		const { link, result } = handleScrapeError(
-			{
-				url: null,
-				error: { name: 'Error', message: 'Unknown error' },
-				shutdown: true,
-				pid: undefined,
-			},
-			linkList,
-			scope,
-			defaultOptions,
-		);
-
-		expect(link).toBeNull();
-		expect(result).toBeUndefined();
-	});
-});

package/src/crawler/handle-scrape-error.ts

@@ -1,58 +0,0 @@
-import type LinkList from './link-list.js';
-import type { CrawlerOptions } from './types.js';
-import type { Link, PageData } from '../utils/index.js';
-import type { ExURL } from '@d-zero/shared/parse-url';
-
-import { crawlerErrorLog } from '../debug.js';
-
-import { linkToPageData } from './link-to-page-data.js';
-
-/**
- * Handle an error that occurred during page scraping.
- *
- * Marks the URL as done and creates a fallback {@link PageData} from the
- * link, regardless of whether the error caused a shutdown. This ensures
- * that errored URLs are recorded in the DB (`status = -1, scraped = 1`)
- * and not re-queued on resume.
- * @param payload - The error payload from the scraper.
- * @param payload.url - The URL being scraped when the error occurred, or `null`.
- * @param payload.error - The error details including name, message, and optional stack.
- * @param payload.error.name
- * @param payload.error.message
- * @param payload.error.stack
- * @param payload.shutdown - Whether the error caused the scraper process to shut down.
- * @param payload.pid - The process ID of the scraper, or `undefined`.
- * @param linkList - The link list managing the crawl queue.
- * @param scope - Map of hostnames to their scope URLs.
- * @param options - Crawler configuration options.
- * @returns An object with the link and an optional fallback PageData result.
- */
-export function handleScrapeError(
-	payload: {
-		url: ExURL | null;
-		error: { name: string; message: string; stack?: string };
-		shutdown: boolean;
-		pid: number | undefined;
-	},
-	linkList: LinkList,
-	scope: ReadonlyMap<string, readonly ExURL[]>,
-	options: CrawlerOptions,
-): { link: Link | null; result?: PageData } {
-	const { url, error, shutdown, pid } = payload;
-	let link: Link | null = null;
-	let result: PageData | undefined;
-
-	if (url) {
-		const updated = linkList.done(url, scope, { error }, options);
-		if (updated) {
-			link = updated;
-			result = linkToPageData(updated);
-		}
-	}
-
-	crawlerErrorLog('From %d(%s)', pid, url?.href ?? 'UNKNOWN_URL');
-	crawlerErrorLog('Then shutdown?: %s', shutdown ? 'Yes' : 'No');
-	crawlerErrorLog('%O', error);
-
-	return { link, result };
-}

package/src/crawler/index.ts

@@ -1,2 +0,0 @@
-export { default as Crawler } from './crawler.js';
-export { clearDestinationCache } from './clear-destination-cache.js';

package/src/crawler/inject-scope-auth.spec.ts

@@ -1,36 +0,0 @@
-import type { ExURL } from '@d-zero/shared/parse-url';
-
-import { tryParseUrl as parseUrl } from '@d-zero/shared/parse-url';
-import { describe, it, expect } from 'vitest';
-
-import { injectScopeAuth } from './inject-scope-auth.js';
-
-/**
- * Create a scope map from hostname-URL pairs for testing.
- * @param entries - Array of [hostname, urls] tuples.
- * @returns A map from hostname to parsed ExURL arrays.
- */
-function createScope(entries: [string, string[]][]): Map<string, ExURL[]> {
-	return new Map(
-		entries.map(([h, urls]) => [h, urls.map((u) => parseUrl(u)!).filter(Boolean)]),
-	);
-}
-
-describe('injectScopeAuth', () => {
-	it('injects auth from matching scope URL', () => {
-		const url = parseUrl('https://example.com/blog/post')!;
-		const scope = createScope([['example.com', ['https://user:pass@example.com/blog']]]);
-		injectScopeAuth(url, scope);
-		expect(url.username).toBe('user');
-		expect(url.password).toBe('pass');
-	});
-
-	it('does not inject auth when hostname does not match', () => {
-		const url = parseUrl('https://other.com/page')!;
-		const scope = createScope([['example.com', ['https://user:pass@example.com/']]]);
-		injectScopeAuth(url, scope);
-		// username/password are null for URLs parsed without auth
-		expect(url.username).toBeNull();
-		expect(url.password).toBeNull();
-	});
-});

package/src/crawler/inject-scope-auth.ts

@@ -1,27 +0,0 @@
-import type { ExURL } from '@d-zero/shared/parse-url';
-
-import { findBestMatchingScope } from './find-best-matching-scope.js';
-
-/**
- * Inject authentication credentials from a matching scope URL into the target URL.
- *
- * Finds the best-matching scope URL (deepest path match) for the given URL's
- * hostname and copies its `username` and `password` properties. This mutates
- * the `url` parameter in place.
- * @param url - The parsed URL to receive authentication credentials (mutated in place).
- * @param scope - Map of hostnames to their scope URLs.
- */
-export function injectScopeAuth(
-	url: ExURL,
-	scope: ReadonlyMap<string, readonly ExURL[]>,
-): void {
-	const scopes = scope.get(url.hostname);
-	if (!scopes) {
-		return;
-	}
-	const matchedScope = findBestMatchingScope(url, scopes);
-	if (matchedScope) {
-		url.username = matchedScope.username;
-		url.password = matchedScope.password;
-	}
-}

package/src/crawler/is-external-url.spec.ts

@@ -1,31 +0,0 @@
-import type { ExURL } from '@d-zero/shared/parse-url';
-
-import { tryParseUrl as parseUrl } from '@d-zero/shared/parse-url';
-import { describe, it, expect } from 'vitest';
-
-import { isExternalUrl } from './is-external-url.js';
-
-/**
- * Create a scope map from hostname-URL pairs for testing.
- * @param entries - Array of [hostname, urls] tuples.
- * @returns A map from hostname to parsed ExURL arrays.
- */
-function createScope(entries: [string, string[]][]): Map<string, ExURL[]> {
-	return new Map(
-		entries.map(([h, urls]) => [h, urls.map((u) => parseUrl(u)!).filter(Boolean)]),
-	);
-}
-
-describe('isExternalUrl', () => {
-	it('returns false when hostname is in scope', () => {
-		const url = parseUrl('https://example.com/page')!;
-		const scope = createScope([['example.com', ['https://example.com/']]]);
-		expect(isExternalUrl(url, scope)).toBe(false);
-	});
-
-	it('returns true when hostname is not in scope', () => {
-		const url = parseUrl('https://other.com/page')!;
-		const scope = createScope([['example.com', ['https://example.com/']]]);
-		expect(isExternalUrl(url, scope)).toBe(true);
-	});
-});

package/src/crawler/is-external-url.ts

@@ -1,17 +0,0 @@
-import type { ExURL } from '@d-zero/shared/parse-url';
-
-/**
- * Determine whether a URL is external to the crawl scope.
- *
- * A URL is considered external if its hostname does not appear
- * as a key in the scope map.
- * @param url - The parsed URL to check.
- * @param scope - Map of hostnames to their scope URLs.
- * @returns `true` if the URL is outside the crawl scope.
- */
-export function isExternalUrl(
-	url: ExURL,
-	scope: ReadonlyMap<string, readonly ExURL[]>,
-): boolean {
-	return !scope.has(url.hostname);
-}

package/src/crawler/is-in-any-lower-layer.spec.ts

@@ -1,31 +0,0 @@
-import type { ParseURLOptions } from '@d-zero/shared/parse-url';
-
-import { tryParseUrl as parseUrl } from '@d-zero/shared/parse-url';
-import { describe, it, expect } from 'vitest';
-
-import { isInAnyLowerLayer } from './is-in-any-lower-layer.js';
-
-const defaultOptions: ParseURLOptions = {};
-
-describe('isInAnyLowerLayer', () => {
-	it('returns true when URL is in a lower layer of a scope', () => {
-		const url = parseUrl('https://example.com/blog/post')!;
-		const scopes = [parseUrl('https://example.com/blog/')!];
-		expect(isInAnyLowerLayer(url, scopes, defaultOptions)).toBe(true);
-	});
-
-	it('returns false when URL is not in a lower layer', () => {
-		const url = parseUrl('https://example.com/about')!;
-		const scopes = [parseUrl('https://example.com/blog/')!];
-		expect(isInAnyLowerLayer(url, scopes, defaultOptions)).toBe(false);
-	});
-
-	it('returns true when URL matches one of multiple scopes', () => {
-		const url = parseUrl('https://example.com/docs/api')!;
-		const scopes = [
-			parseUrl('https://example.com/blog/')!,
-			parseUrl('https://example.com/docs/')!,
-		];
-		expect(isInAnyLowerLayer(url, scopes, defaultOptions)).toBe(true);
-	});
-});

package/src/crawler/is-in-any-lower-layer.ts

@@ -1,22 +0,0 @@
-import type { ExURL, ParseURLOptions } from '@d-zero/shared/parse-url';
-
-import { isLowerLayer } from '@d-zero/shared/is-lower-layer';
-
-/**
- * Check whether a URL is in a lower layer (subdirectory) of any scope URL.
- *
- * Tests the URL against each scope URL using the `isLowerLayer` utility,
- * which checks if the URL's path is at the same level or deeper than
- * the scope URL's path.
- * @param url - The parsed URL to check.
- * @param scopes - The list of scope URLs to test against.
- * @param options - URL parsing options used for layer comparison.
- * @returns `true` if the URL is in a lower layer of at least one scope URL.
- */
-export function isInAnyLowerLayer(
-	url: ExURL,
-	scopes: readonly ExURL[],
-	options: ParseURLOptions,
-): boolean {
-	return scopes.some((scope) => isLowerLayer(url.href, scope.href, options));
-}