@nitpicker/crawler 0.4.2 → 0.4.3

package/src/crawler/link-to-page-data.spec.ts

@@ -1,133 +0,0 @@
-import { tryParseUrl as parseUrl } from '@d-zero/shared/parse-url';
-import { describe, it, expect } from 'vitest';
-
-import { linkToPageData } from './link-to-page-data.js';
-
-/**
- *
- * @param overrides
- */
-function createLink(overrides: Record<string, unknown> = {}) {
-	const url = parseUrl('https://example.com/page')!;
-	return {
-		url,
-		isExternal: false,
-		isLowerLayer: true,
-		...overrides,
-	};
-}
-
-describe('linkToPageData', () => {
-	it('converts link without dest to default PageData', () => {
-		const link = createLink();
-		const result = linkToPageData(link);
-		expect(result.url).toBe(link.url);
-		expect(result.status).toBe(-1);
-		expect(result.statusText).toBe('UnknownError');
-		expect(result.contentType).toBeNull();
-		expect(result.contentLength).toBeNull();
-		expect(result.responseHeaders).toBeNull();
-		expect(result.redirectPaths).toStrictEqual([]);
-		expect(result.meta.title).toBe('');
-		expect(result.anchorList).toStrictEqual([]);
-		expect(result.imageList).toStrictEqual([]);
-		expect(result.html).toBe('');
-		expect(result.isSkipped).toBe(false);
-	});
-
-	it('sets isTarget=true for internal link', () => {
-		const link = createLink({ isExternal: false });
-		const result = linkToPageData(link);
-		expect(result.isTarget).toBe(true);
-		expect(result.isExternal).toBe(false);
-	});
-
-	it('sets isTarget=false for external link', () => {
-		const link = createLink({ isExternal: true });
-		const result = linkToPageData(link);
-		expect(result.isTarget).toBe(false);
-		expect(result.isExternal).toBe(true);
-	});
-
-	it('uses dest data when available', () => {
-		const link = createLink({
-			dest: {
-				redirectPaths: ['/redirect'],
-				status: 200,
-				statusText: 'OK',
-				contentType: 'text/html',
-				contentLength: 1024,
-				responseHeaders: { 'content-type': 'text/html' },
-				title: 'Test Page',
-			},
-		});
-		const result = linkToPageData(link);
-		expect(result.status).toBe(200);
-		expect(result.statusText).toBe('OK');
-		expect(result.contentType).toBe('text/html');
-		expect(result.contentLength).toBe(1024);
-		expect(result.redirectPaths).toStrictEqual(['/redirect']);
-		expect(result.meta.title).toBe('Test Page');
-	});
-
-	it('uses dest title when provided', () => {
-		const link = createLink({
-			dest: {
-				redirectPaths: [],
-				status: 200,
-				statusText: 'OK',
-				contentType: 'text/html',
-				contentLength: 0,
-				responseHeaders: {},
-				title: 'My Title',
-			},
-		});
-		const result = linkToPageData(link);
-		expect(result.meta.title).toBe('My Title');
-	});
-
-	it('falls back to empty title when dest has no title', () => {
-		const link = createLink({
-			dest: {
-				redirectPaths: [],
-				status: 200,
-				statusText: 'OK',
-				contentType: null,
-				contentLength: null,
-				responseHeaders: null,
-			},
-		});
-		const result = linkToPageData(link);
-		expect(result.meta.title).toBe('');
-	});
-
-	it('records current behavior: status 0 becomes -1 via || operator', () => {
-		const link = createLink({
-			dest: {
-				redirectPaths: [],
-				status: 0,
-				statusText: 'OK',
-				contentType: null,
-				contentLength: null,
-				responseHeaders: null,
-			},
-		});
-		const result = linkToPageData(link);
-		expect(result.status).toBe(-1);
-	});
-
-	it('records current behavior: contentLength 0 becomes null via || operator', () => {
-		const link = createLink({
-			dest: {
-				redirectPaths: [],
-				status: 200,
-				statusText: 'OK',
-				contentType: null,
-				contentLength: 0,
-				responseHeaders: null,
-			},
-		});
-		const result = linkToPageData(link);
-		expect(result.contentLength).toBeNull();
-	});
-});

package/src/crawler/link-to-page-data.ts

@@ -1,34 +0,0 @@
-import type { Link, PageData } from '../utils/index.js';
-
-/**
- * Convert a {@link Link} object into a {@link PageData} structure.
- *
- * Creates a minimal PageData from the link's destination metadata. This is used
- * when a full scrape is not performed (e.g., for external pages when
- * `fetchExternal` is disabled, or when a scrape error produces a fallback result).
- *
- * Missing destination fields are filled with sensible defaults (e.g., status -1
- * for unknown, empty arrays for anchors/images, empty string for HTML).
- * @param link - The link to convert, containing URL and optional destination metadata.
- * @returns A PageData object populated from the link's available data.
- */
-export function linkToPageData(link: Link): PageData {
-	return {
-		url: link.url,
-		redirectPaths: link.dest?.redirectPaths || [],
-		isTarget: !link.isExternal,
-		isExternal: link.isExternal,
-		status: link.dest?.status || -1,
-		statusText: link.dest?.statusText || 'UnknownError',
-		contentType: link.dest?.contentType || null,
-		contentLength: link.dest?.contentLength || null,
-		responseHeaders: link.dest?.responseHeaders || null,
-		meta: {
-			title: link.dest?.title || '',
-		},
-		anchorList: [],
-		imageList: [],
-		html: '',
-		isSkipped: false,
-	};
-}

package/src/crawler/net-timeout-error.spec.ts

@@ -1,25 +0,0 @@
-import { describe, it, expect } from 'vitest';
-
-import NetTimeoutError from './net-timeout-error.js';
-
-describe('NetTimeoutError', () => {
-	it('has the name NetTimeoutError', () => {
-		const error = new NetTimeoutError();
-		expect(error.name).toBe('NetTimeoutError');
-	});
-
-	it('includes the URL in the message when provided', () => {
-		const error = new NetTimeoutError('https://example.com/');
-		expect(error.message).toBe('Timeout: https://example.com/');
-	});
-
-	it('uses a generic message when no URL is provided', () => {
-		const error = new NetTimeoutError();
-		expect(error.message).toBe('Timeout');
-	});
-
-	it('is an instance of Error', () => {
-		const error = new NetTimeoutError();
-		expect(error).toBeInstanceOf(Error);
-	});
-});

package/src/crawler/net-timeout-error.ts

@@ -1,11 +0,0 @@
-/**
- * Error thrown when a network request (typically an HTTP HEAD check)
- * exceeds the allowed timeout duration. Used by `fetchDestination`
- * to signal that the destination server did not respond in time.
- */
-export default class NetTimeoutError extends Error {
-	constructor(url?: string) {
-		super(url ? `Timeout: ${url}` : 'Timeout');
-	}
-	override name = 'NetTimeoutError';
-}

package/src/crawler/protocol-agnostic-key.spec.ts

@@ -1,40 +0,0 @@
-import { describe, it, expect } from 'vitest';
-
-import { protocolAgnosticKey } from './protocol-agnostic-key.js';
-
-describe('protocolAgnosticKey', () => {
-	it('strips https: from URL', () => {
-		expect(protocolAgnosticKey('https://example.com/page')).toBe('//example.com/page');
-	});
-
-	it('strips http: from URL', () => {
-		expect(protocolAgnosticKey('http://example.com/page')).toBe('//example.com/page');
-	});
-
-	it('produces the same key for http and https', () => {
-		expect(protocolAgnosticKey('http://example.com/page')).toBe(
-			protocolAgnosticKey('https://example.com/page'),
-		);
-	});
-
-	it('handles URLs with ports', () => {
-		expect(protocolAgnosticKey('https://example.com:8080/page')).toBe(
-			'//example.com:8080/page',
-		);
-	});
-
-	it('does not strip non-HTTP protocols', () => {
-		expect(protocolAgnosticKey('ftp://example.com/file')).toBe('ftp://example.com/file');
-	});
-
-	it('handles URL without path', () => {
-		expect(protocolAgnosticKey('https://example.com')).toBe('//example.com');
-	});
-
-	it('handles withoutHashAndAuth style strings', () => {
-		// withoutHashAndAuth produces strings like "https://example.com/path"
-		const httpKey = protocolAgnosticKey('http://example.com/path');
-		const httpsKey = protocolAgnosticKey('https://example.com/path');
-		expect(httpKey).toBe(httpsKey);
-	});
-});

package/src/crawler/protocol-agnostic-key.ts

@@ -1,11 +0,0 @@
-/**
- * Returns a URL string with the protocol prefix (`http:` / `https:`) stripped.
- *
- * Used as a deduplication key so that HTTP and HTTPS variants of the
- * same URL are treated as identical during crawling.
- * @param url - A URL string (e.g. `"https://example.com/page"`)
- * @returns The URL without its protocol prefix (e.g. `"//example.com/page"`)
- */
-export function protocolAgnosticKey(url: string): string {
-	return url.replace(/^https?:/, '');
-}

package/src/crawler/reconstruct-url.spec.ts

@@ -1,37 +0,0 @@
-import { describe, it, expect } from 'vitest';
-
-import { decomposeUrl } from './decompose-url.js';
-import { reconstructUrl } from './reconstruct-url.js';
-
-describe('reconstructUrl', () => {
-	it('replaces a path segment token', () => {
-		const decomposed = decomposeUrl('//example.com/page/3')!;
-		const result = reconstructUrl(decomposed, 1, '4');
-		expect(result).toBe('//example.com/page/4');
-	});
-
-	it('replaces a query value token', () => {
-		const decomposed = decomposeUrl('//example.com/list?p=2&sort=name')!;
-		const result = reconstructUrl(decomposed, 1, '3');
-		expect(result).toBe('//example.com/list?p=3&sort=name');
-	});
-
-	it('preserves protocol', () => {
-		const decomposed = decomposeUrl('https://example.com/page/1')!;
-		const result = reconstructUrl(decomposed, 1, '2');
-		expect(result).toBe('https://example.com/page/2');
-	});
-
-	it('encodes special characters in query keys and values', () => {
-		const decomposed = decomposeUrl('//example.com/search?q=hello&lang=en')!;
-		// queryKeys are sorted: ['lang', 'q'], tokenIndex 1 replaces lang's value
-		const result = reconstructUrl(decomposed, 1, 'a&b=c');
-		expect(result).toContain('lang=a%26b%3Dc');
-	});
-
-	it('encodes unicode characters in query values', () => {
-		const decomposed = decomposeUrl('//example.com/search?q=test')!;
-		const result = reconstructUrl(decomposed, 1, '日本語');
-		expect(result).toContain(encodeURIComponent('日本語'));
-	});
-});

package/src/crawler/reconstruct-url.ts

@@ -1,37 +0,0 @@
-import type { DecomposedUrl } from './decompose-url.js';
-
-/**
- * Reconstructs a URL string from a decomposed representation with one
- * token replaced at the specified index.
- * @param decomposed - The decomposed URL to reconstruct
- * @param tokenIndex - Index in the combined token array (path segments + query values)
- * @param newValue - The replacement value for the token at `tokenIndex`
- * @returns The reconstructed URL string
- */
-export function reconstructUrl(
-	decomposed: DecomposedUrl,
-	tokenIndex: number,
-	newValue: string,
-): string {
-	const { host, pathSegments, queryKeys, queryValues, protocol } = decomposed;
-	const newPathSegments = [...pathSegments];
-	const newQueryValues = [...queryValues];
-
-	if (tokenIndex < pathSegments.length) {
-		newPathSegments[tokenIndex] = newValue;
-	} else {
-		newQueryValues[tokenIndex - pathSegments.length] = newValue;
-	}
-
-	let url = `${protocol}//${host}`;
-	if (newPathSegments.length > 0) {
-		url += `/${newPathSegments.join('/')}`;
-	}
-	if (queryKeys.length > 0) {
-		const pairs = queryKeys.map(
-			(k, i) => `${encodeURIComponent(k)}=${encodeURIComponent(newQueryValues[i] ?? '')}`,
-		);
-		url += `?${pairs.join('&')}`;
-	}
-	return url;
-}

package/src/crawler/robots-checker.spec.ts

@@ -1,104 +0,0 @@
-import http from 'node:http';
-
-import { tryParseUrl as parseUrl } from '@d-zero/shared/parse-url';
-import { describe, it, expect, beforeAll, afterAll } from 'vitest';
-
-import { RobotsChecker } from './robots-checker.js';
-
-const ROBOTS_TXT = `
-User-agent: *
-Disallow: /secret/
-Allow: /
-
-User-agent: Nitpicker
-Disallow: /admin/
-`;
-
-let server: http.Server;
-let port: number;
-
-beforeAll(async () => {
-	server = http.createServer((req, res) => {
-		if (req.url === '/robots.txt') {
-			res.writeHead(200, { 'Content-Type': 'text/plain' });
-			res.end(ROBOTS_TXT);
-		} else {
-			res.writeHead(200);
-			res.end('ok');
-		}
-	});
-	await new Promise<void>((resolve) => {
-		server.listen(0, () => resolve());
-	});
-	const address = server.address();
-	port = typeof address === 'object' && address ? address.port : 0;
-});
-
-afterAll(async () => {
-	await new Promise<void>((resolve) => {
-		server.close(() => resolve());
-	});
-});
-
-describe('RobotsChecker', () => {
-	it('blocks URLs disallowed by robots.txt', async () => {
-		const checker = new RobotsChecker('Nitpicker', true);
-		const url = parseUrl(`http://127.0.0.1:${port}/admin/settings`)!;
-		const result = await checker.isAllowed(url);
-		expect(result).toBe(false);
-	});
-
-	it('allows URLs permitted by robots.txt', async () => {
-		const checker = new RobotsChecker('Nitpicker', true);
-		const url = parseUrl(`http://127.0.0.1:${port}/public/page`)!;
-		const result = await checker.isAllowed(url);
-		expect(result).toBe(true);
-	});
-
-	it('always allows when disabled', async () => {
-		const checker = new RobotsChecker('Nitpicker', false);
-		const url = parseUrl(`http://127.0.0.1:${port}/admin/settings`)!;
-		const result = await checker.isAllowed(url);
-		expect(result).toBe(true);
-	});
-
-	it('allows non-HTTP URLs', async () => {
-		const checker = new RobotsChecker('Nitpicker', true);
-		const url = parseUrl('mailto:test@example.com')!;
-		const result = await checker.isAllowed(url);
-		expect(result).toBe(true);
-	});
-
-	it('allows all URLs when robots.txt is absent', async () => {
-		const noRobotsServer = http.createServer((_req, res) => {
-			res.writeHead(404);
-			res.end();
-		});
-		await new Promise<void>((resolve) => {
-			noRobotsServer.listen(0, () => resolve());
-		});
-		const address = noRobotsServer.address();
-		const noRobotsPort = typeof address === 'object' && address ? address.port : 0;
-
-		try {
-			const checker = new RobotsChecker('Nitpicker', true);
-			const url = parseUrl(`http://127.0.0.1:${noRobotsPort}/secret/page`)!;
-			const result = await checker.isAllowed(url);
-			expect(result).toBe(true);
-		} finally {
-			await new Promise<void>((resolve) => {
-				noRobotsServer.close(() => resolve());
-			});
-		}
-	});
-
-	it('caches robots.txt per origin', async () => {
-		const checker = new RobotsChecker('Nitpicker', true);
-		const url1 = parseUrl(`http://127.0.0.1:${port}/admin/page1`)!;
-		const url2 = parseUrl(`http://127.0.0.1:${port}/admin/page2`)!;
-
-		await checker.isAllowed(url1);
-		const result = await checker.isAllowed(url2);
-		expect(result).toBe(false);
-	});
-});

package/src/crawler/robots-checker.ts

@@ -1,73 +0,0 @@
-import type { ExURL } from '@d-zero/shared/parse-url';
-
-import { crawlerLog } from '../debug.js';
-
-import { fetchRobotsTxt } from './fetch-robots-txt.js';
-
-/**
- * Derives the origin string from an ExURL (e.g., `https://example.com:8080`).
- * @param url - The extended URL.
- * @returns The origin string.
- */
-function getOrigin(url: ExURL): string {
-	return `${url.protocol}//${url.hostname}${url.port ? `:${url.port}` : ''}`;
-}
-
-/**
- * Checks whether a URL is allowed by the site's robots.txt rules.
- *
- * Caches robots.txt per origin so each origin is fetched at most once.
- * When disabled (i.e., `ignoreRobots` mode), all URLs are allowed.
- */
-export class RobotsChecker {
-	/** Cache of parsed robots.txt per origin. `null` means no robots.txt or fetch failed. */
-	readonly #cache = new Map<string, Awaited<ReturnType<typeof fetchRobotsTxt>>>();
-	/** When `false`, robots.txt checking is disabled and all URLs are allowed. */
-	readonly #enabled: boolean;
-	/** User-Agent string used for robots.txt rule matching and HTTP requests. */
-	readonly #userAgent: string;
-
-	/**
-	 * Create a new RobotsChecker.
-	 * @param userAgent - User-Agent string for rule matching and fetching robots.txt.
-	 * @param enabled - Whether robots.txt checking is enabled. When `false`, {@link isAllowed} always returns `true`.
-	 */
-	constructor(userAgent: string, enabled: boolean) {
-		this.#userAgent = userAgent;
-		this.#enabled = enabled;
-	}
-
-	/**
-	 * Check whether the given URL is allowed by the site's robots.txt.
-	 *
-	 * Fetches and caches robots.txt per origin on first access.
-	 * Returns `true` if robots.txt checking is disabled, if no robots.txt
-	 * exists, or if the URL is explicitly allowed.
-	 * @param url - The URL to check.
-	 * @returns `true` if the URL is allowed, `false` if blocked.
-	 */
-	async isAllowed(url: ExURL): Promise<boolean> {
-		if (!this.#enabled) {
-			return true;
-		}
-
-		if (!url.isHTTP) {
-			return true;
-		}
-
-		const origin = getOrigin(url);
-		if (!this.#cache.has(origin)) {
-			crawlerLog('Fetching robots.txt for %s', origin);
-			const robot = await fetchRobotsTxt(origin, this.#userAgent);
-			this.#cache.set(origin, robot);
-		}
-
-		const robot = this.#cache.get(origin);
-		if (!robot) {
-			return true;
-		}
-
-		const allowed = robot.isAllowed(url.href, this.#userAgent);
-		return allowed !== false;
-	}
-}

package/src/crawler/should-discard-predicted.spec.ts

@@ -1,125 +0,0 @@
-import type { ScrapeResult } from '@d-zero/beholder';
-
-import { describe, expect, it } from 'vitest';
-
-import { shouldDiscardPredicted } from './should-discard-predicted.js';
-
-describe('shouldDiscardPredicted', () => {
-	it('type=error は破棄する', () => {
-		const result: ScrapeResult = {
-			type: 'error',
-			resources: [],
-			error: { name: 'Error', message: 'connection refused', shutdown: false },
-		};
-		expect(shouldDiscardPredicted(result)).toBe(true);
-	});
-
-	it('type=success, status=404 は破棄する', () => {
-		const result: ScrapeResult = {
-			type: 'success',
-			resources: [],
-			pageData: {
-				url: {} as never,
-				redirectPaths: [],
-				isTarget: false,
-				isExternal: false,
-				status: 404,
-				statusText: 'Not Found',
-				contentType: 'text/html',
-				contentLength: 0,
-				responseHeaders: {},
-				meta: { title: '' },
-				anchorList: [],
-				imageList: [],
-				html: '',
-				isSkipped: false,
-			},
-		};
-		expect(shouldDiscardPredicted(result)).toBe(true);
-	});
-
-	it('type=success, status=500 は破棄する', () => {
-		const result: ScrapeResult = {
-			type: 'success',
-			resources: [],
-			pageData: {
-				url: {} as never,
-				redirectPaths: [],
-				isTarget: false,
-				isExternal: false,
-				status: 500,
-				statusText: 'Internal Server Error',
-				contentType: 'text/html',
-				contentLength: 0,
-				responseHeaders: {},
-				meta: { title: '' },
-				anchorList: [],
-				imageList: [],
-				html: '',
-				isSkipped: false,
-			},
-		};
-		expect(shouldDiscardPredicted(result)).toBe(true);
-	});
-
-	it('type=success, status=200 は保持する', () => {
-		const result: ScrapeResult = {
-			type: 'success',
-			resources: [],
-			pageData: {
-				url: {} as never,
-				redirectPaths: [],
-				isTarget: true,
-				isExternal: false,
-				status: 200,
-				statusText: 'OK',
-				contentType: 'text/html',
-				contentLength: 1024,
-				responseHeaders: {},
-				meta: { title: 'Test' },
-				anchorList: [],
-				imageList: [],
-				html: '<html></html>',
-				isSkipped: false,
-			},
-		};
-		expect(shouldDiscardPredicted(result)).toBe(false);
-	});
-
-	it('type=success, status=301 は保持する', () => {
-		const result: ScrapeResult = {
-			type: 'success',
-			resources: [],
-			pageData: {
-				url: {} as never,
-				redirectPaths: [],
-				isTarget: false,
-				isExternal: false,
-				status: 301,
-				statusText: 'Moved Permanently',
-				contentType: 'text/html',
-				contentLength: 0,
-				responseHeaders: {},
-				meta: { title: '' },
-				anchorList: [],
-				imageList: [],
-				html: '',
-				isSkipped: false,
-			},
-		};
-		expect(shouldDiscardPredicted(result)).toBe(false);
-	});
-
-	it('type=skipped は破棄する', () => {
-		const result: ScrapeResult = {
-			type: 'skipped',
-			resources: [],
-			ignored: {
-				url: {} as never,
-				matchedText: 'keyword',
-				excludeKeywords: ['keyword'],
-			},
-		};
-		expect(shouldDiscardPredicted(result)).toBe(true);
-	});
-});

package/src/crawler/should-discard-predicted.ts

@@ -1,33 +0,0 @@
-import type { ScrapeResult } from '@d-zero/beholder';
-
-import { isError } from '@d-zero/beholder';
-
-/**
- * Determines whether a predicted URL's scrape result should be discarded.
- *
- * Predicted URLs are pre-emptively pushed into the crawl queue before
- * knowing if they exist. This function filters out invalid results:
- * - `error` type → discard (server unreachable, timeout, etc.)
- * - `skipped` type → discard (matched exclusion rule)
- * - `success` with HTTP error status (4xx/5xx) → discard
- * - `success` with 2xx/3xx → keep
- * @param result - The scrape result for the predicted URL
- * @returns `true` if the result should be discarded (not saved to archive)
- */
-export function shouldDiscardPredicted(result: ScrapeResult): boolean {
-	switch (result.type) {
-		case 'error': {
-			return true;
-		}
-		case 'skipped': {
-			return true;
-		}
-		case 'success': {
-			if (!result.pageData) return true;
-			return isError(result.pageData.status);
-		}
-		default: {
-			return true;
-		}
-	}
-}