@d-zero/beholder 0.1.29 → 2.0.0

package/src/scraper.ts

@@ -1,130 +1,119 @@
 import type {
-	ScrapeEventTypes,
+	ChangePhaseEvent,
+	ResourceEntry,
+	ScraperEventTypes,
+	ScraperOptions,
+	ScrapeResult,
+	ExURL,
 	ImageElement,
 	NetworkLog,
 	PageData,
 	ParseURLOptions,
 	Resource,
 	SkippedPageData,
-	ExURL,
 } from './types.js';
-import type { Browser, Page } from 'puppeteer';
+import type { PageScanPhase } from '@d-zero/puppeteer-page-scan';
+import type { Page } from 'puppeteer';
 
-import { beforePageScan } from '@d-zero/puppeteer-page-scan';
-import { parseUrl } from '@d-zero/shared/parse-url';
-import { retry } from '@d-zero/shared/retry';
-import { TypedAwaitEventEmitter } from '@d-zero/shared/typed-await-event-emitter';
-import { launch } from 'puppeteer';
+import { beforePageScan, devicePresets } from '@d-zero/puppeteer-page-scan';
+import { detectCDN } from '@d-zero/shared/detect-cdn';
+import { detectCompress } from '@d-zero/shared/detect-compress';
+import { retry as retryable } from '@d-zero/shared/retry';
+import { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-await-event-emitter';
 
 import { resourceLog, scraperLog } from './debug.js';
 import { getAnchorList, getImageList, getMeta } from './dom-evaluation.js';
-import { fetchDestination } from './fetch-destination.js';
+import { isError } from './is-error.js';
 import { keywordCheck } from './keyword-check.js';
-import { detectCDN, detectCompress, isError } from './utils.js';
+import { parseUrl } from './parse-url.js';
 
 const pid = `${process.pid}`;
 const log = scraperLog.extend(pid);
 const rLog = resourceLog.extend(pid);
 
-const LAUNCH_BROWSER_TIMEOUT = 1000 * 30;
-
-export type ScraperOptions = {
-	isExternal: boolean;
-	isGettingImages: boolean;
-	excludeKeywords: string[];
-	executablePath: string | null;
-	isTitleOnly: boolean;
-	screenshot: string | null;
-} & ParseURLOptions;
-
-export default class Scraper extends TypedAwaitEventEmitter<ScrapeEventTypes> {
-	#browser: Browser | null = null;
-	#url: ExURL | null = null;
-
-	async destroy(isExternal: boolean) {
-		log('Scraper destroys self');
-		if (!this.#url) {
-			throw new Error('The instance is already destroyed.');
-		}
-		if (!this.#browser) {
-			void this.emit('destroyed', {
-				pid: process.pid,
-			});
-			void this.emit('changePhase', {
-				pid: process.pid,
-				name: 'destroyed',
-				url: this.#url,
-				isExternal,
-				message: '',
-			});
-			return;
-		}
-		while (!this.#browser.isConnected()) {
-			log('Browser closes all pages');
-			const pages = await this.#browser.pages();
-			for (const page of pages) {
-				page.removeAllListeners();
-				if (!page.isClosed) {
-					await page.close();
-				}
-			}
-			log('Browser closes self');
-			await this.#browser.close();
-			log('Browser disconnects');
-			await this.#browser.disconnect();
-		}
-		log('Scraper discards browser');
-		this.#browser = null;
-		void this.emit('destroyed', {
-			pid: process.pid,
-		});
-		void this.emit('changePhase', {
-			pid: process.pid,
-			name: 'destroyed',
-			url: this.#url,
-			isExternal,
-			message: '',
-		});
-	}
-
-	async scrapeStart(url: ExURL, options?: Partial<ScraperOptions>, isSkip = false) {
+/**
+ * Page-level scraper that extracts data from a single browser page.
+ *
+ * The scraper returns results as values from `scrapeStart()` rather than
+ * emitting them as events. Only streaming events (changePhase, resourceResponse)
+ * are emitted for progress monitoring.
+ *
+ * The Puppeteer `Page` object is injected externally, and page lifecycle
+ * (including `page.close()`) is managed by the caller.
+ * @example
+ * ```ts
+ * const scraper = new Scraper();
+ * scraper.on('changePhase', (e) => console.log(e.name));
+ * const result = await scraper.scrapeStart(page, url, { isExternal: false });
+ * ```
+ */
+// eslint-disable-next-line unicorn/prefer-event-target -- TypedAwaitEventEmitter is a project-specific typed wrapper, not Node.js EventEmitter
+export default class Scraper extends EventEmitter<ScraperEventTypes> {
+	/** Number of retries for `@retryable`-decorated methods. Set per-scrape from options. */
+	retries?: number;
+
+	/**
+	 * Begins the scraping process for a given URL on the provided Puppeteer page.
+	 *
+	 * Returns a `ScrapeResult` containing the outcome:
+	 * - `type: "success"` with `pageData` on success
+	 * - `type: "skipped"` with `ignored` details when the page is excluded
+	 * - `type: "error"` with `error` details when scraping fails
+	 *
+	 * Sub-resources are collected via the `resourceResponse` event and
+	 * included in the returned `ScrapeResult.resources`.
+	 * @param page - The Puppeteer page instance to use for navigation and DOM evaluation.
+	 * @param url - The extended URL to scrape.
+	 * @param options - Optional scraper configuration overriding defaults.
+	 * @param isSkip - When `true`, the page is immediately skipped without any network requests.
+	 * @returns The scrape result containing the outcome and captured resources.
+	 */
+	async scrapeStart(
+		page: Page,
+		url: ExURL,
+		options?: Partial<ScraperOptions>,
+		isSkip = false,
+	): Promise<ScrapeResult> {
+		this.retries = options?.retries;
 		const isExternal = options?.isExternal ?? false;
-		const isGettingImages = options?.isGettingImages ?? true;
+		const captureImages = options?.captureImages ?? true;
 		const excludeKeywords = options?.excludeKeywords ?? [];
-		const executablePath = options?.executablePath ?? null;
-		const isTitleOnly = options?.isTitleOnly ?? false;
+		const metadataOnly = options?.metadataOnly ?? false;
+		const imageLoadTimeout = options?.imageLoadTimeout ?? 5000;
+		const resources: ResourceEntry[] = [];
 
-		this.#url = url;
 		void this.emit('changePhase', {
 			pid: process.pid,
 			name: 'scrapeStart',
-			url: this.#url,
+			url,
 			isExternal,
 			message: '',
 		});
 
+		// Path-excluded: return SkippedPageData
 		if (isSkip) {
-			void this.emit('ignoreAndSkip', {
-				pid: process.pid,
-				url: this.#url,
-				reason: {
-					matchedText: this.#url.pathname || '',
-					excludeKeywords,
-				},
-			});
 			void this.emit('changePhase', {
 				pid: process.pid,
-				name: 'ignoreAndSkip',
-				url: this.#url,
+				name: 'pageSkipped',
+				url,
 				isExternal,
 				message: 'Matched: excluded path',
 			});
-			return;
+			return {
+				type: 'skipped',
+				resources,
+				ignored: {
+					url,
+					matchedText: url.pathname || '',
+					excludeKeywords,
+				},
+			};
 		}
 
-		if (!this.#url.isHTTP) {
+		// Non-HTTP protocol: return minimal PageData
+		if (!url.isHTTP) {
 			const result: PageData = {
-				url: this.#url,
+				url,
 				isTarget: false,
 				isExternal,
 				redirectPaths: [],
@@ -142,69 +131,44 @@ export default class Scraper extends TypedAwaitEventEmitter<ScrapeEventTypes> {
 				isSkipped: false,
 			};
 
-			void this.emit('scrapeEnd', {
-				pid: process.pid,
-				url: this.#url,
-				timestamp: Date.now(),
-				result,
-			});
-
 			void this.emit('changePhase', {
 				pid: process.pid,
 				name: 'scrapeEnd',
-				url: this.#url,
+				url,
 				isExternal,
 				message: '',
 			});
-			return;
+			return { type: 'success', pageData: result, resources };
 		}
 
-		void this.emit('changePhase', {
-			pid: process.pid,
-			name: 'touchHead',
-			url: this.#url,
-			isExternal,
-			message: '',
-		});
+		let headResult: PageData | SkippedPageData | null = options?.headCheckResult ?? null;
 
-		let result: PageData | SkippedPageData | Error | null = await this.#fetchHead(
-			url,
-			isExternal,
-		);
-
-		if (result instanceof Error) {
-			log('Error(FETCH_HEAD): %s', url.href);
-			void this.emit('error', {
+		if (headResult && metadataOnly) {
+			void this.emit('changePhase', {
 				pid: process.pid,
-				url: this.#url,
-				shutdown: false,
-				error: result,
+				name: 'scrapeEnd',
+				url,
+				isExternal,
+				message: '',
 			});
-			result = null;
-		}
-
-		if (result && isTitleOnly) {
-			void this.emit('scrapeEnd', {
-				pid: process.pid,
-				url: this.#url,
-				timestamp: Date.now(),
-				result: {
-					...result,
+			return {
+				type: 'success',
+				pageData: {
+					...headResult,
 					isTarget: false,
 				},
-			});
-			return;
+				resources,
+			};
 		}
 
-		if (result === null || result.contentType === 'text/html') {
-			const headlessMode: true | 'shell' = url.isSecure ? true : 'shell';
-			const page = await this.#createPage(isExternal, executablePath, headlessMode);
-
-			result = await this.#fetchData(
+		if (headResult === null || headResult.contentType === 'text/html') {
+			const fetchResult = await this.#fetchData(
 				page,
 				url,
 				isExternal,
-				isGettingImages,
+				captureImages,
+				imageLoadTimeout,
+				resources,
 				options,
 			).catch((error) => {
 				if (error instanceof Error) {
@@ -213,28 +177,29 @@ export default class Scraper extends TypedAwaitEventEmitter<ScrapeEventTypes> {
 				return new Error(error);
 			});
 
-			if (result instanceof Error) {
+			if (fetchResult instanceof Error) {
 				log('Error(FETCH_DATA): %s', url.href);
-				void this.emit('error', {
-					pid: process.pid,
-					url: this.#url,
-					shutdown: true,
-					error: result,
-				});
-				await this.destroy(isExternal);
-				return;
+				page.removeAllListeners();
+				return {
+					type: 'error',
+					resources,
+					error: {
+						name: fetchResult.name,
+						message: fetchResult.message,
+						stack: fetchResult.stack,
+						shutdown: true,
+					},
+				};
 			}
 
 			page.removeAllListeners();
-			if (!page.isClosed) {
-				await page.close();
-			}
+			headResult = fetchResult;
 
-			if (!result.isSkipped) {
-				const checkedKeyword = keywordCheck(result.html, excludeKeywords);
+			if (!headResult.isSkipped) {
+				const checkedKeyword = keywordCheck(headResult.html, excludeKeywords);
 
 				if (checkedKeyword) {
-					result = {
+					headResult = {
 						url,
 						isSkipped: true,
 						matched: {
@@ -246,136 +211,135 @@ export default class Scraper extends TypedAwaitEventEmitter<ScrapeEventTypes> {
 				}
 			}
 
-			if (result.isSkipped) {
-				if (result.matched.type === 'path') {
-					return;
+			if (headResult.isSkipped) {
+				if (headResult.matched.type === 'path') {
+					return {
+						type: 'skipped',
+						resources,
+						ignored: {
+							url,
+							matchedText: url.pathname || '',
+							excludeKeywords,
+						},
+					};
 				}
-				void this.emit('ignoreAndSkip', {
-					pid: process.pid,
-					url: this.#url,
-					reason: {
-						matchedText: result.matched.text,
-						excludeKeywords,
-					},
-				});
 				void this.emit('changePhase', {
 					pid: process.pid,
-					name: 'ignoreAndSkip',
-					url: this.#url,
+					name: 'pageSkipped',
+					url,
 					isExternal,
-					message: `Matched: "${result.matched.text}"`,
+					message: `Matched: "${headResult.matched.text}"`,
 				});
-				return;
+				return {
+					type: 'skipped',
+					resources,
+					ignored: {
+						url,
+						matchedText: headResult.matched.text,
+						excludeKeywords,
+					},
+				};
 			}
 		}
 
-		void this.emit('scrapeEnd', {
-			pid: process.pid,
-			url: this.#url,
-			timestamp: Date.now(),
-			result,
-		});
-
 		void this.emit('changePhase', {
 			pid: process.pid,
 			name: 'scrapeEnd',
-			url: this.#url,
+			url,
 			isExternal,
 			message: '',
 		});
 
-		return result;
+		return { type: 'success', pageData: headResult, resources };
 	}
 
-	@retry()
-	async #bootBrowser(
+	/**
+	 * Creates a callback for `@d-zero/puppeteer-page-scan`'s `beforePageScan` listener.
+	 *
+	 * WHY a separate factory: The listener must capture `isExternal` for phase events
+	 * while conforming to the `beforePageScan` listener signature.
+	 * Currently only handles the `scroll` phase to report scroll progress.
+	 * @param isExternal - Whether the current page is external to the crawl scope
+	 * @returns A listener function compatible with `beforePageScan`'s `listener` option
+	 */
+	#createPageScanListener(
 		isExternal: boolean,
-		executablePath: string | null,
-		headless: boolean | 'shell',
-	) {
-		if (!this.#browser) {
+	): (phase: keyof PageScanPhase, data: PageScanPhase[keyof PageScanPhase]) => void {
+		return (phase, data) => {
+			switch (phase) {
+				case 'scroll': {
+					const d = data as PageScanPhase['scroll'];
+					const scrollMsg = Number.isNaN(d.scrollHeight)
+						? `%propeller% ${d.message}`
+						: `%propeller% ${d.scrollY}px/${d.scrollHeight}px (${Math.round((d.scrollY / d.scrollHeight) * 100)}%) ${d.message}`;
+					void this.emit('changePhase', {
+						pid: process.pid,
+						name: 'scrollToBottom',
+						url: null,
+						isExternal,
+						message: scrollMsg,
+					} satisfies ChangePhaseEvent);
+					break;
+				}
+			}
+		};
+	}
+	/**
+	 * Navigates the page to the target URL and extracts full page data.
+	 *
+	 * WHY retryable with 3-min timeout: Page navigation can fail due to transient
+	 * network issues or slow-loading pages. The decorator retries automatically,
+	 * emitting `retryWait` / `retryExhausted` phase events for progress monitoring.
+	 *
+	 * Flow:
+	 * 1. Register request/response listeners to capture sub-resources (internal pages only)
+	 * 2. Navigate to URL via `page.goto()` and track redirect chain
+	 * 3. Wait for DOM content and network idle
+	 * 4. Extract anchors, meta, and optionally images
+	 * 5. Check for keyword exclusion in HTML content
+	 * @param page - Puppeteer page instance
+	 * @param url - Target URL to navigate to
+	 * @param isExternal - Whether the URL is external to the crawl scope
+	 * @param captureImages - Whether to run the image extraction pipeline
+	 * @param imageLoadTimeout - Timeout (ms) for waiting lazy-loaded images to complete
+	 * @param resources - Mutable array to collect captured sub-resources into
+	 * @param options - Additional scraper options (e.g. `disableQueries`, `navigationTimeout`)
+	 * @returns Full page data or skipped page data if an exclusion rule matched
+	 */
+	@retryable({
+		timeout: 3 * 60 * 1000,
+		onWait(this: Scraper, determinedInterval, retryCount, methodName, error) {
 			void this.emit('changePhase', {
 				pid: process.pid,
-				name: 'launchBrowser',
-				url: this.#url,
-				isExternal,
-				message: executablePath || '(executablePath is default)',
+				name: 'retryWait',
+				url: null,
+				isExternal: false,
+				message: `${methodName}: ${error.message} — %countdown(${determinedInterval},${methodName}_${retryCount},s)%s (retry #${retryCount + 1})`,
 			});
-
-			const browser = await launch({
-				headless,
-				timeout: LAUNCH_BROWSER_TIMEOUT,
-				executablePath: executablePath ?? undefined,
-				args: [
-					// TODO: Optional lang
-					'--lang=ja',
-					'--no-zygote',
-					'--ignore-certificate-errors',
-				],
-			}).catch((error) => {
-				if (error instanceof Error) {
-					return error;
-				}
-				throw error;
+		},
+		onGiveUp(this: Scraper, retryCount, error, methodName) {
+			void this.emit('changePhase', {
+				pid: process.pid,
+				name: 'retryExhausted',
+				url: null,
+				isExternal: false,
+				message: `${methodName}: gave up after ${retryCount} retries — ${error.message}`,
 			});
-
-			if (browser instanceof Error) {
-				void this.emit('error', {
-					pid: process.pid,
-					url: this.#url!,
-					shutdown: false,
-					error: browser,
-				});
-				throw browser;
-			}
-
-			this.#browser = browser;
-		} else if (!this.#browser.isConnected()) {
-			await this.#browser.close();
-		}
-
-		return this.#browser;
-	}
-
-	@retry()
-	async #createPage(
-		isExternal: boolean,
-		executablePath: string | null,
-		headless: boolean | 'shell',
-	) {
-		const browser = await this.#bootBrowser(isExternal, executablePath, headless);
-
-		void this.emit('changePhase', {
-			pid: process.pid,
-			name: 'newPage',
-			url: this.#url,
-			isExternal,
-			message: '',
-		});
-
-		const page = await browser.newPage();
-		page.setDefaultNavigationTimeout(0);
-		await page.setUserAgent(
-			'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36',
-		);
-		await page.setExtraHTTPHeaders({
-			// TODO: Optional lang
-			'Accept-Language': 'ja-JP',
-		});
-
-		return page;
-	}
-
-	@retry({
-		timeout: 1 * 60 * 1000, // 1sec,
+		},
 	})
 	async #fetchData(
 		page: Page,
 		url: ExURL,
 		isExternal: boolean,
-		isGettingImages: boolean,
-		options?: ParseURLOptions,
+		captureImages: boolean,
+		imageLoadTimeout: number,
+		resources: ResourceEntry[],
+		options?: Partial<ScraperOptions>,
 	): Promise<PageData | SkippedPageData> {
+		const parseOpts: ParseURLOptions | undefined =
+			options?.disableQueries == null
+				? undefined
+				: { disableQueries: options.disableQueries };
 		const networkLogs: Record<string, NetworkLog> = {};
 
 		page.on('dialog', async (dialog) => {
@@ -390,7 +354,7 @@ export default class Scraper extends TypedAwaitEventEmitter<ScrapeEventTypes> {
 
 		if (!isExternal) {
 			page.on('request', (request) => {
-				const url = parseUrl(request.url(), options);
+				const url = parseUrl(request.url(), parseOpts)!;
 				networkLogs[request.url()] = {
 					url,
 					status: null,
@@ -407,7 +371,7 @@ export default class Scraper extends TypedAwaitEventEmitter<ScrapeEventTypes> {
 
 			const uniqueRes = new Set<string>();
 			page.on('response', (response) => {
-				const resURL = parseUrl(response.url(), options);
+				const resURL = parseUrl(response.url(), parseOpts)!;
 
 				if (uniqueRes.has(resURL.withoutHash)) {
 					return;
@@ -453,6 +417,11 @@ export default class Scraper extends TypedAwaitEventEmitter<ScrapeEventTypes> {
 				};
 
 				rLog('Fetched: %s', resURL.href);
+
+				// Collect resource into the results array
+				resources.push({ log, resource: referredLink, pageUrl: url.withoutHash });
+
+				// Also emit for streaming consumers
 				void this.emit('resourceResponse', {
 					pid: process.pid,
 					url,
@@ -462,12 +431,14 @@ export default class Scraper extends TypedAwaitEventEmitter<ScrapeEventTypes> {
 			});
 		}
 
+		const navigationTimeout = options?.navigationTimeout ?? 60_000;
+
 		void this.emit('changePhase', {
 			pid: process.pid,
 			name: 'openPage',
-			url: this.#url,
+			url,
 			isExternal,
-			message: '',
+			message: `%countdown(${navigationTimeout},openPage_${url.withoutHash},s)%s`,
 		});
 
 		if (url.username && url.password) {
@@ -476,19 +447,24 @@ export default class Scraper extends TypedAwaitEventEmitter<ScrapeEventTypes> {
 			});
 		}
 
-		const res = await page.goto(url.withoutHashAndAuth);
+		const res = await page.goto(url.withoutHashAndAuth, { timeout: navigationTimeout });
 
 		if (!res) {
 			throw new Error('The method Page.goto returned null');
 		}
 
-		const destUrl = parseUrl(page.url(), options);
-		const redirectPaths = res
-			.request()
-			.redirectChain()
-			.map((req) => req.url());
-		if (destUrl.withoutHash !== url.withoutHash) {
-			redirectPaths.push(destUrl.withoutHash);
+		const destUrl = parseUrl(page.url(), parseOpts)!;
+		const redirectPaths = new Set<string>();
+
+		if (url.withoutHash !== destUrl.withoutHash) {
+			const redirectChain = res
+				.request()
+				.redirectChain()
+				.map((req) => req.url());
+			for (const redirectPath of redirectChain) {
+				redirectPaths.add(redirectPath);
+			}
+			redirectPaths.add(destUrl.withoutHash);
 		}
 
 		if (destUrl.hostname !== url.hostname) {
@@ -507,7 +483,7 @@ export default class Scraper extends TypedAwaitEventEmitter<ScrapeEventTypes> {
 				url,
 				isTarget: false,
 				isExternal,
-				redirectPaths,
+				redirectPaths: [...redirectPaths],
 				status,
 				statusText,
 				contentType,
@@ -526,7 +502,7 @@ export default class Scraper extends TypedAwaitEventEmitter<ScrapeEventTypes> {
 		void this.emit('changePhase', {
 			pid: process.pid,
 			name: 'loadDOMContent',
-			url: this.#url,
+			url,
 			isExternal,
 			message: '',
 		});
@@ -538,7 +514,7 @@ export default class Scraper extends TypedAwaitEventEmitter<ScrapeEventTypes> {
 		void this.emit('changePhase', {
 			pid: process.pid,
 			name: 'getHTML',
-			url: this.#url,
+			url,
 			isExternal,
 			message: '',
 		});
@@ -556,7 +532,7 @@ export default class Scraper extends TypedAwaitEventEmitter<ScrapeEventTypes> {
 				url,
 				isTarget: false,
 				isExternal,
-				redirectPaths,
+				redirectPaths: [...redirectPaths],
 				status,
 				statusText,
 				contentType,
@@ -574,8 +550,8 @@ export default class Scraper extends TypedAwaitEventEmitter<ScrapeEventTypes> {
 
 		void this.emit('changePhase', {
 			pid: process.pid,
-			name: 'waitNetworkIdleZero',
-			url: this.#url,
+			name: 'waitNetworkIdle',
+			url,
 			isExternal,
 			message: '',
 		});
@@ -587,28 +563,44 @@ export default class Scraper extends TypedAwaitEventEmitter<ScrapeEventTypes> {
 		void this.emit('changePhase', {
 			pid: process.pid,
 			name: 'getAnchors',
-			url: this.#url,
+			url,
 			isExternal,
 			message: '',
 		});
-		const anchorList = await getAnchorList(page, options);
+		const anchorList = await getAnchorList(page, parseOpts);
 
 		void this.emit('changePhase', {
 			pid: process.pid,
 			name: 'getMeta',
-			url: this.#url,
+			url,
 			isExternal,
 			message: '',
 		});
 		const meta = await getMeta(page);
 
-		const imageList = isGettingImages ? await this.#fetchImages(page, isExternal) : [];
+		const imageList = captureImages
+			? await (async () => {
+					void this.emit('changePhase', {
+						pid: process.pid,
+						name: 'extractImages',
+						url,
+						isExternal,
+						message: '',
+					});
+					return this.#fetchImages(
+						page,
+						url.withoutHashAndAuth,
+						isExternal,
+						imageLoadTimeout,
+					);
+				})()
+			: [];
 
 		return {
 			url,
 			isTarget: true,
 			isExternal,
-			redirectPaths,
+			redirectPaths: [...redirectPaths],
 			status,
 			statusText,
 			contentType,
@@ -621,50 +613,96 @@ export default class Scraper extends TypedAwaitEventEmitter<ScrapeEventTypes> {
 			isSkipped: false,
 		};
 	}
-
-	@retry()
-	async #fetchHead(url: ExURL, isExternal: boolean) {
-		return await fetchDestination(url, isExternal);
-	}
-
-	@retry({
-		timeout: 5 * 60 * 1000, // 5sec
+	/**
+	 * Extracts image data from the page across multiple device presets.
+	 *
+	 * WHY multiple device presets: Images may differ between desktop and mobile
+	 * due to responsive `<picture>` / `srcset`. Capturing both `desktop-compact`
+	 * and `mobile-small` viewports reveals responsive image issues.
+	 *
+	 * WHY retryable with 5-min timeout and `fallback: []`: Image extraction is
+	 * best-effort. If all retries fail, an empty array is returned rather than
+	 * failing the entire page scrape.
+	 * @param page - Puppeteer page instance
+	 * @param url - The page URL string (without hash and auth)
+	 * @param isExternal - Whether the page is external
+	 * @param imageLoadTimeout - Timeout (ms) for waiting images to complete loading
+	 * @returns Array of image elements from all device presets
+	 */
+	@retryable({
+		timeout: 5 * 60 * 1000,
 		fallback: [],
+		onWait(this: Scraper, determinedInterval, retryCount, methodName, error) {
+			void this.emit('changePhase', {
+				pid: process.pid,
+				name: 'retryWait',
+				url: null,
+				isExternal: false,
+				message: `${methodName}: ${error.message} — %countdown(${determinedInterval},${methodName}_${retryCount},s)%s (retry #${retryCount + 1} / images)`,
+			});
+		},
+		onGiveUp(this: Scraper, retryCount, error, methodName) {
+			void this.emit('changePhase', {
+				pid: process.pid,
+				name: 'retryExhausted',
+				url: null,
+				isExternal: false,
+				message: `${methodName}: gave up after ${retryCount} retries — ${error.message}`,
+			});
+		},
 	})
-	async #fetchImages(page: Page, isExternal: boolean): Promise<ImageElement[]> {
-		const url = this.#url!.withoutHashAndAuth;
-		const imageList: ImageElement[] = [];
-
-		const devices: { name: string; width: number; resolution?: number }[] = [
-			{ name: 'desktop', width: 1280 },
-			{ name: 'mobile', width: 320, resolution: 2 },
+	async #fetchImages(
+		page: Page,
+		url: string,
+		isExternal: boolean,
+		imageLoadTimeout: number,
+	): Promise<ImageElement[]> {
+		const listener = this.#createPageScanListener(isExternal);
+		const devices: { key: string; preset: { width: number; resolution?: number } }[] = [
+			{ key: 'desktop-compact', preset: devicePresets['desktop-compact'] },
+			{ key: 'mobile-small', preset: devicePresets['mobile-small'] },
 		];
+		const imageList: ImageElement[] = [];
 
-		for (const device of devices) {
+		for (const { key, preset } of devices) {
 			void this.emit('changePhase', {
 				pid: process.pid,
 				name: 'setViewport',
-				url: this.#url,
+				url: null,
 				isExternal,
-				message: device.name,
+				message: `📷 ${key} ↔️ ${preset.width}px`,
 			});
 
 			await beforePageScan(page, url, {
-				name: device.name,
-				width: device.width,
-				resolution: device.resolution,
+				name: key,
+				width: preset.width,
+				resolution: preset.resolution,
+				listener,
 				timeout: 5000,
 			});
 
 			void this.emit('changePhase', {
 				pid: process.pid,
-				name: 'getImages',
-				url: this.#url,
+				name: 'waitImageLoad',
+				url: null,
 				isExternal,
-				message: device.name,
+				message: `📷 ${key}: Waiting for images%dots%`,
 			});
 
-			const images = await getImageList(page, device.width);
+			await page
+				.waitForFunction(() => [...document.images].every((img) => img.complete), {
+					timeout: imageLoadTimeout,
+				})
+				.catch(() => {});
+
+			void this.emit('changePhase', {
+				pid: process.pid,
+				name: 'getImages',
+				url: null,
+				isExternal,
+				message: `📸 ${key}: Extracting images%dots%`,
+			});
+			const images = await getImageList(page, preset.width);
 			imageList.push(...images);
 		}