@d-zero/beholder 4.2.2 → 5.0.0

package/src/dom-evaluation.ts

@@ -371,6 +371,17 @@ export async function getAnchorList(
 		return [];
 	}
 
+	// `$anchors` の各 ElementHandle は CDP 側のリモートオブジェクト参照を保持する
+	// ため、解放しないとページごとにハンドルがリークする。AsyncDisposableStack に
+	// まとめて登録して解放するが、`await using` でこの関数のスコープ脱出時に
+	// 即座に解放してはいけない — タイムアウトで打ち切られた後もバックグラウンドで
+	// resolveAnchor が動き続けている間にハンドルを解放すると、実行中の CDP 呼び出しが
+	// 失敗する。解放は work() の完了（成功・失敗いずれも）を待って初めて行う。
+	const anchorHandles = new AsyncDisposableStack();
+	for (const $anchor of $anchors) {
+		anchorHandles.use($anchor);
+	}
+
 	const collected: AnchorData[] = [];
 	let axHits = 0;
 	let textFallbacks = 0;
@@ -410,7 +421,14 @@ export async function getAnchorList(
 		);
 	};
 
-	const { timeout: timedOut } = await raceWithTimeout(work, timeout);
+	const workPromise = work();
+	// work() が真に完了した時点（タイムアウトで打ち切られた場合は、その後
+	// バックグラウンドで走り続ける resolveAnchor がすべて解決した時点）で
+	// ハンドルを解放する。disposeAsync() 自体の失敗も無視してよい
+	// （ベストエフォートの後始末であり、呼び出し元への影響はない）。
+	void workPromise.finally(() => anchorHandles.disposeAsync()).catch(() => {});
+
+	const { timeout: timedOut } = await raceWithTimeout(() => workPromise, timeout);
 	cancelled = true;
 	if (timedOut) {
 		log(

package/src/image-scan-code.ts

@@ -0,0 +1,34 @@
+/**
+ * Numeric outcome codes for a single device-preset image scan performed by
+ * `Scraper#fetchImages`. Small integers (fit a `Uint8`) so callers that
+ * persist per-page scan results to a database column can use an integer
+ * rather than a string enum. A `null` value (not part of this const) means
+ * the device preset was never attempted — non-HTML/external/non-HTTP pages,
+ * or `captureImages: false`.
+ * @example
+ * ```ts
+ * import { IMAGE_SCAN_CODE } from '@d-zero/beholder';
+ *
+ * db.insert({ imageScanMobile: pageData.imageScan.mobile ?? null });
+ * if (pageData.imageScan.mobile === IMAGE_SCAN_CODE.SCROLL_HEIGHT_EXCEEDED) {
+ *   // mobile images were skipped because scrollHeight exceeded the limit
+ * }
+ * ```
+ */
+export const IMAGE_SCAN_CODE = {
+	/** `beforePageScan` reached `settled: 'idle'`; images extracted normally. */
+	OK: 0,
+	/** `beforePageScan` reached `settled: 'degraded'`; images extracted, but navigation never went network-idle so the result carries lower confidence. */
+	DEGRADED: 1,
+	/** Navigation never settled and the post-timeout frame/URL/readyState check also failed (`NavigationUnsettledError`). */
+	NAV_UNSETTLED: 2,
+	/** The main frame or session was lost mid-scan (detached Frame, session closed, execution context destroyed, or "Not attached to an active page"). */
+	FRAME_LOST: 3,
+	/** `document.body.scrollHeight` exceeded the scan's `maxScrollHeight` guard; scroll and extraction were skipped. */
+	SCROLL_HEIGHT_EXCEEDED: 4,
+	/** An error occurred that does not match any of the categories above. */
+	UNKNOWN: 255,
+} as const;
+
+/** Numeric outcome code for one device preset's image scan. See {@link IMAGE_SCAN_CODE}. */
+export type ImageScanCode = (typeof IMAGE_SCAN_CODE)[keyof typeof IMAGE_SCAN_CODE];

package/src/index.ts

@@ -22,6 +22,8 @@ export { detectCompress } from '@d-zero/shared/detect-compress';
 export type { CompressType } from '@d-zero/shared/detect-compress';
 export { detectCDN } from '@d-zero/shared/detect-cdn';
 export type { CDNType } from '@d-zero/shared/detect-cdn';
+export { IMAGE_SCAN_CODE } from './image-scan-code.js';
+export type { ImageScanCode } from './image-scan-code.js';
 export type { ScrapeResult, ResourceEntry, ConsoleLogEntry, PageData } from './types.js';
 export type { ScraperOptions, ChangePhaseEvent, ScraperEventTypes } from './types.js';
 export type {
@@ -32,6 +34,7 @@ export type {
 	SkippedPageData,
 	NetworkLog,
 	ScrollHeightData,
+	ImageScanData,
 	MainContentsData,
 	MainContentsMainTag,
 	MainContentsHeading,

package/src/scraper.ts

@@ -7,6 +7,7 @@ import type {
 	ScrapeResult,
 	ExURL,
 	ImageElement,
+	ImageScanData,
 	NetworkLog,
 	PageData,
 	ParseURLOptions,
@@ -23,6 +24,7 @@ import { detectCompress } from '@d-zero/shared/detect-compress';
 import { retry as retryable } from '@d-zero/shared/retry';
 import { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-await-event-emitter';
 
+import { classifyImageScanError } from './classify-image-scan-error.js';
 import { resourceLog, scraperLog } from './debug.js';
 import {
 	DEFAULT_DOM_EVALUATION_TIMEOUT,
@@ -31,6 +33,7 @@ import {
 	getMeta,
 } from './dom-evaluation.js';
 import { getMainContents } from './get-main-contents.js';
+import { IMAGE_SCAN_CODE } from './image-scan-code.js';
 import { isError } from './is-error.js';
 import { isHtmlContentType } from './is-html-content-type.js';
 import { keywordCheck } from './keyword-check.js';
@@ -57,6 +60,9 @@ const rLog = resourceLog.extend(pid);
  */
 const MAX_SCROLL_HEIGHT = 1_000_000;
 
+/** Shared "not attempted" value for `PageData.imageScan`, reused everywhere a page is returned without an image scan (non-HTML/external/non-HTTP, or the `@retryable` fallback). */
+const EMPTY_IMAGE_SCAN: ImageScanData = { desktop: null, mobile: null };
+
 /**
  * Page-level scraper that extracts data from a single browser page.
  *
@@ -162,6 +168,7 @@ export default class Scraper extends EventEmitter<ScraperEventTypes> {
 				html: '',
 				mainContents: null,
 				scrollHeight: null,
+				imageScan: EMPTY_IMAGE_SCAN,
 				isSkipped: false,
 			};
 
@@ -633,6 +640,7 @@ export default class Scraper extends EventEmitter<ScraperEventTypes> {
 					html: '',
 					mainContents: null,
 					scrollHeight: null,
+					imageScan: EMPTY_IMAGE_SCAN,
 					isSkipped: false,
 				};
 			}
@@ -684,6 +692,7 @@ export default class Scraper extends EventEmitter<ScraperEventTypes> {
 					html,
 					mainContents: null,
 					scrollHeight: null,
+					imageScan: EMPTY_IMAGE_SCAN,
 					isSkipped: false,
 				};
 			}
@@ -744,6 +753,7 @@ export default class Scraper extends EventEmitter<ScraperEventTypes> {
 
 			let imageList: ImageElement[] = [];
 			let scrollHeight: ScrollHeightData | null = null;
+			let imageScan: ImageScanData = EMPTY_IMAGE_SCAN;
 
 			if (captureImages) {
 				void this.emit('changePhase', {
@@ -762,6 +772,7 @@ export default class Scraper extends EventEmitter<ScraperEventTypes> {
 				);
 				imageList = fetched.imageList;
 				scrollHeight = fetched.scrollHeight;
+				imageScan = fetched.imageScan;
 			} else {
 				scrollHeight = await measureScrollHeight(page);
 			}
@@ -782,6 +793,7 @@ export default class Scraper extends EventEmitter<ScraperEventTypes> {
 				html,
 				mainContents,
 				scrollHeight,
+				imageScan,
 				isSkipped: false,
 			};
 		} finally {
@@ -798,7 +810,12 @@ export default class Scraper extends EventEmitter<ScraperEventTypes> {
 	 * WHY per-device try-catch: Some pages (e.g. those using fullpage.js or
 	 * scroll-jacking libraries) destroy the execution context when the viewport
 	 * changes and triggers a reload. Isolating each device preset allows partial
-	 * results — if one viewport fails, the other can still succeed.
+	 * results — if one viewport fails, the other can still succeed. The outcome
+	 * of each device preset (success, degraded, or one of the failure kinds) is
+	 * recorded per-preset in the returned `imageScan`, classified by
+	 * {@link classifyImageScanError} — a page that never goes network-idle
+	 * (analytics beacons, chat widgets, open WebSocket connections) still
+	 * yields images, just flagged `IMAGE_SCAN_CODE.DEGRADED` instead of `OK`.
 	 *
 	 * WHY retryable with 20-min timeout and empty fallback: Image extraction is
 	 * best-effort. If all retries fail, empty images and null scroll heights are
@@ -825,13 +842,14 @@ export default class Scraper extends EventEmitter<ScraperEventTypes> {
 	 * @param isExternal - Whether the page is external
 	 * @param imageLoadTimeout - Timeout (ms) for waiting images to complete loading
 	 * @param domEvaluationTimeout - Timeout (ms) for the in-page image extraction `page.evaluate`
-	 * @returns Image elements plus desktop/mobile scroll heights from the scan path
+	 * @returns Image elements, desktop/mobile scroll heights, and the per-device `imageScan` outcome codes
 	 */
 	@retryable({
 		timeout: 20 * 60 * 1000,
 		fallback: {
 			imageList: [],
 			scrollHeight: { desktop: null, mobile: null },
+			imageScan: EMPTY_IMAGE_SCAN,
 		},
 		onWait(this: Scraper, determinedInterval, retryCount, methodName, error) {
 			void this.emit('changePhase', {
@@ -858,7 +876,11 @@ export default class Scraper extends EventEmitter<ScraperEventTypes> {
 		isExternal: boolean,
 		imageLoadTimeout: number,
 		domEvaluationTimeout: number,
-	): Promise<{ imageList: ImageElement[]; scrollHeight: ScrollHeightData }> {
+	): Promise<{
+		imageList: ImageElement[];
+		scrollHeight: ScrollHeightData;
+		imageScan: ImageScanData;
+	}> {
 		const listener = this.#createPageScanListener(isExternal);
 		const devices: {
 			key: 'desktop-compact' | 'mobile-small';
@@ -869,6 +891,7 @@ export default class Scraper extends EventEmitter<ScraperEventTypes> {
 		];
 		const imageList: ImageElement[] = [];
 		const scrollHeight: ScrollHeightData = { desktop: null, mobile: null };
+		const imageScan: ImageScanData = { desktop: null, mobile: null };
 
 		for (const { key, preset } of devices) {
 			const scrollKey = key === 'desktop-compact' ? 'desktop' : 'mobile';
@@ -888,11 +911,17 @@ export default class Scraper extends EventEmitter<ScraperEventTypes> {
 					listener,
 					timeout: 5000,
 					maxScrollHeight: MAX_SCROLL_HEIGHT,
+					// This per-device try/catch already isolates and classifies
+					// failures (see `classifyImageScanError`), so a page whose
+					// network never settles should degrade gracefully here
+					// rather than losing the whole device preset's images.
+					continueOnDegradedNetwork: true,
 				});
 
 				scrollHeight[scrollKey] = scanResult.scrollHeight;
 
 				if (!scanResult.scrolled) {
+					imageScan[scrollKey] = IMAGE_SCAN_CODE.SCROLL_HEIGHT_EXCEEDED;
 					void this.emit('changePhase', {
 						pid: process.pid,
 						name: 'retryExhausted',
@@ -926,9 +955,12 @@ export default class Scraper extends EventEmitter<ScraperEventTypes> {
 				});
 				const images = await getImageList(page, preset.width, domEvaluationTimeout);
 				imageList.push(...images);
+				imageScan[scrollKey] =
+					scanResult.settled === 'idle' ? IMAGE_SCAN_CODE.OK : IMAGE_SCAN_CODE.DEGRADED;
 			} catch (error) {
 				const errorMessage = error instanceof Error ? error.message : String(error);
 				log('Error(FETCH_IMAGES/%s): %s', key, errorMessage);
+				imageScan[scrollKey] = classifyImageScanError(error);
 				void this.emit('changePhase', {
 					pid: process.pid,
 					name: 'retryExhausted',
@@ -939,6 +971,6 @@ export default class Scraper extends EventEmitter<ScraperEventTypes> {
 			}
 		}
 
-		return { imageList, scrollHeight };
+		return { imageList, scrollHeight, imageScan };
 	}
 }

package/src/to-console-log-entry.ts

@@ -23,6 +23,10 @@ export async function toConsoleLogEntry(
 			} catch {
 				return;
 			} finally {
+				// Why not `await using`: dispose() が reject すると SuppressedError に
+				// 包まれ、本来無視したいだけの dispose 失敗が呼び出し元へ伝播してしまう。
+				// ここでは jsonValue() の結果を最優先し、dispose の失敗は握りつぶす
+				// 現状の挙動を維持する。
 				await arg.dispose().catch(() => {});
 			}
 		}),

package/src/types.ts

@@ -5,6 +5,7 @@
  * @module
  */
 
+export type { ImageScanCode } from './image-scan-code.js';
 export type { ExURL, ParseURLOptions } from '@d-zero/shared/parse-url';
 export type { CompressType } from '@d-zero/shared/detect-compress';
 export type { CDNType } from '@d-zero/shared/detect-cdn';
@@ -55,6 +56,7 @@ export type {
 	RawHeadEntry,
 } from './meta/types.js';
 
+import type { ImageScanCode } from './image-scan-code.js';
 import type { Meta } from './meta/types.js';
 import type { CDNType } from '@d-zero/shared/detect-cdn';
 import type { CompressType } from '@d-zero/shared/detect-compress';
@@ -117,6 +119,22 @@ export type PageData = {
 	 */
 	scrollHeight: ScrollHeightData | null;
 
+	/**
+	 * Per-device-preset outcome of the `<img>` element scan performed by
+	 * `Scraper#fetchImages`, keyed the same way as {@link ScrollHeightData}.
+	 * A field is `null` when that device preset's image scan was never
+	 * attempted (non-HTML, external, non-HTTP page, or `captureImages: false`);
+	 * otherwise it is an {@link ImageScanCode} recording why the scan
+	 * succeeded, degraded, or was abandoned. See `IMAGE_SCAN_CODE`.
+	 * @example
+	 * ```ts
+	 * if (pageData.imageScan.mobile === IMAGE_SCAN_CODE.SCROLL_HEIGHT_EXCEEDED) {
+	 *   // mobile images were skipped due to an oversized scrollHeight
+	 * }
+	 * ```
+	 */
+	imageScan: ImageScanData;
+
 	/** Always `false` for successfully scraped pages. See {@link SkippedPageData} for skipped pages. */
 	isSkipped: false;
 };
@@ -131,6 +149,17 @@ export type ScrollHeightData = {
 	mobile: number | null;
 };
 
+/**
+ * Per-device-preset {@link ImageScanCode} outcome of `Scraper#fetchImages`,
+ * mirroring {@link ScrollHeightData}'s desktop/mobile shape.
+ */
+export type ImageScanData = {
+	/** Outcome for `desktop-compact` (width 1280), or `null` if not attempted. */
+	desktop: ImageScanCode | null;
+	/** Outcome for `mobile-small` (width 320 @ 2x), or `null` if not attempted. */
+	mobile: ImageScanCode | null;
+};
+
 /**
  * Quantitative metrics for the detected main content region of a page.
  * @example

package/tsconfig.json

@@ -3,7 +3,18 @@
 	"compilerOptions": {
 		"composite": true,
 		"outDir": "./dist",
-		"rootDir": "./src"
+		"rootDir": "./src",
+		// Why not the repo-wide `target: "ESNext"`: `scraper.ts`'s `@retryable`
+		// decorator wraps a private method (`#fetchData`), and esbuild's test
+		// transform (vitest) emits invalid output for that combination at
+		// target ESNext ("Private field '#fetchData' must be declared in an
+		// enclosing class"). `tsc` itself has no issue with it — only vitest's
+		// esbuild-based transform does — so this package alone stays on the
+		// `@d-zero/tsconfig` base target until that esbuild limitation is fixed.
+		// TODO: esbuild がデコレータ + private メソッドを target=esnext で正しく
+		// 変換できるようになったら、この override を撤去して root の ESNext
+		// （ネイティブ using / await using emit）に揃える。
+		"target": "es2024"
 	},
 	"references": [
 		{