@d-zero/beholder 4.2.2 → 5.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +35 -0
- package/dist/classify-image-scan-error.d.ts +20 -0
- package/dist/classify-image-scan-error.js +41 -0
- package/dist/dom-evaluation.js +17 -1
- package/dist/image-scan-code.d.ts +33 -0
- package/dist/image-scan-code.js +31 -0
- package/dist/index.d.ts +3 -1
- package/dist/index.js +1 -0
- package/dist/scraper.js +29 -3
- package/dist/to-console-log-entry.js +4 -0
- package/dist/types.d.ts +27 -0
- package/package.json +9 -5
- package/src/classify-image-scan-error.spec.ts +34 -0
- package/src/classify-image-scan-error.ts +46 -0
- package/src/dom-evaluation.spec.ts +41 -0
- package/src/dom-evaluation.ts +19 -1
- package/src/image-scan-code.ts +34 -0
- package/src/index.ts +3 -0
- package/src/scraper.ts +36 -4
- package/src/to-console-log-entry.ts +4 -0
- package/src/types.ts +29 -0
- package/tsconfig.json +12 -1
- package/tsconfig.tsbuildinfo +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -3,6 +3,41 @@
|
|
|
3
3
|
All notable changes to this project will be documented in this file.
|
|
4
4
|
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
|
|
5
5
|
|
|
6
|
+
# [5.0.0](https://github.com/d-zero-dev/tools/compare/@d-zero/beholder@4.2.3...@d-zero/beholder@5.0.0) (2026-09-10)
|
|
7
|
+
|
|
8
|
+
- feat(beholder)!: report per-viewport image-scan outcome in PageData ([1e69228](https://github.com/d-zero-dev/tools/commit/1e692285f1ef3e0a8a89db695d635c7a2f88ba7c))
|
|
9
|
+
|
|
10
|
+
### BREAKING CHANGES
|
|
11
|
+
|
|
12
|
+
- PageData.imageScan is a new required field ({ desktop,
|
|
13
|
+
mobile }: ImageScanCode | null per viewport), following the same
|
|
14
|
+
precedent as the 4.0.0 addition of mainContents/scrollHeight. Anyone
|
|
15
|
+
constructing a PageData object directly (rather than only reading
|
|
16
|
+
scrapeStart()'s result) must add this field.
|
|
17
|
+
|
|
18
|
+
Scraper#fetchImages now classifies each device preset's outcome via the
|
|
19
|
+
new IMAGE_SCAN_CODE vocabulary (0=ok, 1=degraded, 2=nav-unsettled,
|
|
20
|
+
3=frame-lost, 4=scroll-height-exceeded, 255=unknown) instead of only
|
|
21
|
+
emitting a free-text changePhase message: success records ok/degraded
|
|
22
|
+
based on @d-zero/puppeteer-page-scan's new settled result, a
|
|
23
|
+
scrollHeight-limit skip records scroll-height-exceeded, and a caught
|
|
24
|
+
exception is classified by the new classifyImageScanError (reusing
|
|
25
|
+
puppeteer-scroll's isTransientFrameError, plus "Not attached to an
|
|
26
|
+
active page"/"Target closed" for reload/goto-specific failures).
|
|
27
|
+
|
|
28
|
+
#fetchImages opts into @d-zero/puppeteer-page-scan's new
|
|
29
|
+
continueOnDegradedNetwork so a page whose network never settles still
|
|
30
|
+
yields images (flagged degraded) instead of losing that viewport
|
|
31
|
+
outright.
|
|
32
|
+
|
|
33
|
+
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
|
|
34
|
+
|
|
35
|
+
## [4.2.3](https://github.com/d-zero-dev/tools/compare/@d-zero/beholder@4.2.2...@d-zero/beholder@4.2.3) (2026-08-14)
|
|
36
|
+
|
|
37
|
+
### Bug Fixes
|
|
38
|
+
|
|
39
|
+
- **beholder:** dispose anchor ElementHandles via AsyncDisposableStack ([c47a97b](https://github.com/d-zero-dev/tools/commit/c47a97b75af919c1dec634526146f2726b1be1e6))
|
|
40
|
+
|
|
6
41
|
## [4.2.2](https://github.com/d-zero-dev/tools/compare/@d-zero/beholder@4.2.1...@d-zero/beholder@4.2.2) (2026-08-11)
|
|
7
42
|
|
|
8
43
|
**Note:** Version bump only for package @d-zero/beholder
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { type ImageScanCode } from './image-scan-code.js';
|
|
2
|
+
/**
|
|
3
|
+
* Classifies an error caught in `Scraper#fetchImages`'s per-device try/catch
|
|
4
|
+
* into an {@link ImageScanCode}, so the caught-exception path and the
|
|
5
|
+
* successful-scan path (`settled` / `scrolled`) can be recorded with the
|
|
6
|
+
* same small integer vocabulary for persistence.
|
|
7
|
+
* @param error - The value caught from the per-device `beforePageScan` call.
|
|
8
|
+
* @returns {@link IMAGE_SCAN_CODE.NAV_UNSETTLED} for a `NavigationUnsettledError`,
|
|
9
|
+
* {@link IMAGE_SCAN_CODE.FRAME_LOST} for a known frame/session-loss error,
|
|
10
|
+
* otherwise {@link IMAGE_SCAN_CODE.UNKNOWN}.
|
|
11
|
+
* @example
|
|
12
|
+
* ```ts
|
|
13
|
+
* try {
|
|
14
|
+
* await beforePageScan(page, url, opts);
|
|
15
|
+
* } catch (error) {
|
|
16
|
+
* imageScan[key] = classifyImageScanError(error);
|
|
17
|
+
* }
|
|
18
|
+
* ```
|
|
19
|
+
*/
|
|
20
|
+
export declare function classifyImageScanError(error: unknown): ImageScanCode;
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { NavigationUnsettledError } from '@d-zero/puppeteer-page-scan';
|
|
2
|
+
import { isTransientFrameError } from '@d-zero/puppeteer-scroll';
|
|
3
|
+
import { IMAGE_SCAN_CODE } from './image-scan-code.js';
|
|
4
|
+
/**
|
|
5
|
+
* Matches session/target-loss error messages observed in `#fetchImages`'s
|
|
6
|
+
* per-device retry that `isTransientFrameError` (`@d-zero/puppeteer-scroll`)
|
|
7
|
+
* does not already cover — specific `reload`/`goto` failures rather than
|
|
8
|
+
* `page.evaluate` failures. Deliberately does NOT match a bare `Protocol
|
|
9
|
+
* error` prefix: that wraps many unrelated CDP failures (e.g. `Protocol
|
|
10
|
+
* error (Runtime.evaluate): stack overflow`) that are not frame/session
|
|
11
|
+
* loss and should fall through to `UNKNOWN` for accurate diagnosis.
|
|
12
|
+
*/
|
|
13
|
+
const FRAME_LOST_PATTERN = /Not attached to an active page|Target closed/i;
|
|
14
|
+
/**
|
|
15
|
+
* Classifies an error caught in `Scraper#fetchImages`'s per-device try/catch
|
|
16
|
+
* into an {@link ImageScanCode}, so the caught-exception path and the
|
|
17
|
+
* successful-scan path (`settled` / `scrolled`) can be recorded with the
|
|
18
|
+
* same small integer vocabulary for persistence.
|
|
19
|
+
* @param error - The value caught from the per-device `beforePageScan` call.
|
|
20
|
+
* @returns {@link IMAGE_SCAN_CODE.NAV_UNSETTLED} for a `NavigationUnsettledError`,
|
|
21
|
+
* {@link IMAGE_SCAN_CODE.FRAME_LOST} for a known frame/session-loss error,
|
|
22
|
+
* otherwise {@link IMAGE_SCAN_CODE.UNKNOWN}.
|
|
23
|
+
* @example
|
|
24
|
+
* ```ts
|
|
25
|
+
* try {
|
|
26
|
+
* await beforePageScan(page, url, opts);
|
|
27
|
+
* } catch (error) {
|
|
28
|
+
* imageScan[key] = classifyImageScanError(error);
|
|
29
|
+
* }
|
|
30
|
+
* ```
|
|
31
|
+
*/
|
|
32
|
+
export function classifyImageScanError(error) {
|
|
33
|
+
if (error instanceof NavigationUnsettledError) {
|
|
34
|
+
return IMAGE_SCAN_CODE.NAV_UNSETTLED;
|
|
35
|
+
}
|
|
36
|
+
if (isTransientFrameError(error) ||
|
|
37
|
+
(error instanceof Error && FRAME_LOST_PATTERN.test(error.message))) {
|
|
38
|
+
return IMAGE_SCAN_CODE.FRAME_LOST;
|
|
39
|
+
}
|
|
40
|
+
return IMAGE_SCAN_CODE.UNKNOWN;
|
|
41
|
+
}
|
package/dist/dom-evaluation.js
CHANGED
|
@@ -257,6 +257,16 @@ export async function getAnchorList(page, options, timeout = DEFAULT_DOM_EVALUAT
|
|
|
257
257
|
log('Got 0 anchors');
|
|
258
258
|
return [];
|
|
259
259
|
}
|
|
260
|
+
// `$anchors` の各 ElementHandle は CDP 側のリモートオブジェクト参照を保持する
|
|
261
|
+
// ため、解放しないとページごとにハンドルがリークする。AsyncDisposableStack に
|
|
262
|
+
// まとめて登録して解放するが、`await using` でこの関数のスコープ脱出時に
|
|
263
|
+
// 即座に解放してはいけない — タイムアウトで打ち切られた後もバックグラウンドで
|
|
264
|
+
// resolveAnchor が動き続けている間にハンドルを解放すると、実行中の CDP 呼び出しが
|
|
265
|
+
// 失敗する。解放は work() の完了(成功・失敗いずれも)を待って初めて行う。
|
|
266
|
+
const anchorHandles = new AsyncDisposableStack();
|
|
267
|
+
for (const $anchor of $anchors) {
|
|
268
|
+
anchorHandles.use($anchor);
|
|
269
|
+
}
|
|
260
270
|
const collected = [];
|
|
261
271
|
let axHits = 0;
|
|
262
272
|
let textFallbacks = 0;
|
|
@@ -289,7 +299,13 @@ export async function getAnchorList(page, options, timeout = DEFAULT_DOM_EVALUAT
|
|
|
289
299
|
collected.push(resolved.anchor);
|
|
290
300
|
}));
|
|
291
301
|
};
|
|
292
|
-
const
|
|
302
|
+
const workPromise = work();
|
|
303
|
+
// work() が真に完了した時点(タイムアウトで打ち切られた場合は、その後
|
|
304
|
+
// バックグラウンドで走り続ける resolveAnchor がすべて解決した時点)で
|
|
305
|
+
// ハンドルを解放する。disposeAsync() 自体の失敗も無視してよい
|
|
306
|
+
// (ベストエフォートの後始末であり、呼び出し元への影響はない)。
|
|
307
|
+
void workPromise.finally(() => anchorHandles.disposeAsync()).catch(() => { });
|
|
308
|
+
const { timeout: timedOut } = await raceWithTimeout(() => workPromise, timeout);
|
|
293
309
|
cancelled = true;
|
|
294
310
|
if (timedOut) {
|
|
295
311
|
log('getAnchorList timed out after %dms; returning %d anchors collected so far', timeout, collected.length);
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Numeric outcome codes for a single device-preset image scan performed by
|
|
3
|
+
* `Scraper#fetchImages`. Small integers (fit a `Uint8`) so callers that
|
|
4
|
+
* persist per-page scan results to a database column can use an integer
|
|
5
|
+
* rather than a string enum. A `null` value (not part of this const) means
|
|
6
|
+
* the device preset was never attempted — non-HTML/external/non-HTTP pages,
|
|
7
|
+
* or `captureImages: false`.
|
|
8
|
+
* @example
|
|
9
|
+
* ```ts
|
|
10
|
+
* import { IMAGE_SCAN_CODE } from '@d-zero/beholder';
|
|
11
|
+
*
|
|
12
|
+
* db.insert({ imageScanMobile: pageData.imageScan.mobile ?? null });
|
|
13
|
+
* if (pageData.imageScan.mobile === IMAGE_SCAN_CODE.SCROLL_HEIGHT_EXCEEDED) {
|
|
14
|
+
* // mobile images were skipped because scrollHeight exceeded the limit
|
|
15
|
+
* }
|
|
16
|
+
* ```
|
|
17
|
+
*/
|
|
18
|
+
export declare const IMAGE_SCAN_CODE: {
|
|
19
|
+
/** `beforePageScan` reached `settled: 'idle'`; images extracted normally. */
|
|
20
|
+
readonly OK: 0;
|
|
21
|
+
/** `beforePageScan` reached `settled: 'degraded'`; images extracted, but navigation never went network-idle so the result carries lower confidence. */
|
|
22
|
+
readonly DEGRADED: 1;
|
|
23
|
+
/** Navigation never settled and the post-timeout frame/URL/readyState check also failed (`NavigationUnsettledError`). */
|
|
24
|
+
readonly NAV_UNSETTLED: 2;
|
|
25
|
+
/** The main frame or session was lost mid-scan (detached Frame, session closed, execution context destroyed, or "Not attached to an active page"). */
|
|
26
|
+
readonly FRAME_LOST: 3;
|
|
27
|
+
/** `document.body.scrollHeight` exceeded the scan's `maxScrollHeight` guard; scroll and extraction were skipped. */
|
|
28
|
+
readonly SCROLL_HEIGHT_EXCEEDED: 4;
|
|
29
|
+
/** An error occurred that does not match any of the categories above. */
|
|
30
|
+
readonly UNKNOWN: 255;
|
|
31
|
+
};
|
|
32
|
+
/** Numeric outcome code for one device preset's image scan. See {@link IMAGE_SCAN_CODE}. */
|
|
33
|
+
export type ImageScanCode = (typeof IMAGE_SCAN_CODE)[keyof typeof IMAGE_SCAN_CODE];
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Numeric outcome codes for a single device-preset image scan performed by
|
|
3
|
+
* `Scraper#fetchImages`. Small integers (fit a `Uint8`) so callers that
|
|
4
|
+
* persist per-page scan results to a database column can use an integer
|
|
5
|
+
* rather than a string enum. A `null` value (not part of this const) means
|
|
6
|
+
* the device preset was never attempted — non-HTML/external/non-HTTP pages,
|
|
7
|
+
* or `captureImages: false`.
|
|
8
|
+
* @example
|
|
9
|
+
* ```ts
|
|
10
|
+
* import { IMAGE_SCAN_CODE } from '@d-zero/beholder';
|
|
11
|
+
*
|
|
12
|
+
* db.insert({ imageScanMobile: pageData.imageScan.mobile ?? null });
|
|
13
|
+
* if (pageData.imageScan.mobile === IMAGE_SCAN_CODE.SCROLL_HEIGHT_EXCEEDED) {
|
|
14
|
+
* // mobile images were skipped because scrollHeight exceeded the limit
|
|
15
|
+
* }
|
|
16
|
+
* ```
|
|
17
|
+
*/
|
|
18
|
+
export const IMAGE_SCAN_CODE = {
|
|
19
|
+
/** `beforePageScan` reached `settled: 'idle'`; images extracted normally. */
|
|
20
|
+
OK: 0,
|
|
21
|
+
/** `beforePageScan` reached `settled: 'degraded'`; images extracted, but navigation never went network-idle so the result carries lower confidence. */
|
|
22
|
+
DEGRADED: 1,
|
|
23
|
+
/** Navigation never settled and the post-timeout frame/URL/readyState check also failed (`NavigationUnsettledError`). */
|
|
24
|
+
NAV_UNSETTLED: 2,
|
|
25
|
+
/** The main frame or session was lost mid-scan (detached Frame, session closed, execution context destroyed, or "Not attached to an active page"). */
|
|
26
|
+
FRAME_LOST: 3,
|
|
27
|
+
/** `document.body.scrollHeight` exceeded the scan's `maxScrollHeight` guard; scroll and extraction were skipped. */
|
|
28
|
+
SCROLL_HEIGHT_EXCEEDED: 4,
|
|
29
|
+
/** An error occurred that does not match any of the categories above. */
|
|
30
|
+
UNKNOWN: 255,
|
|
31
|
+
};
|
package/dist/index.d.ts
CHANGED
|
@@ -19,6 +19,8 @@ export { detectCompress } from '@d-zero/shared/detect-compress';
|
|
|
19
19
|
export type { CompressType } from '@d-zero/shared/detect-compress';
|
|
20
20
|
export { detectCDN } from '@d-zero/shared/detect-cdn';
|
|
21
21
|
export type { CDNType } from '@d-zero/shared/detect-cdn';
|
|
22
|
+
export { IMAGE_SCAN_CODE } from './image-scan-code.js';
|
|
23
|
+
export type { ImageScanCode } from './image-scan-code.js';
|
|
22
24
|
export type { ScrapeResult, ResourceEntry, ConsoleLogEntry, PageData } from './types.js';
|
|
23
25
|
export type { ScraperOptions, ChangePhaseEvent, ScraperEventTypes } from './types.js';
|
|
24
|
-
export type { Resource, AnchorData, Meta, ImageElement, SkippedPageData, NetworkLog, ScrollHeightData, MainContentsData, MainContentsMainTag, MainContentsHeading, MainContentsImage, MainContentsTable, MainContentsButton, MainContentsIframe, MainContentsVideo, MainContentsAudio, MainContentsCanvas, OpenGraphMeta, OgArticleMeta, OgBookMeta, OgProfileMeta, OgMusicMeta, OgVideoNsMeta, TwitterMeta, FbMeta, FediverseMeta, AppleMeta, MsApplicationMeta, VerificationMeta, GoogleMeta, GeoMeta, CitationMeta, RdfaMeta, MicrodataMeta, AmpMeta, LegacyMeta, MobileMeta, MicroformatsMeta, PinterestMeta, SlackMeta, LinkedInMeta, ExperimentalMeta, WikiMeta, LinkMeta, LinkEntry, JsonLdEntry, OthersBucket, ScriptEntry, IframeEntry, TagsMeta, TagDetail, TagEntry, TagSource, ViewportMeta, RobotsMeta, ReferrerMeta, FormatDetectionMeta, HttpEquivMeta, HttpEquivRefresh, RawHeadEntry, } from './types.js';
|
|
26
|
+
export type { Resource, AnchorData, Meta, ImageElement, SkippedPageData, NetworkLog, ScrollHeightData, ImageScanData, MainContentsData, MainContentsMainTag, MainContentsHeading, MainContentsImage, MainContentsTable, MainContentsButton, MainContentsIframe, MainContentsVideo, MainContentsAudio, MainContentsCanvas, OpenGraphMeta, OgArticleMeta, OgBookMeta, OgProfileMeta, OgMusicMeta, OgVideoNsMeta, TwitterMeta, FbMeta, FediverseMeta, AppleMeta, MsApplicationMeta, VerificationMeta, GoogleMeta, GeoMeta, CitationMeta, RdfaMeta, MicrodataMeta, AmpMeta, LegacyMeta, MobileMeta, MicroformatsMeta, PinterestMeta, SlackMeta, LinkedInMeta, ExperimentalMeta, WikiMeta, LinkMeta, LinkEntry, JsonLdEntry, OthersBucket, ScriptEntry, IframeEntry, TagsMeta, TagDetail, TagEntry, TagSource, ViewportMeta, RobotsMeta, ReferrerMeta, FormatDetectionMeta, HttpEquivMeta, HttpEquivRefresh, RawHeadEntry, } from './types.js';
|
package/dist/index.js
CHANGED
|
@@ -16,3 +16,4 @@ export { extractMetaFromDocument } from './extract-meta.js';
|
|
|
16
16
|
export { MAIN_CONTENT_FALLBACK_SELECTORS, MAIN_CONTENT_SELECTORS, } from './main-content-selectors.js';
|
|
17
17
|
export { detectCompress } from '@d-zero/shared/detect-compress';
|
|
18
18
|
export { detectCDN } from '@d-zero/shared/detect-cdn';
|
|
19
|
+
export { IMAGE_SCAN_CODE } from './image-scan-code.js';
|
package/dist/scraper.js
CHANGED
|
@@ -41,9 +41,11 @@ import { detectCDN } from '@d-zero/shared/detect-cdn';
|
|
|
41
41
|
import { detectCompress } from '@d-zero/shared/detect-compress';
|
|
42
42
|
import { retry as retryable } from '@d-zero/shared/retry';
|
|
43
43
|
import { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-await-event-emitter';
|
|
44
|
+
import { classifyImageScanError } from './classify-image-scan-error.js';
|
|
44
45
|
import { resourceLog, scraperLog } from './debug.js';
|
|
45
46
|
import { DEFAULT_DOM_EVALUATION_TIMEOUT, getAnchorList, getImageList, getMeta, } from './dom-evaluation.js';
|
|
46
47
|
import { getMainContents } from './get-main-contents.js';
|
|
48
|
+
import { IMAGE_SCAN_CODE } from './image-scan-code.js';
|
|
47
49
|
import { isError } from './is-error.js';
|
|
48
50
|
import { isHtmlContentType } from './is-html-content-type.js';
|
|
49
51
|
import { keywordCheck } from './keyword-check.js';
|
|
@@ -67,6 +69,8 @@ const rLog = resourceLog.extend(pid);
|
|
|
67
69
|
* normal responsive sites complete well within the 20 min retry budget.
|
|
68
70
|
*/
|
|
69
71
|
const MAX_SCROLL_HEIGHT = 1_000_000;
|
|
72
|
+
/** Shared "not attempted" value for `PageData.imageScan`, reused everywhere a page is returned without an image scan (non-HTML/external/non-HTTP, or the `@retryable` fallback). */
|
|
73
|
+
const EMPTY_IMAGE_SCAN = { desktop: null, mobile: null };
|
|
70
74
|
let Scraper = (() => {
|
|
71
75
|
let _classSuper = EventEmitter;
|
|
72
76
|
let _instanceExtraInitializers = [];
|
|
@@ -103,6 +107,7 @@ let Scraper = (() => {
|
|
|
103
107
|
fallback: {
|
|
104
108
|
imageList: [],
|
|
105
109
|
scrollHeight: { desktop: null, mobile: null },
|
|
110
|
+
imageScan: EMPTY_IMAGE_SCAN,
|
|
106
111
|
},
|
|
107
112
|
onWait(determinedInterval, retryCount, methodName, error) {
|
|
108
113
|
void this.emit('changePhase', {
|
|
@@ -323,6 +328,7 @@ let Scraper = (() => {
|
|
|
323
328
|
html: '',
|
|
324
329
|
mainContents: null,
|
|
325
330
|
scrollHeight: null,
|
|
331
|
+
imageScan: EMPTY_IMAGE_SCAN,
|
|
326
332
|
isSkipped: false,
|
|
327
333
|
};
|
|
328
334
|
}
|
|
@@ -369,6 +375,7 @@ let Scraper = (() => {
|
|
|
369
375
|
html,
|
|
370
376
|
mainContents: null,
|
|
371
377
|
scrollHeight: null,
|
|
378
|
+
imageScan: EMPTY_IMAGE_SCAN,
|
|
372
379
|
isSkipped: false,
|
|
373
380
|
};
|
|
374
381
|
}
|
|
@@ -416,6 +423,7 @@ let Scraper = (() => {
|
|
|
416
423
|
}, domEvaluationTimeout);
|
|
417
424
|
let imageList = [];
|
|
418
425
|
let scrollHeight = null;
|
|
426
|
+
let imageScan = EMPTY_IMAGE_SCAN;
|
|
419
427
|
if (captureImages) {
|
|
420
428
|
void this.emit('changePhase', {
|
|
421
429
|
pid: process.pid,
|
|
@@ -427,6 +435,7 @@ let Scraper = (() => {
|
|
|
427
435
|
const fetched = await this.#fetchImages(page, url.withoutHashAndAuth, isExternal, imageLoadTimeout, domEvaluationTimeout);
|
|
428
436
|
imageList = fetched.imageList;
|
|
429
437
|
scrollHeight = fetched.scrollHeight;
|
|
438
|
+
imageScan = fetched.imageScan;
|
|
430
439
|
}
|
|
431
440
|
else {
|
|
432
441
|
scrollHeight = await measureScrollHeight(page);
|
|
@@ -447,6 +456,7 @@ let Scraper = (() => {
|
|
|
447
456
|
html,
|
|
448
457
|
mainContents,
|
|
449
458
|
scrollHeight,
|
|
459
|
+
imageScan,
|
|
450
460
|
isSkipped: false,
|
|
451
461
|
};
|
|
452
462
|
}
|
|
@@ -462,6 +472,7 @@ let Scraper = (() => {
|
|
|
462
472
|
];
|
|
463
473
|
const imageList = [];
|
|
464
474
|
const scrollHeight = { desktop: null, mobile: null };
|
|
475
|
+
const imageScan = { desktop: null, mobile: null };
|
|
465
476
|
for (const { key, preset } of devices) {
|
|
466
477
|
const scrollKey = key === 'desktop-compact' ? 'desktop' : 'mobile';
|
|
467
478
|
try {
|
|
@@ -479,9 +490,15 @@ let Scraper = (() => {
|
|
|
479
490
|
listener,
|
|
480
491
|
timeout: 5000,
|
|
481
492
|
maxScrollHeight: MAX_SCROLL_HEIGHT,
|
|
493
|
+
// This per-device try/catch already isolates and classifies
|
|
494
|
+
// failures (see `classifyImageScanError`), so a page whose
|
|
495
|
+
// network never settles should degrade gracefully here
|
|
496
|
+
// rather than losing the whole device preset's images.
|
|
497
|
+
continueOnDegradedNetwork: true,
|
|
482
498
|
});
|
|
483
499
|
scrollHeight[scrollKey] = scanResult.scrollHeight;
|
|
484
500
|
if (!scanResult.scrolled) {
|
|
501
|
+
imageScan[scrollKey] = IMAGE_SCAN_CODE.SCROLL_HEIGHT_EXCEEDED;
|
|
485
502
|
void this.emit('changePhase', {
|
|
486
503
|
pid: process.pid,
|
|
487
504
|
name: 'retryExhausted',
|
|
@@ -512,10 +529,13 @@ let Scraper = (() => {
|
|
|
512
529
|
});
|
|
513
530
|
const images = await getImageList(page, preset.width, domEvaluationTimeout);
|
|
514
531
|
imageList.push(...images);
|
|
532
|
+
imageScan[scrollKey] =
|
|
533
|
+
scanResult.settled === 'idle' ? IMAGE_SCAN_CODE.OK : IMAGE_SCAN_CODE.DEGRADED;
|
|
515
534
|
}
|
|
516
535
|
catch (error) {
|
|
517
536
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
518
537
|
log('Error(FETCH_IMAGES/%s): %s', key, errorMessage);
|
|
538
|
+
imageScan[scrollKey] = classifyImageScanError(error);
|
|
519
539
|
void this.emit('changePhase', {
|
|
520
540
|
pid: process.pid,
|
|
521
541
|
name: 'retryExhausted',
|
|
@@ -525,7 +545,7 @@ let Scraper = (() => {
|
|
|
525
545
|
});
|
|
526
546
|
}
|
|
527
547
|
}
|
|
528
|
-
return { imageList, scrollHeight };
|
|
548
|
+
return { imageList, scrollHeight, imageScan };
|
|
529
549
|
}, "#fetchImages") }, _private_fetchImages_decorators, { kind: "method", name: "#fetchImages", static: false, private: true, access: { has: obj => #fetchImages in obj, get: obj => obj.#fetchImages }, metadata: _metadata }, null, _instanceExtraInitializers);
|
|
530
550
|
if (_metadata) Object.defineProperty(this, Symbol.metadata, { enumerable: true, configurable: true, writable: true, value: _metadata });
|
|
531
551
|
}
|
|
@@ -607,6 +627,7 @@ let Scraper = (() => {
|
|
|
607
627
|
html: '',
|
|
608
628
|
mainContents: null,
|
|
609
629
|
scrollHeight: null,
|
|
630
|
+
imageScan: EMPTY_IMAGE_SCAN,
|
|
610
631
|
isSkipped: false,
|
|
611
632
|
};
|
|
612
633
|
void this.emit('changePhase', {
|
|
@@ -806,7 +827,12 @@ let Scraper = (() => {
|
|
|
806
827
|
* WHY per-device try-catch: Some pages (e.g. those using fullpage.js or
|
|
807
828
|
* scroll-jacking libraries) destroy the execution context when the viewport
|
|
808
829
|
* changes and triggers a reload. Isolating each device preset allows partial
|
|
809
|
-
* results — if one viewport fails, the other can still succeed.
|
|
830
|
+
* results — if one viewport fails, the other can still succeed. The outcome
|
|
831
|
+
* of each device preset (success, degraded, or one of the failure kinds) is
|
|
832
|
+
* recorded per-preset in the returned `imageScan`, classified by
|
|
833
|
+
* {@link classifyImageScanError} — a page that never goes network-idle
|
|
834
|
+
* (analytics beacons, chat widgets, open WebSocket connections) still
|
|
835
|
+
* yields images, just flagged `IMAGE_SCAN_CODE.DEGRADED` instead of `OK`.
|
|
810
836
|
*
|
|
811
837
|
* WHY retryable with 20-min timeout and empty fallback: Image extraction is
|
|
812
838
|
* best-effort. If all retries fail, empty images and null scroll heights are
|
|
@@ -833,7 +859,7 @@ let Scraper = (() => {
|
|
|
833
859
|
* @param isExternal - Whether the page is external
|
|
834
860
|
* @param imageLoadTimeout - Timeout (ms) for waiting images to complete loading
|
|
835
861
|
* @param domEvaluationTimeout - Timeout (ms) for the in-page image extraction `page.evaluate`
|
|
836
|
-
* @returns Image elements
|
|
862
|
+
* @returns Image elements, desktop/mobile scroll heights, and the per-device `imageScan` outcome codes
|
|
837
863
|
*/
|
|
838
864
|
get #fetchImages() { return _private_fetchImages_descriptor.value; }
|
|
839
865
|
};
|
|
@@ -18,6 +18,10 @@ export async function toConsoleLogEntry(msg, pageUrl) {
|
|
|
18
18
|
return;
|
|
19
19
|
}
|
|
20
20
|
finally {
|
|
21
|
+
// Why not `await using`: dispose() が reject すると SuppressedError に
|
|
22
|
+
// 包まれ、本来無視したいだけの dispose 失敗が呼び出し元へ伝播してしまう。
|
|
23
|
+
// ここでは jsonValue() の結果を最優先し、dispose の失敗は握りつぶす
|
|
24
|
+
// 現状の挙動を維持する。
|
|
21
25
|
await arg.dispose().catch(() => { });
|
|
22
26
|
}
|
|
23
27
|
}));
|
package/dist/types.d.ts
CHANGED
|
@@ -4,10 +4,12 @@
|
|
|
4
4
|
* @see {@link ./dom-evaluation.ts} for DOM extraction functions (anchors, images, meta)
|
|
5
5
|
* @module
|
|
6
6
|
*/
|
|
7
|
+
export type { ImageScanCode } from './image-scan-code.js';
|
|
7
8
|
export type { ExURL, ParseURLOptions } from '@d-zero/shared/parse-url';
|
|
8
9
|
export type { CompressType } from '@d-zero/shared/detect-compress';
|
|
9
10
|
export type { CDNType } from '@d-zero/shared/detect-cdn';
|
|
10
11
|
export type { Meta, OpenGraphMeta, OgArticleMeta, OgBookMeta, OgProfileMeta, OgMusicMeta, OgVideoNsMeta, TwitterMeta, FbMeta, FediverseMeta, AppleMeta, MsApplicationMeta, VerificationMeta, GoogleMeta, GeoMeta, CitationMeta, RdfaMeta, MicrodataMeta, AmpMeta, LegacyMeta, MobileMeta, MicroformatsMeta, PinterestMeta, SlackMeta, LinkedInMeta, ExperimentalMeta, WikiMeta, LinkMeta, LinkEntry, JsonLdEntry, OthersBucket, ScriptEntry, IframeEntry, TagsMeta, TagDetail, TagEntry, TagSource, ViewportMeta, RobotsMeta, ReferrerMeta, FormatDetectionMeta, HttpEquivMeta, HttpEquivRefresh, RawHeadEntry, } from './meta/types.js';
|
|
12
|
+
import type { ImageScanCode } from './image-scan-code.js';
|
|
11
13
|
import type { Meta } from './meta/types.js';
|
|
12
14
|
import type { CDNType } from '@d-zero/shared/detect-cdn';
|
|
13
15
|
import type { CompressType } from '@d-zero/shared/detect-compress';
|
|
@@ -54,6 +56,21 @@ export type PageData = {
|
|
|
54
56
|
* `null` when not measured (non-HTML, external, non-HTTP, or skipped).
|
|
55
57
|
*/
|
|
56
58
|
scrollHeight: ScrollHeightData | null;
|
|
59
|
+
/**
|
|
60
|
+
* Per-device-preset outcome of the `<img>` element scan performed by
|
|
61
|
+
* `Scraper#fetchImages`, keyed the same way as {@link ScrollHeightData}.
|
|
62
|
+
* A field is `null` when that device preset's image scan was never
|
|
63
|
+
* attempted (non-HTML, external, non-HTTP page, or `captureImages: false`);
|
|
64
|
+
* otherwise it is an {@link ImageScanCode} recording why the scan
|
|
65
|
+
* succeeded, degraded, or was abandoned. See `IMAGE_SCAN_CODE`.
|
|
66
|
+
* @example
|
|
67
|
+
* ```ts
|
|
68
|
+
* if (pageData.imageScan.mobile === IMAGE_SCAN_CODE.SCROLL_HEIGHT_EXCEEDED) {
|
|
69
|
+
* // mobile images were skipped due to an oversized scrollHeight
|
|
70
|
+
* }
|
|
71
|
+
* ```
|
|
72
|
+
*/
|
|
73
|
+
imageScan: ImageScanData;
|
|
57
74
|
/** Always `false` for successfully scraped pages. See {@link SkippedPageData} for skipped pages. */
|
|
58
75
|
isSkipped: false;
|
|
59
76
|
};
|
|
@@ -66,6 +83,16 @@ export type ScrollHeightData = {
|
|
|
66
83
|
/** Height at `mobile-small` (width 320 @ 2x), or `null` if that preset failed. */
|
|
67
84
|
mobile: number | null;
|
|
68
85
|
};
|
|
86
|
+
/**
|
|
87
|
+
* Per-device-preset {@link ImageScanCode} outcome of `Scraper#fetchImages`,
|
|
88
|
+
* mirroring {@link ScrollHeightData}'s desktop/mobile shape.
|
|
89
|
+
*/
|
|
90
|
+
export type ImageScanData = {
|
|
91
|
+
/** Outcome for `desktop-compact` (width 1280), or `null` if not attempted. */
|
|
92
|
+
desktop: ImageScanCode | null;
|
|
93
|
+
/** Outcome for `mobile-small` (width 320 @ 2x), or `null` if not attempted. */
|
|
94
|
+
mobile: ImageScanCode | null;
|
|
95
|
+
};
|
|
69
96
|
/**
|
|
70
97
|
* Quantitative metrics for the detected main content region of a page.
|
|
71
98
|
* @example
|
package/package.json
CHANGED
|
@@ -1,12 +1,15 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@d-zero/beholder",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "5.0.0",
|
|
4
4
|
"description": "Page-level scraper for web crawling and auditing",
|
|
5
5
|
"author": "D-ZERO",
|
|
6
6
|
"license": "MIT",
|
|
7
7
|
"publishConfig": {
|
|
8
8
|
"access": "public"
|
|
9
9
|
},
|
|
10
|
+
"engines": {
|
|
11
|
+
"node": ">=24.11.0"
|
|
12
|
+
},
|
|
10
13
|
"type": "module",
|
|
11
14
|
"exports": {
|
|
12
15
|
".": {
|
|
@@ -20,10 +23,11 @@
|
|
|
20
23
|
"clean": "tsc --build --clean"
|
|
21
24
|
},
|
|
22
25
|
"dependencies": {
|
|
23
|
-
"@d-zero/puppeteer-page-scan": "4.
|
|
24
|
-
"@d-zero/
|
|
26
|
+
"@d-zero/puppeteer-page-scan": "4.7.0",
|
|
27
|
+
"@d-zero/puppeteer-scroll": "4.0.12",
|
|
28
|
+
"@d-zero/shared": "0.23.0",
|
|
25
29
|
"debug": "4.4.3",
|
|
26
|
-
"puppeteer": "25.
|
|
30
|
+
"puppeteer": "25.5.0",
|
|
27
31
|
"simple-wappalyzer": "1.1.100"
|
|
28
32
|
},
|
|
29
33
|
"devDependencies": {
|
|
@@ -36,5 +40,5 @@
|
|
|
36
40
|
"url": "https://github.com/d-zero-dev/tools.git",
|
|
37
41
|
"directory": "packages/@d-zero/beholder"
|
|
38
42
|
},
|
|
39
|
-
"gitHead": "
|
|
43
|
+
"gitHead": "41dfa4dc685a038fb1ce391a305617cca7e37de3"
|
|
40
44
|
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { NavigationUnsettledError } from '@d-zero/puppeteer-page-scan';
|
|
2
|
+
import { describe, expect, it } from 'vitest';
|
|
3
|
+
|
|
4
|
+
import { classifyImageScanError } from './classify-image-scan-error.js';
|
|
5
|
+
import { IMAGE_SCAN_CODE } from './image-scan-code.js';
|
|
6
|
+
|
|
7
|
+
describe('classifyImageScanError', () => {
|
|
8
|
+
it('NavigationUnsettledError は NAV_UNSETTLED に分類する', () => {
|
|
9
|
+
expect(classifyImageScanError(new NavigationUnsettledError('never settled'))).toBe(
|
|
10
|
+
IMAGE_SCAN_CODE.NAV_UNSETTLED,
|
|
11
|
+
);
|
|
12
|
+
});
|
|
13
|
+
|
|
14
|
+
it.each([
|
|
15
|
+
"Attempted to use detached Frame 'XXX'.",
|
|
16
|
+
'Session closed.',
|
|
17
|
+
'Execution context was destroyed.',
|
|
18
|
+
'Protocol error (Page.reload): Not attached to an active page',
|
|
19
|
+
'Protocol error (Page.reload): Target closed',
|
|
20
|
+
])('%s は FRAME_LOST に分類する', (message) => {
|
|
21
|
+
expect(classifyImageScanError(new Error(message))).toBe(IMAGE_SCAN_CODE.FRAME_LOST);
|
|
22
|
+
});
|
|
23
|
+
|
|
24
|
+
it('分類不能な Error は UNKNOWN に分類する', () => {
|
|
25
|
+
expect(classifyImageScanError(new Error('TypeError: foo is not a function'))).toBe(
|
|
26
|
+
IMAGE_SCAN_CODE.UNKNOWN,
|
|
27
|
+
);
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
it('Error インスタンスでない値も UNKNOWN に分類する', () => {
|
|
31
|
+
expect(classifyImageScanError('plain string error')).toBe(IMAGE_SCAN_CODE.UNKNOWN);
|
|
32
|
+
expect(classifyImageScanError(null)).toBe(IMAGE_SCAN_CODE.UNKNOWN);
|
|
33
|
+
});
|
|
34
|
+
});
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { NavigationUnsettledError } from '@d-zero/puppeteer-page-scan';
|
|
2
|
+
import { isTransientFrameError } from '@d-zero/puppeteer-scroll';
|
|
3
|
+
|
|
4
|
+
import { IMAGE_SCAN_CODE, type ImageScanCode } from './image-scan-code.js';
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Matches session/target-loss error messages observed in `#fetchImages`'s
|
|
8
|
+
* per-device retry that `isTransientFrameError` (`@d-zero/puppeteer-scroll`)
|
|
9
|
+
* does not already cover — specific `reload`/`goto` failures rather than
|
|
10
|
+
* `page.evaluate` failures. Deliberately does NOT match a bare `Protocol
|
|
11
|
+
* error` prefix: that wraps many unrelated CDP failures (e.g. `Protocol
|
|
12
|
+
* error (Runtime.evaluate): stack overflow`) that are not frame/session
|
|
13
|
+
* loss and should fall through to `UNKNOWN` for accurate diagnosis.
|
|
14
|
+
*/
|
|
15
|
+
const FRAME_LOST_PATTERN = /Not attached to an active page|Target closed/i;
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Classifies an error caught in `Scraper#fetchImages`'s per-device try/catch
|
|
19
|
+
* into an {@link ImageScanCode}, so the caught-exception path and the
|
|
20
|
+
* successful-scan path (`settled` / `scrolled`) can be recorded with the
|
|
21
|
+
* same small integer vocabulary for persistence.
|
|
22
|
+
* @param error - The value caught from the per-device `beforePageScan` call.
|
|
23
|
+
* @returns {@link IMAGE_SCAN_CODE.NAV_UNSETTLED} for a `NavigationUnsettledError`,
|
|
24
|
+
* {@link IMAGE_SCAN_CODE.FRAME_LOST} for a known frame/session-loss error,
|
|
25
|
+
* otherwise {@link IMAGE_SCAN_CODE.UNKNOWN}.
|
|
26
|
+
* @example
|
|
27
|
+
* ```ts
|
|
28
|
+
* try {
|
|
29
|
+
* await beforePageScan(page, url, opts);
|
|
30
|
+
* } catch (error) {
|
|
31
|
+
* imageScan[key] = classifyImageScanError(error);
|
|
32
|
+
* }
|
|
33
|
+
* ```
|
|
34
|
+
*/
|
|
35
|
+
export function classifyImageScanError(error: unknown): ImageScanCode {
|
|
36
|
+
if (error instanceof NavigationUnsettledError) {
|
|
37
|
+
return IMAGE_SCAN_CODE.NAV_UNSETTLED;
|
|
38
|
+
}
|
|
39
|
+
if (
|
|
40
|
+
isTransientFrameError(error) ||
|
|
41
|
+
(error instanceof Error && FRAME_LOST_PATTERN.test(error.message))
|
|
42
|
+
) {
|
|
43
|
+
return IMAGE_SCAN_CODE.FRAME_LOST;
|
|
44
|
+
}
|
|
45
|
+
return IMAGE_SCAN_CODE.UNKNOWN;
|
|
46
|
+
}
|
|
@@ -213,6 +213,9 @@ function mockAnchorHandle(
|
|
|
213
213
|
Promise.resolve({
|
|
214
214
|
jsonValue: () => Promise.resolve(props[propName] ?? ''),
|
|
215
215
|
}),
|
|
216
|
+
// getAnchorList() が AsyncDisposableStack へ登録するため、実際の
|
|
217
|
+
// ElementHandle と同様 Symbol.asyncDispose を実装しておく必要がある
|
|
218
|
+
[Symbol.asyncDispose]: () => Promise.resolve(),
|
|
216
219
|
} as unknown as ElementHandle<Element>;
|
|
217
220
|
}
|
|
218
221
|
|
|
@@ -291,6 +294,7 @@ describe('getAnchorList', () => {
|
|
|
291
294
|
textContent();
|
|
292
295
|
return Promise.resolve({ jsonValue: () => Promise.resolve('text fallback') });
|
|
293
296
|
},
|
|
297
|
+
[Symbol.asyncDispose]: () => Promise.resolve(),
|
|
294
298
|
} as unknown as ElementHandle<Element>;
|
|
295
299
|
const page = mockPageForAnchors({
|
|
296
300
|
anchors: [$anchor],
|
|
@@ -329,6 +333,7 @@ describe('getAnchorList', () => {
|
|
|
329
333
|
remoteObject: () => {
|
|
330
334
|
throw new Error('Handle is detached');
|
|
331
335
|
},
|
|
336
|
+
[Symbol.asyncDispose]: () => Promise.resolve(),
|
|
332
337
|
} as unknown as ElementHandle<Element>;
|
|
333
338
|
const $good = mockAnchorHandle('obj-1', { href: 'https://example.com/page' });
|
|
334
339
|
const page = mockPageForAnchors({
|
|
@@ -436,6 +441,7 @@ describe('getAnchorList', () => {
|
|
|
436
441
|
const $slow = {
|
|
437
442
|
remoteObject: () => ({ objectId: 'obj-slow' }),
|
|
438
443
|
getProperty: () => new Promise(() => {}), // never resolves
|
|
444
|
+
[Symbol.asyncDispose]: () => Promise.resolve(),
|
|
439
445
|
} as unknown as ElementHandle<Element>;
|
|
440
446
|
const page = mockPageForAnchors({
|
|
441
447
|
anchors: [$fast, $slow],
|
|
@@ -491,6 +497,41 @@ describe('getAnchorList', () => {
|
|
|
491
497
|
|
|
492
498
|
expect(anchors).toStrictEqual([]);
|
|
493
499
|
});
|
|
500
|
+
|
|
501
|
+
it('disposes every anchor handle after the list is collected (no CDP handle leak)', async () => {
|
|
502
|
+
const disposeA = vi.fn(async () => {});
|
|
503
|
+
const disposeB = vi.fn(async () => {});
|
|
504
|
+
const $a = {
|
|
505
|
+
remoteObject: () => ({ objectId: 'obj-a' }),
|
|
506
|
+
getProperty: () =>
|
|
507
|
+
Promise.resolve({ jsonValue: () => Promise.resolve('https://example.com/a') }),
|
|
508
|
+
[Symbol.asyncDispose]: disposeA,
|
|
509
|
+
} as unknown as ElementHandle<Element>;
|
|
510
|
+
const $b = {
|
|
511
|
+
remoteObject: () => ({ objectId: 'obj-b' }),
|
|
512
|
+
getProperty: () =>
|
|
513
|
+
Promise.resolve({ jsonValue: () => Promise.resolve('https://example.com/b') }),
|
|
514
|
+
[Symbol.asyncDispose]: disposeB,
|
|
515
|
+
} as unknown as ElementHandle<Element>;
|
|
516
|
+
const page = mockPageForAnchors({
|
|
517
|
+
anchors: [$a, $b],
|
|
518
|
+
axNodes: [
|
|
519
|
+
{ backendDOMNodeId: 1, name: { value: 'A' } },
|
|
520
|
+
{ backendDOMNodeId: 2, name: { value: 'B' } },
|
|
521
|
+
],
|
|
522
|
+
describeNodes: { 'obj-a': 1, 'obj-b': 2 },
|
|
523
|
+
});
|
|
524
|
+
|
|
525
|
+
const anchors = await getAnchorList(page);
|
|
526
|
+
expect(anchors).toHaveLength(2);
|
|
527
|
+
|
|
528
|
+
// 解放は work() 完了後に detached な .finally() で行われるため、
|
|
529
|
+
// getAnchorList の resolve と同期しない。1 tick 待ってから観測する
|
|
530
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
531
|
+
|
|
532
|
+
expect(disposeA).toHaveBeenCalledTimes(1);
|
|
533
|
+
expect(disposeB).toHaveBeenCalledTimes(1);
|
|
534
|
+
});
|
|
494
535
|
});
|
|
495
536
|
|
|
496
537
|
describe('DEFAULT_DOM_EVALUATION_TIMEOUT', () => {
|