@d-zero/beholder 0.1.29 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/README.md +172 -477
- package/dist/debug.d.ts +4 -1
- package/dist/debug.js +5 -2
- package/dist/dom-evaluation.d.ts +72 -14
- package/dist/dom-evaluation.js +169 -43
- package/dist/index.d.ts +20 -3
- package/dist/index.js +15 -3
- package/dist/is-error.d.ts +8 -0
- package/dist/is-error.js +10 -0
- package/dist/keyword-check.d.ts +5 -3
- package/dist/keyword-check.js +5 -3
- package/dist/parse-url.d.ts +14 -0
- package/dist/parse-url.js +23 -0
- package/dist/scraper.d.ts +39 -13
- package/dist/scraper.js +300 -263
- package/dist/types.d.ts +286 -214
- package/dist/types.js +6 -0
- package/package.json +7 -10
- package/src/debug.ts +5 -2
- package/src/dom-evaluation.ts +195 -65
- package/src/index.ts +27 -3
- package/src/is-error.spec.ts +33 -0
- package/src/is-error.ts +10 -0
- package/src/keyword-check.spec.ts +45 -4
- package/src/keyword-check.ts +5 -3
- package/src/parse-url.spec.ts +35 -0
- package/src/parse-url.ts +26 -0
- package/src/scraper.ts +338 -300
- package/src/types.ts +345 -258
- package/tsconfig.tsbuildinfo +1 -1
- package/dist/events.d.ts +0 -32
- package/dist/events.js +0 -15
- package/dist/fetch-destination.d.ts +0 -8
- package/dist/fetch-destination.js +0 -145
- package/dist/net-timeout-error.d.ts +0 -3
- package/dist/net-timeout-error.js +0 -3
- package/dist/sub-process-runner.d.ts +0 -12
- package/dist/sub-process-runner.js +0 -180
- package/dist/sub-process.d.ts +0 -1
- package/dist/sub-process.js +0 -67
- package/dist/utils.d.ts +0 -16
- package/dist/utils.js +0 -69
- package/src/events.ts +0 -21
- package/src/fetch-destination.ts +0 -173
- package/src/net-timeout-error.ts +0 -3
- package/src/sub-process-runner.ts +0 -220
- package/src/sub-process.ts +0 -86
- package/src/utils.ts +0 -89
package/dist/debug.d.ts
CHANGED
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
import debug from 'debug';
|
|
2
|
-
|
|
2
|
+
/** Root debug logger for the beholder package. */
|
|
3
3
|
export declare const scraperLog: debug.Debugger;
|
|
4
|
+
/** Debug logger for resource fetching. */
|
|
4
5
|
export declare const resourceLog: debug.Debugger;
|
|
6
|
+
/** Debug logger for DOM evaluation. */
|
|
5
7
|
export declare const domLog: debug.Debugger;
|
|
8
|
+
/** Debug logger for detailed DOM evaluation output. */
|
|
6
9
|
export declare const domDetailsLog: debug.Debugger;
|
package/dist/debug.js
CHANGED
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
import debug from 'debug';
|
|
2
|
-
|
|
3
|
-
export const scraperLog =
|
|
2
|
+
/** Root debug logger for the beholder package. */
|
|
3
|
+
export const scraperLog = debug('Beholder');
|
|
4
|
+
/** Debug logger for resource fetching. */
|
|
4
5
|
export const resourceLog = scraperLog.extend('Resource');
|
|
6
|
+
/** Debug logger for DOM evaluation. */
|
|
5
7
|
export const domLog = scraperLog.extend('DOM');
|
|
8
|
+
/** Debug logger for detailed DOM evaluation output. */
|
|
6
9
|
export const domDetailsLog = domLog.extend('Details');
|
package/dist/dom-evaluation.d.ts
CHANGED
|
@@ -1,35 +1,93 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* DOM evaluation functions for extracting structured data from Puppeteer pages.
|
|
3
|
+
*
|
|
4
|
+
* These functions are called by {@link ./scraper.ts | Scraper.#fetchData} to extract
|
|
5
|
+
* anchors, images, and meta information after page navigation completes.
|
|
6
|
+
* @see {@link ./types.ts} for the data types returned by these functions
|
|
7
|
+
*/
|
|
1
8
|
import type { AnchorData, ImageElement, ParseURLOptions } from './types.js';
|
|
2
9
|
import type { ElementHandle, Page } from 'puppeteer';
|
|
3
10
|
/**
|
|
11
|
+
* Parameters for {@link getProp}.
|
|
12
|
+
* @template T - The expected type of the property value.
|
|
13
|
+
*/
|
|
14
|
+
export interface GetPropParams<T> {
|
|
15
|
+
/** The Puppeteer element handle to read the property from. */
|
|
16
|
+
readonly $el: ElementHandle<Element>;
|
|
17
|
+
/** The name of the DOM property to retrieve (e.g., `"href"`, `"textContent"`). */
|
|
18
|
+
readonly propName: string;
|
|
19
|
+
/** The default value to return if the property cannot be read or times out. */
|
|
20
|
+
readonly fallback: T;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Retrieves a DOM property value from a Puppeteer element handle with a timeout.
|
|
4
24
|
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
* @
|
|
25
|
+
* Races the actual property retrieval against a 10-second timeout.
|
|
26
|
+
* If the property cannot be read or the timeout expires, the fallback value is returned.
|
|
27
|
+
* @template T - The expected type of the property value.
|
|
28
|
+
* @param params - Parameters containing the element, property name, and fallback.
|
|
29
|
+
* @returns The property value, or the fallback if retrieval fails.
|
|
30
|
+
*/
|
|
31
|
+
export declare function getProp<T>(params: GetPropParams<T>): Promise<T>;
|
|
32
|
+
/**
|
|
33
|
+
* Parameters for {@link getPropBySelector}.
|
|
34
|
+
* @template T - The expected type of the property value.
|
|
8
35
|
*/
|
|
9
|
-
export
|
|
36
|
+
export interface GetPropBySelectorParams<T> {
|
|
37
|
+
/** The Puppeteer page to query. */
|
|
38
|
+
readonly page: Page;
|
|
39
|
+
/** A CSS selector to find the target element. */
|
|
40
|
+
readonly selector: string;
|
|
41
|
+
/** The DOM property name to read from the matched element. */
|
|
42
|
+
readonly propName: string;
|
|
43
|
+
/** The default value if no element matches or the property cannot be read. */
|
|
44
|
+
readonly fallback: T;
|
|
45
|
+
}
|
|
10
46
|
/**
|
|
47
|
+
* Retrieves a DOM property value from the first element matching a CSS selector.
|
|
11
48
|
*
|
|
12
|
-
* @
|
|
13
|
-
* @
|
|
14
|
-
* @param
|
|
15
|
-
* @
|
|
49
|
+
* Combines `page.$()` with {@link getProp} for convenient single-element lookups.
|
|
50
|
+
* @template T - The expected type of the property value.
|
|
51
|
+
* @param params - Parameters containing the page, selector, property name, and fallback.
|
|
52
|
+
* @returns The property value, or the fallback if the element is not found or retrieval fails.
|
|
16
53
|
*/
|
|
17
|
-
export declare function getPropBySelector<T>(
|
|
54
|
+
export declare function getPropBySelector<T>(params: GetPropBySelectorParams<T>): Promise<T>;
|
|
18
55
|
/**
|
|
56
|
+
* Extracts all `<img>` elements from the page and returns their properties.
|
|
19
57
|
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
58
|
+
* For each image, collects the `src`, `currentSrc`, `alt`, bounding box dimensions,
|
|
59
|
+
* natural dimensions, lazy-loading status, and the outer HTML source code.
|
|
60
|
+
* @param page - The Puppeteer page to extract images from.
|
|
61
|
+
* @param viewportWidth - The current viewport width in pixels, recorded alongside each image entry.
|
|
62
|
+
* @returns An array of {@link ImageElement} objects describing each image on the page.
|
|
22
63
|
*/
|
|
23
64
|
export declare function getImageList(page: Page, viewportWidth: number): Promise<ImageElement[]>;
|
|
24
65
|
/**
|
|
66
|
+
* Extracts all anchor (`<a>` and `<area>`) elements with `href` attributes from the page.
|
|
25
67
|
*
|
|
26
|
-
*
|
|
27
|
-
*
|
|
68
|
+
* For each anchor, resolves the `href` to an `ExURL` via `parseUrl`, retrieves
|
|
69
|
+
* the accessible name (from the accessibility tree, falling back to `textContent`),
|
|
70
|
+
* and filters out non-HTTP links.
|
|
71
|
+
* @param page - The Puppeteer page to extract anchors from.
|
|
72
|
+
* @param options - Optional URL parsing options (e.g., `disableQueries`).
|
|
73
|
+
* @returns An array of {@link AnchorData} objects for all HTTP(S) links found on the page.
|
|
28
74
|
*/
|
|
29
75
|
export declare function getAnchorList(page: Page, options?: ParseURLOptions): Promise<AnchorData[]>;
|
|
30
76
|
/**
|
|
77
|
+
* Extracts comprehensive meta information from the page's `<head>`.
|
|
31
78
|
*
|
|
32
|
-
*
|
|
79
|
+
* Collects the following metadata:
|
|
80
|
+
* - `title` - The document title.
|
|
81
|
+
* - `lang` - The `lang` attribute of the `<html>` element.
|
|
82
|
+
* - `description` - The `<meta name="description">` content.
|
|
83
|
+
* - `keywords` - The `<meta name="keywords">` content.
|
|
84
|
+
* - `noindex` / `nofollow` / `noarchive` - Parsed from the `<meta name="robots">` directives.
|
|
85
|
+
* - `canonical` - The `<link rel="canonical">` content.
|
|
86
|
+
* - `alternate` - The `<link rel="alternate">` content.
|
|
87
|
+
* - Open Graph tags: `og:type`, `og:title`, `og:site_name`, `og:description`, `og:url`, `og:image`.
|
|
88
|
+
* - `twitter:card` - The Twitter Card type.
|
|
89
|
+
* @param page - The Puppeteer page to extract meta information from.
|
|
90
|
+
* @returns An object containing all extracted meta properties.
|
|
33
91
|
*/
|
|
34
92
|
export declare function getMeta(page: Page): Promise<{
|
|
35
93
|
title: string;
|
package/dist/dom-evaluation.js
CHANGED
|
@@ -1,25 +1,38 @@
|
|
|
1
|
-
|
|
1
|
+
/**
|
|
2
|
+
* DOM evaluation functions for extracting structured data from Puppeteer pages.
|
|
3
|
+
*
|
|
4
|
+
* These functions are called by {@link ./scraper.ts | Scraper.#fetchData} to extract
|
|
5
|
+
* anchors, images, and meta information after page navigation completes.
|
|
6
|
+
* @see {@link ./types.ts} for the data types returned by these functions
|
|
7
|
+
*/
|
|
2
8
|
import { domDetailsLog, domLog } from './debug.js';
|
|
9
|
+
import { parseUrl } from './parse-url.js';
|
|
3
10
|
const pid = `${process.pid}`;
|
|
4
11
|
const log = domLog.extend(pid);
|
|
5
12
|
const dLog = domDetailsLog.extend(pid);
|
|
6
13
|
/**
|
|
14
|
+
* Retrieves a DOM property value from a Puppeteer element handle with a timeout.
|
|
7
15
|
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
* @
|
|
16
|
+
* Races the actual property retrieval against a 10-second timeout.
|
|
17
|
+
* If the property cannot be read or the timeout expires, the fallback value is returned.
|
|
18
|
+
* @template T - The expected type of the property value.
|
|
19
|
+
* @param params - Parameters containing the element, property name, and fallback.
|
|
20
|
+
* @returns The property value, or the fallback if retrieval fails.
|
|
11
21
|
*/
|
|
12
|
-
export async function getProp(
|
|
22
|
+
export async function getProp(params) {
|
|
23
|
+
const { $el, propName, fallback } = params;
|
|
13
24
|
return Promise.race([
|
|
14
25
|
_getProp($el, propName, fallback),
|
|
15
26
|
new Promise((res) => setTimeout(() => res(fallback), 10 * 1000)),
|
|
16
27
|
]);
|
|
17
28
|
}
|
|
18
29
|
/**
|
|
19
|
-
*
|
|
20
|
-
* @
|
|
21
|
-
* @param
|
|
22
|
-
* @param
|
|
30
|
+
* Internal implementation of property retrieval without timeout.
|
|
31
|
+
* @template T - The expected type of the property value.
|
|
32
|
+
* @param $el - The Puppeteer element handle.
|
|
33
|
+
* @param propName - The DOM property name.
|
|
34
|
+
* @param fallback - The default value on failure.
|
|
35
|
+
* @returns The property value cast to `T`, or the fallback.
|
|
23
36
|
*/
|
|
24
37
|
async function _getProp($el, propName, fallback) {
|
|
25
38
|
try {
|
|
@@ -35,23 +48,29 @@ async function _getProp($el, propName, fallback) {
|
|
|
35
48
|
}
|
|
36
49
|
}
|
|
37
50
|
/**
|
|
51
|
+
* Retrieves a DOM property value from the first element matching a CSS selector.
|
|
38
52
|
*
|
|
39
|
-
* @
|
|
40
|
-
* @
|
|
41
|
-
* @param
|
|
42
|
-
* @
|
|
53
|
+
* Combines `page.$()` with {@link getProp} for convenient single-element lookups.
|
|
54
|
+
* @template T - The expected type of the property value.
|
|
55
|
+
* @param params - Parameters containing the page, selector, property name, and fallback.
|
|
56
|
+
* @returns The property value, or the fallback if the element is not found or retrieval fails.
|
|
43
57
|
*/
|
|
44
|
-
export async function getPropBySelector(
|
|
58
|
+
export async function getPropBySelector(params) {
|
|
59
|
+
const { page, selector, propName, fallback } = params;
|
|
45
60
|
const $el = await page.$(selector);
|
|
46
61
|
if (!$el) {
|
|
47
62
|
return fallback;
|
|
48
63
|
}
|
|
49
|
-
return getProp($el, propName, fallback);
|
|
64
|
+
return getProp({ $el, propName, fallback });
|
|
50
65
|
}
|
|
51
66
|
/**
|
|
67
|
+
* Extracts all `<img>` elements from the page and returns their properties.
|
|
52
68
|
*
|
|
53
|
-
*
|
|
54
|
-
*
|
|
69
|
+
* For each image, collects the `src`, `currentSrc`, `alt`, bounding box dimensions,
|
|
70
|
+
* natural dimensions, lazy-loading status, and the outer HTML source code.
|
|
71
|
+
* @param page - The Puppeteer page to extract images from.
|
|
72
|
+
* @param viewportWidth - The current viewport width in pixels, recorded alongside each image entry.
|
|
73
|
+
* @returns An array of {@link ImageElement} objects describing each image on the page.
|
|
55
74
|
*/
|
|
56
75
|
export async function getImageList(page, viewportWidth) {
|
|
57
76
|
log('Getting images (Viewport: %dpx)', viewportWidth);
|
|
@@ -61,13 +80,29 @@ export async function getImageList(page, viewportWidth) {
|
|
|
61
80
|
const boundingBox = await $image.boundingBox();
|
|
62
81
|
const width = boundingBox?.width || 0;
|
|
63
82
|
const height = boundingBox?.height || 0;
|
|
64
|
-
const src = await getProp($image, 'src', '');
|
|
65
|
-
const currentSrc = await getProp(
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
const
|
|
83
|
+
const src = await getProp({ $el: $image, propName: 'src', fallback: '' });
|
|
84
|
+
const currentSrc = await getProp({
|
|
85
|
+
$el: $image,
|
|
86
|
+
propName: 'currentSrc',
|
|
87
|
+
fallback: '',
|
|
88
|
+
});
|
|
89
|
+
const alt = await getProp({ $el: $image, propName: 'alt', fallback: '' });
|
|
90
|
+
const naturalWidth = await getProp({
|
|
91
|
+
$el: $image,
|
|
92
|
+
propName: 'naturalWidth',
|
|
93
|
+
fallback: 0,
|
|
94
|
+
});
|
|
95
|
+
const naturalHeight = await getProp({
|
|
96
|
+
$el: $image,
|
|
97
|
+
propName: 'naturalHeight',
|
|
98
|
+
fallback: 0,
|
|
99
|
+
});
|
|
100
|
+
const loading = await getProp({ $el: $image, propName: 'loading', fallback: '' });
|
|
101
|
+
const sourceCode = await getProp({
|
|
102
|
+
$el: $image,
|
|
103
|
+
propName: 'outerHTML',
|
|
104
|
+
fallback: '',
|
|
105
|
+
});
|
|
71
106
|
const isLazy = loading.toLowerCase().trim() === 'lazy';
|
|
72
107
|
imageList.push({
|
|
73
108
|
src,
|
|
@@ -87,23 +122,32 @@ export async function getImageList(page, viewportWidth) {
|
|
|
87
122
|
return imageList;
|
|
88
123
|
}
|
|
89
124
|
/**
|
|
125
|
+
* Extracts all anchor (`<a>` and `<area>`) elements with `href` attributes from the page.
|
|
90
126
|
*
|
|
91
|
-
*
|
|
92
|
-
*
|
|
127
|
+
* For each anchor, resolves the `href` to an `ExURL` via `parseUrl`, retrieves
|
|
128
|
+
* the accessible name (from the accessibility tree, falling back to `textContent`),
|
|
129
|
+
* and filters out non-HTTP links.
|
|
130
|
+
* @param page - The Puppeteer page to extract anchors from.
|
|
131
|
+
* @param options - Optional URL parsing options (e.g., `disableQueries`).
|
|
132
|
+
* @returns An array of {@link AnchorData} objects for all HTTP(S) links found on the page.
|
|
93
133
|
*/
|
|
94
134
|
export async function getAnchorList(page, options) {
|
|
95
135
|
log('Getting anchors');
|
|
96
136
|
const $anchors = await page.$$('a[href], area[href]');
|
|
97
137
|
const anchorList = [];
|
|
98
138
|
for (const $anchor of $anchors) {
|
|
99
|
-
const $href = await getProp($anchor, 'href', '');
|
|
139
|
+
const $href = await getProp({ $el: $anchor, propName: 'href', fallback: '' });
|
|
100
140
|
const hrefVal = $href.toString();
|
|
101
141
|
const href = parseUrl(hrefVal, options);
|
|
102
142
|
if (!href || !href.isHTTP) {
|
|
103
143
|
continue;
|
|
104
144
|
}
|
|
105
145
|
const axNode = await page.accessibility.snapshot({ root: $anchor });
|
|
106
|
-
const textContent = await getProp(
|
|
146
|
+
const textContent = await getProp({
|
|
147
|
+
$el: $anchor,
|
|
148
|
+
propName: 'textContent',
|
|
149
|
+
fallback: '',
|
|
150
|
+
});
|
|
107
151
|
const accessibleName = axNode ? axNode.name || '' : textContent.trim();
|
|
108
152
|
const link = {
|
|
109
153
|
href,
|
|
@@ -116,30 +160,112 @@ export async function getAnchorList(page, options) {
|
|
|
116
160
|
return anchorList;
|
|
117
161
|
}
|
|
118
162
|
/**
|
|
163
|
+
* Extracts comprehensive meta information from the page's `<head>`.
|
|
119
164
|
*
|
|
120
|
-
*
|
|
165
|
+
* Collects the following metadata:
|
|
166
|
+
* - `title` - The document title.
|
|
167
|
+
* - `lang` - The `lang` attribute of the `<html>` element.
|
|
168
|
+
* - `description` - The `<meta name="description">` content.
|
|
169
|
+
* - `keywords` - The `<meta name="keywords">` content.
|
|
170
|
+
* - `noindex` / `nofollow` / `noarchive` - Parsed from the `<meta name="robots">` directives.
|
|
171
|
+
* - `canonical` - The `<link rel="canonical">` content.
|
|
172
|
+
* - `alternate` - The `<link rel="alternate">` content.
|
|
173
|
+
* - Open Graph tags: `og:type`, `og:title`, `og:site_name`, `og:description`, `og:url`, `og:image`.
|
|
174
|
+
* - `twitter:card` - The Twitter Card type.
|
|
175
|
+
* @param page - The Puppeteer page to extract meta information from.
|
|
176
|
+
* @returns An object containing all extracted meta properties.
|
|
121
177
|
*/
|
|
122
178
|
export async function getMeta(page) {
|
|
123
179
|
log('Getting Meta');
|
|
124
|
-
const robotsVal = await getPropBySelector(
|
|
180
|
+
const robotsVal = await getPropBySelector({
|
|
181
|
+
page,
|
|
182
|
+
selector: 'meta[name="robots"]',
|
|
183
|
+
propName: 'content',
|
|
184
|
+
fallback: '',
|
|
185
|
+
});
|
|
125
186
|
const robots = new Set(robotsVal.split(',').map((robot) => robot.trim().toLowerCase()));
|
|
126
187
|
const meta = {
|
|
127
|
-
title: await getPropBySelector(
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
188
|
+
title: await getPropBySelector({
|
|
189
|
+
page,
|
|
190
|
+
selector: 'title',
|
|
191
|
+
propName: 'textContent',
|
|
192
|
+
fallback: '',
|
|
193
|
+
}),
|
|
194
|
+
lang: await getPropBySelector({
|
|
195
|
+
page,
|
|
196
|
+
selector: 'html',
|
|
197
|
+
propName: 'lang',
|
|
198
|
+
fallback: '',
|
|
199
|
+
}),
|
|
200
|
+
description: await getPropBySelector({
|
|
201
|
+
page,
|
|
202
|
+
selector: 'meta[name="description"]',
|
|
203
|
+
propName: 'content',
|
|
204
|
+
fallback: '',
|
|
205
|
+
}),
|
|
206
|
+
keywords: await getPropBySelector({
|
|
207
|
+
page,
|
|
208
|
+
selector: 'meta[name="keywords"]',
|
|
209
|
+
propName: 'content',
|
|
210
|
+
fallback: '',
|
|
211
|
+
}),
|
|
131
212
|
noindex: robots.has('noindex'),
|
|
132
213
|
nofollow: robots.has('nofollow'),
|
|
133
214
|
noarchive: robots.has('noarchive'),
|
|
134
|
-
canonical: await getPropBySelector(
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
215
|
+
canonical: await getPropBySelector({
|
|
216
|
+
page,
|
|
217
|
+
selector: 'link[rel="canonical"]',
|
|
218
|
+
propName: 'href',
|
|
219
|
+
fallback: '',
|
|
220
|
+
}),
|
|
221
|
+
alternate: await getPropBySelector({
|
|
222
|
+
page,
|
|
223
|
+
selector: 'link[rel="alternate"]',
|
|
224
|
+
propName: 'href',
|
|
225
|
+
fallback: '',
|
|
226
|
+
}),
|
|
227
|
+
'og:type': await getPropBySelector({
|
|
228
|
+
page,
|
|
229
|
+
selector: 'meta[property="og:type"]',
|
|
230
|
+
propName: 'content',
|
|
231
|
+
fallback: '',
|
|
232
|
+
}),
|
|
233
|
+
'og:title': await getPropBySelector({
|
|
234
|
+
page,
|
|
235
|
+
selector: 'meta[property="og:title"]',
|
|
236
|
+
propName: 'content',
|
|
237
|
+
fallback: '',
|
|
238
|
+
}),
|
|
239
|
+
'og:site_name': await getPropBySelector({
|
|
240
|
+
page,
|
|
241
|
+
selector: 'meta[property="og:site_name"]',
|
|
242
|
+
propName: 'content',
|
|
243
|
+
fallback: '',
|
|
244
|
+
}),
|
|
245
|
+
'og:description': await getPropBySelector({
|
|
246
|
+
page,
|
|
247
|
+
selector: 'meta[property="og:description"]',
|
|
248
|
+
propName: 'content',
|
|
249
|
+
fallback: '',
|
|
250
|
+
}),
|
|
251
|
+
'og:url': await getPropBySelector({
|
|
252
|
+
page,
|
|
253
|
+
selector: 'meta[property="og:url"]',
|
|
254
|
+
propName: 'content',
|
|
255
|
+
fallback: '',
|
|
256
|
+
}),
|
|
257
|
+
'og:image': await getPropBySelector({
|
|
258
|
+
page,
|
|
259
|
+
selector: 'meta[property="og:image"]',
|
|
260
|
+
propName: 'content',
|
|
261
|
+
fallback: '',
|
|
262
|
+
}),
|
|
263
|
+
'twitter:card': await getPropBySelector({
|
|
264
|
+
page,
|
|
265
|
+
selector: 'meta[name="twitter:card"]',
|
|
266
|
+
propName: 'content',
|
|
267
|
+
fallback: '',
|
|
268
|
+
}),
|
|
143
269
|
};
|
|
144
270
|
log('Got meta');
|
|
145
271
|
dLog('Meta data are: %O', meta);
|
package/dist/index.d.ts
CHANGED
|
@@ -1,4 +1,21 @@
|
|
|
1
|
-
|
|
1
|
+
/**
|
|
2
|
+
* @module @d-zero/beholder
|
|
3
|
+
*
|
|
4
|
+
* The beholder package provides page-level scraping capabilities for web crawlers.
|
|
5
|
+
* It handles browser page navigation, DOM data extraction (anchors, images, meta tags),
|
|
6
|
+
* network resource monitoring, and keyword-based page exclusion.
|
|
7
|
+
*
|
|
8
|
+
* Results are returned as values from `scrapeStart()`, not emitted as events.
|
|
9
|
+
* Only streaming events (changePhase, resourceResponse) are emitted for progress monitoring.
|
|
10
|
+
*
|
|
11
|
+
* The main entry point is the `Scraper` class (default export).
|
|
12
|
+
*/
|
|
2
13
|
export { default as default } from './scraper.js';
|
|
3
|
-
export
|
|
4
|
-
export
|
|
14
|
+
export { isError } from './is-error.js';
|
|
15
|
+
export { detectCompress } from '@d-zero/shared/detect-compress';
|
|
16
|
+
export type { CompressType } from '@d-zero/shared/detect-compress';
|
|
17
|
+
export { detectCDN } from '@d-zero/shared/detect-cdn';
|
|
18
|
+
export type { CDNType } from '@d-zero/shared/detect-cdn';
|
|
19
|
+
export type { ScrapeResult, ResourceEntry, PageData } from './types.js';
|
|
20
|
+
export type { ScraperOptions, ChangePhaseEvent, ScraperEventTypes } from './types.js';
|
|
21
|
+
export type { Resource, AnchorData, Meta, ImageElement, SkippedPageData, NetworkLog, } from './types.js';
|
package/dist/index.js
CHANGED
|
@@ -1,4 +1,16 @@
|
|
|
1
|
-
|
|
1
|
+
/**
|
|
2
|
+
* @module @d-zero/beholder
|
|
3
|
+
*
|
|
4
|
+
* The beholder package provides page-level scraping capabilities for web crawlers.
|
|
5
|
+
* It handles browser page navigation, DOM data extraction (anchors, images, meta tags),
|
|
6
|
+
* network resource monitoring, and keyword-based page exclusion.
|
|
7
|
+
*
|
|
8
|
+
* Results are returned as values from `scrapeStart()`, not emitted as events.
|
|
9
|
+
* Only streaming events (changePhase, resourceResponse) are emitted for progress monitoring.
|
|
10
|
+
*
|
|
11
|
+
* The main entry point is the `Scraper` class (default export).
|
|
12
|
+
*/
|
|
2
13
|
export { default as default } from './scraper.js';
|
|
3
|
-
export
|
|
4
|
-
export
|
|
14
|
+
export { isError } from './is-error.js';
|
|
15
|
+
export { detectCompress } from '@d-zero/shared/detect-compress';
|
|
16
|
+
export { detectCDN } from '@d-zero/shared/detect-cdn';
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Determines whether an HTTP status code represents an error.
|
|
3
|
+
* Status codes in the range 200-399 (inclusive) are considered successful;
|
|
4
|
+
* all others are considered errors.
|
|
5
|
+
* @param status - HTTP status code to evaluate
|
|
6
|
+
* @returns `true` if the status code indicates an error (< 200 or >= 400)
|
|
7
|
+
*/
|
|
8
|
+
export declare function isError(status: number): boolean;
|
package/dist/is-error.js
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Determines whether an HTTP status code represents an error.
|
|
3
|
+
* Status codes in the range 200-399 (inclusive) are considered successful;
|
|
4
|
+
* all others are considered errors.
|
|
5
|
+
* @param status - HTTP status code to evaluate
|
|
6
|
+
* @returns `true` if the status code indicates an error (< 200 or >= 400)
|
|
7
|
+
*/
|
|
8
|
+
export function isError(status) {
|
|
9
|
+
return !(200 <= status && status < 400);
|
|
10
|
+
}
|
package/dist/keyword-check.d.ts
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
3
|
-
*
|
|
4
|
-
* @param
|
|
2
|
+
* Checks whether the given HTML content contains any of the specified exclude keywords.
|
|
3
|
+
* Each keyword is converted to a regular expression via `strToRegex` before testing.
|
|
4
|
+
* @param html - The raw HTML string to search within.
|
|
5
|
+
* @param excludeKeywords - An array of keyword strings or regex patterns to match against the HTML.
|
|
6
|
+
* @returns The first matched keyword string if a match is found, or `false` if none match.
|
|
5
7
|
*/
|
|
6
8
|
export declare function keywordCheck(html: string, excludeKeywords: string[]): string | false;
|
package/dist/keyword-check.js
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import { strToRegex } from '@d-zero/shared/str-to-regex';
|
|
2
2
|
/**
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
* @param
|
|
3
|
+
* Checks whether the given HTML content contains any of the specified exclude keywords.
|
|
4
|
+
* Each keyword is converted to a regular expression via `strToRegex` before testing.
|
|
5
|
+
* @param html - The raw HTML string to search within.
|
|
6
|
+
* @param excludeKeywords - An array of keyword strings or regex patterns to match against the HTML.
|
|
7
|
+
* @returns The first matched keyword string if a match is found, or `false` if none match.
|
|
6
8
|
*/
|
|
7
9
|
export function keywordCheck(html, excludeKeywords) {
|
|
8
10
|
for (const keyword of excludeKeywords) {
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { ExURL, ParseURLOptions } from '@d-zero/shared/parse-url';
|
|
2
|
+
/**
|
|
3
|
+
* Parses a URL string into an ExURL object, filtering out non-HTTP URLs
|
|
4
|
+
* that lack a hostname and protocol. If the input is already an ExURL object,
|
|
5
|
+
* it is returned as-is without re-parsing.
|
|
6
|
+
*
|
|
7
|
+
* WHY null return: Bare fragment-only strings (e.g. `"#section"`) and
|
|
8
|
+
* protocol-relative paths without a host are not meaningful URLs for crawling.
|
|
9
|
+
* @param url - A URL string or an already-parsed ExURL object
|
|
10
|
+
* @param options - URL parsing options (e.g. `disableQueries` to strip query strings)
|
|
11
|
+
* @returns The parsed ExURL, or `null` if the URL is not navigable
|
|
12
|
+
* @see `@d-zero/shared/parse-url` for the underlying parser
|
|
13
|
+
*/
|
|
14
|
+
export declare function parseUrl(url: string | ExURL, options?: ParseURLOptions): ExURL | null;
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { parseUrl as sharedParseUrl } from '@d-zero/shared/parse-url';
|
|
2
|
+
/**
|
|
3
|
+
* Parses a URL string into an ExURL object, filtering out non-HTTP URLs
|
|
4
|
+
* that lack a hostname and protocol. If the input is already an ExURL object,
|
|
5
|
+
* it is returned as-is without re-parsing.
|
|
6
|
+
*
|
|
7
|
+
* WHY null return: Bare fragment-only strings (e.g. `"#section"`) and
|
|
8
|
+
* protocol-relative paths without a host are not meaningful URLs for crawling.
|
|
9
|
+
* @param url - A URL string or an already-parsed ExURL object
|
|
10
|
+
* @param options - URL parsing options (e.g. `disableQueries` to strip query strings)
|
|
11
|
+
* @returns The parsed ExURL, or `null` if the URL is not navigable
|
|
12
|
+
* @see `@d-zero/shared/parse-url` for the underlying parser
|
|
13
|
+
*/
|
|
14
|
+
export function parseUrl(url, options) {
|
|
15
|
+
if (typeof url !== 'string') {
|
|
16
|
+
return url;
|
|
17
|
+
}
|
|
18
|
+
const result = sharedParseUrl(url, options);
|
|
19
|
+
if (!result.isHTTP && !result.hostname && !result.protocol) {
|
|
20
|
+
return null;
|
|
21
|
+
}
|
|
22
|
+
return result;
|
|
23
|
+
}
|
package/dist/scraper.d.ts
CHANGED
|
@@ -1,15 +1,41 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
import {
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
1
|
+
import type { ScraperEventTypes, ScraperOptions, ScrapeResult, ExURL } from './types.js';
|
|
2
|
+
import type { Page } from 'puppeteer';
|
|
3
|
+
import { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-await-event-emitter';
|
|
4
|
+
/**
|
|
5
|
+
* Page-level scraper that extracts data from a single browser page.
|
|
6
|
+
*
|
|
7
|
+
* The scraper returns results as values from `scrapeStart()` rather than
|
|
8
|
+
* emitting them as events. Only streaming events (changePhase, resourceResponse)
|
|
9
|
+
* are emitted for progress monitoring.
|
|
10
|
+
*
|
|
11
|
+
* The Puppeteer `Page` object is injected externally, and page lifecycle
|
|
12
|
+
* (including `page.close()`) is managed by the caller.
|
|
13
|
+
* @example
|
|
14
|
+
* ```ts
|
|
15
|
+
* const scraper = new Scraper();
|
|
16
|
+
* scraper.on('changePhase', (e) => console.log(e.name));
|
|
17
|
+
* const result = await scraper.scrapeStart(page, url, { isExternal: false });
|
|
18
|
+
* ```
|
|
19
|
+
*/
|
|
20
|
+
export default class Scraper extends EventEmitter<ScraperEventTypes> {
|
|
12
21
|
#private;
|
|
13
|
-
|
|
14
|
-
|
|
22
|
+
/** Number of retries for `@retryable`-decorated methods. Set per-scrape from options. */
|
|
23
|
+
retries?: number;
|
|
24
|
+
/**
|
|
25
|
+
* Begins the scraping process for a given URL on the provided Puppeteer page.
|
|
26
|
+
*
|
|
27
|
+
* Returns a `ScrapeResult` containing the outcome:
|
|
28
|
+
* - `type: "success"` with `pageData` on success
|
|
29
|
+
* - `type: "skipped"` with `ignored` details when the page is excluded
|
|
30
|
+
* - `type: "error"` with `error` details when scraping fails
|
|
31
|
+
*
|
|
32
|
+
* Sub-resources are collected via the `resourceResponse` event and
|
|
33
|
+
* included in the returned `ScrapeResult.resources`.
|
|
34
|
+
* @param page - The Puppeteer page instance to use for navigation and DOM evaluation.
|
|
35
|
+
* @param url - The extended URL to scrape.
|
|
36
|
+
* @param options - Optional scraper configuration overriding defaults.
|
|
37
|
+
* @param isSkip - When `true`, the page is immediately skipped without any network requests.
|
|
38
|
+
* @returns The scrape result containing the outcome and captured resources.
|
|
39
|
+
*/
|
|
40
|
+
scrapeStart(page: Page, url: ExURL, options?: Partial<ScraperOptions>, isSkip?: boolean): Promise<ScrapeResult>;
|
|
15
41
|
}
|