@d-zero/beholder 4.0.0 → 4.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +10 -0
- package/README.md +10 -0
- package/dist/index.d.ts +1 -1
- package/dist/scraper.d.ts +4 -1
- package/dist/scraper.js +216 -167
- package/dist/to-console-log-entry.d.ts +14 -0
- package/dist/to-console-log-entry.js +33 -0
- package/dist/to-page-error-entry.d.ts +10 -0
- package/dist/to-page-error-entry.js +19 -0
- package/dist/types.d.ts +35 -0
- package/package.json +4 -4
- package/src/index.ts +1 -1
- package/src/scraper.ts +242 -184
- package/src/to-console-log-entry.spec.ts +108 -0
- package/src/to-console-log-entry.ts +40 -0
- package/src/to-page-error-entry.spec.ts +23 -0
- package/src/to-page-error-entry.ts +22 -0
- package/src/types.ts +32 -0
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Converts an uncaught exception / unhandled Promise rejection (captured via
|
|
3
|
+
* `page.on('pageerror')`) into a `ConsoleLogEntry`.
|
|
4
|
+
* @param error - The value emitted by `page.on('pageerror')`; typed as `unknown`
|
|
5
|
+
* because Puppeteer allows non-`Error` throw values (e.g. `throw 'oops'`)
|
|
6
|
+
* @param pageUrl - The URL (without hash) of the page that produced the error
|
|
7
|
+
* @returns The console log entry, with `type: 'pageerror'`
|
|
8
|
+
*/
|
|
9
|
+
export function toPageErrorEntry(error, pageUrl) {
|
|
10
|
+
const isErr = error instanceof Error;
|
|
11
|
+
return {
|
|
12
|
+
pageUrl,
|
|
13
|
+
type: 'pageerror',
|
|
14
|
+
text: isErr ? error.message : String(error),
|
|
15
|
+
args: [],
|
|
16
|
+
stack: isErr ? error.stack : undefined,
|
|
17
|
+
ts: Date.now(),
|
|
18
|
+
};
|
|
19
|
+
}
|
package/dist/types.d.ts
CHANGED
|
@@ -12,6 +12,7 @@ import type { Meta } from './meta/types.js';
|
|
|
12
12
|
import type { CDNType } from '@d-zero/shared/detect-cdn';
|
|
13
13
|
import type { CompressType } from '@d-zero/shared/detect-compress';
|
|
14
14
|
import type { ExURL } from '@d-zero/shared/parse-url';
|
|
15
|
+
import type { ConsoleMessageType } from 'puppeteer';
|
|
15
16
|
/**
|
|
16
17
|
* Scraped page data returned by the scraper after successfully processing a page.
|
|
17
18
|
*/
|
|
@@ -367,6 +368,8 @@ export type ScrapeResult = {
|
|
|
367
368
|
pageData?: PageData;
|
|
368
369
|
/** All sub-resources captured during the page load. */
|
|
369
370
|
resources: ResourceEntry[];
|
|
371
|
+
/** All console messages and uncaught page errors captured during the page load. */
|
|
372
|
+
consoleLogs: ConsoleLogEntry[];
|
|
370
373
|
/** Details about why the page was ignored, present when `type` is `"skipped"`. */
|
|
371
374
|
ignored?: {
|
|
372
375
|
url: ExURL;
|
|
@@ -398,6 +401,38 @@ export type ResourceEntry = {
|
|
|
398
401
|
/** The URL (without hash) of the page that triggered this resource load. */
|
|
399
402
|
pageUrl: string;
|
|
400
403
|
};
|
|
404
|
+
/**
|
|
405
|
+
* A single console message or uncaught page error captured during page scraping.
|
|
406
|
+
* Captured via Puppeteer's `console` and `pageerror` page events; internal pages only.
|
|
407
|
+
*/
|
|
408
|
+
export type ConsoleLogEntry = {
|
|
409
|
+
/** The URL (without hash) of the page that produced this message. */
|
|
410
|
+
pageUrl: string;
|
|
411
|
+
/**
|
|
412
|
+
* The console message type (Puppeteer's `ConsoleMessageType`, e.g. `"log"`, `"warn"`, `"error"`),
|
|
413
|
+
* or `"pageerror"` for an uncaught exception / unhandled Promise rejection.
|
|
414
|
+
*/
|
|
415
|
+
type: ConsoleMessageType | 'pageerror';
|
|
416
|
+
/** The message text, or the error's `message` for `"pageerror"` entries. */
|
|
417
|
+
text: string;
|
|
418
|
+
/**
|
|
419
|
+
* Arguments passed to the console call, resolved via `JSHandle.jsonValue()`.
|
|
420
|
+
* An individual argument is `undefined` when it could not be resolved (e.g. a
|
|
421
|
+
* destroyed execution context or a value `jsonValue()` cannot serialize).
|
|
422
|
+
* Always empty for `"pageerror"` entries.
|
|
423
|
+
*/
|
|
424
|
+
args: unknown[];
|
|
425
|
+
/** Source location of the message, or `undefined` when unavailable (always `undefined` for `"pageerror"`). */
|
|
426
|
+
location?: {
|
|
427
|
+
url?: string;
|
|
428
|
+
lineNumber?: number;
|
|
429
|
+
columnNumber?: number;
|
|
430
|
+
};
|
|
431
|
+
/** Stack trace text, present only for `"pageerror"` entries. */
|
|
432
|
+
stack?: string;
|
|
433
|
+
/** Timestamp (ms since epoch) when the message was captured. */
|
|
434
|
+
ts: number;
|
|
435
|
+
};
|
|
401
436
|
/**
|
|
402
437
|
* Event payload describing a phase transition in the scraping lifecycle.
|
|
403
438
|
* Phases proceed roughly in order: scrapeStart -> openPage ->
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@d-zero/beholder",
|
|
3
|
-
"version": "4.
|
|
3
|
+
"version": "4.1.1",
|
|
4
4
|
"description": "Page-level scraper for web crawling and auditing",
|
|
5
5
|
"author": "D-ZERO",
|
|
6
6
|
"license": "MIT",
|
|
@@ -20,8 +20,8 @@
|
|
|
20
20
|
"clean": "tsc --build --clean"
|
|
21
21
|
},
|
|
22
22
|
"dependencies": {
|
|
23
|
-
"@d-zero/puppeteer-page-scan": "4.6.
|
|
24
|
-
"@d-zero/shared": "0.22.
|
|
23
|
+
"@d-zero/puppeteer-page-scan": "4.6.6",
|
|
24
|
+
"@d-zero/shared": "0.22.3",
|
|
25
25
|
"debug": "4.4.3",
|
|
26
26
|
"puppeteer": "25.3.0",
|
|
27
27
|
"simple-wappalyzer": "1.1.100"
|
|
@@ -36,5 +36,5 @@
|
|
|
36
36
|
"url": "https://github.com/d-zero-dev/tools.git",
|
|
37
37
|
"directory": "packages/@d-zero/beholder"
|
|
38
38
|
},
|
|
39
|
-
"gitHead": "
|
|
39
|
+
"gitHead": "a2f4e4a764db0930911c5c6f81696228b31926cb"
|
|
40
40
|
}
|
package/src/index.ts
CHANGED
|
@@ -18,7 +18,7 @@ export { detectCompress } from '@d-zero/shared/detect-compress';
|
|
|
18
18
|
export type { CompressType } from '@d-zero/shared/detect-compress';
|
|
19
19
|
export { detectCDN } from '@d-zero/shared/detect-cdn';
|
|
20
20
|
export type { CDNType } from '@d-zero/shared/detect-cdn';
|
|
21
|
-
export type { ScrapeResult, ResourceEntry, PageData } from './types.js';
|
|
21
|
+
export type { ScrapeResult, ResourceEntry, ConsoleLogEntry, PageData } from './types.js';
|
|
22
22
|
export type { ScraperOptions, ChangePhaseEvent, ScraperEventTypes } from './types.js';
|
|
23
23
|
export type {
|
|
24
24
|
Resource,
|