@xberg-io/crawlberg 1.11.1 → 1.12.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/binding.cjs +138 -0
- package/index.d.ts +144 -16
- package/index.js +583 -437
- package/index.mjs +47 -0
- package/package.json +12 -9
package/binding.cjs
ADDED
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
// DO NOT EDIT - auto-generated by alef
|
|
2
|
+
// alef:hash:b5a84836004f3b1787dc4ddf756b1f8dd85684282f90c6b508552bd1861b3f61
|
|
3
|
+
"use strict";
|
|
4
|
+
|
|
5
|
+
// The native binding attaches the structured properties of a failed call to the thrown error as
|
|
6
|
+
// the JSON message of its `cause`: the napi runtime cannot set own properties on an error that an
|
|
7
|
+
// async Rust function returns. This module moves those properties onto the error itself, so a
|
|
8
|
+
// caller reads `error.kind`, `error.statusCode`, `error.retryAfter` and the rest as declared in
|
|
9
|
+
// `index.d.ts`. It wraps every exported function, every static and prototype method of an
|
|
10
|
+
// exported class, and the async iterator a streaming call returns.
|
|
11
|
+
// Constructors are not wrapped: a constructor that throws keeps the plain native error.
|
|
12
|
+
|
|
13
|
+
const MARKER = "alef:error-info:";
|
|
14
|
+
const SKIP = new Set(["constructor", "length", "name", "prototype", "caller", "arguments"]);
|
|
15
|
+
const patched = new WeakSet();
|
|
16
|
+
const iterables = new WeakSet();
|
|
17
|
+
|
|
18
|
+
function decorate(error) {
|
|
19
|
+
if (error === null || typeof error !== "object") return error;
|
|
20
|
+
const cause = error.cause;
|
|
21
|
+
if (cause === null || typeof cause !== "object" || typeof cause.message !== "string") return error;
|
|
22
|
+
if (!cause.message.startsWith(MARKER)) return error;
|
|
23
|
+
let info;
|
|
24
|
+
try {
|
|
25
|
+
info = JSON.parse(cause.message.slice(MARKER.length));
|
|
26
|
+
} catch {
|
|
27
|
+
return error;
|
|
28
|
+
}
|
|
29
|
+
for (const [name, value] of Object.entries(info)) {
|
|
30
|
+
Object.defineProperty(error, name, { value, enumerable: true, writable: true, configurable: true });
|
|
31
|
+
}
|
|
32
|
+
try {
|
|
33
|
+
delete error.cause;
|
|
34
|
+
} catch {
|
|
35
|
+
// The payload stays on the cause when the runtime refuses the delete.
|
|
36
|
+
}
|
|
37
|
+
return error;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function rethrow(error) {
|
|
41
|
+
throw decorate(error);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function wrapIterator(iterator) {
|
|
45
|
+
if (iterator === null || typeof iterator !== "object") return iterator;
|
|
46
|
+
for (const name of ["next", "return", "throw"]) {
|
|
47
|
+
const original = iterator[name];
|
|
48
|
+
if (typeof original !== "function") continue;
|
|
49
|
+
try {
|
|
50
|
+
Object.defineProperty(iterator, name, {
|
|
51
|
+
configurable: true,
|
|
52
|
+
writable: true,
|
|
53
|
+
value: function (...args) {
|
|
54
|
+
return Promise.resolve(original.apply(this, args)).then(undefined, rethrow);
|
|
55
|
+
},
|
|
56
|
+
});
|
|
57
|
+
} catch {
|
|
58
|
+
// A frozen iterator keeps its plain errors.
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
return iterator;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function wrapIterable(value) {
|
|
65
|
+
if (value === null || typeof value !== "object") return value;
|
|
66
|
+
const open = value[Symbol.asyncIterator];
|
|
67
|
+
if (typeof open !== "function" || iterables.has(value)) return value;
|
|
68
|
+
iterables.add(value);
|
|
69
|
+
try {
|
|
70
|
+
Object.defineProperty(value, Symbol.asyncIterator, {
|
|
71
|
+
configurable: true,
|
|
72
|
+
writable: true,
|
|
73
|
+
value: function () {
|
|
74
|
+
return wrapIterator(open.call(this));
|
|
75
|
+
},
|
|
76
|
+
});
|
|
77
|
+
} catch {
|
|
78
|
+
return value;
|
|
79
|
+
}
|
|
80
|
+
wrapIterator(value);
|
|
81
|
+
return value;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function settle(result) {
|
|
85
|
+
if (result !== null && typeof result === "object" && typeof result.then === "function") {
|
|
86
|
+
return result.then(wrapIterable, rethrow);
|
|
87
|
+
}
|
|
88
|
+
return wrapIterable(result);
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
function wrapFunction(fn) {
|
|
92
|
+
return function (...args) {
|
|
93
|
+
let result;
|
|
94
|
+
try {
|
|
95
|
+
result = fn.apply(this, args);
|
|
96
|
+
} catch (error) {
|
|
97
|
+
throw decorate(error);
|
|
98
|
+
}
|
|
99
|
+
return settle(result);
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
function patchMethods(target) {
|
|
104
|
+
if (target === null || (typeof target !== "object" && typeof target !== "function")) return;
|
|
105
|
+
if (patched.has(target)) return;
|
|
106
|
+
patched.add(target);
|
|
107
|
+
for (const key of Reflect.ownKeys(target)) {
|
|
108
|
+
if (typeof key === "string" && SKIP.has(key)) continue;
|
|
109
|
+
const descriptor = Object.getOwnPropertyDescriptor(target, key);
|
|
110
|
+
if (!descriptor || typeof descriptor.value !== "function" || !descriptor.configurable) continue;
|
|
111
|
+
try {
|
|
112
|
+
Object.defineProperty(target, key, { ...descriptor, value: wrapFunction(descriptor.value) });
|
|
113
|
+
} catch {
|
|
114
|
+
// A method the runtime refuses to redefine keeps its plain errors.
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
function wrapExport(value) {
|
|
120
|
+
if (typeof value !== "function") return value;
|
|
121
|
+
patchMethods(value);
|
|
122
|
+
patchMethods(value.prototype);
|
|
123
|
+
// Only `apply` is trapped, so `new`, `instanceof` and static access reach the native class.
|
|
124
|
+
return new Proxy(value, {
|
|
125
|
+
apply(target, thisArg, args) {
|
|
126
|
+
return wrapFunction(target).apply(thisArg, args);
|
|
127
|
+
},
|
|
128
|
+
});
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
module.exports = function decorateBinding(native) {
|
|
132
|
+
const decorated = {};
|
|
133
|
+
for (const key of Object.keys(native)) {
|
|
134
|
+
decorated[key] = wrapExport(native[key]);
|
|
135
|
+
}
|
|
136
|
+
return decorated;
|
|
137
|
+
};
|
|
138
|
+
module.exports.decorate = decorate;
|
package/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// This file is auto-generated by alef — DO NOT EDIT.
|
|
2
|
-
// alef:hash:
|
|
2
|
+
// alef:hash:f58f5415140504047100134aac44d0ee1f666d2775ab43a5ab8921162c1d7c84
|
|
3
3
|
// To regenerate: alef generate
|
|
4
4
|
// To verify freshness: alef verify
|
|
5
5
|
/* eslint-disable */
|
|
@@ -214,7 +214,15 @@ export interface BrowserConfig {
|
|
|
214
214
|
readonly wait?: BrowserWait;
|
|
215
215
|
/** CSS selector to wait for when `wait` is `Selector`. */
|
|
216
216
|
readonly waitSelector?: string;
|
|
217
|
-
/**
|
|
217
|
+
/**
|
|
218
|
+
* Extra time to wait after the wait condition is met.
|
|
219
|
+
*
|
|
220
|
+
* The default `network_idle` wait returns once the page has loaded and its DOM has been
|
|
221
|
+
* unchanged for `dom_quiet_window` (at most `dom_quiet_cap`, and never past `timeout`, counted
|
|
222
|
+
* from the start of the navigation). An app that draws after a longer quiet period needs
|
|
223
|
+
* a larger `dom_quiet_window` or `extra_wait`; for
|
|
224
|
+
* example `3000` waits three more seconds.
|
|
225
|
+
*/
|
|
218
226
|
readonly extraWait?: number;
|
|
219
227
|
/**
|
|
220
228
|
* Proxy for browser fetches. Overrides `CrawlConfig.proxy` when set.
|
|
@@ -274,6 +282,42 @@ export interface BrowserConfig {
|
|
|
274
282
|
* shared browser pool.
|
|
275
283
|
*/
|
|
276
284
|
readonly chromeArgs?: Array<string>;
|
|
285
|
+
/**
|
|
286
|
+
* Scroll overflowing scroll containers (not only the window) and keep the items that a
|
|
287
|
+
* virtualised list mounts as they pass through view, so the page that is read holds items
|
|
288
|
+
* beyond the first screen. Default: `false`.
|
|
289
|
+
*
|
|
290
|
+
* Runs once after the page is ready and `extra_wait` has elapsed, bounded to 8 containers,
|
|
291
|
+
* 200 steps and 5 seconds in total. Each container is scrolled by 80% of its height per step
|
|
292
|
+
* and the items of its most repetitive child list are collected, keyed by `id`, `data-index`,
|
|
293
|
+
* `data-key`, `data-id`, `aria-posinset` or `aria-rowindex` when present and otherwise by
|
|
294
|
+
* text when it is unique among the rendered items, else by position and text. Only
|
|
295
|
+
* containers whose child list repeats an element are considered, most repetitive first. The list is rewritten, in position order, only when items were unmounted
|
|
296
|
+
* while scrolling and items beyond the first screen were seen; a list that keeps all its
|
|
297
|
+
* items mounted is left as it is. Best effort: it does not follow lists that load data over
|
|
298
|
+
* the network slower than about 100 ms per step, does not reach lists inside iframes or
|
|
299
|
+
* shadow roots, and a rewrite replaces the element children of the list. Lists that recycle
|
|
300
|
+
* their DOM nodes (reusing the same elements with new content) are not collected, and
|
|
301
|
+
* scrolling can trigger infinite-scroll load-more requests. When the step, item or time
|
|
302
|
+
* budget stops the scroll, the items collected so far are kept and a warning is logged.
|
|
303
|
+
* Chromiumoxide backend only.
|
|
304
|
+
*/
|
|
305
|
+
readonly scrollContainers?: boolean;
|
|
306
|
+
/**
|
|
307
|
+
* How long the DOM must stay unchanged after page load for `network_idle` to return,
|
|
308
|
+
* in milliseconds when serialized. Default: 150 ms. Every mutation restarts this window;
|
|
309
|
+
* the whole wait stays bounded to `dom_quiet_cap` and the remaining navigation timeout.
|
|
310
|
+
* A page whose first delayed render follows a longer quiet gap needs a larger value
|
|
311
|
+
* or `extra_wait`. Chromiumoxide backend only.
|
|
312
|
+
*/
|
|
313
|
+
readonly domQuietWindow?: number;
|
|
314
|
+
/**
|
|
315
|
+
* Maximum time spent waiting for DOM quiet after page load, in milliseconds when serialized.
|
|
316
|
+
* Default: 4000 ms. Lower this for pages with continuous mutations; zero skips the quiet wait.
|
|
317
|
+
* The remaining navigation timeout also bounds this wait, independently of the quiet window.
|
|
318
|
+
* Chromiumoxide backend only.
|
|
319
|
+
*/
|
|
320
|
+
readonly domQuietCap?: number;
|
|
277
321
|
}
|
|
278
322
|
|
|
279
323
|
/**
|
|
@@ -400,7 +444,11 @@ export interface ContentConfig {
|
|
|
400
444
|
readonly excludeSelectors?: Array<string>;
|
|
401
445
|
/** Skip image elements in output. Default: `false`. */
|
|
402
446
|
readonly skipImages?: boolean;
|
|
403
|
-
/**
|
|
447
|
+
/**
|
|
448
|
+
* Max DOM traversal depth. Prevents stack overflow on deeply nested HTML. Content nested
|
|
449
|
+
* deeper is dropped and reported as a `DepthLimitExceeded` warning. `None` means 1024 in
|
|
450
|
+
* crawlberg, not the converter's own default of 64.
|
|
451
|
+
*/
|
|
404
452
|
readonly maxDepth?: number;
|
|
405
453
|
/** Enable line wrapping. Default: `false`. */
|
|
406
454
|
readonly wrap?: boolean;
|
|
@@ -435,6 +483,14 @@ export interface ContentConfig {
|
|
|
435
483
|
* Scripts and styles remain excluded; explicit exclusions and image settings still apply.
|
|
436
484
|
*/
|
|
437
485
|
readonly keepHidden?: boolean;
|
|
486
|
+
/** Append abbreviation titles in parentheses. Default: `true`. */
|
|
487
|
+
readonly expandAbbreviations?: boolean;
|
|
488
|
+
/** Escape literal Markdown metacharacters such as backticks and brackets. Default: `false`. */
|
|
489
|
+
readonly escapeMisc?: boolean;
|
|
490
|
+
/** Escape literal asterisks in text. Default: `false`. */
|
|
491
|
+
readonly escapeAsterisks?: boolean;
|
|
492
|
+
/** Include names and anchor tooltips of decorative linked SVG icons. Default: `true`. */
|
|
493
|
+
readonly includeIconLabels?: boolean;
|
|
438
494
|
}
|
|
439
495
|
|
|
440
496
|
/** Content filter applied to each crawled page before it reaches the result. */
|
|
@@ -510,15 +566,11 @@ export interface CrawlConfig {
|
|
|
510
566
|
*/
|
|
511
567
|
readonly respectRobotsTxt?: boolean;
|
|
512
568
|
/**
|
|
513
|
-
* When true, HTTP
|
|
514
|
-
*
|
|
515
|
-
*
|
|
516
|
-
*
|
|
517
|
-
*
|
|
518
|
-
* throw-on-error contract for direct fetches. Independently of this flag,
|
|
519
|
-
* 404s reached at the end of a redirect chain are *always* surfaced softly —
|
|
520
|
-
* the user opted into redirect-following, so receiving a 404 there is part of
|
|
521
|
-
* the normal flow rather than an unexpected error.
|
|
569
|
+
* When true, readable HTTP error responses (4xx and 5xx) are surfaced as
|
|
570
|
+
* `ScrapeResult` records with their status and body rather than raised as errors.
|
|
571
|
+
* WAF blocks report their original 4xx/5xx status, or 403 for a block served as 2xx.
|
|
572
|
+
* Default `false` raises errors for recognized error statuses in both `scrape`
|
|
573
|
+
* and `crawl`, including a 404 reached through redirects.
|
|
522
574
|
*/
|
|
523
575
|
readonly softHttpErrors?: boolean;
|
|
524
576
|
/** Custom user-agent string. */
|
|
@@ -542,7 +594,7 @@ export interface CrawlConfig {
|
|
|
542
594
|
readonly allowSubdomains?: boolean;
|
|
543
595
|
/** Regex patterns for paths to include during crawling. */
|
|
544
596
|
readonly includePaths?: Array<string>;
|
|
545
|
-
/** Regex patterns for paths to exclude during crawling. */
|
|
597
|
+
/** Regex patterns for paths to exclude during crawling. The seed URL is never excluded. */
|
|
546
598
|
readonly excludePaths?: Array<string>;
|
|
547
599
|
/**
|
|
548
600
|
* Whether `include_paths`/`exclude_paths` match against `path?query` instead of just
|
|
@@ -583,8 +635,9 @@ export interface CrawlConfig {
|
|
|
583
635
|
*/
|
|
584
636
|
readonly trackingParams?: Array<string>;
|
|
585
637
|
/**
|
|
586
|
-
* Custom HTTP headers
|
|
587
|
-
*
|
|
638
|
+
* Custom HTTP headers sent only to the seed URL's host by default, on any scheme or port.
|
|
639
|
+
* Explicitly authorize individual headers for admitted subdomains with
|
|
640
|
+
* `Self.subdomain_header_allowlist`. Other hosts never receive these headers.
|
|
588
641
|
*/
|
|
589
642
|
readonly customHeaders?: Record<string, string>;
|
|
590
643
|
/** Timeout for individual HTTP requests (in milliseconds when serialized). */
|
|
@@ -610,7 +663,12 @@ export interface CrawlConfig {
|
|
|
610
663
|
* subsequent attempt (capped at `retry_max_delay_ms`). Defaults to 100ms.
|
|
611
664
|
*/
|
|
612
665
|
readonly retryInitialDelayMs?: number;
|
|
613
|
-
/**
|
|
666
|
+
/**
|
|
667
|
+
* Upper bound, in milliseconds, on the exponential retry backoff. A 429 or 503 whose
|
|
668
|
+
* `Retry-After` is longer than the backoff is waited out, but one that exceeds this bound
|
|
669
|
+
* ends the retries with a warning instead (so `0` abandons the retry on any positive
|
|
670
|
+
* `Retry-After`). Defaults to 60s.
|
|
671
|
+
*/
|
|
614
672
|
readonly retryMaxDelayMs?: number;
|
|
615
673
|
/**
|
|
616
674
|
* Fraction of the per-domain rate-limit delay to randomly jitter by, in `[0.0, 1.0]`.
|
|
@@ -750,6 +808,22 @@ export interface CrawlConfig {
|
|
|
750
808
|
* and the environment variable is not consulted for this config.
|
|
751
809
|
*/
|
|
752
810
|
readonly ssrfDenyPrivateExplicit?: boolean;
|
|
811
|
+
/**
|
|
812
|
+
* Accept invalid TLS certificates for page and robots.txt requests.
|
|
813
|
+
* Defaults to false. Enable only for sites whose certificates you explicitly trust.
|
|
814
|
+
* The browser backends also apply this setting before navigation.
|
|
815
|
+
* On WebAssembly, enabling this setting returns `Unsupported`: the host's Fetch
|
|
816
|
+
* implementation controls certificate validation.
|
|
817
|
+
*/
|
|
818
|
+
readonly dangerAcceptInvalidCerts?: boolean;
|
|
819
|
+
/**
|
|
820
|
+
* Custom header names explicitly authorized for proper subdomains when
|
|
821
|
+
* `Self.allow_subdomains` is true. Empty by default; names match case-insensitively.
|
|
822
|
+
* Authorization, cookies, other standard credential headers, and the header named by
|
|
823
|
+
* `Self.auth` remain seed-host-only even when listed. HTTP, Chromiumoxide, and the
|
|
824
|
+
* native browser backend apply this policy to each request and redirect hop.
|
|
825
|
+
*/
|
|
826
|
+
readonly subdomainHeaderAllowlist?: Array<string>;
|
|
753
827
|
}
|
|
754
828
|
|
|
755
829
|
/**
|
|
@@ -892,6 +966,8 @@ export interface CrawlResult {
|
|
|
892
966
|
readonly stayedOnDomain?: boolean;
|
|
893
967
|
/** Whether the browser fallback was used for any page in this crawl. */
|
|
894
968
|
readonly browserUsed?: boolean;
|
|
969
|
+
/** Details of the first WAF block encountered during the crawl. <!-- --> */
|
|
970
|
+
readonly wafBlock?: WafBlockDetails;
|
|
895
971
|
}
|
|
896
972
|
|
|
897
973
|
/**
|
|
@@ -1467,6 +1543,16 @@ export interface SsrfPolicy {
|
|
|
1467
1543
|
readonly schemeAllowlist?: Array<string>;
|
|
1468
1544
|
}
|
|
1469
1545
|
|
|
1546
|
+
/** Structured context for a WAF-blocked response. <!-- --> */
|
|
1547
|
+
export interface WafBlockDetails {
|
|
1548
|
+
/** Lowercase WAF vendor identifier. <!-- --> */
|
|
1549
|
+
readonly vendor?: string;
|
|
1550
|
+
/** Origin HTTP status, when a response caused the block. <!-- --> */
|
|
1551
|
+
readonly status?: number;
|
|
1552
|
+
/** At most 4 KiB of the blocked response body, when it was read. <!-- --> */
|
|
1553
|
+
readonly bodyPrefix?: string;
|
|
1554
|
+
}
|
|
1555
|
+
|
|
1470
1556
|
/** Discover all pages on a website by following links and sitemaps. */
|
|
1471
1557
|
export declare function mapUrls(engine: CrawlEngineHandle, url: string): Promise<MapResult>;
|
|
1472
1558
|
|
|
@@ -1482,3 +1568,45 @@ export declare class CrawlStreamIterator {
|
|
|
1482
1568
|
next(value?: undefined): Promise<IteratorResult<CrawlEvent, void>>;
|
|
1483
1569
|
[Symbol.asyncIterator](): AsyncGenerator<CrawlEvent, void, undefined>;
|
|
1484
1570
|
}
|
|
1571
|
+
|
|
1572
|
+
/** The error a binding call throws for `CrawlError`. */
|
|
1573
|
+
export interface ThrownCrawlError extends Error {
|
|
1574
|
+
kind:
|
|
1575
|
+
| "NotFound"
|
|
1576
|
+
| "Unauthorized"
|
|
1577
|
+
| "Forbidden"
|
|
1578
|
+
| "WafBlocked"
|
|
1579
|
+
| "Timeout"
|
|
1580
|
+
| "RateLimited"
|
|
1581
|
+
| "ServerError"
|
|
1582
|
+
| "BadGateway"
|
|
1583
|
+
| "Gone"
|
|
1584
|
+
| "Connection"
|
|
1585
|
+
| "Dns"
|
|
1586
|
+
| "Ssl"
|
|
1587
|
+
| "DataLoss"
|
|
1588
|
+
| "ConversionFailed"
|
|
1589
|
+
| "BrowserError"
|
|
1590
|
+
| "BrowserTimeout"
|
|
1591
|
+
| "InvalidConfig"
|
|
1592
|
+
| "Unsupported"
|
|
1593
|
+
| "TooManyRedirects"
|
|
1594
|
+
| "SsrfPolicyViolation"
|
|
1595
|
+
| "Other"
|
|
1596
|
+
| "Unknown";
|
|
1597
|
+
errorCode: number;
|
|
1598
|
+
}
|
|
1599
|
+
|
|
1600
|
+
/** The error a binding call throws for `SsrfError`. */
|
|
1601
|
+
export interface ThrownSsrfError extends Error {
|
|
1602
|
+
kind:
|
|
1603
|
+
| "DeniedByPolicy"
|
|
1604
|
+
| "NotOnAllowlist"
|
|
1605
|
+
| "InvalidCidr"
|
|
1606
|
+
| "DnsResolutionFailed"
|
|
1607
|
+
| "InvalidUrl"
|
|
1608
|
+
| "DisallowedScheme"
|
|
1609
|
+
| "TooManyRedirects"
|
|
1610
|
+
| "Unknown";
|
|
1611
|
+
errorCode: number;
|
|
1612
|
+
}
|