@xberg-io/crawlberg 1.12.0 → 1.12.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/binding.cjs +138 -0
- package/index.d.ts +71 -14
- package/index.js +584 -495
- package/index.mjs +47 -0
- package/package.json +11 -8
package/binding.cjs
ADDED
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
// DO NOT EDIT - auto-generated by alef
|
|
2
|
+
// alef:hash:b5a84836004f3b1787dc4ddf756b1f8dd85684282f90c6b508552bd1861b3f61
|
|
3
|
+
"use strict";
|
|
4
|
+
|
|
5
|
+
// The native binding attaches the structured properties of a failed call to the thrown error as
|
|
6
|
+
// the JSON message of its `cause`: the napi runtime cannot set own properties on an error that an
|
|
7
|
+
// async Rust function returns. This module moves those properties onto the error itself, so a
|
|
8
|
+
// caller reads `error.kind`, `error.statusCode`, `error.retryAfter` and the rest as declared in
|
|
9
|
+
// `index.d.ts`. It wraps every exported function, every static and prototype method of an
|
|
10
|
+
// exported class, and the async iterator a streaming call returns.
|
|
11
|
+
// Constructors are not wrapped: a constructor that throws keeps the plain native error.
|
|
12
|
+
|
|
13
|
+
const MARKER = "alef:error-info:";
|
|
14
|
+
const SKIP = new Set(["constructor", "length", "name", "prototype", "caller", "arguments"]);
|
|
15
|
+
const patched = new WeakSet();
|
|
16
|
+
const iterables = new WeakSet();
|
|
17
|
+
|
|
18
|
+
function decorate(error) {
|
|
19
|
+
if (error === null || typeof error !== "object") return error;
|
|
20
|
+
const cause = error.cause;
|
|
21
|
+
if (cause === null || typeof cause !== "object" || typeof cause.message !== "string") return error;
|
|
22
|
+
if (!cause.message.startsWith(MARKER)) return error;
|
|
23
|
+
let info;
|
|
24
|
+
try {
|
|
25
|
+
info = JSON.parse(cause.message.slice(MARKER.length));
|
|
26
|
+
} catch {
|
|
27
|
+
return error;
|
|
28
|
+
}
|
|
29
|
+
for (const [name, value] of Object.entries(info)) {
|
|
30
|
+
Object.defineProperty(error, name, { value, enumerable: true, writable: true, configurable: true });
|
|
31
|
+
}
|
|
32
|
+
try {
|
|
33
|
+
delete error.cause;
|
|
34
|
+
} catch {
|
|
35
|
+
// The payload stays on the cause when the runtime refuses the delete.
|
|
36
|
+
}
|
|
37
|
+
return error;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function rethrow(error) {
|
|
41
|
+
throw decorate(error);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function wrapIterator(iterator) {
|
|
45
|
+
if (iterator === null || typeof iterator !== "object") return iterator;
|
|
46
|
+
for (const name of ["next", "return", "throw"]) {
|
|
47
|
+
const original = iterator[name];
|
|
48
|
+
if (typeof original !== "function") continue;
|
|
49
|
+
try {
|
|
50
|
+
Object.defineProperty(iterator, name, {
|
|
51
|
+
configurable: true,
|
|
52
|
+
writable: true,
|
|
53
|
+
value: function (...args) {
|
|
54
|
+
return Promise.resolve(original.apply(this, args)).then(undefined, rethrow);
|
|
55
|
+
},
|
|
56
|
+
});
|
|
57
|
+
} catch {
|
|
58
|
+
// A frozen iterator keeps its plain errors.
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
return iterator;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function wrapIterable(value) {
|
|
65
|
+
if (value === null || typeof value !== "object") return value;
|
|
66
|
+
const open = value[Symbol.asyncIterator];
|
|
67
|
+
if (typeof open !== "function" || iterables.has(value)) return value;
|
|
68
|
+
iterables.add(value);
|
|
69
|
+
try {
|
|
70
|
+
Object.defineProperty(value, Symbol.asyncIterator, {
|
|
71
|
+
configurable: true,
|
|
72
|
+
writable: true,
|
|
73
|
+
value: function () {
|
|
74
|
+
return wrapIterator(open.call(this));
|
|
75
|
+
},
|
|
76
|
+
});
|
|
77
|
+
} catch {
|
|
78
|
+
return value;
|
|
79
|
+
}
|
|
80
|
+
wrapIterator(value);
|
|
81
|
+
return value;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function settle(result) {
|
|
85
|
+
if (result !== null && typeof result === "object" && typeof result.then === "function") {
|
|
86
|
+
return result.then(wrapIterable, rethrow);
|
|
87
|
+
}
|
|
88
|
+
return wrapIterable(result);
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
function wrapFunction(fn) {
|
|
92
|
+
return function (...args) {
|
|
93
|
+
let result;
|
|
94
|
+
try {
|
|
95
|
+
result = fn.apply(this, args);
|
|
96
|
+
} catch (error) {
|
|
97
|
+
throw decorate(error);
|
|
98
|
+
}
|
|
99
|
+
return settle(result);
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
function patchMethods(target) {
|
|
104
|
+
if (target === null || (typeof target !== "object" && typeof target !== "function")) return;
|
|
105
|
+
if (patched.has(target)) return;
|
|
106
|
+
patched.add(target);
|
|
107
|
+
for (const key of Reflect.ownKeys(target)) {
|
|
108
|
+
if (typeof key === "string" && SKIP.has(key)) continue;
|
|
109
|
+
const descriptor = Object.getOwnPropertyDescriptor(target, key);
|
|
110
|
+
if (!descriptor || typeof descriptor.value !== "function" || !descriptor.configurable) continue;
|
|
111
|
+
try {
|
|
112
|
+
Object.defineProperty(target, key, { ...descriptor, value: wrapFunction(descriptor.value) });
|
|
113
|
+
} catch {
|
|
114
|
+
// A method the runtime refuses to redefine keeps its plain errors.
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
function wrapExport(value) {
|
|
120
|
+
if (typeof value !== "function") return value;
|
|
121
|
+
patchMethods(value);
|
|
122
|
+
patchMethods(value.prototype);
|
|
123
|
+
// Only `apply` is trapped, so `new`, `instanceof` and static access reach the native class.
|
|
124
|
+
return new Proxy(value, {
|
|
125
|
+
apply(target, thisArg, args) {
|
|
126
|
+
return wrapFunction(target).apply(thisArg, args);
|
|
127
|
+
},
|
|
128
|
+
});
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
module.exports = function decorateBinding(native) {
|
|
132
|
+
const decorated = {};
|
|
133
|
+
for (const key of Object.keys(native)) {
|
|
134
|
+
decorated[key] = wrapExport(native[key]);
|
|
135
|
+
}
|
|
136
|
+
return decorated;
|
|
137
|
+
};
|
|
138
|
+
module.exports.decorate = decorate;
|
package/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// This file is auto-generated by alef — DO NOT EDIT.
|
|
2
|
-
// alef:hash:
|
|
2
|
+
// alef:hash:f58f5415140504047100134aac44d0ee1f666d2775ab43a5ab8921162c1d7c84
|
|
3
3
|
// To regenerate: alef generate
|
|
4
4
|
// To verify freshness: alef verify
|
|
5
5
|
/* eslint-disable */
|
|
@@ -218,7 +218,7 @@ export interface BrowserConfig {
|
|
|
218
218
|
* Extra time to wait after the wait condition is met.
|
|
219
219
|
*
|
|
220
220
|
* The default `network_idle` wait returns once the page has loaded and its DOM has been
|
|
221
|
-
* unchanged for `dom_quiet_window` (at most
|
|
221
|
+
* unchanged for `dom_quiet_window` (at most `dom_quiet_cap`, and never past `timeout`, counted
|
|
222
222
|
* from the start of the navigation). An app that draws after a longer quiet period needs
|
|
223
223
|
* a larger `dom_quiet_window` or `extra_wait`; for
|
|
224
224
|
* example `3000` waits three more seconds.
|
|
@@ -306,11 +306,18 @@ export interface BrowserConfig {
|
|
|
306
306
|
/**
|
|
307
307
|
* How long the DOM must stay unchanged after page load for `network_idle` to return,
|
|
308
308
|
* in milliseconds when serialized. Default: 150 ms. Every mutation restarts this window;
|
|
309
|
-
* the whole wait stays bounded to
|
|
309
|
+
* the whole wait stays bounded to `dom_quiet_cap` and the remaining navigation timeout.
|
|
310
310
|
* A page whose first delayed render follows a longer quiet gap needs a larger value
|
|
311
311
|
* or `extra_wait`. Chromiumoxide backend only.
|
|
312
312
|
*/
|
|
313
313
|
readonly domQuietWindow?: number;
|
|
314
|
+
/**
|
|
315
|
+
* Maximum time spent waiting for DOM quiet after page load, in milliseconds when serialized.
|
|
316
|
+
* Default: 4000 ms. Lower this for pages with continuous mutations; zero skips the quiet wait.
|
|
317
|
+
* The remaining navigation timeout also bounds this wait, independently of the quiet window.
|
|
318
|
+
* Chromiumoxide backend only.
|
|
319
|
+
*/
|
|
320
|
+
readonly domQuietCap?: number;
|
|
314
321
|
}
|
|
315
322
|
|
|
316
323
|
/**
|
|
@@ -476,6 +483,14 @@ export interface ContentConfig {
|
|
|
476
483
|
* Scripts and styles remain excluded; explicit exclusions and image settings still apply.
|
|
477
484
|
*/
|
|
478
485
|
readonly keepHidden?: boolean;
|
|
486
|
+
/** Append abbreviation titles in parentheses. Default: `true`. */
|
|
487
|
+
readonly expandAbbreviations?: boolean;
|
|
488
|
+
/** Escape literal Markdown metacharacters such as backticks and brackets. Default: `false`. */
|
|
489
|
+
readonly escapeMisc?: boolean;
|
|
490
|
+
/** Escape literal asterisks in text. Default: `false`. */
|
|
491
|
+
readonly escapeAsterisks?: boolean;
|
|
492
|
+
/** Include names and anchor tooltips of decorative linked SVG icons. Default: `true`. */
|
|
493
|
+
readonly includeIconLabels?: boolean;
|
|
479
494
|
}
|
|
480
495
|
|
|
481
496
|
/** Content filter applied to each crawled page before it reaches the result. */
|
|
@@ -620,17 +635,9 @@ export interface CrawlConfig {
|
|
|
620
635
|
*/
|
|
621
636
|
readonly trackingParams?: Array<string>;
|
|
622
637
|
/**
|
|
623
|
-
* Custom HTTP headers
|
|
624
|
-
*
|
|
625
|
-
*
|
|
626
|
-
* seed host: `Authorization`, `Proxy-Authorization`, `Cookie`, `Set-Cookie`, `X-Api-Key`,
|
|
627
|
-
* `X-Amz-Security-Token`, `Authentication-Info`, the header `Self.auth` names, and any
|
|
628
|
-
* header whose name contains `auth`, `token`, `key`, `secret`, `session`, `cookie`, `csrf`,
|
|
629
|
-
* `xsrf`, `password`, `signature`, `credential`, `passwd`, `pwd`, `pass`, `jwt`, `bearer`,
|
|
630
|
-
* `otp`, `assertion`, `hmac`, `private`, `sso` or `sig`. A secret in a header whose name
|
|
631
|
-
* matches none of these reaches subdomains, so name secret headers recognisably. A request
|
|
632
|
-
* to any other host does not carry any of them. Subdomain forwarding applies to the HTTP and
|
|
633
|
-
* chromiumoxide backends; the native browser backend sends them to the seed host only.
|
|
638
|
+
* Custom HTTP headers sent only to the seed URL's host by default, on any scheme or port.
|
|
639
|
+
* Explicitly authorize individual headers for admitted subdomains with
|
|
640
|
+
* `Self.subdomain_header_allowlist`. Other hosts never receive these headers.
|
|
634
641
|
*/
|
|
635
642
|
readonly customHeaders?: Record<string, string>;
|
|
636
643
|
/** Timeout for individual HTTP requests (in milliseconds when serialized). */
|
|
@@ -809,6 +816,14 @@ export interface CrawlConfig {
|
|
|
809
816
|
* implementation controls certificate validation.
|
|
810
817
|
*/
|
|
811
818
|
readonly dangerAcceptInvalidCerts?: boolean;
|
|
819
|
+
/**
|
|
820
|
+
* Custom header names explicitly authorized for proper subdomains when
|
|
821
|
+
* `Self.allow_subdomains` is true. Empty by default; names match case-insensitively.
|
|
822
|
+
* Authorization, cookies, other standard credential headers, and the header named by
|
|
823
|
+
* `Self.auth` remain seed-host-only even when listed. HTTP, Chromiumoxide, and the
|
|
824
|
+
* native browser backend apply this policy to each request and redirect hop.
|
|
825
|
+
*/
|
|
826
|
+
readonly subdomainHeaderAllowlist?: Array<string>;
|
|
812
827
|
}
|
|
813
828
|
|
|
814
829
|
/**
|
|
@@ -1553,3 +1568,45 @@ export declare class CrawlStreamIterator {
|
|
|
1553
1568
|
next(value?: undefined): Promise<IteratorResult<CrawlEvent, void>>;
|
|
1554
1569
|
[Symbol.asyncIterator](): AsyncGenerator<CrawlEvent, void, undefined>;
|
|
1555
1570
|
}
|
|
1571
|
+
|
|
1572
|
+
/** The error a binding call throws for `CrawlError`. */
|
|
1573
|
+
export interface ThrownCrawlError extends Error {
|
|
1574
|
+
kind:
|
|
1575
|
+
| "NotFound"
|
|
1576
|
+
| "Unauthorized"
|
|
1577
|
+
| "Forbidden"
|
|
1578
|
+
| "WafBlocked"
|
|
1579
|
+
| "Timeout"
|
|
1580
|
+
| "RateLimited"
|
|
1581
|
+
| "ServerError"
|
|
1582
|
+
| "BadGateway"
|
|
1583
|
+
| "Gone"
|
|
1584
|
+
| "Connection"
|
|
1585
|
+
| "Dns"
|
|
1586
|
+
| "Ssl"
|
|
1587
|
+
| "DataLoss"
|
|
1588
|
+
| "ConversionFailed"
|
|
1589
|
+
| "BrowserError"
|
|
1590
|
+
| "BrowserTimeout"
|
|
1591
|
+
| "InvalidConfig"
|
|
1592
|
+
| "Unsupported"
|
|
1593
|
+
| "TooManyRedirects"
|
|
1594
|
+
| "SsrfPolicyViolation"
|
|
1595
|
+
| "Other"
|
|
1596
|
+
| "Unknown";
|
|
1597
|
+
errorCode: number;
|
|
1598
|
+
}
|
|
1599
|
+
|
|
1600
|
+
/** The error a binding call throws for `SsrfError`. */
|
|
1601
|
+
export interface ThrownSsrfError extends Error {
|
|
1602
|
+
kind:
|
|
1603
|
+
| "DeniedByPolicy"
|
|
1604
|
+
| "NotOnAllowlist"
|
|
1605
|
+
| "InvalidCidr"
|
|
1606
|
+
| "DnsResolutionFailed"
|
|
1607
|
+
| "InvalidUrl"
|
|
1608
|
+
| "DisallowedScheme"
|
|
1609
|
+
| "TooManyRedirects"
|
|
1610
|
+
| "Unknown";
|
|
1611
|
+
errorCode: number;
|
|
1612
|
+
}
|