@xberg-io/crawlberg 1.12.0 → 1.12.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/binding.cjs +138 -0
- package/index.d.ts +103 -46
- package/index.js +584 -495
- package/index.mjs +47 -0
- package/package.json +11 -8
package/binding.cjs
ADDED
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
// DO NOT EDIT - auto-generated by alef
|
|
2
|
+
// alef:hash:b5a84836004f3b1787dc4ddf756b1f8dd85684282f90c6b508552bd1861b3f61
|
|
3
|
+
"use strict";
|
|
4
|
+
|
|
5
|
+
// The native binding attaches the structured properties of a failed call to the thrown error as
|
|
6
|
+
// the JSON message of its `cause`: the napi runtime cannot set own properties on an error that an
|
|
7
|
+
// async Rust function returns. This module moves those properties onto the error itself, so a
|
|
8
|
+
// caller reads `error.kind`, `error.statusCode`, `error.retryAfter` and the rest as declared in
|
|
9
|
+
// `index.d.ts`. It wraps every exported function, every static and prototype method of an
|
|
10
|
+
// exported class, and the async iterator a streaming call returns.
|
|
11
|
+
// Constructors are not wrapped: a constructor that throws keeps the plain native error.
|
|
12
|
+
|
|
13
|
+
const MARKER = "alef:error-info:";
|
|
14
|
+
const SKIP = new Set(["constructor", "length", "name", "prototype", "caller", "arguments"]);
|
|
15
|
+
const patched = new WeakSet();
|
|
16
|
+
const iterables = new WeakSet();
|
|
17
|
+
|
|
18
|
+
function decorate(error) {
|
|
19
|
+
if (error === null || typeof error !== "object") return error;
|
|
20
|
+
const cause = error.cause;
|
|
21
|
+
if (cause === null || typeof cause !== "object" || typeof cause.message !== "string") return error;
|
|
22
|
+
if (!cause.message.startsWith(MARKER)) return error;
|
|
23
|
+
let info;
|
|
24
|
+
try {
|
|
25
|
+
info = JSON.parse(cause.message.slice(MARKER.length));
|
|
26
|
+
} catch {
|
|
27
|
+
return error;
|
|
28
|
+
}
|
|
29
|
+
for (const [name, value] of Object.entries(info)) {
|
|
30
|
+
Object.defineProperty(error, name, { value, enumerable: true, writable: true, configurable: true });
|
|
31
|
+
}
|
|
32
|
+
try {
|
|
33
|
+
delete error.cause;
|
|
34
|
+
} catch {
|
|
35
|
+
// The payload stays on the cause when the runtime refuses the delete.
|
|
36
|
+
}
|
|
37
|
+
return error;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function rethrow(error) {
|
|
41
|
+
throw decorate(error);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function wrapIterator(iterator) {
|
|
45
|
+
if (iterator === null || typeof iterator !== "object") return iterator;
|
|
46
|
+
for (const name of ["next", "return", "throw"]) {
|
|
47
|
+
const original = iterator[name];
|
|
48
|
+
if (typeof original !== "function") continue;
|
|
49
|
+
try {
|
|
50
|
+
Object.defineProperty(iterator, name, {
|
|
51
|
+
configurable: true,
|
|
52
|
+
writable: true,
|
|
53
|
+
value: function (...args) {
|
|
54
|
+
return Promise.resolve(original.apply(this, args)).then(undefined, rethrow);
|
|
55
|
+
},
|
|
56
|
+
});
|
|
57
|
+
} catch {
|
|
58
|
+
// A frozen iterator keeps its plain errors.
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
return iterator;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function wrapIterable(value) {
|
|
65
|
+
if (value === null || typeof value !== "object") return value;
|
|
66
|
+
const open = value[Symbol.asyncIterator];
|
|
67
|
+
if (typeof open !== "function" || iterables.has(value)) return value;
|
|
68
|
+
iterables.add(value);
|
|
69
|
+
try {
|
|
70
|
+
Object.defineProperty(value, Symbol.asyncIterator, {
|
|
71
|
+
configurable: true,
|
|
72
|
+
writable: true,
|
|
73
|
+
value: function () {
|
|
74
|
+
return wrapIterator(open.call(this));
|
|
75
|
+
},
|
|
76
|
+
});
|
|
77
|
+
} catch {
|
|
78
|
+
return value;
|
|
79
|
+
}
|
|
80
|
+
wrapIterator(value);
|
|
81
|
+
return value;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function settle(result) {
|
|
85
|
+
if (result !== null && typeof result === "object" && typeof result.then === "function") {
|
|
86
|
+
return result.then(wrapIterable, rethrow);
|
|
87
|
+
}
|
|
88
|
+
return wrapIterable(result);
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
function wrapFunction(fn) {
|
|
92
|
+
return function (...args) {
|
|
93
|
+
let result;
|
|
94
|
+
try {
|
|
95
|
+
result = fn.apply(this, args);
|
|
96
|
+
} catch (error) {
|
|
97
|
+
throw decorate(error);
|
|
98
|
+
}
|
|
99
|
+
return settle(result);
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
function patchMethods(target) {
|
|
104
|
+
if (target === null || (typeof target !== "object" && typeof target !== "function")) return;
|
|
105
|
+
if (patched.has(target)) return;
|
|
106
|
+
patched.add(target);
|
|
107
|
+
for (const key of Reflect.ownKeys(target)) {
|
|
108
|
+
if (typeof key === "string" && SKIP.has(key)) continue;
|
|
109
|
+
const descriptor = Object.getOwnPropertyDescriptor(target, key);
|
|
110
|
+
if (!descriptor || typeof descriptor.value !== "function" || !descriptor.configurable) continue;
|
|
111
|
+
try {
|
|
112
|
+
Object.defineProperty(target, key, { ...descriptor, value: wrapFunction(descriptor.value) });
|
|
113
|
+
} catch {
|
|
114
|
+
// A method the runtime refuses to redefine keeps its plain errors.
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
function wrapExport(value) {
|
|
120
|
+
if (typeof value !== "function") return value;
|
|
121
|
+
patchMethods(value);
|
|
122
|
+
patchMethods(value.prototype);
|
|
123
|
+
// Only `apply` is trapped, so `new`, `instanceof` and static access reach the native class.
|
|
124
|
+
return new Proxy(value, {
|
|
125
|
+
apply(target, thisArg, args) {
|
|
126
|
+
return wrapFunction(target).apply(thisArg, args);
|
|
127
|
+
},
|
|
128
|
+
});
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
module.exports = function decorateBinding(native) {
|
|
132
|
+
const decorated = {};
|
|
133
|
+
for (const key of Object.keys(native)) {
|
|
134
|
+
decorated[key] = wrapExport(native[key]);
|
|
135
|
+
}
|
|
136
|
+
return decorated;
|
|
137
|
+
};
|
|
138
|
+
module.exports.decorate = decorate;
|
package/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// This file is auto-generated by alef — DO NOT EDIT.
|
|
2
|
-
// alef:hash:
|
|
2
|
+
// alef:hash:0d4aec5266b8d358067295a8183a3dd48d59d5a2aedf733d0af70004d5e79b88
|
|
3
3
|
// To regenerate: alef generate
|
|
4
4
|
// To verify freshness: alef verify
|
|
5
5
|
/* eslint-disable */
|
|
@@ -218,7 +218,7 @@ export interface BrowserConfig {
|
|
|
218
218
|
* Extra time to wait after the wait condition is met.
|
|
219
219
|
*
|
|
220
220
|
* The default `network_idle` wait returns once the page has loaded and its DOM has been
|
|
221
|
-
* unchanged for `dom_quiet_window` (at most
|
|
221
|
+
* unchanged for `dom_quiet_window` (at most `dom_quiet_cap`, and never past `timeout`, counted
|
|
222
222
|
* from the start of the navigation). An app that draws after a longer quiet period needs
|
|
223
223
|
* a larger `dom_quiet_window` or `extra_wait`; for
|
|
224
224
|
* example `3000` waits three more seconds.
|
|
@@ -306,11 +306,18 @@ export interface BrowserConfig {
|
|
|
306
306
|
/**
|
|
307
307
|
* How long the DOM must stay unchanged after page load for `network_idle` to return,
|
|
308
308
|
* in milliseconds when serialized. Default: 150 ms. Every mutation restarts this window;
|
|
309
|
-
* the whole wait stays bounded to
|
|
309
|
+
* the whole wait stays bounded to `dom_quiet_cap` and the remaining navigation timeout.
|
|
310
310
|
* A page whose first delayed render follows a longer quiet gap needs a larger value
|
|
311
311
|
* or `extra_wait`. Chromiumoxide backend only.
|
|
312
312
|
*/
|
|
313
313
|
readonly domQuietWindow?: number;
|
|
314
|
+
/**
|
|
315
|
+
* Maximum time spent waiting for DOM quiet after page load, in milliseconds when serialized.
|
|
316
|
+
* Default: 4000 ms. Lower this for pages with continuous mutations; zero skips the quiet wait.
|
|
317
|
+
* The remaining navigation timeout also bounds this wait, independently of the quiet window.
|
|
318
|
+
* Chromiumoxide backend only.
|
|
319
|
+
*/
|
|
320
|
+
readonly domQuietCap?: number;
|
|
314
321
|
}
|
|
315
322
|
|
|
316
323
|
/**
|
|
@@ -476,6 +483,14 @@ export interface ContentConfig {
|
|
|
476
483
|
* Scripts and styles remain excluded; explicit exclusions and image settings still apply.
|
|
477
484
|
*/
|
|
478
485
|
readonly keepHidden?: boolean;
|
|
486
|
+
/** Append abbreviation titles in parentheses. Default: `true`. */
|
|
487
|
+
readonly expandAbbreviations?: boolean;
|
|
488
|
+
/** Escape literal Markdown metacharacters such as backticks and brackets. Default: `false`. */
|
|
489
|
+
readonly escapeMisc?: boolean;
|
|
490
|
+
/** Escape literal asterisks in text. Default: `false`. */
|
|
491
|
+
readonly escapeAsterisks?: boolean;
|
|
492
|
+
/** Include names and anchor tooltips of decorative linked SVG icons. Default: `true`. */
|
|
493
|
+
readonly includeIconLabels?: boolean;
|
|
479
494
|
}
|
|
480
495
|
|
|
481
496
|
/** Content filter applied to each crawled page before it reaches the result. */
|
|
@@ -620,17 +635,9 @@ export interface CrawlConfig {
|
|
|
620
635
|
*/
|
|
621
636
|
readonly trackingParams?: Array<string>;
|
|
622
637
|
/**
|
|
623
|
-
* Custom HTTP headers
|
|
624
|
-
*
|
|
625
|
-
*
|
|
626
|
-
* seed host: `Authorization`, `Proxy-Authorization`, `Cookie`, `Set-Cookie`, `X-Api-Key`,
|
|
627
|
-
* `X-Amz-Security-Token`, `Authentication-Info`, the header `Self.auth` names, and any
|
|
628
|
-
* header whose name contains `auth`, `token`, `key`, `secret`, `session`, `cookie`, `csrf`,
|
|
629
|
-
* `xsrf`, `password`, `signature`, `credential`, `passwd`, `pwd`, `pass`, `jwt`, `bearer`,
|
|
630
|
-
* `otp`, `assertion`, `hmac`, `private`, `sso` or `sig`. A secret in a header whose name
|
|
631
|
-
* matches none of these reaches subdomains, so name secret headers recognisably. A request
|
|
632
|
-
* to any other host does not carry any of them. Subdomain forwarding applies to the HTTP and
|
|
633
|
-
* chromiumoxide backends; the native browser backend sends them to the seed host only.
|
|
638
|
+
* Custom HTTP headers sent only to the seed URL's host by default, on any scheme or port.
|
|
639
|
+
* Explicitly authorize individual headers for admitted subdomains with
|
|
640
|
+
* `Self.subdomain_header_allowlist`. Other hosts never receive these headers.
|
|
634
641
|
*/
|
|
635
642
|
readonly customHeaders?: Record<string, string>;
|
|
636
643
|
/** Timeout for individual HTTP requests (in milliseconds when serialized). */
|
|
@@ -809,6 +816,14 @@ export interface CrawlConfig {
|
|
|
809
816
|
* implementation controls certificate validation.
|
|
810
817
|
*/
|
|
811
818
|
readonly dangerAcceptInvalidCerts?: boolean;
|
|
819
|
+
/**
|
|
820
|
+
* Custom header names explicitly authorized for proper subdomains when
|
|
821
|
+
* `Self.allow_subdomains` is true. Empty by default; names match case-insensitively.
|
|
822
|
+
* Authorization, cookies, other standard credential headers, and the header named by
|
|
823
|
+
* `Self.auth` remain seed-host-only even when listed. HTTP, Chromiumoxide, and the
|
|
824
|
+
* native browser backend apply this policy to each request and redirect hop.
|
|
825
|
+
*/
|
|
826
|
+
readonly subdomainHeaderAllowlist?: Array<string>;
|
|
812
827
|
}
|
|
813
828
|
|
|
814
829
|
/**
|
|
@@ -862,35 +877,35 @@ export type CrawlEvent =
|
|
|
862
877
|
/** The result of crawling a single page during a crawl operation. */
|
|
863
878
|
export interface CrawlPageResult {
|
|
864
879
|
/** The original URL of the page. */
|
|
865
|
-
readonly url
|
|
880
|
+
readonly url: string;
|
|
866
881
|
/** The normalized URL of the page. */
|
|
867
|
-
readonly normalizedUrl
|
|
882
|
+
readonly normalizedUrl: string;
|
|
868
883
|
/** The HTTP status code of the response. */
|
|
869
|
-
readonly statusCode
|
|
884
|
+
readonly statusCode: number;
|
|
870
885
|
/** The Content-Type header value. */
|
|
871
|
-
readonly contentType
|
|
886
|
+
readonly contentType: string;
|
|
872
887
|
/** The HTML body of the response. */
|
|
873
|
-
readonly html
|
|
888
|
+
readonly html: string;
|
|
874
889
|
/** The size of the response body in bytes. */
|
|
875
|
-
readonly bodySize
|
|
890
|
+
readonly bodySize: number;
|
|
876
891
|
/** Extracted metadata from the page. */
|
|
877
|
-
readonly metadata
|
|
892
|
+
readonly metadata: PageMetadata;
|
|
878
893
|
/** Links found on the page. */
|
|
879
|
-
readonly links
|
|
894
|
+
readonly links: Array<LinkInfo>;
|
|
880
895
|
/** Images found on the page. */
|
|
881
|
-
readonly images
|
|
896
|
+
readonly images: Array<ImageInfo>;
|
|
882
897
|
/** Feed links found on the page. */
|
|
883
|
-
readonly feeds
|
|
898
|
+
readonly feeds: Array<FeedInfo>;
|
|
884
899
|
/** JSON-LD entries found on the page. */
|
|
885
|
-
readonly jsonLd
|
|
900
|
+
readonly jsonLd: Array<JsonLdEntry>;
|
|
886
901
|
/** The depth of this page from the start URL. */
|
|
887
|
-
readonly depth
|
|
902
|
+
readonly depth: number;
|
|
888
903
|
/** Whether this page is on the same domain as the start URL. */
|
|
889
|
-
readonly stayedOnDomain
|
|
904
|
+
readonly stayedOnDomain: boolean;
|
|
890
905
|
/** Whether this page was skipped (binary or PDF content). */
|
|
891
|
-
readonly wasSkipped
|
|
906
|
+
readonly wasSkipped: boolean;
|
|
892
907
|
/** Whether the content is a PDF. */
|
|
893
|
-
readonly isPdf
|
|
908
|
+
readonly isPdf: boolean;
|
|
894
909
|
/** The detected character set encoding. */
|
|
895
910
|
readonly detectedCharset?: string;
|
|
896
911
|
/** Markdown conversion of the page content. */
|
|
@@ -902,30 +917,30 @@ export interface CrawlPageResult {
|
|
|
902
917
|
/** Downloaded non-HTML document (PDF, DOCX, image, code, etc.). */
|
|
903
918
|
readonly downloadedDocument?: DownloadedDocument;
|
|
904
919
|
/** Whether the browser fallback was used to fetch this page. */
|
|
905
|
-
readonly browserUsed
|
|
920
|
+
readonly browserUsed: boolean;
|
|
906
921
|
/**
|
|
907
922
|
* The URL this page's content was actually fetched from, after following any HTTP,
|
|
908
923
|
* `Refresh` header, or `<meta http-equiv="refresh">` redirect `url` pointed at.
|
|
909
924
|
* Equal to `url` when the fetch did not redirect.
|
|
910
925
|
*/
|
|
911
|
-
readonly finalUrl
|
|
926
|
+
readonly finalUrl: string;
|
|
912
927
|
/** Redirect hops taken to reach `final_url` from `url`. */
|
|
913
|
-
readonly redirectCount
|
|
928
|
+
readonly redirectCount: number;
|
|
914
929
|
/**
|
|
915
930
|
* Whether the page asked not to be indexed, by its robots meta tag or `X-Robots-Tag`
|
|
916
931
|
* header. The page is still crawled and its links still followed.
|
|
917
932
|
*/
|
|
918
|
-
readonly noindexDetected
|
|
933
|
+
readonly noindexDetected: boolean;
|
|
919
934
|
/**
|
|
920
935
|
* Whether the page asked that its links not be followed, by its robots meta tag or
|
|
921
936
|
* `X-Robots-Tag` header. When the crawl respects robots, its links are not followed.
|
|
922
937
|
*/
|
|
923
|
-
readonly nofollowDetected
|
|
938
|
+
readonly nofollowDetected: boolean;
|
|
924
939
|
/**
|
|
925
940
|
* URLs of the requests the page sent in browser mode that the SSRF policy refused, without
|
|
926
941
|
* their credentials, each listed once. The page is kept; only the refused requests failed.
|
|
927
942
|
*/
|
|
928
|
-
readonly ssrfRefusedUrls
|
|
943
|
+
readonly ssrfRefusedUrls: Array<string>;
|
|
929
944
|
/**
|
|
930
945
|
* Seed address as supplied, or the discovered link address resolved against its base.
|
|
931
946
|
* Preserves source spelling through redirects; URL credentials are removed. <!-- -->
|
|
@@ -936,21 +951,21 @@ export interface CrawlPageResult {
|
|
|
936
951
|
/** The result of a multi-page crawl operation. */
|
|
937
952
|
export interface CrawlResult {
|
|
938
953
|
/** The list of crawled pages. */
|
|
939
|
-
readonly pages
|
|
954
|
+
readonly pages: Array<CrawlPageResult>;
|
|
940
955
|
/** The final URL after following redirects. */
|
|
941
|
-
readonly finalUrl
|
|
956
|
+
readonly finalUrl: string;
|
|
942
957
|
/** The number of redirects followed. */
|
|
943
|
-
readonly redirectCount
|
|
958
|
+
readonly redirectCount: number;
|
|
944
959
|
/** Whether any page was skipped during crawling. */
|
|
945
|
-
readonly wasSkipped
|
|
960
|
+
readonly wasSkipped: boolean;
|
|
946
961
|
/** An error message, if the crawl encountered an issue. */
|
|
947
962
|
readonly error?: string;
|
|
948
963
|
/** Cookies collected during the crawl. */
|
|
949
|
-
readonly cookies
|
|
964
|
+
readonly cookies: Array<CookieInfo>;
|
|
950
965
|
/** Whether all crawled pages stayed on the same domain as the start URL. */
|
|
951
|
-
readonly stayedOnDomain
|
|
966
|
+
readonly stayedOnDomain: boolean;
|
|
952
967
|
/** Whether the browser fallback was used for any page in this crawl. */
|
|
953
|
-
readonly browserUsed
|
|
968
|
+
readonly browserUsed: boolean;
|
|
954
969
|
/** Details of the first WAF block encountered during the crawl. <!-- --> */
|
|
955
970
|
readonly wafBlock?: WafBlockDetails;
|
|
956
971
|
}
|
|
@@ -1196,15 +1211,15 @@ export interface JsonLdEntry {
|
|
|
1196
1211
|
/** Information about a link found on a page. */
|
|
1197
1212
|
export interface LinkInfo {
|
|
1198
1213
|
/** The resolved URL of the link. */
|
|
1199
|
-
readonly url
|
|
1214
|
+
readonly url: string;
|
|
1200
1215
|
/** The visible text of the link. */
|
|
1201
|
-
readonly text
|
|
1216
|
+
readonly text: string;
|
|
1202
1217
|
/** The classification of the link. */
|
|
1203
|
-
readonly linkType
|
|
1218
|
+
readonly linkType: LinkType;
|
|
1204
1219
|
/** The `rel` attribute value, if present. */
|
|
1205
1220
|
readonly rel?: string;
|
|
1206
1221
|
/** Whether the link has `rel="nofollow"`. */
|
|
1207
|
-
readonly nofollow
|
|
1222
|
+
readonly nofollow: boolean;
|
|
1208
1223
|
/**
|
|
1209
1224
|
* Link address resolved against the page's base without changing path spelling.
|
|
1210
1225
|
* Percent escapes, Unicode, spaces, backslashes and dot segments are preserved.
|
|
@@ -1553,3 +1568,45 @@ export declare class CrawlStreamIterator {
|
|
|
1553
1568
|
next(value?: undefined): Promise<IteratorResult<CrawlEvent, void>>;
|
|
1554
1569
|
[Symbol.asyncIterator](): AsyncGenerator<CrawlEvent, void, undefined>;
|
|
1555
1570
|
}
|
|
1571
|
+
|
|
1572
|
+
/** The error a binding call throws for `CrawlError`. */
|
|
1573
|
+
export interface ThrownCrawlError extends Error {
|
|
1574
|
+
kind:
|
|
1575
|
+
| "NotFound"
|
|
1576
|
+
| "Unauthorized"
|
|
1577
|
+
| "Forbidden"
|
|
1578
|
+
| "WafBlocked"
|
|
1579
|
+
| "Timeout"
|
|
1580
|
+
| "RateLimited"
|
|
1581
|
+
| "ServerError"
|
|
1582
|
+
| "BadGateway"
|
|
1583
|
+
| "Gone"
|
|
1584
|
+
| "Connection"
|
|
1585
|
+
| "Dns"
|
|
1586
|
+
| "Ssl"
|
|
1587
|
+
| "DataLoss"
|
|
1588
|
+
| "ConversionFailed"
|
|
1589
|
+
| "BrowserError"
|
|
1590
|
+
| "BrowserTimeout"
|
|
1591
|
+
| "InvalidConfig"
|
|
1592
|
+
| "Unsupported"
|
|
1593
|
+
| "TooManyRedirects"
|
|
1594
|
+
| "SsrfPolicyViolation"
|
|
1595
|
+
| "Other"
|
|
1596
|
+
| "Unknown";
|
|
1597
|
+
errorCode: number;
|
|
1598
|
+
}
|
|
1599
|
+
|
|
1600
|
+
/** The error a binding call throws for `SsrfError`. */
|
|
1601
|
+
export interface ThrownSsrfError extends Error {
|
|
1602
|
+
kind:
|
|
1603
|
+
| "DeniedByPolicy"
|
|
1604
|
+
| "NotOnAllowlist"
|
|
1605
|
+
| "InvalidCidr"
|
|
1606
|
+
| "DnsResolutionFailed"
|
|
1607
|
+
| "InvalidUrl"
|
|
1608
|
+
| "DisallowedScheme"
|
|
1609
|
+
| "TooManyRedirects"
|
|
1610
|
+
| "Unknown";
|
|
1611
|
+
errorCode: number;
|
|
1612
|
+
}
|