@xberg-io/crawlberg 1.12.0 → 1.12.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (5) hide show
  1. package/binding.cjs +138 -0
  2. package/index.d.ts +71 -14
  3. package/index.js +584 -495
  4. package/index.mjs +47 -0
  5. package/package.json +11 -8
package/binding.cjs ADDED
@@ -0,0 +1,138 @@
1
+ // DO NOT EDIT - auto-generated by alef
2
+ // alef:hash:b5a84836004f3b1787dc4ddf756b1f8dd85684282f90c6b508552bd1861b3f61
3
+ "use strict";
4
+
5
+ // The native binding attaches the structured properties of a failed call to the thrown error as
6
+ // the JSON message of its `cause`: the napi runtime cannot set own properties on an error that an
7
+ // async Rust function returns. This module moves those properties onto the error itself, so a
8
+ // caller reads `error.kind`, `error.statusCode`, `error.retryAfter` and the rest as declared in
9
+ // `index.d.ts`. It wraps every exported function, every static and prototype method of an
10
+ // exported class, and the async iterator a streaming call returns.
11
+ // Constructors are not wrapped: a constructor that throws keeps the plain native error.
12
+
13
+ const MARKER = "alef:error-info:";
14
+ const SKIP = new Set(["constructor", "length", "name", "prototype", "caller", "arguments"]);
15
+ const patched = new WeakSet();
16
+ const iterables = new WeakSet();
17
+
18
+ function decorate(error) {
19
+ if (error === null || typeof error !== "object") return error;
20
+ const cause = error.cause;
21
+ if (cause === null || typeof cause !== "object" || typeof cause.message !== "string") return error;
22
+ if (!cause.message.startsWith(MARKER)) return error;
23
+ let info;
24
+ try {
25
+ info = JSON.parse(cause.message.slice(MARKER.length));
26
+ } catch {
27
+ return error;
28
+ }
29
+ for (const [name, value] of Object.entries(info)) {
30
+ Object.defineProperty(error, name, { value, enumerable: true, writable: true, configurable: true });
31
+ }
32
+ try {
33
+ delete error.cause;
34
+ } catch {
35
+ // The payload stays on the cause when the runtime refuses the delete.
36
+ }
37
+ return error;
38
+ }
39
+
40
+ function rethrow(error) {
41
+ throw decorate(error);
42
+ }
43
+
44
+ function wrapIterator(iterator) {
45
+ if (iterator === null || typeof iterator !== "object") return iterator;
46
+ for (const name of ["next", "return", "throw"]) {
47
+ const original = iterator[name];
48
+ if (typeof original !== "function") continue;
49
+ try {
50
+ Object.defineProperty(iterator, name, {
51
+ configurable: true,
52
+ writable: true,
53
+ value: function (...args) {
54
+ return Promise.resolve(original.apply(this, args)).then(undefined, rethrow);
55
+ },
56
+ });
57
+ } catch {
58
+ // A frozen iterator keeps its plain errors.
59
+ }
60
+ }
61
+ return iterator;
62
+ }
63
+
64
+ function wrapIterable(value) {
65
+ if (value === null || typeof value !== "object") return value;
66
+ const open = value[Symbol.asyncIterator];
67
+ if (typeof open !== "function" || iterables.has(value)) return value;
68
+ iterables.add(value);
69
+ try {
70
+ Object.defineProperty(value, Symbol.asyncIterator, {
71
+ configurable: true,
72
+ writable: true,
73
+ value: function () {
74
+ return wrapIterator(open.call(this));
75
+ },
76
+ });
77
+ } catch {
78
+ return value;
79
+ }
80
+ wrapIterator(value);
81
+ return value;
82
+ }
83
+
84
+ function settle(result) {
85
+ if (result !== null && typeof result === "object" && typeof result.then === "function") {
86
+ return result.then(wrapIterable, rethrow);
87
+ }
88
+ return wrapIterable(result);
89
+ }
90
+
91
+ function wrapFunction(fn) {
92
+ return function (...args) {
93
+ let result;
94
+ try {
95
+ result = fn.apply(this, args);
96
+ } catch (error) {
97
+ throw decorate(error);
98
+ }
99
+ return settle(result);
100
+ };
101
+ }
102
+
103
+ function patchMethods(target) {
104
+ if (target === null || (typeof target !== "object" && typeof target !== "function")) return;
105
+ if (patched.has(target)) return;
106
+ patched.add(target);
107
+ for (const key of Reflect.ownKeys(target)) {
108
+ if (typeof key === "string" && SKIP.has(key)) continue;
109
+ const descriptor = Object.getOwnPropertyDescriptor(target, key);
110
+ if (!descriptor || typeof descriptor.value !== "function" || !descriptor.configurable) continue;
111
+ try {
112
+ Object.defineProperty(target, key, { ...descriptor, value: wrapFunction(descriptor.value) });
113
+ } catch {
114
+ // A method the runtime refuses to redefine keeps its plain errors.
115
+ }
116
+ }
117
+ }
118
+
119
+ function wrapExport(value) {
120
+ if (typeof value !== "function") return value;
121
+ patchMethods(value);
122
+ patchMethods(value.prototype);
123
+ // Only `apply` is trapped, so `new`, `instanceof` and static access reach the native class.
124
+ return new Proxy(value, {
125
+ apply(target, thisArg, args) {
126
+ return wrapFunction(target).apply(thisArg, args);
127
+ },
128
+ });
129
+ }
130
+
131
+ module.exports = function decorateBinding(native) {
132
+ const decorated = {};
133
+ for (const key of Object.keys(native)) {
134
+ decorated[key] = wrapExport(native[key]);
135
+ }
136
+ return decorated;
137
+ };
138
+ module.exports.decorate = decorate;
package/index.d.ts CHANGED
@@ -1,5 +1,5 @@
1
1
  // This file is auto-generated by alef — DO NOT EDIT.
2
- // alef:hash:970b01dd0b98d7d53a619b3b8702083b17b8050bd3ed2122e6bbee5cef6a5676
2
+ // alef:hash:f58f5415140504047100134aac44d0ee1f666d2775ab43a5ab8921162c1d7c84
3
3
  // To regenerate: alef generate
4
4
  // To verify freshness: alef verify
5
5
  /* eslint-disable */
@@ -218,7 +218,7 @@ export interface BrowserConfig {
218
218
  * Extra time to wait after the wait condition is met.
219
219
  *
220
220
  * The default `network_idle` wait returns once the page has loaded and its DOM has been
221
- * unchanged for `dom_quiet_window` (at most four seconds, and never past `timeout`, counted
221
+ * unchanged for `dom_quiet_window` (at most `dom_quiet_cap`, and never past `timeout`, counted
222
222
  * from the start of the navigation). An app that draws after a longer quiet period needs
223
223
  * a larger `dom_quiet_window` or `extra_wait`; for
224
224
  * example `3000` waits three more seconds.
@@ -306,11 +306,18 @@ export interface BrowserConfig {
306
306
  /**
307
307
  * How long the DOM must stay unchanged after page load for `network_idle` to return,
308
308
  * in milliseconds when serialized. Default: 150 ms. Every mutation restarts this window;
309
- * the whole wait stays bounded to four seconds and the remaining navigation timeout.
309
+ * the whole wait stays bounded to `dom_quiet_cap` and the remaining navigation timeout.
310
310
  * A page whose first delayed render follows a longer quiet gap needs a larger value
311
311
  * or `extra_wait`. Chromiumoxide backend only.
312
312
  */
313
313
  readonly domQuietWindow?: number;
314
+ /**
315
+ * Maximum time spent waiting for DOM quiet after page load, in milliseconds when serialized.
316
+ * Default: 4000 ms. Lower this for pages with continuous mutations; zero skips the quiet wait.
317
+ * The remaining navigation timeout also bounds this wait, independently of the quiet window.
318
+ * Chromiumoxide backend only.
319
+ */
320
+ readonly domQuietCap?: number;
314
321
  }
315
322
 
316
323
  /**
@@ -476,6 +483,14 @@ export interface ContentConfig {
476
483
  * Scripts and styles remain excluded; explicit exclusions and image settings still apply.
477
484
  */
478
485
  readonly keepHidden?: boolean;
486
+ /** Append abbreviation titles in parentheses. Default: `true`. */
487
+ readonly expandAbbreviations?: boolean;
488
+ /** Escape literal Markdown metacharacters such as backticks and brackets. Default: `false`. */
489
+ readonly escapeMisc?: boolean;
490
+ /** Escape literal asterisks in text. Default: `false`. */
491
+ readonly escapeAsterisks?: boolean;
492
+ /** Include names and anchor tooltips of decorative linked SVG icons. Default: `true`. */
493
+ readonly includeIconLabels?: boolean;
479
494
  }
480
495
 
481
496
  /** Content filter applied to each crawled page before it reaches the result. */
@@ -620,17 +635,9 @@ export interface CrawlConfig {
620
635
  */
621
636
  readonly trackingParams?: Array<string>;
622
637
  /**
623
- * Custom HTTP headers to send with each request to the seed URL's host. With
624
- * `Self.allow_subdomains`, a request to a subdomain of the seed host carries the
625
- * non-credential headers too, whatever its scheme or port. Credential headers stay with the
626
- * seed host: `Authorization`, `Proxy-Authorization`, `Cookie`, `Set-Cookie`, `X-Api-Key`,
627
- * `X-Amz-Security-Token`, `Authentication-Info`, the header `Self.auth` names, and any
628
- * header whose name contains `auth`, `token`, `key`, `secret`, `session`, `cookie`, `csrf`,
629
- * `xsrf`, `password`, `signature`, `credential`, `passwd`, `pwd`, `pass`, `jwt`, `bearer`,
630
- * `otp`, `assertion`, `hmac`, `private`, `sso` or `sig`. A secret in a header whose name
631
- * matches none of these reaches subdomains, so name secret headers recognisably. A request
632
- * to any other host does not carry any of them. Subdomain forwarding applies to the HTTP and
633
- * chromiumoxide backends; the native browser backend sends them to the seed host only.
638
+ * Custom HTTP headers sent only to the seed URL's host by default, on any scheme or port.
639
+ * Explicitly authorize individual headers for admitted subdomains with
640
+ * `Self.subdomain_header_allowlist`. Other hosts never receive these headers.
634
641
  */
635
642
  readonly customHeaders?: Record<string, string>;
636
643
  /** Timeout for individual HTTP requests (in milliseconds when serialized). */
@@ -809,6 +816,14 @@ export interface CrawlConfig {
809
816
  * implementation controls certificate validation.
810
817
  */
811
818
  readonly dangerAcceptInvalidCerts?: boolean;
819
+ /**
820
+ * Custom header names explicitly authorized for proper subdomains when
821
+ * `Self.allow_subdomains` is true. Empty by default; names match case-insensitively.
822
+ * Authorization, cookies, other standard credential headers, and the header named by
823
+ * `Self.auth` remain seed-host-only even when listed. HTTP, Chromiumoxide, and the
824
+ * native browser backend apply this policy to each request and redirect hop.
825
+ */
826
+ readonly subdomainHeaderAllowlist?: Array<string>;
812
827
  }
813
828
 
814
829
  /**
@@ -1553,3 +1568,45 @@ export declare class CrawlStreamIterator {
1553
1568
  next(value?: undefined): Promise<IteratorResult<CrawlEvent, void>>;
1554
1569
  [Symbol.asyncIterator](): AsyncGenerator<CrawlEvent, void, undefined>;
1555
1570
  }
1571
+
1572
+ /** The error a binding call throws for `CrawlError`. */
1573
+ export interface ThrownCrawlError extends Error {
1574
+ kind:
1575
+ | "NotFound"
1576
+ | "Unauthorized"
1577
+ | "Forbidden"
1578
+ | "WafBlocked"
1579
+ | "Timeout"
1580
+ | "RateLimited"
1581
+ | "ServerError"
1582
+ | "BadGateway"
1583
+ | "Gone"
1584
+ | "Connection"
1585
+ | "Dns"
1586
+ | "Ssl"
1587
+ | "DataLoss"
1588
+ | "ConversionFailed"
1589
+ | "BrowserError"
1590
+ | "BrowserTimeout"
1591
+ | "InvalidConfig"
1592
+ | "Unsupported"
1593
+ | "TooManyRedirects"
1594
+ | "SsrfPolicyViolation"
1595
+ | "Other"
1596
+ | "Unknown";
1597
+ errorCode: number;
1598
+ }
1599
+
1600
+ /** The error a binding call throws for `SsrfError`. */
1601
+ export interface ThrownSsrfError extends Error {
1602
+ kind:
1603
+ | "DeniedByPolicy"
1604
+ | "NotOnAllowlist"
1605
+ | "InvalidCidr"
1606
+ | "DnsResolutionFailed"
1607
+ | "InvalidUrl"
1608
+ | "DisallowedScheme"
1609
+ | "TooManyRedirects"
1610
+ | "Unknown";
1611
+ errorCode: number;
1612
+ }