@xberg-io/crawlberg 1.11.1 → 1.12.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (5) hide show
  1. package/binding.cjs +138 -0
  2. package/index.d.ts +144 -16
  3. package/index.js +583 -437
  4. package/index.mjs +47 -0
  5. package/package.json +12 -9
package/binding.cjs ADDED
@@ -0,0 +1,138 @@
1
+ // DO NOT EDIT - auto-generated by alef
2
+ // alef:hash:b5a84836004f3b1787dc4ddf756b1f8dd85684282f90c6b508552bd1861b3f61
3
+ "use strict";
4
+
5
+ // The native binding attaches the structured properties of a failed call to the thrown error as
6
+ // the JSON message of its `cause`: the napi runtime cannot set own properties on an error that an
7
+ // async Rust function returns. This module moves those properties onto the error itself, so a
8
+ // caller reads `error.kind`, `error.statusCode`, `error.retryAfter` and the rest as declared in
9
+ // `index.d.ts`. It wraps every exported function, every static and prototype method of an
10
+ // exported class, and the async iterator a streaming call returns.
11
+ // Constructors are not wrapped: a constructor that throws keeps the plain native error.
12
+
13
+ const MARKER = "alef:error-info:";
14
+ const SKIP = new Set(["constructor", "length", "name", "prototype", "caller", "arguments"]);
15
+ const patched = new WeakSet();
16
+ const iterables = new WeakSet();
17
+
18
+ function decorate(error) {
19
+ if (error === null || typeof error !== "object") return error;
20
+ const cause = error.cause;
21
+ if (cause === null || typeof cause !== "object" || typeof cause.message !== "string") return error;
22
+ if (!cause.message.startsWith(MARKER)) return error;
23
+ let info;
24
+ try {
25
+ info = JSON.parse(cause.message.slice(MARKER.length));
26
+ } catch {
27
+ return error;
28
+ }
29
+ for (const [name, value] of Object.entries(info)) {
30
+ Object.defineProperty(error, name, { value, enumerable: true, writable: true, configurable: true });
31
+ }
32
+ try {
33
+ delete error.cause;
34
+ } catch {
35
+ // The payload stays on the cause when the runtime refuses the delete.
36
+ }
37
+ return error;
38
+ }
39
+
40
+ function rethrow(error) {
41
+ throw decorate(error);
42
+ }
43
+
44
+ function wrapIterator(iterator) {
45
+ if (iterator === null || typeof iterator !== "object") return iterator;
46
+ for (const name of ["next", "return", "throw"]) {
47
+ const original = iterator[name];
48
+ if (typeof original !== "function") continue;
49
+ try {
50
+ Object.defineProperty(iterator, name, {
51
+ configurable: true,
52
+ writable: true,
53
+ value: function (...args) {
54
+ return Promise.resolve(original.apply(this, args)).then(undefined, rethrow);
55
+ },
56
+ });
57
+ } catch {
58
+ // A frozen iterator keeps its plain errors.
59
+ }
60
+ }
61
+ return iterator;
62
+ }
63
+
64
+ function wrapIterable(value) {
65
+ if (value === null || typeof value !== "object") return value;
66
+ const open = value[Symbol.asyncIterator];
67
+ if (typeof open !== "function" || iterables.has(value)) return value;
68
+ iterables.add(value);
69
+ try {
70
+ Object.defineProperty(value, Symbol.asyncIterator, {
71
+ configurable: true,
72
+ writable: true,
73
+ value: function () {
74
+ return wrapIterator(open.call(this));
75
+ },
76
+ });
77
+ } catch {
78
+ return value;
79
+ }
80
+ wrapIterator(value);
81
+ return value;
82
+ }
83
+
84
+ function settle(result) {
85
+ if (result !== null && typeof result === "object" && typeof result.then === "function") {
86
+ return result.then(wrapIterable, rethrow);
87
+ }
88
+ return wrapIterable(result);
89
+ }
90
+
91
+ function wrapFunction(fn) {
92
+ return function (...args) {
93
+ let result;
94
+ try {
95
+ result = fn.apply(this, args);
96
+ } catch (error) {
97
+ throw decorate(error);
98
+ }
99
+ return settle(result);
100
+ };
101
+ }
102
+
103
+ function patchMethods(target) {
104
+ if (target === null || (typeof target !== "object" && typeof target !== "function")) return;
105
+ if (patched.has(target)) return;
106
+ patched.add(target);
107
+ for (const key of Reflect.ownKeys(target)) {
108
+ if (typeof key === "string" && SKIP.has(key)) continue;
109
+ const descriptor = Object.getOwnPropertyDescriptor(target, key);
110
+ if (!descriptor || typeof descriptor.value !== "function" || !descriptor.configurable) continue;
111
+ try {
112
+ Object.defineProperty(target, key, { ...descriptor, value: wrapFunction(descriptor.value) });
113
+ } catch {
114
+ // A method the runtime refuses to redefine keeps its plain errors.
115
+ }
116
+ }
117
+ }
118
+
119
+ function wrapExport(value) {
120
+ if (typeof value !== "function") return value;
121
+ patchMethods(value);
122
+ patchMethods(value.prototype);
123
+ // Only `apply` is trapped, so `new`, `instanceof` and static access reach the native class.
124
+ return new Proxy(value, {
125
+ apply(target, thisArg, args) {
126
+ return wrapFunction(target).apply(thisArg, args);
127
+ },
128
+ });
129
+ }
130
+
131
+ module.exports = function decorateBinding(native) {
132
+ const decorated = {};
133
+ for (const key of Object.keys(native)) {
134
+ decorated[key] = wrapExport(native[key]);
135
+ }
136
+ return decorated;
137
+ };
138
+ module.exports.decorate = decorate;
package/index.d.ts CHANGED
@@ -1,5 +1,5 @@
1
1
  // This file is auto-generated by alef — DO NOT EDIT.
2
- // alef:hash:760ec0215fe97398ca13eabd5625cdc092e885d8a65d635b7fb1d3a6044f1005
2
+ // alef:hash:f58f5415140504047100134aac44d0ee1f666d2775ab43a5ab8921162c1d7c84
3
3
  // To regenerate: alef generate
4
4
  // To verify freshness: alef verify
5
5
  /* eslint-disable */
@@ -214,7 +214,15 @@ export interface BrowserConfig {
214
214
  readonly wait?: BrowserWait;
215
215
  /** CSS selector to wait for when `wait` is `Selector`. */
216
216
  readonly waitSelector?: string;
217
- /** Extra time to wait after the wait condition is met. */
217
+ /**
218
+ * Extra time to wait after the wait condition is met.
219
+ *
220
+ * The default `network_idle` wait returns once the page has loaded and its DOM has been
221
+ * unchanged for `dom_quiet_window` (at most `dom_quiet_cap`, and never past `timeout`, counted
222
+ * from the start of the navigation). An app that draws after a longer quiet period needs
223
+ * a larger `dom_quiet_window` or `extra_wait`; for
224
+ * example `3000` waits three more seconds.
225
+ */
218
226
  readonly extraWait?: number;
219
227
  /**
220
228
  * Proxy for browser fetches. Overrides `CrawlConfig.proxy` when set.
@@ -274,6 +282,42 @@ export interface BrowserConfig {
274
282
  * shared browser pool.
275
283
  */
276
284
  readonly chromeArgs?: Array<string>;
285
+ /**
286
+ * Scroll overflowing scroll containers (not only the window) and keep the items that a
287
+ * virtualised list mounts as they pass through view, so the page that is read holds items
288
+ * beyond the first screen. Default: `false`.
289
+ *
290
+ * Runs once after the page is ready and `extra_wait` has elapsed, bounded to 8 containers,
291
+ * 200 steps and 5 seconds in total. Each container is scrolled by 80% of its height per step
292
+ * and the items of its most repetitive child list are collected, keyed by `id`, `data-index`,
293
+ * `data-key`, `data-id`, `aria-posinset` or `aria-rowindex` when present and otherwise by
294
+ * text when it is unique among the rendered items, else by position and text. Only
295
+ * containers whose child list repeats an element are considered, most repetitive first. The list is rewritten, in position order, only when items were unmounted
296
+ * while scrolling and items beyond the first screen were seen; a list that keeps all its
297
+ * items mounted is left as it is. Best effort: it does not follow lists that load data over
298
+ * the network slower than about 100 ms per step, does not reach lists inside iframes or
299
+ * shadow roots, and a rewrite replaces the element children of the list. Lists that recycle
300
+ * their DOM nodes (reusing the same elements with new content) are not collected, and
301
+ * scrolling can trigger infinite-scroll load-more requests. When the step, item or time
302
+ * budget stops the scroll, the items collected so far are kept and a warning is logged.
303
+ * Chromiumoxide backend only.
304
+ */
305
+ readonly scrollContainers?: boolean;
306
+ /**
307
+ * How long the DOM must stay unchanged after page load for `network_idle` to return,
308
+ * in milliseconds when serialized. Default: 150 ms. Every mutation restarts this window;
309
+ * the whole wait stays bounded to `dom_quiet_cap` and the remaining navigation timeout.
310
+ * A page whose first delayed render follows a longer quiet gap needs a larger value
311
+ * or `extra_wait`. Chromiumoxide backend only.
312
+ */
313
+ readonly domQuietWindow?: number;
314
+ /**
315
+ * Maximum time spent waiting for DOM quiet after page load, in milliseconds when serialized.
316
+ * Default: 4000 ms. Lower this for pages with continuous mutations; zero skips the quiet wait.
317
+ * The remaining navigation timeout also bounds this wait, independently of the quiet window.
318
+ * Chromiumoxide backend only.
319
+ */
320
+ readonly domQuietCap?: number;
277
321
  }
278
322
 
279
323
  /**
@@ -400,7 +444,11 @@ export interface ContentConfig {
400
444
  readonly excludeSelectors?: Array<string>;
401
445
  /** Skip image elements in output. Default: `false`. */
402
446
  readonly skipImages?: boolean;
403
- /** Max DOM traversal depth. Prevents stack overflow on deeply nested HTML. */
447
+ /**
448
+ * Max DOM traversal depth. Prevents stack overflow on deeply nested HTML. Content nested
449
+ * deeper is dropped and reported as a `DepthLimitExceeded` warning. `None` means 1024 in
450
+ * crawlberg, not the converter's own default of 64.
451
+ */
404
452
  readonly maxDepth?: number;
405
453
  /** Enable line wrapping. Default: `false`. */
406
454
  readonly wrap?: boolean;
@@ -435,6 +483,14 @@ export interface ContentConfig {
435
483
  * Scripts and styles remain excluded; explicit exclusions and image settings still apply.
436
484
  */
437
485
  readonly keepHidden?: boolean;
486
+ /** Append abbreviation titles in parentheses. Default: `true`. */
487
+ readonly expandAbbreviations?: boolean;
488
+ /** Escape literal Markdown metacharacters such as backticks and brackets. Default: `false`. */
489
+ readonly escapeMisc?: boolean;
490
+ /** Escape literal asterisks in text. Default: `false`. */
491
+ readonly escapeAsterisks?: boolean;
492
+ /** Include names and anchor tooltips of decorative linked SVG icons. Default: `true`. */
493
+ readonly includeIconLabels?: boolean;
438
494
  }
439
495
 
440
496
  /** Content filter applied to each crawled page before it reaches the result. */
@@ -510,15 +566,11 @@ export interface CrawlConfig {
510
566
  */
511
567
  readonly respectRobotsTxt?: boolean;
512
568
  /**
513
- * When true, HTTP-level error responses (404 NotFound, 403 Forbidden, WAF blocks)
514
- * are surfaced as `ScrapeResult` records with the matching `status_code` rather
515
- * than raised as `CrawlError`. A WAF block reports the status of the refused
516
- * response when it is a 4xx or 5xx, and 403 when the block page came with a
517
- * 2xx status. Default `false` preserves the historical
518
- * throw-on-error contract for direct fetches. Independently of this flag,
519
- * 404s reached at the end of a redirect chain are *always* surfaced softly —
520
- * the user opted into redirect-following, so receiving a 404 there is part of
521
- * the normal flow rather than an unexpected error.
569
+ * When true, readable HTTP error responses (4xx and 5xx) are surfaced as
570
+ * `ScrapeResult` records with their status and body rather than raised as errors.
571
+ * WAF blocks report their original 4xx/5xx status, or 403 for a block served as 2xx.
572
+ * Default `false` raises errors for recognized error statuses in both `scrape`
573
+ * and `crawl`, including a 404 reached through redirects.
522
574
  */
523
575
  readonly softHttpErrors?: boolean;
524
576
  /** Custom user-agent string. */
@@ -542,7 +594,7 @@ export interface CrawlConfig {
542
594
  readonly allowSubdomains?: boolean;
543
595
  /** Regex patterns for paths to include during crawling. */
544
596
  readonly includePaths?: Array<string>;
545
- /** Regex patterns for paths to exclude during crawling. */
597
+ /** Regex patterns for paths to exclude during crawling. The seed URL is never excluded. */
546
598
  readonly excludePaths?: Array<string>;
547
599
  /**
548
600
  * Whether `include_paths`/`exclude_paths` match against `path?query` instead of just
@@ -583,8 +635,9 @@ export interface CrawlConfig {
583
635
  */
584
636
  readonly trackingParams?: Array<string>;
585
637
  /**
586
- * Custom HTTP headers to send with each request to the seed URL's host. A request to another host
587
- * does not carry them.
638
+ * Custom HTTP headers sent only to the seed URL's host by default, on any scheme or port.
639
+ * Explicitly authorize individual headers for admitted subdomains with
640
+ * `Self.subdomain_header_allowlist`. Other hosts never receive these headers.
588
641
  */
589
642
  readonly customHeaders?: Record<string, string>;
590
643
  /** Timeout for individual HTTP requests (in milliseconds when serialized). */
@@ -610,7 +663,12 @@ export interface CrawlConfig {
610
663
  * subsequent attempt (capped at `retry_max_delay_ms`). Defaults to 100ms.
611
664
  */
612
665
  readonly retryInitialDelayMs?: number;
613
- /** Upper bound, in milliseconds, on the exponential retry backoff. Defaults to 60s. */
666
+ /**
667
+ * Upper bound, in milliseconds, on the exponential retry backoff. A 429 or 503 whose
668
+ * `Retry-After` is longer than the backoff is waited out, but one that exceeds this bound
669
+ * ends the retries with a warning instead (so `0` abandons the retry on any positive
670
+ * `Retry-After`). Defaults to 60s.
671
+ */
614
672
  readonly retryMaxDelayMs?: number;
615
673
  /**
616
674
  * Fraction of the per-domain rate-limit delay to randomly jitter by, in `[0.0, 1.0]`.
@@ -750,6 +808,22 @@ export interface CrawlConfig {
750
808
  * and the environment variable is not consulted for this config.
751
809
  */
752
810
  readonly ssrfDenyPrivateExplicit?: boolean;
811
+ /**
812
+ * Accept invalid TLS certificates for page and robots.txt requests.
813
+ * Defaults to false. Enable only for sites whose certificates you explicitly trust.
814
+ * The browser backends also apply this setting before navigation.
815
+ * On WebAssembly, enabling this setting returns `Unsupported`: the host's Fetch
816
+ * implementation controls certificate validation.
817
+ */
818
+ readonly dangerAcceptInvalidCerts?: boolean;
819
+ /**
820
+ * Custom header names explicitly authorized for proper subdomains when
821
+ * `Self.allow_subdomains` is true. Empty by default; names match case-insensitively.
822
+ * Authorization, cookies, other standard credential headers, and the header named by
823
+ * `Self.auth` remain seed-host-only even when listed. HTTP, Chromiumoxide, and the
824
+ * native browser backend apply this policy to each request and redirect hop.
825
+ */
826
+ readonly subdomainHeaderAllowlist?: Array<string>;
753
827
  }
754
828
 
755
829
  /**
@@ -892,6 +966,8 @@ export interface CrawlResult {
892
966
  readonly stayedOnDomain?: boolean;
893
967
  /** Whether the browser fallback was used for any page in this crawl. */
894
968
  readonly browserUsed?: boolean;
969
+ /** Details of the first WAF block encountered during the crawl. <!-- --> */
970
+ readonly wafBlock?: WafBlockDetails;
895
971
  }
896
972
 
897
973
  /**
@@ -1467,6 +1543,16 @@ export interface SsrfPolicy {
1467
1543
  readonly schemeAllowlist?: Array<string>;
1468
1544
  }
1469
1545
 
1546
+ /** Structured context for a WAF-blocked response. <!-- --> */
1547
+ export interface WafBlockDetails {
1548
+ /** Lowercase WAF vendor identifier. <!-- --> */
1549
+ readonly vendor?: string;
1550
+ /** Origin HTTP status, when a response caused the block. <!-- --> */
1551
+ readonly status?: number;
1552
+ /** At most 4 KiB of the blocked response body, when it was read. <!-- --> */
1553
+ readonly bodyPrefix?: string;
1554
+ }
1555
+
1470
1556
  /** Discover all pages on a website by following links and sitemaps. */
1471
1557
  export declare function mapUrls(engine: CrawlEngineHandle, url: string): Promise<MapResult>;
1472
1558
 
@@ -1482,3 +1568,45 @@ export declare class CrawlStreamIterator {
1482
1568
  next(value?: undefined): Promise<IteratorResult<CrawlEvent, void>>;
1483
1569
  [Symbol.asyncIterator](): AsyncGenerator<CrawlEvent, void, undefined>;
1484
1570
  }
1571
+
1572
+ /** The error a binding call throws for `CrawlError`. */
1573
+ export interface ThrownCrawlError extends Error {
1574
+ kind:
1575
+ | "NotFound"
1576
+ | "Unauthorized"
1577
+ | "Forbidden"
1578
+ | "WafBlocked"
1579
+ | "Timeout"
1580
+ | "RateLimited"
1581
+ | "ServerError"
1582
+ | "BadGateway"
1583
+ | "Gone"
1584
+ | "Connection"
1585
+ | "Dns"
1586
+ | "Ssl"
1587
+ | "DataLoss"
1588
+ | "ConversionFailed"
1589
+ | "BrowserError"
1590
+ | "BrowserTimeout"
1591
+ | "InvalidConfig"
1592
+ | "Unsupported"
1593
+ | "TooManyRedirects"
1594
+ | "SsrfPolicyViolation"
1595
+ | "Other"
1596
+ | "Unknown";
1597
+ errorCode: number;
1598
+ }
1599
+
1600
+ /** The error a binding call throws for `SsrfError`. */
1601
+ export interface ThrownSsrfError extends Error {
1602
+ kind:
1603
+ | "DeniedByPolicy"
1604
+ | "NotOnAllowlist"
1605
+ | "InvalidCidr"
1606
+ | "DnsResolutionFailed"
1607
+ | "InvalidUrl"
1608
+ | "DisallowedScheme"
1609
+ | "TooManyRedirects"
1610
+ | "Unknown";
1611
+ errorCode: number;
1612
+ }