@xberg-io/crawlberg 1.11.1 → 1.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/index.d.ts +87 -16
  2. package/index.js +112 -55
  3. package/package.json +8 -8
package/index.d.ts CHANGED
@@ -1,5 +1,5 @@
1
1
  // This file is auto-generated by alef — DO NOT EDIT.
2
- // alef:hash:760ec0215fe97398ca13eabd5625cdc092e885d8a65d635b7fb1d3a6044f1005
2
+ // alef:hash:970b01dd0b98d7d53a619b3b8702083b17b8050bd3ed2122e6bbee5cef6a5676
3
3
  // To regenerate: alef generate
4
4
  // To verify freshness: alef verify
5
5
  /* eslint-disable */
@@ -214,7 +214,15 @@ export interface BrowserConfig {
214
214
  readonly wait?: BrowserWait;
215
215
  /** CSS selector to wait for when `wait` is `Selector`. */
216
216
  readonly waitSelector?: string;
217
- /** Extra time to wait after the wait condition is met. */
217
+ /**
218
+ * Extra time to wait after the wait condition is met.
219
+ *
220
+ * The default `network_idle` wait returns once the page has loaded and its DOM has been
221
+ * unchanged for `dom_quiet_window` (at most four seconds, and never past `timeout`, counted
222
+ * from the start of the navigation). An app that draws after a longer quiet period needs
223
+ * a larger `dom_quiet_window` or `extra_wait`; for
224
+ * example `3000` waits three more seconds.
225
+ */
218
226
  readonly extraWait?: number;
219
227
  /**
220
228
  * Proxy for browser fetches. Overrides `CrawlConfig.proxy` when set.
@@ -274,6 +282,35 @@ export interface BrowserConfig {
274
282
  * shared browser pool.
275
283
  */
276
284
  readonly chromeArgs?: Array<string>;
285
+ /**
286
+ * Scroll overflowing scroll containers (not only the window) and keep the items that a
287
+ * virtualised list mounts as they pass through view, so the page that is read holds items
288
+ * beyond the first screen. Default: `false`.
289
+ *
290
+ * Runs once after the page is ready and `extra_wait` has elapsed, bounded to 8 containers,
291
+ * 200 steps and 5 seconds in total. Each container is scrolled by 80% of its height per step
292
+ * and the items of its most repetitive child list are collected, keyed by `id`, `data-index`,
293
+ * `data-key`, `data-id`, `aria-posinset` or `aria-rowindex` when present and otherwise by
294
+ * text when it is unique among the rendered items, else by position and text. Only
295
+ * containers whose child list repeats an element are considered, most repetitive first. The list is rewritten, in position order, only when items were unmounted
296
+ * while scrolling and items beyond the first screen were seen; a list that keeps all its
297
+ * items mounted is left as it is. Best effort: it does not follow lists that load data over
298
+ * the network slower than about 100 ms per step, does not reach lists inside iframes or
299
+ * shadow roots, and a rewrite replaces the element children of the list. Lists that recycle
300
+ * their DOM nodes (reusing the same elements with new content) are not collected, and
301
+ * scrolling can trigger infinite-scroll load-more requests. When the step, item or time
302
+ * budget stops the scroll, the items collected so far are kept and a warning is logged.
303
+ * Chromiumoxide backend only.
304
+ */
305
+ readonly scrollContainers?: boolean;
306
+ /**
307
+ * How long the DOM must stay unchanged after page load for `network_idle` to return,
308
+ * in milliseconds when serialized. Default: 150 ms. Every mutation restarts this window;
309
+ * the whole wait stays bounded to four seconds and the remaining navigation timeout.
310
+ * A page whose first delayed render follows a longer quiet gap needs a larger value
311
+ * or `extra_wait`. Chromiumoxide backend only.
312
+ */
313
+ readonly domQuietWindow?: number;
277
314
  }
278
315
 
279
316
  /**
@@ -400,7 +437,11 @@ export interface ContentConfig {
400
437
  readonly excludeSelectors?: Array<string>;
401
438
  /** Skip image elements in output. Default: `false`. */
402
439
  readonly skipImages?: boolean;
403
- /** Max DOM traversal depth. Prevents stack overflow on deeply nested HTML. */
440
+ /**
441
+ * Max DOM traversal depth. Prevents stack overflow on deeply nested HTML. Content nested
442
+ * deeper is dropped and reported as a `DepthLimitExceeded` warning. `None` means 1024 in
443
+ * crawlberg, not the converter's own default of 64.
444
+ */
404
445
  readonly maxDepth?: number;
405
446
  /** Enable line wrapping. Default: `false`. */
406
447
  readonly wrap?: boolean;
@@ -510,15 +551,11 @@ export interface CrawlConfig {
510
551
  */
511
552
  readonly respectRobotsTxt?: boolean;
512
553
  /**
513
- * When true, HTTP-level error responses (404 NotFound, 403 Forbidden, WAF blocks)
514
- * are surfaced as `ScrapeResult` records with the matching `status_code` rather
515
- * than raised as `CrawlError`. A WAF block reports the status of the refused
516
- * response when it is a 4xx or 5xx, and 403 when the block page came with a
517
- * 2xx status. Default `false` preserves the historical
518
- * throw-on-error contract for direct fetches. Independently of this flag,
519
- * 404s reached at the end of a redirect chain are *always* surfaced softly —
520
- * the user opted into redirect-following, so receiving a 404 there is part of
521
- * the normal flow rather than an unexpected error.
554
+ * When true, readable HTTP error responses (4xx and 5xx) are surfaced as
555
+ * `ScrapeResult` records with their status and body rather than raised as errors.
556
+ * WAF blocks report their original 4xx/5xx status, or 403 for a block served as 2xx.
557
+ * Default `false` raises errors for recognized error statuses in both `scrape`
558
+ * and `crawl`, including a 404 reached through redirects.
522
559
  */
523
560
  readonly softHttpErrors?: boolean;
524
561
  /** Custom user-agent string. */
@@ -542,7 +579,7 @@ export interface CrawlConfig {
542
579
  readonly allowSubdomains?: boolean;
543
580
  /** Regex patterns for paths to include during crawling. */
544
581
  readonly includePaths?: Array<string>;
545
- /** Regex patterns for paths to exclude during crawling. */
582
+ /** Regex patterns for paths to exclude during crawling. The seed URL is never excluded. */
546
583
  readonly excludePaths?: Array<string>;
547
584
  /**
548
585
  * Whether `include_paths`/`exclude_paths` match against `path?query` instead of just
@@ -583,8 +620,17 @@ export interface CrawlConfig {
583
620
  */
584
621
  readonly trackingParams?: Array<string>;
585
622
  /**
586
- * Custom HTTP headers to send with each request to the seed URL's host. A request to another host
587
- * does not carry them.
623
+ * Custom HTTP headers to send with each request to the seed URL's host. With
624
+ * `Self.allow_subdomains`, a request to a subdomain of the seed host carries the
625
+ * non-credential headers too, whatever its scheme or port. Credential headers stay with the
626
+ * seed host: `Authorization`, `Proxy-Authorization`, `Cookie`, `Set-Cookie`, `X-Api-Key`,
627
+ * `X-Amz-Security-Token`, `Authentication-Info`, the header `Self.auth` names, and any
628
+ * header whose name contains `auth`, `token`, `key`, `secret`, `session`, `cookie`, `csrf`,
629
+ * `xsrf`, `password`, `signature`, `credential`, `passwd`, `pwd`, `pass`, `jwt`, `bearer`,
630
+ * `otp`, `assertion`, `hmac`, `private`, `sso` or `sig`. A secret in a header whose name
631
+ * matches none of these reaches subdomains, so name secret headers recognisably. A request
632
+ * to any other host does not carry any of them. Subdomain forwarding applies to the HTTP and
633
+ * chromiumoxide backends; the native browser backend sends them to the seed host only.
588
634
  */
589
635
  readonly customHeaders?: Record<string, string>;
590
636
  /** Timeout for individual HTTP requests (in milliseconds when serialized). */
@@ -610,7 +656,12 @@ export interface CrawlConfig {
610
656
  * subsequent attempt (capped at `retry_max_delay_ms`). Defaults to 100ms.
611
657
  */
612
658
  readonly retryInitialDelayMs?: number;
613
- /** Upper bound, in milliseconds, on the exponential retry backoff. Defaults to 60s. */
659
+ /**
660
+ * Upper bound, in milliseconds, on the exponential retry backoff. A 429 or 503 whose
661
+ * `Retry-After` is longer than the backoff is waited out, but one that exceeds this bound
662
+ * ends the retries with a warning instead (so `0` abandons the retry on any positive
663
+ * `Retry-After`). Defaults to 60s.
664
+ */
614
665
  readonly retryMaxDelayMs?: number;
615
666
  /**
616
667
  * Fraction of the per-domain rate-limit delay to randomly jitter by, in `[0.0, 1.0]`.
@@ -750,6 +801,14 @@ export interface CrawlConfig {
750
801
  * and the environment variable is not consulted for this config.
751
802
  */
752
803
  readonly ssrfDenyPrivateExplicit?: boolean;
804
+ /**
805
+ * Accept invalid TLS certificates for page and robots.txt requests.
806
+ * Defaults to false. Enable only for sites whose certificates you explicitly trust.
807
+ * The browser backends also apply this setting before navigation.
808
+ * On WebAssembly, enabling this setting returns `Unsupported`: the host's Fetch
809
+ * implementation controls certificate validation.
810
+ */
811
+ readonly dangerAcceptInvalidCerts?: boolean;
753
812
  }
754
813
 
755
814
  /**
@@ -892,6 +951,8 @@ export interface CrawlResult {
892
951
  readonly stayedOnDomain?: boolean;
893
952
  /** Whether the browser fallback was used for any page in this crawl. */
894
953
  readonly browserUsed?: boolean;
954
+ /** Details of the first WAF block encountered during the crawl. <!-- --> */
955
+ readonly wafBlock?: WafBlockDetails;
895
956
  }
896
957
 
897
958
  /**
@@ -1467,6 +1528,16 @@ export interface SsrfPolicy {
1467
1528
  readonly schemeAllowlist?: Array<string>;
1468
1529
  }
1469
1530
 
1531
+ /** Structured context for a WAF-blocked response. <!-- --> */
1532
+ export interface WafBlockDetails {
1533
+ /** Lowercase WAF vendor identifier. <!-- --> */
1534
+ readonly vendor?: string;
1535
+ /** Origin HTTP status, when a response caused the block. <!-- --> */
1536
+ readonly status?: number;
1537
+ /** At most 4 KiB of the blocked response body, when it was read. <!-- --> */
1538
+ readonly bodyPrefix?: string;
1539
+ }
1540
+
1470
1541
  /** Discover all pages on a website by following links and sitemaps. */
1471
1542
  export declare function mapUrls(engine: CrawlEngineHandle, url: string): Promise<MapResult>;
1472
1543
 
package/index.js CHANGED
@@ -6,8 +6,13 @@ const { readFileSync } = require('fs')
6
6
  let nativeBinding = null
7
7
  // Which artifact actually loaded. The WASI fallback chain overwrites it with
8
8
  // the flavor it resolved; the late native retry below leaves it alone because
9
- // it only runs while no WASI candidate has been loaded.
9
+ // it only runs while no WASI candidate has been loaded. The take-over check
10
+ // after the load-failure block can also set it.
10
11
  let __napiLoadedBindingTarget = 'native'
12
+ // What this loader's own `requireNative()` calls returned. The check after
13
+ // the load-failure block compares it with `nativeBinding` to tell a binding
14
+ // this loader loaded from one that code outside it put in its place.
15
+ let __napiRequiredNativeBinding = null
11
16
  const loadErrors = []
12
17
 
13
18
  const isMusl = () => {
@@ -89,8 +94,8 @@ function requireNative() {
89
94
  try {
90
95
  const binding = require('@xberg-io/crawlberg-android-arm64')
91
96
  const bindingPackageVersion = require('@xberg-io/crawlberg-android-arm64/package.json').version
92
- if (bindingPackageVersion !== '1.11.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
93
- throw new Error(`Native binding package version mismatch, expected 1.11.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
97
+ if (bindingPackageVersion !== '1.12.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
98
+ throw new Error(`Native binding package version mismatch, expected 1.12.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
94
99
  }
95
100
  return binding
96
101
  } catch (e) {
@@ -105,8 +110,8 @@ function requireNative() {
105
110
  try {
106
111
  const binding = require('@xberg-io/crawlberg-android-arm-eabi')
107
112
  const bindingPackageVersion = require('@xberg-io/crawlberg-android-arm-eabi/package.json').version
108
- if (bindingPackageVersion !== '1.11.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
109
- throw new Error(`Native binding package version mismatch, expected 1.11.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
113
+ if (bindingPackageVersion !== '1.12.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
114
+ throw new Error(`Native binding package version mismatch, expected 1.12.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
110
115
  }
111
116
  return binding
112
117
  } catch (e) {
@@ -126,8 +131,8 @@ function requireNative() {
126
131
  try {
127
132
  const binding = require('@xberg-io/crawlberg-win32-x64-gnu')
128
133
  const bindingPackageVersion = require('@xberg-io/crawlberg-win32-x64-gnu/package.json').version
129
- if (bindingPackageVersion !== '1.11.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
130
- throw new Error(`Native binding package version mismatch, expected 1.11.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
134
+ if (bindingPackageVersion !== '1.12.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
135
+ throw new Error(`Native binding package version mismatch, expected 1.12.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
131
136
  }
132
137
  return binding
133
138
  } catch (e) {
@@ -142,8 +147,8 @@ function requireNative() {
142
147
  try {
143
148
  const binding = require('@xberg-io/crawlberg-win32-x64-msvc')
144
149
  const bindingPackageVersion = require('@xberg-io/crawlberg-win32-x64-msvc/package.json').version
145
- if (bindingPackageVersion !== '1.11.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
146
- throw new Error(`Native binding package version mismatch, expected 1.11.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
150
+ if (bindingPackageVersion !== '1.12.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
151
+ throw new Error(`Native binding package version mismatch, expected 1.12.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
147
152
  }
148
153
  return binding
149
154
  } catch (e) {
@@ -159,8 +164,8 @@ function requireNative() {
159
164
  try {
160
165
  const binding = require('@xberg-io/crawlberg-win32-ia32-msvc')
161
166
  const bindingPackageVersion = require('@xberg-io/crawlberg-win32-ia32-msvc/package.json').version
162
- if (bindingPackageVersion !== '1.11.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
163
- throw new Error(`Native binding package version mismatch, expected 1.11.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
167
+ if (bindingPackageVersion !== '1.12.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
168
+ throw new Error(`Native binding package version mismatch, expected 1.12.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
164
169
  }
165
170
  return binding
166
171
  } catch (e) {
@@ -175,8 +180,8 @@ function requireNative() {
175
180
  try {
176
181
  const binding = require('@xberg-io/crawlberg-win32-arm64-msvc')
177
182
  const bindingPackageVersion = require('@xberg-io/crawlberg-win32-arm64-msvc/package.json').version
178
- if (bindingPackageVersion !== '1.11.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
179
- throw new Error(`Native binding package version mismatch, expected 1.11.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
183
+ if (bindingPackageVersion !== '1.12.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
184
+ throw new Error(`Native binding package version mismatch, expected 1.12.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
180
185
  }
181
186
  return binding
182
187
  } catch (e) {
@@ -194,8 +199,8 @@ function requireNative() {
194
199
  try {
195
200
  const binding = require('@xberg-io/crawlberg-darwin-universal')
196
201
  const bindingPackageVersion = require('@xberg-io/crawlberg-darwin-universal/package.json').version
197
- if (bindingPackageVersion !== '1.11.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
198
- throw new Error(`Native binding package version mismatch, expected 1.11.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
202
+ if (bindingPackageVersion !== '1.12.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
203
+ throw new Error(`Native binding package version mismatch, expected 1.12.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
199
204
  }
200
205
  return binding
201
206
  } catch (e) {
@@ -210,8 +215,8 @@ function requireNative() {
210
215
  try {
211
216
  const binding = require('@xberg-io/crawlberg-darwin-x64')
212
217
  const bindingPackageVersion = require('@xberg-io/crawlberg-darwin-x64/package.json').version
213
- if (bindingPackageVersion !== '1.11.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
214
- throw new Error(`Native binding package version mismatch, expected 1.11.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
218
+ if (bindingPackageVersion !== '1.12.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
219
+ throw new Error(`Native binding package version mismatch, expected 1.12.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
215
220
  }
216
221
  return binding
217
222
  } catch (e) {
@@ -226,8 +231,8 @@ function requireNative() {
226
231
  try {
227
232
  const binding = require('@xberg-io/crawlberg-darwin-arm64')
228
233
  const bindingPackageVersion = require('@xberg-io/crawlberg-darwin-arm64/package.json').version
229
- if (bindingPackageVersion !== '1.11.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
230
- throw new Error(`Native binding package version mismatch, expected 1.11.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
234
+ if (bindingPackageVersion !== '1.12.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
235
+ throw new Error(`Native binding package version mismatch, expected 1.12.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
231
236
  }
232
237
  return binding
233
238
  } catch (e) {
@@ -246,8 +251,8 @@ function requireNative() {
246
251
  try {
247
252
  const binding = require('@xberg-io/crawlberg-freebsd-x64')
248
253
  const bindingPackageVersion = require('@xberg-io/crawlberg-freebsd-x64/package.json').version
249
- if (bindingPackageVersion !== '1.11.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
250
- throw new Error(`Native binding package version mismatch, expected 1.11.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
254
+ if (bindingPackageVersion !== '1.12.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
255
+ throw new Error(`Native binding package version mismatch, expected 1.12.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
251
256
  }
252
257
  return binding
253
258
  } catch (e) {
@@ -262,8 +267,24 @@ function requireNative() {
262
267
  try {
263
268
  const binding = require('@xberg-io/crawlberg-freebsd-arm64')
264
269
  const bindingPackageVersion = require('@xberg-io/crawlberg-freebsd-arm64/package.json').version
265
- if (bindingPackageVersion !== '1.11.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
266
- throw new Error(`Native binding package version mismatch, expected 1.11.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
270
+ if (bindingPackageVersion !== '1.12.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
271
+ throw new Error(`Native binding package version mismatch, expected 1.12.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
272
+ }
273
+ return binding
274
+ } catch (e) {
275
+ loadErrors.push(e)
276
+ }
277
+ } else if (process.arch === 'ppc64') {
278
+ try {
279
+ return require('./crawlberg-node.freebsd-ppc64.node')
280
+ } catch (e) {
281
+ loadErrors.push(e)
282
+ }
283
+ try {
284
+ const binding = require('@xberg-io/crawlberg-freebsd-ppc64')
285
+ const bindingPackageVersion = require('@xberg-io/crawlberg-freebsd-ppc64/package.json').version
286
+ if (bindingPackageVersion !== '1.12.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
287
+ throw new Error(`Native binding package version mismatch, expected 1.12.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
267
288
  }
268
289
  return binding
269
290
  } catch (e) {
@@ -283,8 +304,8 @@ function requireNative() {
283
304
  try {
284
305
  const binding = require('@xberg-io/crawlberg-linux-x64-musl')
285
306
  const bindingPackageVersion = require('@xberg-io/crawlberg-linux-x64-musl/package.json').version
286
- if (bindingPackageVersion !== '1.11.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
287
- throw new Error(`Native binding package version mismatch, expected 1.11.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
307
+ if (bindingPackageVersion !== '1.12.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
308
+ throw new Error(`Native binding package version mismatch, expected 1.12.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
288
309
  }
289
310
  return binding
290
311
  } catch (e) {
@@ -299,8 +320,8 @@ function requireNative() {
299
320
  try {
300
321
  const binding = require('@xberg-io/crawlberg-linux-x64-gnu')
301
322
  const bindingPackageVersion = require('@xberg-io/crawlberg-linux-x64-gnu/package.json').version
302
- if (bindingPackageVersion !== '1.11.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
303
- throw new Error(`Native binding package version mismatch, expected 1.11.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
323
+ if (bindingPackageVersion !== '1.12.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
324
+ throw new Error(`Native binding package version mismatch, expected 1.12.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
304
325
  }
305
326
  return binding
306
327
  } catch (e) {
@@ -317,8 +338,8 @@ function requireNative() {
317
338
  try {
318
339
  const binding = require('@xberg-io/crawlberg-linux-arm64-musl')
319
340
  const bindingPackageVersion = require('@xberg-io/crawlberg-linux-arm64-musl/package.json').version
320
- if (bindingPackageVersion !== '1.11.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
321
- throw new Error(`Native binding package version mismatch, expected 1.11.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
341
+ if (bindingPackageVersion !== '1.12.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
342
+ throw new Error(`Native binding package version mismatch, expected 1.12.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
322
343
  }
323
344
  return binding
324
345
  } catch (e) {
@@ -333,8 +354,8 @@ function requireNative() {
333
354
  try {
334
355
  const binding = require('@xberg-io/crawlberg-linux-arm64-gnu')
335
356
  const bindingPackageVersion = require('@xberg-io/crawlberg-linux-arm64-gnu/package.json').version
336
- if (bindingPackageVersion !== '1.11.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
337
- throw new Error(`Native binding package version mismatch, expected 1.11.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
357
+ if (bindingPackageVersion !== '1.12.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
358
+ throw new Error(`Native binding package version mismatch, expected 1.12.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
338
359
  }
339
360
  return binding
340
361
  } catch (e) {
@@ -351,8 +372,8 @@ function requireNative() {
351
372
  try {
352
373
  const binding = require('@xberg-io/crawlberg-linux-arm-musleabihf')
353
374
  const bindingPackageVersion = require('@xberg-io/crawlberg-linux-arm-musleabihf/package.json').version
354
- if (bindingPackageVersion !== '1.11.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
355
- throw new Error(`Native binding package version mismatch, expected 1.11.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
375
+ if (bindingPackageVersion !== '1.12.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
376
+ throw new Error(`Native binding package version mismatch, expected 1.12.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
356
377
  }
357
378
  return binding
358
379
  } catch (e) {
@@ -367,8 +388,8 @@ function requireNative() {
367
388
  try {
368
389
  const binding = require('@xberg-io/crawlberg-linux-arm-gnueabihf')
369
390
  const bindingPackageVersion = require('@xberg-io/crawlberg-linux-arm-gnueabihf/package.json').version
370
- if (bindingPackageVersion !== '1.11.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
371
- throw new Error(`Native binding package version mismatch, expected 1.11.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
391
+ if (bindingPackageVersion !== '1.12.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
392
+ throw new Error(`Native binding package version mismatch, expected 1.12.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
372
393
  }
373
394
  return binding
374
395
  } catch (e) {
@@ -385,8 +406,8 @@ function requireNative() {
385
406
  try {
386
407
  const binding = require('@xberg-io/crawlberg-linux-loong64-musl')
387
408
  const bindingPackageVersion = require('@xberg-io/crawlberg-linux-loong64-musl/package.json').version
388
- if (bindingPackageVersion !== '1.11.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
389
- throw new Error(`Native binding package version mismatch, expected 1.11.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
409
+ if (bindingPackageVersion !== '1.12.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
410
+ throw new Error(`Native binding package version mismatch, expected 1.12.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
390
411
  }
391
412
  return binding
392
413
  } catch (e) {
@@ -401,8 +422,8 @@ function requireNative() {
401
422
  try {
402
423
  const binding = require('@xberg-io/crawlberg-linux-loong64-gnu')
403
424
  const bindingPackageVersion = require('@xberg-io/crawlberg-linux-loong64-gnu/package.json').version
404
- if (bindingPackageVersion !== '1.11.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
405
- throw new Error(`Native binding package version mismatch, expected 1.11.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
425
+ if (bindingPackageVersion !== '1.12.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
426
+ throw new Error(`Native binding package version mismatch, expected 1.12.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
406
427
  }
407
428
  return binding
408
429
  } catch (e) {
@@ -419,8 +440,8 @@ function requireNative() {
419
440
  try {
420
441
  const binding = require('@xberg-io/crawlberg-linux-riscv64-musl')
421
442
  const bindingPackageVersion = require('@xberg-io/crawlberg-linux-riscv64-musl/package.json').version
422
- if (bindingPackageVersion !== '1.11.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
423
- throw new Error(`Native binding package version mismatch, expected 1.11.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
443
+ if (bindingPackageVersion !== '1.12.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
444
+ throw new Error(`Native binding package version mismatch, expected 1.12.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
424
445
  }
425
446
  return binding
426
447
  } catch (e) {
@@ -435,8 +456,8 @@ function requireNative() {
435
456
  try {
436
457
  const binding = require('@xberg-io/crawlberg-linux-riscv64-gnu')
437
458
  const bindingPackageVersion = require('@xberg-io/crawlberg-linux-riscv64-gnu/package.json').version
438
- if (bindingPackageVersion !== '1.11.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
439
- throw new Error(`Native binding package version mismatch, expected 1.11.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
459
+ if (bindingPackageVersion !== '1.12.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
460
+ throw new Error(`Native binding package version mismatch, expected 1.12.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
440
461
  }
441
462
  return binding
442
463
  } catch (e) {
@@ -452,8 +473,8 @@ function requireNative() {
452
473
  try {
453
474
  const binding = require('@xberg-io/crawlberg-linux-ppc64-gnu')
454
475
  const bindingPackageVersion = require('@xberg-io/crawlberg-linux-ppc64-gnu/package.json').version
455
- if (bindingPackageVersion !== '1.11.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
456
- throw new Error(`Native binding package version mismatch, expected 1.11.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
476
+ if (bindingPackageVersion !== '1.12.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
477
+ throw new Error(`Native binding package version mismatch, expected 1.12.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
457
478
  }
458
479
  return binding
459
480
  } catch (e) {
@@ -468,8 +489,8 @@ function requireNative() {
468
489
  try {
469
490
  const binding = require('@xberg-io/crawlberg-linux-s390x-gnu')
470
491
  const bindingPackageVersion = require('@xberg-io/crawlberg-linux-s390x-gnu/package.json').version
471
- if (bindingPackageVersion !== '1.11.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
472
- throw new Error(`Native binding package version mismatch, expected 1.11.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
492
+ if (bindingPackageVersion !== '1.12.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
493
+ throw new Error(`Native binding package version mismatch, expected 1.12.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
473
494
  }
474
495
  return binding
475
496
  } catch (e) {
@@ -488,8 +509,8 @@ function requireNative() {
488
509
  try {
489
510
  const binding = require('@xberg-io/crawlberg-openharmony-arm64')
490
511
  const bindingPackageVersion = require('@xberg-io/crawlberg-openharmony-arm64/package.json').version
491
- if (bindingPackageVersion !== '1.11.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
492
- throw new Error(`Native binding package version mismatch, expected 1.11.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
512
+ if (bindingPackageVersion !== '1.12.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
513
+ throw new Error(`Native binding package version mismatch, expected 1.12.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
493
514
  }
494
515
  return binding
495
516
  } catch (e) {
@@ -504,8 +525,8 @@ function requireNative() {
504
525
  try {
505
526
  const binding = require('@xberg-io/crawlberg-openharmony-x64')
506
527
  const bindingPackageVersion = require('@xberg-io/crawlberg-openharmony-x64/package.json').version
507
- if (bindingPackageVersion !== '1.11.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
508
- throw new Error(`Native binding package version mismatch, expected 1.11.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
528
+ if (bindingPackageVersion !== '1.12.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
529
+ throw new Error(`Native binding package version mismatch, expected 1.12.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
509
530
  }
510
531
  return binding
511
532
  } catch (e) {
@@ -520,8 +541,8 @@ function requireNative() {
520
541
  try {
521
542
  const binding = require('@xberg-io/crawlberg-openharmony-arm')
522
543
  const bindingPackageVersion = require('@xberg-io/crawlberg-openharmony-arm/package.json').version
523
- if (bindingPackageVersion !== '1.11.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
524
- throw new Error(`Native binding package version mismatch, expected 1.11.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
544
+ if (bindingPackageVersion !== '1.12.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
545
+ throw new Error(`Native binding package version mismatch, expected 1.12.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
525
546
  }
526
547
  return binding
527
548
  } catch (e) {
@@ -585,6 +606,7 @@ const forceWasi =
585
606
 
586
607
  if (!forceWasi) {
587
608
  nativeBinding = requireNative()
609
+ __napiRequiredNativeBinding = nativeBinding
588
610
  }
589
611
 
590
612
  if (!nativeBinding || forceWasi) {
@@ -661,8 +683,8 @@ if (!nativeBinding || forceWasi) {
661
683
  if (!candidateFailed) {
662
684
  if (process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
663
685
  const bindingPackageVersion = require('@xberg-io/crawlberg-wasm32-wasi/package.json').version
664
- if (bindingPackageVersion !== '1.11.1') {
665
- throw new Error(`WASI binding package version mismatch, expected 1.11.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
686
+ if (bindingPackageVersion !== '1.12.0') {
687
+ throw new Error(`WASI binding package version mismatch, expected 1.12.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
666
688
  }
667
689
  }
668
690
  wasiBinding = require('@xberg-io/crawlberg-wasm32-wasi')
@@ -686,6 +708,7 @@ if (!nativeBinding || forceWasi) {
686
708
  !__napiWasiFlavorRequested
687
709
  ) {
688
710
  nativeBinding = requireNative()
711
+ __napiRequiredNativeBinding = nativeBinding
689
712
  }
690
713
  if ((forceWasiError || __napiWasiFlavorRequested) && !wasiBindingLoaded) {
691
714
  const error = new Error(
@@ -713,6 +736,40 @@ if (!nativeBinding) {
713
736
  throw new Error(`Failed to load native binding`)
714
737
  }
715
738
 
739
+ // A package can patch its own fallback in front of the load-failure check
740
+ // above, e.g. a WebContainer download through `napi-postinstall/fallback`.
741
+ // That fallback swaps `nativeBinding` for a generated WASI loader, which has
742
+ // already stamped its own flavor, while the target above still says 'native'.
743
+ // Take that flavor over, or the CommonJS stamp throws a conflict and the ESM
744
+ // export reports the wrong artifact. Three limits keep the conflict check:
745
+ // - only a binding this loader did not load itself, so a native addon whose
746
+ // `#[napi(module_exports)]` hook claims the name is still rejected;
747
+ // - only an own data property, read through its descriptor, so no addon
748
+ // getter runs here, and it must read back the same, as the stamp reads it;
749
+ // - only a flavor in `__napiWasiFlavors`, the ones this loader's own WASI
750
+ // chain can report ('wasm32-wasi' for a package without a WASI target).
751
+ // This is metadata and never fails a load: a throwing Proxy trap keeps 'native'.
752
+ if (
753
+ __napiLoadedBindingTarget === 'native' &&
754
+ nativeBinding !== __napiRequiredNativeBinding
755
+ ) {
756
+ try {
757
+ const __napiBindingTargetDescriptor = Object.getOwnPropertyDescriptor(
758
+ nativeBinding,
759
+ '__napiBindingTarget',
760
+ )
761
+ if (
762
+ __napiBindingTargetDescriptor &&
763
+ __napiWasiFlavors.indexOf(__napiBindingTargetDescriptor.value) !== -1 &&
764
+ nativeBinding.__napiBindingTarget === __napiBindingTargetDescriptor.value
765
+ ) {
766
+ __napiLoadedBindingTarget = __napiBindingTargetDescriptor.value
767
+ }
768
+ } catch {
769
+ // keep 'native'
770
+ }
771
+ }
772
+
716
773
  function __napiStampBindingTarget(exportsObject, target) {
717
774
  if (
718
775
  Object.prototype.hasOwnProperty.call(exportsObject, '__napiBindingTarget')
package/package.json CHANGED
@@ -10,7 +10,7 @@
10
10
  }
11
11
  ],
12
12
  "name": "@xberg-io/crawlberg",
13
- "version": "1.11.1",
13
+ "version": "1.12.0",
14
14
  "description": "High-performance web crawling engine",
15
15
  "license": "MIT",
16
16
  "repository": {
@@ -32,12 +32,12 @@
32
32
  "*.node"
33
33
  ],
34
34
  "optionalDependencies": {
35
- "@xberg-io/crawlberg-darwin-arm64": "1.11.1",
36
- "@xberg-io/crawlberg-darwin-x64": "1.11.1",
37
- "@xberg-io/crawlberg-linux-arm64-gnu": "1.11.1",
38
- "@xberg-io/crawlberg-linux-x64-gnu": "1.11.1",
39
- "@xberg-io/crawlberg-win32-arm64-msvc": "1.11.1",
40
- "@xberg-io/crawlberg-win32-x64-msvc": "1.11.1"
35
+ "@xberg-io/crawlberg-darwin-arm64": "1.12.0",
36
+ "@xberg-io/crawlberg-darwin-x64": "1.12.0",
37
+ "@xberg-io/crawlberg-linux-arm64-gnu": "1.12.0",
38
+ "@xberg-io/crawlberg-linux-x64-gnu": "1.12.0",
39
+ "@xberg-io/crawlberg-win32-arm64-msvc": "1.12.0",
40
+ "@xberg-io/crawlberg-win32-x64-msvc": "1.12.0"
41
41
  },
42
42
  "napi": {
43
43
  "packageName": "@xberg-io/crawlberg",
@@ -58,7 +58,7 @@
58
58
  "access": "public"
59
59
  },
60
60
  "devDependencies": {
61
- "@napi-rs/cli": "^3.10.8"
61
+ "@napi-rs/cli": "^3.10.9"
62
62
  },
63
63
  "scripts": {
64
64
  "build": "pnpm --package @napi-rs/cli@^3.7.3 dlx napi build --platform --dts index.native.d.ts --release",