@xberg-io/crawlberg 1.11.1 → 1.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.d.ts +87 -16
- package/index.js +112 -55
- package/package.json +8 -8
package/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// This file is auto-generated by alef — DO NOT EDIT.
|
|
2
|
-
// alef:hash:
|
|
2
|
+
// alef:hash:970b01dd0b98d7d53a619b3b8702083b17b8050bd3ed2122e6bbee5cef6a5676
|
|
3
3
|
// To regenerate: alef generate
|
|
4
4
|
// To verify freshness: alef verify
|
|
5
5
|
/* eslint-disable */
|
|
@@ -214,7 +214,15 @@ export interface BrowserConfig {
|
|
|
214
214
|
readonly wait?: BrowserWait;
|
|
215
215
|
/** CSS selector to wait for when `wait` is `Selector`. */
|
|
216
216
|
readonly waitSelector?: string;
|
|
217
|
-
/**
|
|
217
|
+
/**
|
|
218
|
+
* Extra time to wait after the wait condition is met.
|
|
219
|
+
*
|
|
220
|
+
* The default `network_idle` wait returns once the page has loaded and its DOM has been
|
|
221
|
+
* unchanged for `dom_quiet_window` (at most four seconds, and never past `timeout`, counted
|
|
222
|
+
* from the start of the navigation). An app that draws after a longer quiet period needs
|
|
223
|
+
* a larger `dom_quiet_window` or `extra_wait`; for
|
|
224
|
+
* example `3000` waits three more seconds.
|
|
225
|
+
*/
|
|
218
226
|
readonly extraWait?: number;
|
|
219
227
|
/**
|
|
220
228
|
* Proxy for browser fetches. Overrides `CrawlConfig.proxy` when set.
|
|
@@ -274,6 +282,35 @@ export interface BrowserConfig {
|
|
|
274
282
|
* shared browser pool.
|
|
275
283
|
*/
|
|
276
284
|
readonly chromeArgs?: Array<string>;
|
|
285
|
+
/**
|
|
286
|
+
* Scroll overflowing scroll containers (not only the window) and keep the items that a
|
|
287
|
+
* virtualised list mounts as they pass through view, so the page that is read holds items
|
|
288
|
+
* beyond the first screen. Default: `false`.
|
|
289
|
+
*
|
|
290
|
+
* Runs once after the page is ready and `extra_wait` has elapsed, bounded to 8 containers,
|
|
291
|
+
* 200 steps and 5 seconds in total. Each container is scrolled by 80% of its height per step
|
|
292
|
+
* and the items of its most repetitive child list are collected, keyed by `id`, `data-index`,
|
|
293
|
+
* `data-key`, `data-id`, `aria-posinset` or `aria-rowindex` when present and otherwise by
|
|
294
|
+
* text when it is unique among the rendered items, else by position and text. Only
|
|
295
|
+
* containers whose child list repeats an element are considered, most repetitive first. The list is rewritten, in position order, only when items were unmounted
|
|
296
|
+
* while scrolling and items beyond the first screen were seen; a list that keeps all its
|
|
297
|
+
* items mounted is left as it is. Best effort: it does not follow lists that load data over
|
|
298
|
+
* the network slower than about 100 ms per step, does not reach lists inside iframes or
|
|
299
|
+
* shadow roots, and a rewrite replaces the element children of the list. Lists that recycle
|
|
300
|
+
* their DOM nodes (reusing the same elements with new content) are not collected, and
|
|
301
|
+
* scrolling can trigger infinite-scroll load-more requests. When the step, item or time
|
|
302
|
+
* budget stops the scroll, the items collected so far are kept and a warning is logged.
|
|
303
|
+
* Chromiumoxide backend only.
|
|
304
|
+
*/
|
|
305
|
+
readonly scrollContainers?: boolean;
|
|
306
|
+
/**
|
|
307
|
+
* How long the DOM must stay unchanged after page load for `network_idle` to return,
|
|
308
|
+
* in milliseconds when serialized. Default: 150 ms. Every mutation restarts this window;
|
|
309
|
+
* the whole wait stays bounded to four seconds and the remaining navigation timeout.
|
|
310
|
+
* A page whose first delayed render follows a longer quiet gap needs a larger value
|
|
311
|
+
* or `extra_wait`. Chromiumoxide backend only.
|
|
312
|
+
*/
|
|
313
|
+
readonly domQuietWindow?: number;
|
|
277
314
|
}
|
|
278
315
|
|
|
279
316
|
/**
|
|
@@ -400,7 +437,11 @@ export interface ContentConfig {
|
|
|
400
437
|
readonly excludeSelectors?: Array<string>;
|
|
401
438
|
/** Skip image elements in output. Default: `false`. */
|
|
402
439
|
readonly skipImages?: boolean;
|
|
403
|
-
/**
|
|
440
|
+
/**
|
|
441
|
+
* Max DOM traversal depth. Prevents stack overflow on deeply nested HTML. Content nested
|
|
442
|
+
* deeper is dropped and reported as a `DepthLimitExceeded` warning. `None` means 1024 in
|
|
443
|
+
* crawlberg, not the converter's own default of 64.
|
|
444
|
+
*/
|
|
404
445
|
readonly maxDepth?: number;
|
|
405
446
|
/** Enable line wrapping. Default: `false`. */
|
|
406
447
|
readonly wrap?: boolean;
|
|
@@ -510,15 +551,11 @@ export interface CrawlConfig {
|
|
|
510
551
|
*/
|
|
511
552
|
readonly respectRobotsTxt?: boolean;
|
|
512
553
|
/**
|
|
513
|
-
* When true, HTTP
|
|
514
|
-
*
|
|
515
|
-
*
|
|
516
|
-
*
|
|
517
|
-
*
|
|
518
|
-
* throw-on-error contract for direct fetches. Independently of this flag,
|
|
519
|
-
* 404s reached at the end of a redirect chain are *always* surfaced softly —
|
|
520
|
-
* the user opted into redirect-following, so receiving a 404 there is part of
|
|
521
|
-
* the normal flow rather than an unexpected error.
|
|
554
|
+
* When true, readable HTTP error responses (4xx and 5xx) are surfaced as
|
|
555
|
+
* `ScrapeResult` records with their status and body rather than raised as errors.
|
|
556
|
+
* WAF blocks report their original 4xx/5xx status, or 403 for a block served as 2xx.
|
|
557
|
+
* Default `false` raises errors for recognized error statuses in both `scrape`
|
|
558
|
+
* and `crawl`, including a 404 reached through redirects.
|
|
522
559
|
*/
|
|
523
560
|
readonly softHttpErrors?: boolean;
|
|
524
561
|
/** Custom user-agent string. */
|
|
@@ -542,7 +579,7 @@ export interface CrawlConfig {
|
|
|
542
579
|
readonly allowSubdomains?: boolean;
|
|
543
580
|
/** Regex patterns for paths to include during crawling. */
|
|
544
581
|
readonly includePaths?: Array<string>;
|
|
545
|
-
/** Regex patterns for paths to exclude during crawling. */
|
|
582
|
+
/** Regex patterns for paths to exclude during crawling. The seed URL is never excluded. */
|
|
546
583
|
readonly excludePaths?: Array<string>;
|
|
547
584
|
/**
|
|
548
585
|
* Whether `include_paths`/`exclude_paths` match against `path?query` instead of just
|
|
@@ -583,8 +620,17 @@ export interface CrawlConfig {
|
|
|
583
620
|
*/
|
|
584
621
|
readonly trackingParams?: Array<string>;
|
|
585
622
|
/**
|
|
586
|
-
* Custom HTTP headers to send with each request to the seed URL's host.
|
|
587
|
-
*
|
|
623
|
+
* Custom HTTP headers to send with each request to the seed URL's host. With
|
|
624
|
+
* `Self.allow_subdomains`, a request to a subdomain of the seed host carries the
|
|
625
|
+
* non-credential headers too, whatever its scheme or port. Credential headers stay with the
|
|
626
|
+
* seed host: `Authorization`, `Proxy-Authorization`, `Cookie`, `Set-Cookie`, `X-Api-Key`,
|
|
627
|
+
* `X-Amz-Security-Token`, `Authentication-Info`, the header `Self.auth` names, and any
|
|
628
|
+
* header whose name contains `auth`, `token`, `key`, `secret`, `session`, `cookie`, `csrf`,
|
|
629
|
+
* `xsrf`, `password`, `signature`, `credential`, `passwd`, `pwd`, `pass`, `jwt`, `bearer`,
|
|
630
|
+
* `otp`, `assertion`, `hmac`, `private`, `sso` or `sig`. A secret in a header whose name
|
|
631
|
+
* matches none of these reaches subdomains, so name secret headers recognisably. A request
|
|
632
|
+
* to any other host does not carry any of them. Subdomain forwarding applies to the HTTP and
|
|
633
|
+
* chromiumoxide backends; the native browser backend sends them to the seed host only.
|
|
588
634
|
*/
|
|
589
635
|
readonly customHeaders?: Record<string, string>;
|
|
590
636
|
/** Timeout for individual HTTP requests (in milliseconds when serialized). */
|
|
@@ -610,7 +656,12 @@ export interface CrawlConfig {
|
|
|
610
656
|
* subsequent attempt (capped at `retry_max_delay_ms`). Defaults to 100ms.
|
|
611
657
|
*/
|
|
612
658
|
readonly retryInitialDelayMs?: number;
|
|
613
|
-
/**
|
|
659
|
+
/**
|
|
660
|
+
* Upper bound, in milliseconds, on the exponential retry backoff. A 429 or 503 whose
|
|
661
|
+
* `Retry-After` is longer than the backoff is waited out, but one that exceeds this bound
|
|
662
|
+
* ends the retries with a warning instead (so `0` abandons the retry on any positive
|
|
663
|
+
* `Retry-After`). Defaults to 60s.
|
|
664
|
+
*/
|
|
614
665
|
readonly retryMaxDelayMs?: number;
|
|
615
666
|
/**
|
|
616
667
|
* Fraction of the per-domain rate-limit delay to randomly jitter by, in `[0.0, 1.0]`.
|
|
@@ -750,6 +801,14 @@ export interface CrawlConfig {
|
|
|
750
801
|
* and the environment variable is not consulted for this config.
|
|
751
802
|
*/
|
|
752
803
|
readonly ssrfDenyPrivateExplicit?: boolean;
|
|
804
|
+
/**
|
|
805
|
+
* Accept invalid TLS certificates for page and robots.txt requests.
|
|
806
|
+
* Defaults to false. Enable only for sites whose certificates you explicitly trust.
|
|
807
|
+
* The browser backends also apply this setting before navigation.
|
|
808
|
+
* On WebAssembly, enabling this setting returns `Unsupported`: the host's Fetch
|
|
809
|
+
* implementation controls certificate validation.
|
|
810
|
+
*/
|
|
811
|
+
readonly dangerAcceptInvalidCerts?: boolean;
|
|
753
812
|
}
|
|
754
813
|
|
|
755
814
|
/**
|
|
@@ -892,6 +951,8 @@ export interface CrawlResult {
|
|
|
892
951
|
readonly stayedOnDomain?: boolean;
|
|
893
952
|
/** Whether the browser fallback was used for any page in this crawl. */
|
|
894
953
|
readonly browserUsed?: boolean;
|
|
954
|
+
/** Details of the first WAF block encountered during the crawl. <!-- --> */
|
|
955
|
+
readonly wafBlock?: WafBlockDetails;
|
|
895
956
|
}
|
|
896
957
|
|
|
897
958
|
/**
|
|
@@ -1467,6 +1528,16 @@ export interface SsrfPolicy {
|
|
|
1467
1528
|
readonly schemeAllowlist?: Array<string>;
|
|
1468
1529
|
}
|
|
1469
1530
|
|
|
1531
|
+
/** Structured context for a WAF-blocked response. <!-- --> */
|
|
1532
|
+
export interface WafBlockDetails {
|
|
1533
|
+
/** Lowercase WAF vendor identifier. <!-- --> */
|
|
1534
|
+
readonly vendor?: string;
|
|
1535
|
+
/** Origin HTTP status, when a response caused the block. <!-- --> */
|
|
1536
|
+
readonly status?: number;
|
|
1537
|
+
/** At most 4 KiB of the blocked response body, when it was read. <!-- --> */
|
|
1538
|
+
readonly bodyPrefix?: string;
|
|
1539
|
+
}
|
|
1540
|
+
|
|
1470
1541
|
/** Discover all pages on a website by following links and sitemaps. */
|
|
1471
1542
|
export declare function mapUrls(engine: CrawlEngineHandle, url: string): Promise<MapResult>;
|
|
1472
1543
|
|
package/index.js
CHANGED
|
@@ -6,8 +6,13 @@ const { readFileSync } = require('fs')
|
|
|
6
6
|
let nativeBinding = null
|
|
7
7
|
// Which artifact actually loaded. The WASI fallback chain overwrites it with
|
|
8
8
|
// the flavor it resolved; the late native retry below leaves it alone because
|
|
9
|
-
// it only runs while no WASI candidate has been loaded.
|
|
9
|
+
// it only runs while no WASI candidate has been loaded. The take-over check
|
|
10
|
+
// after the load-failure block can also set it.
|
|
10
11
|
let __napiLoadedBindingTarget = 'native'
|
|
12
|
+
// What this loader's own `requireNative()` calls returned. The check after
|
|
13
|
+
// the load-failure block compares it with `nativeBinding` to tell a binding
|
|
14
|
+
// this loader loaded from one that code outside it put in its place.
|
|
15
|
+
let __napiRequiredNativeBinding = null
|
|
11
16
|
const loadErrors = []
|
|
12
17
|
|
|
13
18
|
const isMusl = () => {
|
|
@@ -89,8 +94,8 @@ function requireNative() {
|
|
|
89
94
|
try {
|
|
90
95
|
const binding = require('@xberg-io/crawlberg-android-arm64')
|
|
91
96
|
const bindingPackageVersion = require('@xberg-io/crawlberg-android-arm64/package.json').version
|
|
92
|
-
if (bindingPackageVersion !== '1.
|
|
93
|
-
throw new Error(`Native binding package version mismatch, expected 1.
|
|
97
|
+
if (bindingPackageVersion !== '1.12.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
98
|
+
throw new Error(`Native binding package version mismatch, expected 1.12.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
94
99
|
}
|
|
95
100
|
return binding
|
|
96
101
|
} catch (e) {
|
|
@@ -105,8 +110,8 @@ function requireNative() {
|
|
|
105
110
|
try {
|
|
106
111
|
const binding = require('@xberg-io/crawlberg-android-arm-eabi')
|
|
107
112
|
const bindingPackageVersion = require('@xberg-io/crawlberg-android-arm-eabi/package.json').version
|
|
108
|
-
if (bindingPackageVersion !== '1.
|
|
109
|
-
throw new Error(`Native binding package version mismatch, expected 1.
|
|
113
|
+
if (bindingPackageVersion !== '1.12.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
114
|
+
throw new Error(`Native binding package version mismatch, expected 1.12.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
110
115
|
}
|
|
111
116
|
return binding
|
|
112
117
|
} catch (e) {
|
|
@@ -126,8 +131,8 @@ function requireNative() {
|
|
|
126
131
|
try {
|
|
127
132
|
const binding = require('@xberg-io/crawlberg-win32-x64-gnu')
|
|
128
133
|
const bindingPackageVersion = require('@xberg-io/crawlberg-win32-x64-gnu/package.json').version
|
|
129
|
-
if (bindingPackageVersion !== '1.
|
|
130
|
-
throw new Error(`Native binding package version mismatch, expected 1.
|
|
134
|
+
if (bindingPackageVersion !== '1.12.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
135
|
+
throw new Error(`Native binding package version mismatch, expected 1.12.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
131
136
|
}
|
|
132
137
|
return binding
|
|
133
138
|
} catch (e) {
|
|
@@ -142,8 +147,8 @@ function requireNative() {
|
|
|
142
147
|
try {
|
|
143
148
|
const binding = require('@xberg-io/crawlberg-win32-x64-msvc')
|
|
144
149
|
const bindingPackageVersion = require('@xberg-io/crawlberg-win32-x64-msvc/package.json').version
|
|
145
|
-
if (bindingPackageVersion !== '1.
|
|
146
|
-
throw new Error(`Native binding package version mismatch, expected 1.
|
|
150
|
+
if (bindingPackageVersion !== '1.12.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
151
|
+
throw new Error(`Native binding package version mismatch, expected 1.12.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
147
152
|
}
|
|
148
153
|
return binding
|
|
149
154
|
} catch (e) {
|
|
@@ -159,8 +164,8 @@ function requireNative() {
|
|
|
159
164
|
try {
|
|
160
165
|
const binding = require('@xberg-io/crawlberg-win32-ia32-msvc')
|
|
161
166
|
const bindingPackageVersion = require('@xberg-io/crawlberg-win32-ia32-msvc/package.json').version
|
|
162
|
-
if (bindingPackageVersion !== '1.
|
|
163
|
-
throw new Error(`Native binding package version mismatch, expected 1.
|
|
167
|
+
if (bindingPackageVersion !== '1.12.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
168
|
+
throw new Error(`Native binding package version mismatch, expected 1.12.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
164
169
|
}
|
|
165
170
|
return binding
|
|
166
171
|
} catch (e) {
|
|
@@ -175,8 +180,8 @@ function requireNative() {
|
|
|
175
180
|
try {
|
|
176
181
|
const binding = require('@xberg-io/crawlberg-win32-arm64-msvc')
|
|
177
182
|
const bindingPackageVersion = require('@xberg-io/crawlberg-win32-arm64-msvc/package.json').version
|
|
178
|
-
if (bindingPackageVersion !== '1.
|
|
179
|
-
throw new Error(`Native binding package version mismatch, expected 1.
|
|
183
|
+
if (bindingPackageVersion !== '1.12.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
184
|
+
throw new Error(`Native binding package version mismatch, expected 1.12.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
180
185
|
}
|
|
181
186
|
return binding
|
|
182
187
|
} catch (e) {
|
|
@@ -194,8 +199,8 @@ function requireNative() {
|
|
|
194
199
|
try {
|
|
195
200
|
const binding = require('@xberg-io/crawlberg-darwin-universal')
|
|
196
201
|
const bindingPackageVersion = require('@xberg-io/crawlberg-darwin-universal/package.json').version
|
|
197
|
-
if (bindingPackageVersion !== '1.
|
|
198
|
-
throw new Error(`Native binding package version mismatch, expected 1.
|
|
202
|
+
if (bindingPackageVersion !== '1.12.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
203
|
+
throw new Error(`Native binding package version mismatch, expected 1.12.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
199
204
|
}
|
|
200
205
|
return binding
|
|
201
206
|
} catch (e) {
|
|
@@ -210,8 +215,8 @@ function requireNative() {
|
|
|
210
215
|
try {
|
|
211
216
|
const binding = require('@xberg-io/crawlberg-darwin-x64')
|
|
212
217
|
const bindingPackageVersion = require('@xberg-io/crawlberg-darwin-x64/package.json').version
|
|
213
|
-
if (bindingPackageVersion !== '1.
|
|
214
|
-
throw new Error(`Native binding package version mismatch, expected 1.
|
|
218
|
+
if (bindingPackageVersion !== '1.12.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
219
|
+
throw new Error(`Native binding package version mismatch, expected 1.12.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
215
220
|
}
|
|
216
221
|
return binding
|
|
217
222
|
} catch (e) {
|
|
@@ -226,8 +231,8 @@ function requireNative() {
|
|
|
226
231
|
try {
|
|
227
232
|
const binding = require('@xberg-io/crawlberg-darwin-arm64')
|
|
228
233
|
const bindingPackageVersion = require('@xberg-io/crawlberg-darwin-arm64/package.json').version
|
|
229
|
-
if (bindingPackageVersion !== '1.
|
|
230
|
-
throw new Error(`Native binding package version mismatch, expected 1.
|
|
234
|
+
if (bindingPackageVersion !== '1.12.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
235
|
+
throw new Error(`Native binding package version mismatch, expected 1.12.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
231
236
|
}
|
|
232
237
|
return binding
|
|
233
238
|
} catch (e) {
|
|
@@ -246,8 +251,8 @@ function requireNative() {
|
|
|
246
251
|
try {
|
|
247
252
|
const binding = require('@xberg-io/crawlberg-freebsd-x64')
|
|
248
253
|
const bindingPackageVersion = require('@xberg-io/crawlberg-freebsd-x64/package.json').version
|
|
249
|
-
if (bindingPackageVersion !== '1.
|
|
250
|
-
throw new Error(`Native binding package version mismatch, expected 1.
|
|
254
|
+
if (bindingPackageVersion !== '1.12.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
255
|
+
throw new Error(`Native binding package version mismatch, expected 1.12.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
251
256
|
}
|
|
252
257
|
return binding
|
|
253
258
|
} catch (e) {
|
|
@@ -262,8 +267,24 @@ function requireNative() {
|
|
|
262
267
|
try {
|
|
263
268
|
const binding = require('@xberg-io/crawlberg-freebsd-arm64')
|
|
264
269
|
const bindingPackageVersion = require('@xberg-io/crawlberg-freebsd-arm64/package.json').version
|
|
265
|
-
if (bindingPackageVersion !== '1.
|
|
266
|
-
throw new Error(`Native binding package version mismatch, expected 1.
|
|
270
|
+
if (bindingPackageVersion !== '1.12.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
271
|
+
throw new Error(`Native binding package version mismatch, expected 1.12.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
272
|
+
}
|
|
273
|
+
return binding
|
|
274
|
+
} catch (e) {
|
|
275
|
+
loadErrors.push(e)
|
|
276
|
+
}
|
|
277
|
+
} else if (process.arch === 'ppc64') {
|
|
278
|
+
try {
|
|
279
|
+
return require('./crawlberg-node.freebsd-ppc64.node')
|
|
280
|
+
} catch (e) {
|
|
281
|
+
loadErrors.push(e)
|
|
282
|
+
}
|
|
283
|
+
try {
|
|
284
|
+
const binding = require('@xberg-io/crawlberg-freebsd-ppc64')
|
|
285
|
+
const bindingPackageVersion = require('@xberg-io/crawlberg-freebsd-ppc64/package.json').version
|
|
286
|
+
if (bindingPackageVersion !== '1.12.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
287
|
+
throw new Error(`Native binding package version mismatch, expected 1.12.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
267
288
|
}
|
|
268
289
|
return binding
|
|
269
290
|
} catch (e) {
|
|
@@ -283,8 +304,8 @@ function requireNative() {
|
|
|
283
304
|
try {
|
|
284
305
|
const binding = require('@xberg-io/crawlberg-linux-x64-musl')
|
|
285
306
|
const bindingPackageVersion = require('@xberg-io/crawlberg-linux-x64-musl/package.json').version
|
|
286
|
-
if (bindingPackageVersion !== '1.
|
|
287
|
-
throw new Error(`Native binding package version mismatch, expected 1.
|
|
307
|
+
if (bindingPackageVersion !== '1.12.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
308
|
+
throw new Error(`Native binding package version mismatch, expected 1.12.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
288
309
|
}
|
|
289
310
|
return binding
|
|
290
311
|
} catch (e) {
|
|
@@ -299,8 +320,8 @@ function requireNative() {
|
|
|
299
320
|
try {
|
|
300
321
|
const binding = require('@xberg-io/crawlberg-linux-x64-gnu')
|
|
301
322
|
const bindingPackageVersion = require('@xberg-io/crawlberg-linux-x64-gnu/package.json').version
|
|
302
|
-
if (bindingPackageVersion !== '1.
|
|
303
|
-
throw new Error(`Native binding package version mismatch, expected 1.
|
|
323
|
+
if (bindingPackageVersion !== '1.12.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
324
|
+
throw new Error(`Native binding package version mismatch, expected 1.12.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
304
325
|
}
|
|
305
326
|
return binding
|
|
306
327
|
} catch (e) {
|
|
@@ -317,8 +338,8 @@ function requireNative() {
|
|
|
317
338
|
try {
|
|
318
339
|
const binding = require('@xberg-io/crawlberg-linux-arm64-musl')
|
|
319
340
|
const bindingPackageVersion = require('@xberg-io/crawlberg-linux-arm64-musl/package.json').version
|
|
320
|
-
if (bindingPackageVersion !== '1.
|
|
321
|
-
throw new Error(`Native binding package version mismatch, expected 1.
|
|
341
|
+
if (bindingPackageVersion !== '1.12.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
342
|
+
throw new Error(`Native binding package version mismatch, expected 1.12.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
322
343
|
}
|
|
323
344
|
return binding
|
|
324
345
|
} catch (e) {
|
|
@@ -333,8 +354,8 @@ function requireNative() {
|
|
|
333
354
|
try {
|
|
334
355
|
const binding = require('@xberg-io/crawlberg-linux-arm64-gnu')
|
|
335
356
|
const bindingPackageVersion = require('@xberg-io/crawlberg-linux-arm64-gnu/package.json').version
|
|
336
|
-
if (bindingPackageVersion !== '1.
|
|
337
|
-
throw new Error(`Native binding package version mismatch, expected 1.
|
|
357
|
+
if (bindingPackageVersion !== '1.12.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
358
|
+
throw new Error(`Native binding package version mismatch, expected 1.12.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
338
359
|
}
|
|
339
360
|
return binding
|
|
340
361
|
} catch (e) {
|
|
@@ -351,8 +372,8 @@ function requireNative() {
|
|
|
351
372
|
try {
|
|
352
373
|
const binding = require('@xberg-io/crawlberg-linux-arm-musleabihf')
|
|
353
374
|
const bindingPackageVersion = require('@xberg-io/crawlberg-linux-arm-musleabihf/package.json').version
|
|
354
|
-
if (bindingPackageVersion !== '1.
|
|
355
|
-
throw new Error(`Native binding package version mismatch, expected 1.
|
|
375
|
+
if (bindingPackageVersion !== '1.12.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
376
|
+
throw new Error(`Native binding package version mismatch, expected 1.12.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
356
377
|
}
|
|
357
378
|
return binding
|
|
358
379
|
} catch (e) {
|
|
@@ -367,8 +388,8 @@ function requireNative() {
|
|
|
367
388
|
try {
|
|
368
389
|
const binding = require('@xberg-io/crawlberg-linux-arm-gnueabihf')
|
|
369
390
|
const bindingPackageVersion = require('@xberg-io/crawlberg-linux-arm-gnueabihf/package.json').version
|
|
370
|
-
if (bindingPackageVersion !== '1.
|
|
371
|
-
throw new Error(`Native binding package version mismatch, expected 1.
|
|
391
|
+
if (bindingPackageVersion !== '1.12.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
392
|
+
throw new Error(`Native binding package version mismatch, expected 1.12.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
372
393
|
}
|
|
373
394
|
return binding
|
|
374
395
|
} catch (e) {
|
|
@@ -385,8 +406,8 @@ function requireNative() {
|
|
|
385
406
|
try {
|
|
386
407
|
const binding = require('@xberg-io/crawlberg-linux-loong64-musl')
|
|
387
408
|
const bindingPackageVersion = require('@xberg-io/crawlberg-linux-loong64-musl/package.json').version
|
|
388
|
-
if (bindingPackageVersion !== '1.
|
|
389
|
-
throw new Error(`Native binding package version mismatch, expected 1.
|
|
409
|
+
if (bindingPackageVersion !== '1.12.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
410
|
+
throw new Error(`Native binding package version mismatch, expected 1.12.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
390
411
|
}
|
|
391
412
|
return binding
|
|
392
413
|
} catch (e) {
|
|
@@ -401,8 +422,8 @@ function requireNative() {
|
|
|
401
422
|
try {
|
|
402
423
|
const binding = require('@xberg-io/crawlberg-linux-loong64-gnu')
|
|
403
424
|
const bindingPackageVersion = require('@xberg-io/crawlberg-linux-loong64-gnu/package.json').version
|
|
404
|
-
if (bindingPackageVersion !== '1.
|
|
405
|
-
throw new Error(`Native binding package version mismatch, expected 1.
|
|
425
|
+
if (bindingPackageVersion !== '1.12.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
426
|
+
throw new Error(`Native binding package version mismatch, expected 1.12.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
406
427
|
}
|
|
407
428
|
return binding
|
|
408
429
|
} catch (e) {
|
|
@@ -419,8 +440,8 @@ function requireNative() {
|
|
|
419
440
|
try {
|
|
420
441
|
const binding = require('@xberg-io/crawlberg-linux-riscv64-musl')
|
|
421
442
|
const bindingPackageVersion = require('@xberg-io/crawlberg-linux-riscv64-musl/package.json').version
|
|
422
|
-
if (bindingPackageVersion !== '1.
|
|
423
|
-
throw new Error(`Native binding package version mismatch, expected 1.
|
|
443
|
+
if (bindingPackageVersion !== '1.12.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
444
|
+
throw new Error(`Native binding package version mismatch, expected 1.12.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
424
445
|
}
|
|
425
446
|
return binding
|
|
426
447
|
} catch (e) {
|
|
@@ -435,8 +456,8 @@ function requireNative() {
|
|
|
435
456
|
try {
|
|
436
457
|
const binding = require('@xberg-io/crawlberg-linux-riscv64-gnu')
|
|
437
458
|
const bindingPackageVersion = require('@xberg-io/crawlberg-linux-riscv64-gnu/package.json').version
|
|
438
|
-
if (bindingPackageVersion !== '1.
|
|
439
|
-
throw new Error(`Native binding package version mismatch, expected 1.
|
|
459
|
+
if (bindingPackageVersion !== '1.12.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
460
|
+
throw new Error(`Native binding package version mismatch, expected 1.12.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
440
461
|
}
|
|
441
462
|
return binding
|
|
442
463
|
} catch (e) {
|
|
@@ -452,8 +473,8 @@ function requireNative() {
|
|
|
452
473
|
try {
|
|
453
474
|
const binding = require('@xberg-io/crawlberg-linux-ppc64-gnu')
|
|
454
475
|
const bindingPackageVersion = require('@xberg-io/crawlberg-linux-ppc64-gnu/package.json').version
|
|
455
|
-
if (bindingPackageVersion !== '1.
|
|
456
|
-
throw new Error(`Native binding package version mismatch, expected 1.
|
|
476
|
+
if (bindingPackageVersion !== '1.12.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
477
|
+
throw new Error(`Native binding package version mismatch, expected 1.12.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
457
478
|
}
|
|
458
479
|
return binding
|
|
459
480
|
} catch (e) {
|
|
@@ -468,8 +489,8 @@ function requireNative() {
|
|
|
468
489
|
try {
|
|
469
490
|
const binding = require('@xberg-io/crawlberg-linux-s390x-gnu')
|
|
470
491
|
const bindingPackageVersion = require('@xberg-io/crawlberg-linux-s390x-gnu/package.json').version
|
|
471
|
-
if (bindingPackageVersion !== '1.
|
|
472
|
-
throw new Error(`Native binding package version mismatch, expected 1.
|
|
492
|
+
if (bindingPackageVersion !== '1.12.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
493
|
+
throw new Error(`Native binding package version mismatch, expected 1.12.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
473
494
|
}
|
|
474
495
|
return binding
|
|
475
496
|
} catch (e) {
|
|
@@ -488,8 +509,8 @@ function requireNative() {
|
|
|
488
509
|
try {
|
|
489
510
|
const binding = require('@xberg-io/crawlberg-openharmony-arm64')
|
|
490
511
|
const bindingPackageVersion = require('@xberg-io/crawlberg-openharmony-arm64/package.json').version
|
|
491
|
-
if (bindingPackageVersion !== '1.
|
|
492
|
-
throw new Error(`Native binding package version mismatch, expected 1.
|
|
512
|
+
if (bindingPackageVersion !== '1.12.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
513
|
+
throw new Error(`Native binding package version mismatch, expected 1.12.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
493
514
|
}
|
|
494
515
|
return binding
|
|
495
516
|
} catch (e) {
|
|
@@ -504,8 +525,8 @@ function requireNative() {
|
|
|
504
525
|
try {
|
|
505
526
|
const binding = require('@xberg-io/crawlberg-openharmony-x64')
|
|
506
527
|
const bindingPackageVersion = require('@xberg-io/crawlberg-openharmony-x64/package.json').version
|
|
507
|
-
if (bindingPackageVersion !== '1.
|
|
508
|
-
throw new Error(`Native binding package version mismatch, expected 1.
|
|
528
|
+
if (bindingPackageVersion !== '1.12.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
529
|
+
throw new Error(`Native binding package version mismatch, expected 1.12.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
509
530
|
}
|
|
510
531
|
return binding
|
|
511
532
|
} catch (e) {
|
|
@@ -520,8 +541,8 @@ function requireNative() {
|
|
|
520
541
|
try {
|
|
521
542
|
const binding = require('@xberg-io/crawlberg-openharmony-arm')
|
|
522
543
|
const bindingPackageVersion = require('@xberg-io/crawlberg-openharmony-arm/package.json').version
|
|
523
|
-
if (bindingPackageVersion !== '1.
|
|
524
|
-
throw new Error(`Native binding package version mismatch, expected 1.
|
|
544
|
+
if (bindingPackageVersion !== '1.12.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
545
|
+
throw new Error(`Native binding package version mismatch, expected 1.12.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
525
546
|
}
|
|
526
547
|
return binding
|
|
527
548
|
} catch (e) {
|
|
@@ -585,6 +606,7 @@ const forceWasi =
|
|
|
585
606
|
|
|
586
607
|
if (!forceWasi) {
|
|
587
608
|
nativeBinding = requireNative()
|
|
609
|
+
__napiRequiredNativeBinding = nativeBinding
|
|
588
610
|
}
|
|
589
611
|
|
|
590
612
|
if (!nativeBinding || forceWasi) {
|
|
@@ -661,8 +683,8 @@ if (!nativeBinding || forceWasi) {
|
|
|
661
683
|
if (!candidateFailed) {
|
|
662
684
|
if (process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
663
685
|
const bindingPackageVersion = require('@xberg-io/crawlberg-wasm32-wasi/package.json').version
|
|
664
|
-
if (bindingPackageVersion !== '1.
|
|
665
|
-
throw new Error(`WASI binding package version mismatch, expected 1.
|
|
686
|
+
if (bindingPackageVersion !== '1.12.0') {
|
|
687
|
+
throw new Error(`WASI binding package version mismatch, expected 1.12.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
666
688
|
}
|
|
667
689
|
}
|
|
668
690
|
wasiBinding = require('@xberg-io/crawlberg-wasm32-wasi')
|
|
@@ -686,6 +708,7 @@ if (!nativeBinding || forceWasi) {
|
|
|
686
708
|
!__napiWasiFlavorRequested
|
|
687
709
|
) {
|
|
688
710
|
nativeBinding = requireNative()
|
|
711
|
+
__napiRequiredNativeBinding = nativeBinding
|
|
689
712
|
}
|
|
690
713
|
if ((forceWasiError || __napiWasiFlavorRequested) && !wasiBindingLoaded) {
|
|
691
714
|
const error = new Error(
|
|
@@ -713,6 +736,40 @@ if (!nativeBinding) {
|
|
|
713
736
|
throw new Error(`Failed to load native binding`)
|
|
714
737
|
}
|
|
715
738
|
|
|
739
|
+
// A package can patch its own fallback in front of the load-failure check
|
|
740
|
+
// above, e.g. a WebContainer download through `napi-postinstall/fallback`.
|
|
741
|
+
// That fallback swaps `nativeBinding` for a generated WASI loader, which has
|
|
742
|
+
// already stamped its own flavor, while the target above still says 'native'.
|
|
743
|
+
// Take that flavor over, or the CommonJS stamp throws a conflict and the ESM
|
|
744
|
+
// export reports the wrong artifact. Three limits keep the conflict check:
|
|
745
|
+
// - only a binding this loader did not load itself, so a native addon whose
|
|
746
|
+
// `#[napi(module_exports)]` hook claims the name is still rejected;
|
|
747
|
+
// - only an own data property, read through its descriptor, so no addon
|
|
748
|
+
// getter runs here, and it must read back the same, as the stamp reads it;
|
|
749
|
+
// - only a flavor in `__napiWasiFlavors`, the ones this loader's own WASI
|
|
750
|
+
// chain can report ('wasm32-wasi' for a package without a WASI target).
|
|
751
|
+
// This is metadata and never fails a load: a throwing Proxy trap keeps 'native'.
|
|
752
|
+
if (
|
|
753
|
+
__napiLoadedBindingTarget === 'native' &&
|
|
754
|
+
nativeBinding !== __napiRequiredNativeBinding
|
|
755
|
+
) {
|
|
756
|
+
try {
|
|
757
|
+
const __napiBindingTargetDescriptor = Object.getOwnPropertyDescriptor(
|
|
758
|
+
nativeBinding,
|
|
759
|
+
'__napiBindingTarget',
|
|
760
|
+
)
|
|
761
|
+
if (
|
|
762
|
+
__napiBindingTargetDescriptor &&
|
|
763
|
+
__napiWasiFlavors.indexOf(__napiBindingTargetDescriptor.value) !== -1 &&
|
|
764
|
+
nativeBinding.__napiBindingTarget === __napiBindingTargetDescriptor.value
|
|
765
|
+
) {
|
|
766
|
+
__napiLoadedBindingTarget = __napiBindingTargetDescriptor.value
|
|
767
|
+
}
|
|
768
|
+
} catch {
|
|
769
|
+
// keep 'native'
|
|
770
|
+
}
|
|
771
|
+
}
|
|
772
|
+
|
|
716
773
|
function __napiStampBindingTarget(exportsObject, target) {
|
|
717
774
|
if (
|
|
718
775
|
Object.prototype.hasOwnProperty.call(exportsObject, '__napiBindingTarget')
|
package/package.json
CHANGED
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
}
|
|
11
11
|
],
|
|
12
12
|
"name": "@xberg-io/crawlberg",
|
|
13
|
-
"version": "1.
|
|
13
|
+
"version": "1.12.0",
|
|
14
14
|
"description": "High-performance web crawling engine",
|
|
15
15
|
"license": "MIT",
|
|
16
16
|
"repository": {
|
|
@@ -32,12 +32,12 @@
|
|
|
32
32
|
"*.node"
|
|
33
33
|
],
|
|
34
34
|
"optionalDependencies": {
|
|
35
|
-
"@xberg-io/crawlberg-darwin-arm64": "1.
|
|
36
|
-
"@xberg-io/crawlberg-darwin-x64": "1.
|
|
37
|
-
"@xberg-io/crawlberg-linux-arm64-gnu": "1.
|
|
38
|
-
"@xberg-io/crawlberg-linux-x64-gnu": "1.
|
|
39
|
-
"@xberg-io/crawlberg-win32-arm64-msvc": "1.
|
|
40
|
-
"@xberg-io/crawlberg-win32-x64-msvc": "1.
|
|
35
|
+
"@xberg-io/crawlberg-darwin-arm64": "1.12.0",
|
|
36
|
+
"@xberg-io/crawlberg-darwin-x64": "1.12.0",
|
|
37
|
+
"@xberg-io/crawlberg-linux-arm64-gnu": "1.12.0",
|
|
38
|
+
"@xberg-io/crawlberg-linux-x64-gnu": "1.12.0",
|
|
39
|
+
"@xberg-io/crawlberg-win32-arm64-msvc": "1.12.0",
|
|
40
|
+
"@xberg-io/crawlberg-win32-x64-msvc": "1.12.0"
|
|
41
41
|
},
|
|
42
42
|
"napi": {
|
|
43
43
|
"packageName": "@xberg-io/crawlberg",
|
|
@@ -58,7 +58,7 @@
|
|
|
58
58
|
"access": "public"
|
|
59
59
|
},
|
|
60
60
|
"devDependencies": {
|
|
61
|
-
"@napi-rs/cli": "^3.10.
|
|
61
|
+
"@napi-rs/cli": "^3.10.9"
|
|
62
62
|
},
|
|
63
63
|
"scripts": {
|
|
64
64
|
"build": "pnpm --package @napi-rs/cli@^3.7.3 dlx napi build --platform --dts index.native.d.ts --release",
|