@xberg-io/crawlberg 1.2.1 → 1.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.d.ts +59 -11
- package/index.js +56 -52
- package/package.json +9 -9
package/index.d.ts
CHANGED
|
@@ -204,15 +204,15 @@ export declare const enum BrowserBackend {
|
|
|
204
204
|
/** Browser fallback configuration. */
|
|
205
205
|
export interface BrowserConfig {
|
|
206
206
|
/** When to use the headless browser fallback. */
|
|
207
|
-
mode?:
|
|
207
|
+
mode?: BrowserMode | 'auto' | 'always' | 'never' | 'stealth'
|
|
208
208
|
/** Browser backend used to render JavaScript-heavy pages. */
|
|
209
|
-
backend?:
|
|
209
|
+
backend?: BrowserBackend | 'chromiumoxide' | 'native'
|
|
210
210
|
/** CDP WebSocket endpoint for connecting to an external browser instance. */
|
|
211
211
|
endpoint?: string
|
|
212
212
|
/** Timeout for browser page load and rendering (in milliseconds when serialized). */
|
|
213
213
|
timeout?: number
|
|
214
214
|
/** Wait strategy after browser navigation. */
|
|
215
|
-
wait?:
|
|
215
|
+
wait?: BrowserWait | 'network_idle' | 'selector' | 'fixed'
|
|
216
216
|
/** CSS selector to wait for when `wait` is `Selector`. */
|
|
217
217
|
waitSelector?: string
|
|
218
218
|
/** Extra time to wait after the wait condition is met. */
|
|
@@ -373,7 +373,7 @@ export interface ContentConfig {
|
|
|
373
373
|
* tracking pixels, GTM iframes) is meant for browsers with JavaScript
|
|
374
374
|
* disabled, not for a markdown reader, and `strip_tags` cannot drop it —
|
|
375
375
|
* on `preprocessing_preset: "standard"` (crawlberg's only path) it only
|
|
376
|
-
* removes the wrapper and still renders the children.
|
|
376
|
+
* removes the wrapper and still renders the children.
|
|
377
377
|
*
|
|
378
378
|
* Example: `[".cookie-banner", "#ad-container", "[role='complementary']"]`
|
|
379
379
|
*/
|
|
@@ -392,6 +392,12 @@ export interface ContentConfig {
|
|
|
392
392
|
|
|
393
393
|
export declare function contentConfigDefault(): ContentConfig
|
|
394
394
|
|
|
395
|
+
/** Content filter applied to each crawled page before it reaches the result. */
|
|
396
|
+
export declare const enum ContentFilterKind {
|
|
397
|
+
/** Keep only pages scoring at or above `bm25_threshold` for `bm25_query`. */
|
|
398
|
+
Bm25 = 'bm_25'
|
|
399
|
+
}
|
|
400
|
+
|
|
395
401
|
/** Information about an HTTP cookie received from a response. */
|
|
396
402
|
export interface CookieInfo {
|
|
397
403
|
/** The cookie name. */
|
|
@@ -422,6 +428,23 @@ export interface CrawlConfig {
|
|
|
422
428
|
maxLinksPerPage?: number
|
|
423
429
|
/** Maximum number of concurrent requests. */
|
|
424
430
|
maxConcurrent?: number
|
|
431
|
+
/**
|
|
432
|
+
* Traversal order. Defaults to breadth-first.
|
|
433
|
+
*
|
|
434
|
+
* A frontier or strategy set explicitly on `CrawlEngineBuilder` takes precedence over
|
|
435
|
+
* this field.
|
|
436
|
+
*/
|
|
437
|
+
crawlStrategy?: CrawlStrategyKind | 'bfs' | 'dfs' | 'best_first' | 'adaptive'
|
|
438
|
+
/**
|
|
439
|
+
* Content filter applied to each page. `None` keeps every page.
|
|
440
|
+
*
|
|
441
|
+
* A content filter set explicitly on `CrawlEngineBuilder` takes precedence.
|
|
442
|
+
*/
|
|
443
|
+
contentFilter?: ContentFilterKind | 'bm25'
|
|
444
|
+
/** Query the BM25 content filter scores pages against. Required by `ContentFilterKind.Bm25`. */
|
|
445
|
+
bm25Query?: string
|
|
446
|
+
/** Minimum BM25 score a page must reach to be kept. Defaults to `0.0`. */
|
|
447
|
+
bm25Threshold?: number
|
|
425
448
|
/** Whether to respect robots.txt directives. */
|
|
426
449
|
respectRobotsTxt?: boolean
|
|
427
450
|
/**
|
|
@@ -463,7 +486,14 @@ export interface CrawlConfig {
|
|
|
463
486
|
cookiesEnabled?: boolean
|
|
464
487
|
/** Authentication configuration. */
|
|
465
488
|
auth?: JsAuthConfig
|
|
466
|
-
/**
|
|
489
|
+
/**
|
|
490
|
+
* Maximum response body size in bytes.
|
|
491
|
+
*
|
|
492
|
+
* `None` does not mean unbounded: an unset cap falls back to a 100 MiB safety
|
|
493
|
+
* ceiling, because HTTP responses are decompressed while being read and a few
|
|
494
|
+
* hundred compressed bytes can otherwise expand to gigabytes in memory. To read
|
|
495
|
+
* bodies larger than that, set this explicitly.
|
|
496
|
+
*/
|
|
467
497
|
maxBodySize?: number
|
|
468
498
|
/** CSS selectors for tags to remove from HTML before processing. */
|
|
469
499
|
removeTags?: Array<string>
|
|
@@ -476,7 +506,7 @@ export interface CrawlConfig {
|
|
|
476
506
|
/** Whether to download assets (CSS, JS, images, etc.) from the page. */
|
|
477
507
|
downloadAssets?: boolean
|
|
478
508
|
/** Filter for asset categories to download. */
|
|
479
|
-
assetTypes?: Array<
|
|
509
|
+
assetTypes?: Array<AssetCategory | 'document' | 'image' | 'audio' | 'video' | 'font' | 'stylesheet' | 'script' | 'archive' | 'data' | 'other'>
|
|
480
510
|
/** Maximum size in bytes for individual asset downloads. */
|
|
481
511
|
maxAssetSize?: number
|
|
482
512
|
/** Browser configuration. */
|
|
@@ -533,7 +563,7 @@ export interface CrawlConfig {
|
|
|
533
563
|
* `alef(skip)`ed). `None` (default) means no encoding is produced. Independent of
|
|
534
564
|
* `document_output_dir` — set both to get a file on disk and an in-memory copy.
|
|
535
565
|
*/
|
|
536
|
-
documentContentEncoding?:
|
|
566
|
+
documentContentEncoding?: DocumentContentEncoding | 'base64'
|
|
537
567
|
/** Path to write WARC output. If `None`, WARC output is disabled. */
|
|
538
568
|
warcOutput?: string
|
|
539
569
|
/**
|
|
@@ -671,6 +701,24 @@ export interface CrawlResult {
|
|
|
671
701
|
browserUsed?: boolean
|
|
672
702
|
}
|
|
673
703
|
|
|
704
|
+
/**
|
|
705
|
+
* Traversal order for a crawl.
|
|
706
|
+
*
|
|
707
|
+
* Selects both the queue discipline and the selection strategy, because global order is a
|
|
708
|
+
* property of the frontier: the engine hands its bounded selection window to the strategy, so
|
|
709
|
+
* a strategy alone can only reorder URLs that have already been dequeued.
|
|
710
|
+
*/
|
|
711
|
+
export declare const enum CrawlStrategyKind {
|
|
712
|
+
/** Breadth-first: a FIFO frontier visits every URL at one depth before the next. */
|
|
713
|
+
Bfs = 'bfs',
|
|
714
|
+
/** Depth-first: a LIFO frontier descends into a page's children before its siblings. */
|
|
715
|
+
Dfs = 'dfs',
|
|
716
|
+
/** Highest-priority-first within the selection window, scored by `CrawlStrategy.score_url`. */
|
|
717
|
+
BestFirst = 'best_first',
|
|
718
|
+
/** Like `BestFirst`, but stops once newly crawled pages stop contributing new terms. */
|
|
719
|
+
Adaptive = 'adaptive'
|
|
720
|
+
}
|
|
721
|
+
|
|
674
722
|
export declare function crawlStream(engine: CrawlEngineHandle, url: string): Promise<CrawlStreamIterator>
|
|
675
723
|
|
|
676
724
|
/**
|
|
@@ -717,7 +765,7 @@ export interface DownloadedAsset {
|
|
|
717
765
|
/** The size of the asset in bytes. */
|
|
718
766
|
size?: number
|
|
719
767
|
/** The category of the asset. */
|
|
720
|
-
assetCategory?:
|
|
768
|
+
assetCategory?: AssetCategory | 'document' | 'image' | 'audio' | 'video' | 'font' | 'stylesheet' | 'script' | 'archive' | 'data' | 'other'
|
|
721
769
|
/** The HTML tag that referenced this asset (e.g., "link", "script", "img"). */
|
|
722
770
|
htmlTag?: string
|
|
723
771
|
}
|
|
@@ -792,7 +840,7 @@ export interface FeedInfo {
|
|
|
792
840
|
/** The feed title, if present. */
|
|
793
841
|
title?: string
|
|
794
842
|
/** The type of feed. */
|
|
795
|
-
feedType?:
|
|
843
|
+
feedType?: FeedType | 'rss' | 'atom' | 'json_feed'
|
|
796
844
|
}
|
|
797
845
|
|
|
798
846
|
/** The type of a feed (RSS, Atom, or JSON Feed). */
|
|
@@ -863,7 +911,7 @@ export interface ImageInfo {
|
|
|
863
911
|
/** The height attribute, if present and parseable. */
|
|
864
912
|
height?: number
|
|
865
913
|
/** The source of the image reference. */
|
|
866
|
-
source?:
|
|
914
|
+
source?: ImageSource | 'img' | 'picture_source' | 'og:image' | 'twitter:image'
|
|
867
915
|
}
|
|
868
916
|
|
|
869
917
|
/** The source of an image reference. */
|
|
@@ -915,7 +963,7 @@ export interface LinkInfo {
|
|
|
915
963
|
/** The visible text of the link. */
|
|
916
964
|
text?: string
|
|
917
965
|
/** The classification of the link. */
|
|
918
|
-
linkType?:
|
|
966
|
+
linkType?: LinkType | 'internal' | 'external' | 'anchor' | 'document'
|
|
919
967
|
/** The `rel` attribute value, if present. */
|
|
920
968
|
rel?: string
|
|
921
969
|
/** Whether the link has `rel="nofollow"`. */
|
package/index.js
CHANGED
|
@@ -77,8 +77,8 @@ function requireNative() {
|
|
|
77
77
|
try {
|
|
78
78
|
const binding = require('@xberg-io/crawlberg-android-arm64')
|
|
79
79
|
const bindingPackageVersion = require('@xberg-io/crawlberg-android-arm64/package.json').version
|
|
80
|
-
if (bindingPackageVersion !== '1.2
|
|
81
|
-
throw new Error(`Native binding package version mismatch, expected 1.2
|
|
80
|
+
if (bindingPackageVersion !== '1.3.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
81
|
+
throw new Error(`Native binding package version mismatch, expected 1.3.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
82
82
|
}
|
|
83
83
|
return binding
|
|
84
84
|
} catch (e) {
|
|
@@ -93,8 +93,8 @@ function requireNative() {
|
|
|
93
93
|
try {
|
|
94
94
|
const binding = require('@xberg-io/crawlberg-android-arm-eabi')
|
|
95
95
|
const bindingPackageVersion = require('@xberg-io/crawlberg-android-arm-eabi/package.json').version
|
|
96
|
-
if (bindingPackageVersion !== '1.2
|
|
97
|
-
throw new Error(`Native binding package version mismatch, expected 1.2
|
|
96
|
+
if (bindingPackageVersion !== '1.3.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
97
|
+
throw new Error(`Native binding package version mismatch, expected 1.3.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
98
98
|
}
|
|
99
99
|
return binding
|
|
100
100
|
} catch (e) {
|
|
@@ -114,8 +114,8 @@ function requireNative() {
|
|
|
114
114
|
try {
|
|
115
115
|
const binding = require('@xberg-io/crawlberg-win32-x64-gnu')
|
|
116
116
|
const bindingPackageVersion = require('@xberg-io/crawlberg-win32-x64-gnu/package.json').version
|
|
117
|
-
if (bindingPackageVersion !== '1.2
|
|
118
|
-
throw new Error(`Native binding package version mismatch, expected 1.2
|
|
117
|
+
if (bindingPackageVersion !== '1.3.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
118
|
+
throw new Error(`Native binding package version mismatch, expected 1.3.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
119
119
|
}
|
|
120
120
|
return binding
|
|
121
121
|
} catch (e) {
|
|
@@ -130,8 +130,8 @@ function requireNative() {
|
|
|
130
130
|
try {
|
|
131
131
|
const binding = require('@xberg-io/crawlberg-win32-x64-msvc')
|
|
132
132
|
const bindingPackageVersion = require('@xberg-io/crawlberg-win32-x64-msvc/package.json').version
|
|
133
|
-
if (bindingPackageVersion !== '1.2
|
|
134
|
-
throw new Error(`Native binding package version mismatch, expected 1.2
|
|
133
|
+
if (bindingPackageVersion !== '1.3.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
134
|
+
throw new Error(`Native binding package version mismatch, expected 1.3.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
135
135
|
}
|
|
136
136
|
return binding
|
|
137
137
|
} catch (e) {
|
|
@@ -147,8 +147,8 @@ function requireNative() {
|
|
|
147
147
|
try {
|
|
148
148
|
const binding = require('@xberg-io/crawlberg-win32-ia32-msvc')
|
|
149
149
|
const bindingPackageVersion = require('@xberg-io/crawlberg-win32-ia32-msvc/package.json').version
|
|
150
|
-
if (bindingPackageVersion !== '1.2
|
|
151
|
-
throw new Error(`Native binding package version mismatch, expected 1.2
|
|
150
|
+
if (bindingPackageVersion !== '1.3.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
151
|
+
throw new Error(`Native binding package version mismatch, expected 1.3.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
152
152
|
}
|
|
153
153
|
return binding
|
|
154
154
|
} catch (e) {
|
|
@@ -163,8 +163,8 @@ function requireNative() {
|
|
|
163
163
|
try {
|
|
164
164
|
const binding = require('@xberg-io/crawlberg-win32-arm64-msvc')
|
|
165
165
|
const bindingPackageVersion = require('@xberg-io/crawlberg-win32-arm64-msvc/package.json').version
|
|
166
|
-
if (bindingPackageVersion !== '1.2
|
|
167
|
-
throw new Error(`Native binding package version mismatch, expected 1.2
|
|
166
|
+
if (bindingPackageVersion !== '1.3.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
167
|
+
throw new Error(`Native binding package version mismatch, expected 1.3.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
168
168
|
}
|
|
169
169
|
return binding
|
|
170
170
|
} catch (e) {
|
|
@@ -182,8 +182,8 @@ function requireNative() {
|
|
|
182
182
|
try {
|
|
183
183
|
const binding = require('@xberg-io/crawlberg-darwin-universal')
|
|
184
184
|
const bindingPackageVersion = require('@xberg-io/crawlberg-darwin-universal/package.json').version
|
|
185
|
-
if (bindingPackageVersion !== '1.2
|
|
186
|
-
throw new Error(`Native binding package version mismatch, expected 1.2
|
|
185
|
+
if (bindingPackageVersion !== '1.3.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
186
|
+
throw new Error(`Native binding package version mismatch, expected 1.3.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
187
187
|
}
|
|
188
188
|
return binding
|
|
189
189
|
} catch (e) {
|
|
@@ -198,8 +198,8 @@ function requireNative() {
|
|
|
198
198
|
try {
|
|
199
199
|
const binding = require('@xberg-io/crawlberg-darwin-x64')
|
|
200
200
|
const bindingPackageVersion = require('@xberg-io/crawlberg-darwin-x64/package.json').version
|
|
201
|
-
if (bindingPackageVersion !== '1.2
|
|
202
|
-
throw new Error(`Native binding package version mismatch, expected 1.2
|
|
201
|
+
if (bindingPackageVersion !== '1.3.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
202
|
+
throw new Error(`Native binding package version mismatch, expected 1.3.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
203
203
|
}
|
|
204
204
|
return binding
|
|
205
205
|
} catch (e) {
|
|
@@ -214,8 +214,8 @@ function requireNative() {
|
|
|
214
214
|
try {
|
|
215
215
|
const binding = require('@xberg-io/crawlberg-darwin-arm64')
|
|
216
216
|
const bindingPackageVersion = require('@xberg-io/crawlberg-darwin-arm64/package.json').version
|
|
217
|
-
if (bindingPackageVersion !== '1.2
|
|
218
|
-
throw new Error(`Native binding package version mismatch, expected 1.2
|
|
217
|
+
if (bindingPackageVersion !== '1.3.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
218
|
+
throw new Error(`Native binding package version mismatch, expected 1.3.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
219
219
|
}
|
|
220
220
|
return binding
|
|
221
221
|
} catch (e) {
|
|
@@ -234,8 +234,8 @@ function requireNative() {
|
|
|
234
234
|
try {
|
|
235
235
|
const binding = require('@xberg-io/crawlberg-freebsd-x64')
|
|
236
236
|
const bindingPackageVersion = require('@xberg-io/crawlberg-freebsd-x64/package.json').version
|
|
237
|
-
if (bindingPackageVersion !== '1.2
|
|
238
|
-
throw new Error(`Native binding package version mismatch, expected 1.2
|
|
237
|
+
if (bindingPackageVersion !== '1.3.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
238
|
+
throw new Error(`Native binding package version mismatch, expected 1.3.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
239
239
|
}
|
|
240
240
|
return binding
|
|
241
241
|
} catch (e) {
|
|
@@ -250,8 +250,8 @@ function requireNative() {
|
|
|
250
250
|
try {
|
|
251
251
|
const binding = require('@xberg-io/crawlberg-freebsd-arm64')
|
|
252
252
|
const bindingPackageVersion = require('@xberg-io/crawlberg-freebsd-arm64/package.json').version
|
|
253
|
-
if (bindingPackageVersion !== '1.2
|
|
254
|
-
throw new Error(`Native binding package version mismatch, expected 1.2
|
|
253
|
+
if (bindingPackageVersion !== '1.3.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
254
|
+
throw new Error(`Native binding package version mismatch, expected 1.3.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
255
255
|
}
|
|
256
256
|
return binding
|
|
257
257
|
} catch (e) {
|
|
@@ -271,8 +271,8 @@ function requireNative() {
|
|
|
271
271
|
try {
|
|
272
272
|
const binding = require('@xberg-io/crawlberg-linux-x64-musl')
|
|
273
273
|
const bindingPackageVersion = require('@xberg-io/crawlberg-linux-x64-musl/package.json').version
|
|
274
|
-
if (bindingPackageVersion !== '1.2
|
|
275
|
-
throw new Error(`Native binding package version mismatch, expected 1.2
|
|
274
|
+
if (bindingPackageVersion !== '1.3.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
275
|
+
throw new Error(`Native binding package version mismatch, expected 1.3.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
276
276
|
}
|
|
277
277
|
return binding
|
|
278
278
|
} catch (e) {
|
|
@@ -287,8 +287,8 @@ function requireNative() {
|
|
|
287
287
|
try {
|
|
288
288
|
const binding = require('@xberg-io/crawlberg-linux-x64-gnu')
|
|
289
289
|
const bindingPackageVersion = require('@xberg-io/crawlberg-linux-x64-gnu/package.json').version
|
|
290
|
-
if (bindingPackageVersion !== '1.2
|
|
291
|
-
throw new Error(`Native binding package version mismatch, expected 1.2
|
|
290
|
+
if (bindingPackageVersion !== '1.3.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
291
|
+
throw new Error(`Native binding package version mismatch, expected 1.3.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
292
292
|
}
|
|
293
293
|
return binding
|
|
294
294
|
} catch (e) {
|
|
@@ -305,8 +305,8 @@ function requireNative() {
|
|
|
305
305
|
try {
|
|
306
306
|
const binding = require('@xberg-io/crawlberg-linux-arm64-musl')
|
|
307
307
|
const bindingPackageVersion = require('@xberg-io/crawlberg-linux-arm64-musl/package.json').version
|
|
308
|
-
if (bindingPackageVersion !== '1.2
|
|
309
|
-
throw new Error(`Native binding package version mismatch, expected 1.2
|
|
308
|
+
if (bindingPackageVersion !== '1.3.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
309
|
+
throw new Error(`Native binding package version mismatch, expected 1.3.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
310
310
|
}
|
|
311
311
|
return binding
|
|
312
312
|
} catch (e) {
|
|
@@ -321,8 +321,8 @@ function requireNative() {
|
|
|
321
321
|
try {
|
|
322
322
|
const binding = require('@xberg-io/crawlberg-linux-arm64-gnu')
|
|
323
323
|
const bindingPackageVersion = require('@xberg-io/crawlberg-linux-arm64-gnu/package.json').version
|
|
324
|
-
if (bindingPackageVersion !== '1.2
|
|
325
|
-
throw new Error(`Native binding package version mismatch, expected 1.2
|
|
324
|
+
if (bindingPackageVersion !== '1.3.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
325
|
+
throw new Error(`Native binding package version mismatch, expected 1.3.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
326
326
|
}
|
|
327
327
|
return binding
|
|
328
328
|
} catch (e) {
|
|
@@ -339,8 +339,8 @@ function requireNative() {
|
|
|
339
339
|
try {
|
|
340
340
|
const binding = require('@xberg-io/crawlberg-linux-arm-musleabihf')
|
|
341
341
|
const bindingPackageVersion = require('@xberg-io/crawlberg-linux-arm-musleabihf/package.json').version
|
|
342
|
-
if (bindingPackageVersion !== '1.2
|
|
343
|
-
throw new Error(`Native binding package version mismatch, expected 1.2
|
|
342
|
+
if (bindingPackageVersion !== '1.3.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
343
|
+
throw new Error(`Native binding package version mismatch, expected 1.3.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
344
344
|
}
|
|
345
345
|
return binding
|
|
346
346
|
} catch (e) {
|
|
@@ -355,8 +355,8 @@ function requireNative() {
|
|
|
355
355
|
try {
|
|
356
356
|
const binding = require('@xberg-io/crawlberg-linux-arm-gnueabihf')
|
|
357
357
|
const bindingPackageVersion = require('@xberg-io/crawlberg-linux-arm-gnueabihf/package.json').version
|
|
358
|
-
if (bindingPackageVersion !== '1.2
|
|
359
|
-
throw new Error(`Native binding package version mismatch, expected 1.2
|
|
358
|
+
if (bindingPackageVersion !== '1.3.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
359
|
+
throw new Error(`Native binding package version mismatch, expected 1.3.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
360
360
|
}
|
|
361
361
|
return binding
|
|
362
362
|
} catch (e) {
|
|
@@ -373,8 +373,8 @@ function requireNative() {
|
|
|
373
373
|
try {
|
|
374
374
|
const binding = require('@xberg-io/crawlberg-linux-loong64-musl')
|
|
375
375
|
const bindingPackageVersion = require('@xberg-io/crawlberg-linux-loong64-musl/package.json').version
|
|
376
|
-
if (bindingPackageVersion !== '1.2
|
|
377
|
-
throw new Error(`Native binding package version mismatch, expected 1.2
|
|
376
|
+
if (bindingPackageVersion !== '1.3.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
377
|
+
throw new Error(`Native binding package version mismatch, expected 1.3.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
378
378
|
}
|
|
379
379
|
return binding
|
|
380
380
|
} catch (e) {
|
|
@@ -389,8 +389,8 @@ function requireNative() {
|
|
|
389
389
|
try {
|
|
390
390
|
const binding = require('@xberg-io/crawlberg-linux-loong64-gnu')
|
|
391
391
|
const bindingPackageVersion = require('@xberg-io/crawlberg-linux-loong64-gnu/package.json').version
|
|
392
|
-
if (bindingPackageVersion !== '1.2
|
|
393
|
-
throw new Error(`Native binding package version mismatch, expected 1.2
|
|
392
|
+
if (bindingPackageVersion !== '1.3.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
393
|
+
throw new Error(`Native binding package version mismatch, expected 1.3.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
394
394
|
}
|
|
395
395
|
return binding
|
|
396
396
|
} catch (e) {
|
|
@@ -407,8 +407,8 @@ function requireNative() {
|
|
|
407
407
|
try {
|
|
408
408
|
const binding = require('@xberg-io/crawlberg-linux-riscv64-musl')
|
|
409
409
|
const bindingPackageVersion = require('@xberg-io/crawlberg-linux-riscv64-musl/package.json').version
|
|
410
|
-
if (bindingPackageVersion !== '1.2
|
|
411
|
-
throw new Error(`Native binding package version mismatch, expected 1.2
|
|
410
|
+
if (bindingPackageVersion !== '1.3.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
411
|
+
throw new Error(`Native binding package version mismatch, expected 1.3.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
412
412
|
}
|
|
413
413
|
return binding
|
|
414
414
|
} catch (e) {
|
|
@@ -423,8 +423,8 @@ function requireNative() {
|
|
|
423
423
|
try {
|
|
424
424
|
const binding = require('@xberg-io/crawlberg-linux-riscv64-gnu')
|
|
425
425
|
const bindingPackageVersion = require('@xberg-io/crawlberg-linux-riscv64-gnu/package.json').version
|
|
426
|
-
if (bindingPackageVersion !== '1.2
|
|
427
|
-
throw new Error(`Native binding package version mismatch, expected 1.2
|
|
426
|
+
if (bindingPackageVersion !== '1.3.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
427
|
+
throw new Error(`Native binding package version mismatch, expected 1.3.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
428
428
|
}
|
|
429
429
|
return binding
|
|
430
430
|
} catch (e) {
|
|
@@ -440,8 +440,8 @@ function requireNative() {
|
|
|
440
440
|
try {
|
|
441
441
|
const binding = require('@xberg-io/crawlberg-linux-ppc64-gnu')
|
|
442
442
|
const bindingPackageVersion = require('@xberg-io/crawlberg-linux-ppc64-gnu/package.json').version
|
|
443
|
-
if (bindingPackageVersion !== '1.2
|
|
444
|
-
throw new Error(`Native binding package version mismatch, expected 1.2
|
|
443
|
+
if (bindingPackageVersion !== '1.3.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
444
|
+
throw new Error(`Native binding package version mismatch, expected 1.3.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
445
445
|
}
|
|
446
446
|
return binding
|
|
447
447
|
} catch (e) {
|
|
@@ -456,8 +456,8 @@ function requireNative() {
|
|
|
456
456
|
try {
|
|
457
457
|
const binding = require('@xberg-io/crawlberg-linux-s390x-gnu')
|
|
458
458
|
const bindingPackageVersion = require('@xberg-io/crawlberg-linux-s390x-gnu/package.json').version
|
|
459
|
-
if (bindingPackageVersion !== '1.2
|
|
460
|
-
throw new Error(`Native binding package version mismatch, expected 1.2
|
|
459
|
+
if (bindingPackageVersion !== '1.3.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
460
|
+
throw new Error(`Native binding package version mismatch, expected 1.3.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
461
461
|
}
|
|
462
462
|
return binding
|
|
463
463
|
} catch (e) {
|
|
@@ -476,8 +476,8 @@ function requireNative() {
|
|
|
476
476
|
try {
|
|
477
477
|
const binding = require('@xberg-io/crawlberg-openharmony-arm64')
|
|
478
478
|
const bindingPackageVersion = require('@xberg-io/crawlberg-openharmony-arm64/package.json').version
|
|
479
|
-
if (bindingPackageVersion !== '1.2
|
|
480
|
-
throw new Error(`Native binding package version mismatch, expected 1.2
|
|
479
|
+
if (bindingPackageVersion !== '1.3.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
480
|
+
throw new Error(`Native binding package version mismatch, expected 1.3.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
481
481
|
}
|
|
482
482
|
return binding
|
|
483
483
|
} catch (e) {
|
|
@@ -492,8 +492,8 @@ function requireNative() {
|
|
|
492
492
|
try {
|
|
493
493
|
const binding = require('@xberg-io/crawlberg-openharmony-x64')
|
|
494
494
|
const bindingPackageVersion = require('@xberg-io/crawlberg-openharmony-x64/package.json').version
|
|
495
|
-
if (bindingPackageVersion !== '1.2
|
|
496
|
-
throw new Error(`Native binding package version mismatch, expected 1.2
|
|
495
|
+
if (bindingPackageVersion !== '1.3.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
496
|
+
throw new Error(`Native binding package version mismatch, expected 1.3.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
497
497
|
}
|
|
498
498
|
return binding
|
|
499
499
|
} catch (e) {
|
|
@@ -508,8 +508,8 @@ function requireNative() {
|
|
|
508
508
|
try {
|
|
509
509
|
const binding = require('@xberg-io/crawlberg-openharmony-arm')
|
|
510
510
|
const bindingPackageVersion = require('@xberg-io/crawlberg-openharmony-arm/package.json').version
|
|
511
|
-
if (bindingPackageVersion !== '1.2
|
|
512
|
-
throw new Error(`Native binding package version mismatch, expected 1.2
|
|
511
|
+
if (bindingPackageVersion !== '1.3.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
512
|
+
throw new Error(`Native binding package version mismatch, expected 1.3.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
513
513
|
}
|
|
514
514
|
return binding
|
|
515
515
|
} catch (e) {
|
|
@@ -604,8 +604,12 @@ module.exports.JsBrowserMode = nativeBinding.JsBrowserMode
|
|
|
604
604
|
module.exports.BrowserWait = nativeBinding.BrowserWait
|
|
605
605
|
module.exports.JsBrowserWait = nativeBinding.JsBrowserWait
|
|
606
606
|
module.exports.contentConfigDefault = nativeBinding.contentConfigDefault
|
|
607
|
+
module.exports.ContentFilterKind = nativeBinding.ContentFilterKind
|
|
608
|
+
module.exports.JsContentFilterKind = nativeBinding.JsContentFilterKind
|
|
607
609
|
module.exports.crawl = nativeBinding.crawl
|
|
608
610
|
module.exports.crawlConfigDefault = nativeBinding.crawlConfigDefault
|
|
611
|
+
module.exports.CrawlStrategyKind = nativeBinding.CrawlStrategyKind
|
|
612
|
+
module.exports.JsCrawlStrategyKind = nativeBinding.JsCrawlStrategyKind
|
|
609
613
|
module.exports.crawlStream = nativeBinding.crawlStream
|
|
610
614
|
module.exports.createEngine = nativeBinding.createEngine
|
|
611
615
|
module.exports.DocumentContentEncoding = nativeBinding.DocumentContentEncoding
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@xberg-io/crawlberg",
|
|
3
|
-
"version": "1.2
|
|
3
|
+
"version": "1.3.2",
|
|
4
4
|
"description": "High-performance web crawling engine",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -22,14 +22,14 @@
|
|
|
22
22
|
"*.node"
|
|
23
23
|
],
|
|
24
24
|
"optionalDependencies": {
|
|
25
|
-
"@xberg-io/crawlberg-darwin-arm64": "1.2
|
|
26
|
-
"@xberg-io/crawlberg-darwin-x64": "1.2
|
|
27
|
-
"@xberg-io/crawlberg-linux-arm64-gnu": "1.2
|
|
28
|
-
"@xberg-io/crawlberg-linux-arm64-musl": "1.2
|
|
29
|
-
"@xberg-io/crawlberg-linux-x64-gnu": "1.2
|
|
30
|
-
"@xberg-io/crawlberg-linux-x64-musl": "1.2
|
|
31
|
-
"@xberg-io/crawlberg-win32-arm64-msvc": "1.2
|
|
32
|
-
"@xberg-io/crawlberg-win32-x64-msvc": "1.2
|
|
25
|
+
"@xberg-io/crawlberg-darwin-arm64": "1.3.2",
|
|
26
|
+
"@xberg-io/crawlberg-darwin-x64": "1.3.2",
|
|
27
|
+
"@xberg-io/crawlberg-linux-arm64-gnu": "1.3.2",
|
|
28
|
+
"@xberg-io/crawlberg-linux-arm64-musl": "1.3.2",
|
|
29
|
+
"@xberg-io/crawlberg-linux-x64-gnu": "1.3.2",
|
|
30
|
+
"@xberg-io/crawlberg-linux-x64-musl": "1.3.2",
|
|
31
|
+
"@xberg-io/crawlberg-win32-arm64-msvc": "1.3.2",
|
|
32
|
+
"@xberg-io/crawlberg-win32-x64-msvc": "1.3.2"
|
|
33
33
|
},
|
|
34
34
|
"napi": {
|
|
35
35
|
"packageName": "@xberg-io/crawlberg",
|