@xberg-io/crawlberg-wasm 1.7.2 → 1.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Binary file
@@ -124,28 +124,36 @@ export const wasmbatchscraperesults_totalCount: (a: number) => number;
124
124
  export const wasmbrowserconfig_backend: (a: number) => [number, number];
125
125
  export const wasmbrowserconfig_blockUrlPatterns: (a: number) => [number, number];
126
126
  export const wasmbrowserconfig_captureNetworkEvents: (a: number) => number;
127
+ export const wasmbrowserconfig_chromeArgs: (a: number) => [number, number];
128
+ export const wasmbrowserconfig_chromePath: (a: number) => [number, number];
127
129
  export const wasmbrowserconfig_default: () => number;
128
130
  export const wasmbrowserconfig_endpoint: (a: number) => [number, number];
129
131
  export const wasmbrowserconfig_evalScript: (a: number) => [number, number];
130
132
  export const wasmbrowserconfig_extraWait: (a: number) => [number, bigint];
131
133
  export const wasmbrowserconfig_mode: (a: number) => [number, number];
132
- export const wasmbrowserconfig_new: (a: number, b: number, c: number, d: bigint, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number, n: number, o: bigint, p: number, q: number, r: number, s: number, t: number) => number;
134
+ export const wasmbrowserconfig_new: (a: number, b: number, c: number, d: bigint, e: number, f: bigint, g: number, h: bigint, i: number, j: number, k: number, l: number, m: number, n: number, o: number, p: number, q: number, r: number, s: number, t: number, u: bigint, v: number, w: number, x: number, y: number, z: number, a1: number, b1: number) => number;
135
+ export const wasmbrowserconfig_overallTimeout: (a: number) => [number, bigint];
133
136
  export const wasmbrowserconfig_proxy: (a: number) => number;
134
137
  export const wasmbrowserconfig_robotsUserAgent: (a: number) => [number, number];
135
138
  export const wasmbrowserconfig_sessionAffinity: (a: number) => number;
136
139
  export const wasmbrowserconfig_set_backend: (a: number, b: number) => void;
137
140
  export const wasmbrowserconfig_set_blockUrlPatterns: (a: number, b: number, c: number) => void;
138
141
  export const wasmbrowserconfig_set_captureNetworkEvents: (a: number, b: number) => void;
142
+ export const wasmbrowserconfig_set_chromeArgs: (a: number, b: number, c: number) => void;
143
+ export const wasmbrowserconfig_set_chromePath: (a: number, b: number, c: number) => void;
139
144
  export const wasmbrowserconfig_set_endpoint: (a: number, b: number, c: number) => void;
140
145
  export const wasmbrowserconfig_set_evalScript: (a: number, b: number, c: number) => void;
141
146
  export const wasmbrowserconfig_set_extraWait: (a: number, b: number, c: bigint) => void;
142
147
  export const wasmbrowserconfig_set_mode: (a: number, b: number) => void;
148
+ export const wasmbrowserconfig_set_overallTimeout: (a: number, b: number, c: bigint) => void;
143
149
  export const wasmbrowserconfig_set_proxy: (a: number, b: number) => void;
144
150
  export const wasmbrowserconfig_set_robotsUserAgent: (a: number, b: number, c: number) => void;
145
151
  export const wasmbrowserconfig_set_sessionAffinity: (a: number, b: number) => void;
152
+ export const wasmbrowserconfig_set_shutdownTimeout: (a: number, b: number, c: bigint) => void;
146
153
  export const wasmbrowserconfig_set_timeout: (a: number, b: number, c: bigint) => void;
147
154
  export const wasmbrowserconfig_set_wait: (a: number, b: number) => void;
148
155
  export const wasmbrowserconfig_set_waitSelector: (a: number, b: number, c: number) => void;
156
+ export const wasmbrowserconfig_shutdownTimeout: (a: number) => [number, bigint];
149
157
  export const wasmbrowserconfig_timeout: (a: number) => [number, bigint];
150
158
  export const wasmbrowserconfig_wait: (a: number) => [number, number];
151
159
  export const wasmbrowserconfig_waitSelector: (a: number) => [number, number];
@@ -173,15 +181,17 @@ export const wasmcitationresult_set_content: (a: number, b: number, c: number) =
173
181
  export const wasmcitationresult_set_references: (a: number, b: number, c: number) => void;
174
182
  export const wasmcontentconfig_default: () => number;
175
183
  export const wasmcontentconfig_excludeSelectors: (a: number) => [number, number];
184
+ export const wasmcontentconfig_extractMetadata: (a: number) => number;
176
185
  export const wasmcontentconfig_includeDocumentStructure: (a: number) => number;
177
186
  export const wasmcontentconfig_maxDepth: (a: number) => number;
178
- export const wasmcontentconfig_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number, n: number, o: number, p: number, q: number) => number;
187
+ export const wasmcontentconfig_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number, n: number, o: number, p: number, q: number, r: number) => number;
179
188
  export const wasmcontentconfig_outputFormat: (a: number) => [number, number];
180
189
  export const wasmcontentconfig_preprocessingPreset: (a: number) => [number, number];
181
190
  export const wasmcontentconfig_preserveTags: (a: number) => [number, number];
182
191
  export const wasmcontentconfig_removeForms: (a: number) => number;
183
192
  export const wasmcontentconfig_removeNavigation: (a: number) => number;
184
193
  export const wasmcontentconfig_set_excludeSelectors: (a: number, b: number, c: number) => void;
194
+ export const wasmcontentconfig_set_extractMetadata: (a: number, b: number) => void;
185
195
  export const wasmcontentconfig_set_includeDocumentStructure: (a: number, b: number) => void;
186
196
  export const wasmcontentconfig_set_maxDepth: (a: number, b: number) => void;
187
197
  export const wasmcontentconfig_set_outputFormat: (a: number, b: number, c: number) => void;
@@ -220,6 +230,7 @@ export const wasmcrawlconfig_contentFilter: (a: number) => [number, number];
220
230
  export const wasmcrawlconfig_cookiesEnabled: (a: number) => number;
221
231
  export const wasmcrawlconfig_crawlStrategy: (a: number) => [number, number];
222
232
  export const wasmcrawlconfig_customHeaders: (a: number) => any;
233
+ export const wasmcrawlconfig_dedupIncludeQuery: (a: number) => number;
223
234
  export const wasmcrawlconfig_default: () => number;
224
235
  export const wasmcrawlconfig_documentContentEncoding: (a: number) => [number, number];
225
236
  export const wasmcrawlconfig_documentMaxSize: (a: number) => number;
@@ -240,14 +251,19 @@ export const wasmcrawlconfig_maxDepth: (a: number) => number;
240
251
  export const wasmcrawlconfig_maxLinksPerPage: (a: number) => number;
241
252
  export const wasmcrawlconfig_maxPages: (a: number) => number;
242
253
  export const wasmcrawlconfig_maxRedirects: (a: number) => number;
243
- export const wasmcrawlconfig_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: bigint, m: number, n: number, o: number, p: number, q: number, r: number, s: number, t: number, u: number, v: number, w: number, x: number, y: number, z: number, a1: number, b1: number, c1: number, d1: number, e1: number, f1: number, g1: number, h1: number, i1: number, j1: number, k1: number, l1: number, m1: number, n1: number, o1: number, p1: number, q1: number, r1: number, s1: number, t1: bigint, u1: number, v1: number, w1: number, x1: number, y1: number, z1: number, a2: number, b2: number, c2: number, d2: number, e2: number, f2: number, g2: number, h2: number, i2: number, j2: number, k2: number) => number;
254
+ export const wasmcrawlconfig_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number, n: number, o: number, p: number, q: number, r: bigint, s: number, t: number, u: number, v: number, w: number, x: bigint, y: number, z: bigint, a1: number, b1: number, c1: number, d1: number, e1: number, f1: number, g1: number, h1: number, i1: number, j1: number, k1: number, l1: number, m1: number, n1: number, o1: number, p1: number, q1: number, r1: number, s1: number, t1: number, u1: number, v1: number, w1: number, x1: number, y1: number, z1: number, a2: number, b2: number, c2: number, d2: number, e2: number, f2: bigint, g2: number, h2: number, i2: number, j2: number, k2: number, l2: number, m2: number, n2: number, o2: number, p2: number, q2: number, r2: number, s2: number, t2: number, u2: number, v2: number, w2: number) => number;
255
+ export const wasmcrawlconfig_pathPatternsMatchQuery: (a: number) => number;
256
+ export const wasmcrawlconfig_pathPatternsMatchUrl: (a: number) => number;
244
257
  export const wasmcrawlconfig_proxy: (a: number) => number;
258
+ export const wasmcrawlconfig_rateLimitJitterRatio: (a: number) => number;
245
259
  export const wasmcrawlconfig_rateLimitMs: (a: number) => [number, bigint];
246
260
  export const wasmcrawlconfig_removeTags: (a: number) => [number, number];
247
261
  export const wasmcrawlconfig_requestTimeout: (a: number) => [number, bigint];
248
262
  export const wasmcrawlconfig_respectRobotsTxt: (a: number) => number;
249
263
  export const wasmcrawlconfig_retryCodes: (a: number) => [number, number];
250
264
  export const wasmcrawlconfig_retryCount: (a: number) => number;
265
+ export const wasmcrawlconfig_retryInitialDelayMs: (a: number) => bigint;
266
+ export const wasmcrawlconfig_retryMaxDelayMs: (a: number) => bigint;
251
267
  export const wasmcrawlconfig_saveBrowserProfile: (a: number) => number;
252
268
  export const wasmcrawlconfig_set_allowSubdomains: (a: number, b: number) => void;
253
269
  export const wasmcrawlconfig_set_assetTypes: (a: number, b: number, c: number) => void;
@@ -262,6 +278,7 @@ export const wasmcrawlconfig_set_contentFilter: (a: number, b: number) => void;
262
278
  export const wasmcrawlconfig_set_cookiesEnabled: (a: number, b: number) => void;
263
279
  export const wasmcrawlconfig_set_crawlStrategy: (a: number, b: number) => void;
264
280
  export const wasmcrawlconfig_set_customHeaders: (a: number, b: any) => void;
281
+ export const wasmcrawlconfig_set_dedupIncludeQuery: (a: number, b: number) => void;
265
282
  export const wasmcrawlconfig_set_documentContentEncoding: (a: number, b: number) => void;
266
283
  export const wasmcrawlconfig_set_documentMaxSize: (a: number, b: number) => void;
267
284
  export const wasmcrawlconfig_set_documentMimeTypes: (a: number, b: number, c: number) => void;
@@ -281,18 +298,25 @@ export const wasmcrawlconfig_set_maxDepth: (a: number, b: number) => void;
281
298
  export const wasmcrawlconfig_set_maxLinksPerPage: (a: number, b: number) => void;
282
299
  export const wasmcrawlconfig_set_maxPages: (a: number, b: number) => void;
283
300
  export const wasmcrawlconfig_set_maxRedirects: (a: number, b: number) => void;
301
+ export const wasmcrawlconfig_set_pathPatternsMatchQuery: (a: number, b: number) => void;
302
+ export const wasmcrawlconfig_set_pathPatternsMatchUrl: (a: number, b: number) => void;
284
303
  export const wasmcrawlconfig_set_proxy: (a: number, b: number) => void;
304
+ export const wasmcrawlconfig_set_rateLimitJitterRatio: (a: number, b: number) => void;
285
305
  export const wasmcrawlconfig_set_rateLimitMs: (a: number, b: number, c: bigint) => void;
286
306
  export const wasmcrawlconfig_set_removeTags: (a: number, b: number, c: number) => void;
287
307
  export const wasmcrawlconfig_set_requestTimeout: (a: number, b: number, c: bigint) => void;
288
308
  export const wasmcrawlconfig_set_respectRobotsTxt: (a: number, b: number) => void;
289
309
  export const wasmcrawlconfig_set_retryCodes: (a: number, b: number, c: number) => void;
290
310
  export const wasmcrawlconfig_set_retryCount: (a: number, b: number) => void;
311
+ export const wasmcrawlconfig_set_retryInitialDelayMs: (a: number, b: bigint) => void;
312
+ export const wasmcrawlconfig_set_retryMaxDelayMs: (a: number, b: bigint) => void;
291
313
  export const wasmcrawlconfig_set_saveBrowserProfile: (a: number, b: number) => void;
292
314
  export const wasmcrawlconfig_set_softHttpErrors: (a: number, b: number) => void;
293
315
  export const wasmcrawlconfig_set_ssrf: (a: number, b: number) => void;
294
316
  export const wasmcrawlconfig_set_ssrfDenyPrivateExplicit: (a: number, b: number) => void;
295
317
  export const wasmcrawlconfig_set_stayOnDomain: (a: number, b: number) => void;
318
+ export const wasmcrawlconfig_set_stripTrackingParams: (a: number, b: number) => void;
319
+ export const wasmcrawlconfig_set_trackingParams: (a: number, b: number, c: number) => void;
296
320
  export const wasmcrawlconfig_set_userAgent: (a: number, b: number, c: number) => void;
297
321
  export const wasmcrawlconfig_set_userAgents: (a: number, b: number, c: number) => void;
298
322
  export const wasmcrawlconfig_set_warcOutput: (a: number, b: number, c: number) => void;
@@ -300,6 +324,8 @@ export const wasmcrawlconfig_softHttpErrors: (a: number) => number;
300
324
  export const wasmcrawlconfig_ssrf: (a: number) => number;
301
325
  export const wasmcrawlconfig_ssrfDenyPrivateExplicit: (a: number) => number;
302
326
  export const wasmcrawlconfig_stayOnDomain: (a: number) => number;
327
+ export const wasmcrawlconfig_stripTrackingParams: (a: number) => number;
328
+ export const wasmcrawlconfig_trackingParams: (a: number) => [number, number];
303
329
  export const wasmcrawlconfig_userAgent: (a: number) => [number, number];
304
330
  export const wasmcrawlconfig_userAgents: (a: number) => [number, number];
305
331
  export const wasmcrawlconfig_validate: (a: number) => [number, number];
@@ -314,6 +340,7 @@ export const wasmcrawlpageresult_downloadedDocument: (a: number) => number;
314
340
  export const wasmcrawlpageresult_extractedData: (a: number) => any;
315
341
  export const wasmcrawlpageresult_extractionMeta: (a: number) => number;
316
342
  export const wasmcrawlpageresult_feeds: (a: number) => [number, number];
343
+ export const wasmcrawlpageresult_finalUrl: (a: number) => [number, number];
317
344
  export const wasmcrawlpageresult_html: (a: number) => [number, number];
318
345
  export const wasmcrawlpageresult_images: (a: number) => [number, number];
319
346
  export const wasmcrawlpageresult_isPdf: (a: number) => number;
@@ -321,8 +348,11 @@ export const wasmcrawlpageresult_jsonLd: (a: number) => [number, number];
321
348
  export const wasmcrawlpageresult_links: (a: number) => [number, number];
322
349
  export const wasmcrawlpageresult_markdown: (a: number) => number;
323
350
  export const wasmcrawlpageresult_metadata: (a: number) => number;
324
- export const wasmcrawlpageresult_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number, n: number, o: number, p: number, q: number, r: number, s: number, t: number, u: number, v: number, w: number, x: number, y: number, z: number, a1: number, b1: number, c1: number, d1: number) => number;
351
+ export const wasmcrawlpageresult_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number, n: number, o: number, p: number, q: number, r: number, s: number, t: number, u: number, v: number, w: number, x: number, y: number, z: number, a1: number, b1: number, c1: number, d1: number, e1: number, f1: number, g1: number, h1: number, i1: number, j1: number, k1: number) => number;
352
+ export const wasmcrawlpageresult_nofollowDetected: (a: number) => number;
353
+ export const wasmcrawlpageresult_noindexDetected: (a: number) => number;
325
354
  export const wasmcrawlpageresult_normalizedUrl: (a: number) => [number, number];
355
+ export const wasmcrawlpageresult_redirectCount: (a: number) => number;
326
356
  export const wasmcrawlpageresult_set_bodySize: (a: number, b: number) => void;
327
357
  export const wasmcrawlpageresult_set_browserUsed: (a: number, b: number) => void;
328
358
  export const wasmcrawlpageresult_set_contentType: (a: number, b: number, c: number) => void;
@@ -332,6 +362,7 @@ export const wasmcrawlpageresult_set_downloadedDocument: (a: number, b: number)
332
362
  export const wasmcrawlpageresult_set_extractedData: (a: number, b: number) => void;
333
363
  export const wasmcrawlpageresult_set_extractionMeta: (a: number, b: number) => void;
334
364
  export const wasmcrawlpageresult_set_feeds: (a: number, b: number, c: number) => void;
365
+ export const wasmcrawlpageresult_set_finalUrl: (a: number, b: number, c: number) => void;
335
366
  export const wasmcrawlpageresult_set_html: (a: number, b: number, c: number) => void;
336
367
  export const wasmcrawlpageresult_set_images: (a: number, b: number, c: number) => void;
337
368
  export const wasmcrawlpageresult_set_isPdf: (a: number, b: number) => void;
@@ -339,11 +370,16 @@ export const wasmcrawlpageresult_set_jsonLd: (a: number, b: number, c: number) =
339
370
  export const wasmcrawlpageresult_set_links: (a: number, b: number, c: number) => void;
340
371
  export const wasmcrawlpageresult_set_markdown: (a: number, b: number) => void;
341
372
  export const wasmcrawlpageresult_set_metadata: (a: number, b: number) => void;
373
+ export const wasmcrawlpageresult_set_nofollowDetected: (a: number, b: number) => void;
374
+ export const wasmcrawlpageresult_set_noindexDetected: (a: number, b: number) => void;
342
375
  export const wasmcrawlpageresult_set_normalizedUrl: (a: number, b: number, c: number) => void;
376
+ export const wasmcrawlpageresult_set_redirectCount: (a: number, b: number) => void;
377
+ export const wasmcrawlpageresult_set_ssrfRefusedUrls: (a: number, b: number, c: number) => void;
343
378
  export const wasmcrawlpageresult_set_statusCode: (a: number, b: number) => void;
344
379
  export const wasmcrawlpageresult_set_stayedOnDomain: (a: number, b: number) => void;
345
380
  export const wasmcrawlpageresult_set_url: (a: number, b: number, c: number) => void;
346
381
  export const wasmcrawlpageresult_set_wasSkipped: (a: number, b: number) => void;
382
+ export const wasmcrawlpageresult_ssrfRefusedUrls: (a: number) => [number, number];
347
383
  export const wasmcrawlpageresult_statusCode: (a: number) => number;
348
384
  export const wasmcrawlpageresult_stayedOnDomain: (a: number) => number;
349
385
  export const wasmcrawlpageresult_url: (a: number) => [number, number];
@@ -465,12 +501,14 @@ export const wasminteractionresult_actionResults: (a: number) => [number, number
465
501
  export const wasminteractionresult_default: () => number;
466
502
  export const wasminteractionresult_finalHtml: (a: number) => [number, number];
467
503
  export const wasminteractionresult_finalUrl: (a: number) => [number, number];
468
- export const wasminteractionresult_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number) => number;
504
+ export const wasminteractionresult_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number) => number;
469
505
  export const wasminteractionresult_screenshotBase64: (a: number) => [number, number];
470
506
  export const wasminteractionresult_set_actionResults: (a: number, b: number, c: number) => void;
471
507
  export const wasminteractionresult_set_finalHtml: (a: number, b: number, c: number) => void;
472
508
  export const wasminteractionresult_set_finalUrl: (a: number, b: number, c: number) => void;
473
509
  export const wasminteractionresult_set_screenshotBase64: (a: number, b: number, c: number) => void;
510
+ export const wasminteractionresult_set_ssrfRefusedUrls: (a: number, b: number, c: number) => void;
511
+ export const wasminteractionresult_ssrfRefusedUrls: (a: number) => [number, number];
474
512
  export const wasmjsonldentry_default: () => number;
475
513
  export const wasmjsonldentry_name: (a: number) => [number, number];
476
514
  export const wasmjsonldentry_new: (a: number, b: number, c: number, d: number, e: number, f: number) => number;
@@ -664,7 +702,7 @@ export const wasmscraperesult_jsonLd: (a: number) => [number, number];
664
702
  export const wasmscraperesult_links: (a: number) => [number, number];
665
703
  export const wasmscraperesult_markdown: (a: number) => number;
666
704
  export const wasmscraperesult_metadata: (a: number) => number;
667
- export const wasmscraperesult_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number, n: number, o: number, p: number, q: number, r: number, s: number, t: number, u: number, v: number, w: number, x: number, y: number, z: number, a1: number, b1: number, c1: bigint, d1: number, e1: number, f1: number, g1: number, h1: number, i1: number, j1: number, k1: number, l1: number, m1: number, n1: number, o1: number) => number;
705
+ export const wasmscraperesult_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number, n: number, o: number, p: number, q: number, r: number, s: number, t: number, u: number, v: number, w: number, x: number, y: number, z: number, a1: number, b1: number, c1: number, d1: number, e1: bigint, f1: number, g1: number, h1: number, i1: number, j1: number, k1: number, l1: number, m1: number, n1: number, o1: number, p1: number, q1: number) => number;
668
706
  export const wasmscraperesult_nofollowDetected: (a: number) => number;
669
707
  export const wasmscraperesult_noindexDetected: (a: number) => number;
670
708
  export const wasmscraperesult_responseMeta: (a: number) => number;
@@ -695,9 +733,11 @@ export const wasmscraperesult_set_nofollowDetected: (a: number, b: number) => vo
695
733
  export const wasmscraperesult_set_noindexDetected: (a: number, b: number) => void;
696
734
  export const wasmscraperesult_set_responseMeta: (a: number, b: number) => void;
697
735
  export const wasmscraperesult_set_screenshotBase64: (a: number, b: number, c: number) => void;
736
+ export const wasmscraperesult_set_ssrfRefusedUrls: (a: number, b: number, c: number) => void;
698
737
  export const wasmscraperesult_set_statusCode: (a: number, b: number) => void;
699
738
  export const wasmscraperesult_set_wasSkipped: (a: number, b: number) => void;
700
739
  export const wasmscraperesult_set_xRobotsTag: (a: number, b: number, c: number) => void;
740
+ export const wasmscraperesult_ssrfRefusedUrls: (a: number) => [number, number];
701
741
  export const wasmscraperesult_statusCode: (a: number) => number;
702
742
  export const wasmscraperesult_wasSkipped: (a: number) => number;
703
743
  export const wasmscraperesult_xRobotsTag: (a: number) => [number, number];
@@ -722,9 +762,9 @@ export const wasmssrfpolicy_set_allowlist: (a: number, b: any) => void;
722
762
  export const wasmssrfpolicy_set_denyPrivate: (a: number, b: number) => void;
723
763
  export const wasmssrfpolicy_set_maxRedirects: (a: number, b: number) => void;
724
764
  export const wasmssrfpolicy_set_schemeAllowlist: (a: number, b: number, c: number) => void;
725
- export const wasm_bindgen__convert__closures_____invoke__h5567083e9d00da74: (a: number, b: number, c: any) => [number, number];
726
- export const wasm_bindgen__convert__closures_____invoke__h9cb213bb50b5f7d0: (a: number, b: number, c: any, d: any) => void;
727
- export const wasm_bindgen__convert__closures_____invoke__h00e67b488e823f1f: (a: number, b: number) => void;
765
+ export const wasm_bindgen__convert__closures_____invoke__h2eedacf566f3748c: (a: number, b: number, c: any) => [number, number];
766
+ export const wasm_bindgen__convert__closures_____invoke__h8ad11faaa2905452: (a: number, b: number, c: any, d: any) => void;
767
+ export const wasm_bindgen__convert__closures_____invoke__h3a7ccac3e94bbb5c: (a: number, b: number) => void;
728
768
  export const __wbindgen_malloc: (a: number, b: number) => number;
729
769
  export const __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number;
730
770
  export const __wbindgen_exn_store: (a: number) => void;
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "crawlberg-wasm",
3
3
  "description": "High-performance web crawling engine",
4
- "version": "1.7.2",
4
+ "version": "1.9.0",
5
5
  "license": "MIT",
6
6
  "repository": {
7
7
  "type": "git",
@@ -153,11 +153,14 @@ export class WasmBrowserConfig {
153
153
  free(): void;
154
154
  [Symbol.dispose](): void;
155
155
  static default(): WasmBrowserConfig;
156
- constructor(mode?: WasmBrowserMode | null, backend?: WasmBrowserBackend | null, timeout?: bigint | null, wait?: WasmBrowserWait | null, blockUrlPatterns?: string[] | null, captureNetworkEvents?: boolean | null, sessionAffinity?: boolean | null, endpoint?: string | null, waitSelector?: string | null, extraWait?: bigint | null, proxy?: WasmProxyConfig | null, evalScript?: string | null, robotsUserAgent?: string | null);
156
+ constructor(mode?: WasmBrowserMode | null, backend?: WasmBrowserBackend | null, timeout?: bigint | null, overallTimeout?: bigint | null, shutdownTimeout?: bigint | null, wait?: WasmBrowserWait | null, blockUrlPatterns?: string[] | null, captureNetworkEvents?: boolean | null, sessionAffinity?: boolean | null, chromeArgs?: string[] | null, endpoint?: string | null, waitSelector?: string | null, extraWait?: bigint | null, proxy?: WasmProxyConfig | null, evalScript?: string | null, robotsUserAgent?: string | null, chromePath?: string | null);
157
157
  get backend(): string;
158
158
  set backend(value: WasmBrowserBackend);
159
159
  blockUrlPatterns: string[];
160
160
  captureNetworkEvents: boolean;
161
+ chromeArgs: string[];
162
+ get chromePath(): string | undefined;
163
+ set chromePath(value: string | null | undefined);
161
164
  get endpoint(): string | undefined;
162
165
  set endpoint(value: string | null | undefined);
163
166
  get evalScript(): string | undefined;
@@ -166,11 +169,15 @@ export class WasmBrowserConfig {
166
169
  set extraWait(value: bigint | null | undefined);
167
170
  get mode(): string;
168
171
  set mode(value: WasmBrowserMode);
172
+ get overallTimeout(): bigint | undefined;
173
+ set overallTimeout(value: bigint | null | undefined);
169
174
  get proxy(): WasmProxyConfig | undefined;
170
175
  set proxy(value: WasmProxyConfig | null | undefined);
171
176
  get robotsUserAgent(): string | undefined;
172
177
  set robotsUserAgent(value: string | null | undefined);
173
178
  sessionAffinity: boolean;
179
+ get shutdownTimeout(): bigint | undefined;
180
+ set shutdownTimeout(value: bigint | null | undefined);
174
181
  get timeout(): bigint | undefined;
175
182
  set timeout(value: bigint | null | undefined);
176
183
  get waitSelector(): string | undefined;
@@ -251,8 +258,9 @@ export class WasmContentConfig {
251
258
  free(): void;
252
259
  [Symbol.dispose](): void;
253
260
  static default(): WasmContentConfig;
254
- constructor(outputFormat?: string | null, preprocessingPreset?: string | null, removeNavigation?: boolean | null, removeForms?: boolean | null, stripTags?: string[] | null, preserveTags?: string[] | null, excludeSelectors?: string[] | null, skipImages?: boolean | null, wrap?: boolean | null, wrapWidth?: number | null, includeDocumentStructure?: boolean | null, maxDepth?: number | null);
261
+ constructor(outputFormat?: string | null, preprocessingPreset?: string | null, removeNavigation?: boolean | null, removeForms?: boolean | null, stripTags?: string[] | null, preserveTags?: string[] | null, excludeSelectors?: string[] | null, skipImages?: boolean | null, wrap?: boolean | null, wrapWidth?: number | null, includeDocumentStructure?: boolean | null, extractMetadata?: boolean | null, maxDepth?: number | null);
255
262
  excludeSelectors: string[];
263
+ extractMetadata: boolean;
256
264
  includeDocumentStructure: boolean;
257
265
  get maxDepth(): number | undefined;
258
266
  set maxDepth(value: number | null | undefined);
@@ -297,7 +305,7 @@ export class WasmCrawlConfig {
297
305
  free(): void;
298
306
  [Symbol.dispose](): void;
299
307
  static default(): WasmCrawlConfig;
300
- constructor(crawlStrategy?: WasmCrawlStrategyKind | null, respectRobotsTxt?: boolean | null, softHttpErrors?: boolean | null, stayOnDomain?: boolean | null, allowSubdomains?: boolean | null, includePaths?: string[] | null, excludePaths?: string[] | null, customHeaders?: any | null, requestTimeout?: bigint | null, maxRedirects?: number | null, retryCount?: number | null, retryCodes?: Uint16Array | null, cookiesEnabled?: boolean | null, removeTags?: string[] | null, content?: WasmContentConfig | null, downloadAssets?: boolean | null, assetTypes?: WasmAssetCategory[] | null, browser?: WasmBrowserConfig | null, userAgents?: string[] | null, captureScreenshot?: boolean | null, followDocumentUrls?: boolean | null, downloadDocuments?: boolean | null, documentMimeTypes?: string[] | null, saveBrowserProfile?: boolean | null, ssrf?: WasmSsrfPolicy | null, maxDepth?: number | null, maxPages?: number | null, maxLinksPerPage?: number | null, maxConcurrent?: number | null, contentFilter?: WasmContentFilterKind | null, bm25Query?: string | null, bm25Threshold?: number | null, userAgent?: string | null, rateLimitMs?: bigint | null, auth?: any | null, maxBodySize?: number | null, mapLimit?: number | null, mapSearch?: string | null, maxAssetSize?: number | null, proxy?: WasmProxyConfig | null, documentUrlDepth?: number | null, documentMaxSize?: number | null, documentOutputDir?: string | null, documentContentEncoding?: WasmDocumentContentEncoding | null, warcOutput?: string | null, browserProfile?: string | null, ssrfDenyPrivateExplicit?: boolean | null);
308
+ constructor(crawlStrategy?: WasmCrawlStrategyKind | null, respectRobotsTxt?: boolean | null, softHttpErrors?: boolean | null, stayOnDomain?: boolean | null, allowSubdomains?: boolean | null, includePaths?: string[] | null, excludePaths?: string[] | null, pathPatternsMatchQuery?: boolean | null, pathPatternsMatchUrl?: boolean | null, dedupIncludeQuery?: boolean | null, stripTrackingParams?: boolean | null, trackingParams?: string[] | null, customHeaders?: any | null, requestTimeout?: bigint | null, maxRedirects?: number | null, retryCount?: number | null, retryCodes?: Uint16Array | null, retryInitialDelayMs?: bigint | null, retryMaxDelayMs?: bigint | null, rateLimitJitterRatio?: number | null, cookiesEnabled?: boolean | null, removeTags?: string[] | null, content?: WasmContentConfig | null, downloadAssets?: boolean | null, assetTypes?: WasmAssetCategory[] | null, browser?: WasmBrowserConfig | null, userAgents?: string[] | null, captureScreenshot?: boolean | null, followDocumentUrls?: boolean | null, downloadDocuments?: boolean | null, documentMimeTypes?: string[] | null, saveBrowserProfile?: boolean | null, ssrf?: WasmSsrfPolicy | null, maxDepth?: number | null, maxPages?: number | null, maxLinksPerPage?: number | null, maxConcurrent?: number | null, contentFilter?: WasmContentFilterKind | null, bm25Query?: string | null, bm25Threshold?: number | null, userAgent?: string | null, rateLimitMs?: bigint | null, auth?: any | null, maxBodySize?: number | null, mapLimit?: number | null, mapSearch?: string | null, maxAssetSize?: number | null, proxy?: WasmProxyConfig | null, documentUrlDepth?: number | null, documentMaxSize?: number | null, documentOutputDir?: string | null, documentContentEncoding?: WasmDocumentContentEncoding | null, warcOutput?: string | null, browserProfile?: string | null, ssrfDenyPrivateExplicit?: boolean | null);
301
309
  /**
302
310
  * Validate the configuration, returning an error if any values are invalid.
303
311
  *
@@ -324,6 +332,7 @@ export class WasmCrawlConfig {
324
332
  get crawlStrategy(): string;
325
333
  set crawlStrategy(value: WasmCrawlStrategyKind);
326
334
  customHeaders: any;
335
+ dedupIncludeQuery: boolean;
327
336
  get documentContentEncoding(): string | undefined;
328
337
  set documentContentEncoding(value: WasmDocumentContentEncoding | null | undefined);
329
338
  get documentMaxSize(): number | undefined;
@@ -355,8 +364,11 @@ export class WasmCrawlConfig {
355
364
  get maxPages(): number | undefined;
356
365
  set maxPages(value: number | null | undefined);
357
366
  maxRedirects: number;
367
+ pathPatternsMatchQuery: boolean;
368
+ pathPatternsMatchUrl: boolean;
358
369
  get proxy(): WasmProxyConfig | undefined;
359
370
  set proxy(value: WasmProxyConfig | null | undefined);
371
+ rateLimitJitterRatio: number;
360
372
  get rateLimitMs(): bigint | undefined;
361
373
  set rateLimitMs(value: bigint | null | undefined);
362
374
  removeTags: string[];
@@ -365,12 +377,16 @@ export class WasmCrawlConfig {
365
377
  respectRobotsTxt: boolean;
366
378
  retryCodes: Uint16Array;
367
379
  retryCount: number;
380
+ retryInitialDelayMs: bigint;
381
+ retryMaxDelayMs: bigint;
368
382
  saveBrowserProfile: boolean;
369
383
  softHttpErrors: boolean;
370
384
  get ssrfDenyPrivateExplicit(): boolean | undefined;
371
385
  set ssrfDenyPrivateExplicit(value: boolean | null | undefined);
372
386
  ssrf: WasmSsrfPolicy;
373
387
  stayOnDomain: boolean;
388
+ stripTrackingParams: boolean;
389
+ trackingParams: string[];
374
390
  get userAgent(): string | undefined;
375
391
  set userAgent(value: string | null | undefined);
376
392
  userAgents: string[];
@@ -397,7 +413,7 @@ export class WasmCrawlPageResult {
397
413
  free(): void;
398
414
  [Symbol.dispose](): void;
399
415
  static default(): WasmCrawlPageResult;
400
- constructor(url?: string | null, normalizedUrl?: string | null, statusCode?: number | null, contentType?: string | null, html?: string | null, bodySize?: number | null, metadata?: WasmPageMetadata | null, links?: WasmLinkInfo[] | null, images?: WasmImageInfo[] | null, feeds?: WasmFeedInfo[] | null, jsonLd?: WasmJsonLdEntry[] | null, depth?: number | null, stayedOnDomain?: boolean | null, wasSkipped?: boolean | null, isPdf?: boolean | null, browserUsed?: boolean | null, detectedCharset?: string | null, markdown?: WasmMarkdownResult | null, extractedData?: any | null, extractionMeta?: WasmExtractionMeta | null, downloadedDocument?: WasmDownloadedDocument | null);
416
+ constructor(url?: string | null, normalizedUrl?: string | null, statusCode?: number | null, contentType?: string | null, html?: string | null, bodySize?: number | null, metadata?: WasmPageMetadata | null, links?: WasmLinkInfo[] | null, images?: WasmImageInfo[] | null, feeds?: WasmFeedInfo[] | null, jsonLd?: WasmJsonLdEntry[] | null, depth?: number | null, stayedOnDomain?: boolean | null, wasSkipped?: boolean | null, isPdf?: boolean | null, browserUsed?: boolean | null, finalUrl?: string | null, redirectCount?: number | null, noindexDetected?: boolean | null, nofollowDetected?: boolean | null, ssrfRefusedUrls?: string[] | null, detectedCharset?: string | null, markdown?: WasmMarkdownResult | null, extractedData?: any | null, extractionMeta?: WasmExtractionMeta | null, downloadedDocument?: WasmDownloadedDocument | null);
401
417
  bodySize: number;
402
418
  browserUsed: boolean;
403
419
  contentType: string;
@@ -411,6 +427,7 @@ export class WasmCrawlPageResult {
411
427
  get extractionMeta(): WasmExtractionMeta | undefined;
412
428
  set extractionMeta(value: WasmExtractionMeta | null | undefined);
413
429
  feeds: WasmFeedInfo[];
430
+ finalUrl: string;
414
431
  html: string;
415
432
  images: WasmImageInfo[];
416
433
  isPdf: boolean;
@@ -419,7 +436,11 @@ export class WasmCrawlPageResult {
419
436
  get markdown(): WasmMarkdownResult | undefined;
420
437
  set markdown(value: WasmMarkdownResult | null | undefined);
421
438
  metadata: WasmPageMetadata;
439
+ nofollowDetected: boolean;
440
+ noindexDetected: boolean;
422
441
  normalizedUrl: string;
442
+ redirectCount: number;
443
+ ssrfRefusedUrls: string[];
423
444
  statusCode: number;
424
445
  stayedOnDomain: boolean;
425
446
  url: string;
@@ -673,12 +694,13 @@ export class WasmInteractionResult {
673
694
  free(): void;
674
695
  [Symbol.dispose](): void;
675
696
  static default(): WasmInteractionResult;
676
- constructor(actionResults?: WasmActionResult[] | null, finalHtml?: string | null, finalUrl?: string | null, screenshotBase64?: string | null);
697
+ constructor(actionResults?: WasmActionResult[] | null, finalHtml?: string | null, finalUrl?: string | null, ssrfRefusedUrls?: string[] | null, screenshotBase64?: string | null);
677
698
  actionResults: WasmActionResult[];
678
699
  finalHtml: string;
679
700
  finalUrl: string;
680
701
  get screenshotBase64(): string | undefined;
681
702
  set screenshotBase64(value: string | null | undefined);
703
+ ssrfRefusedUrls: string[];
682
704
  }
683
705
 
684
706
  /**
@@ -923,7 +945,7 @@ export class WasmScrapeResult {
923
945
  free(): void;
924
946
  [Symbol.dispose](): void;
925
947
  static default(): WasmScrapeResult;
926
- constructor(statusCode?: number | null, finalUrl?: string | null, contentType?: string | null, html?: string | null, bodySize?: number | null, metadata?: WasmPageMetadata | null, links?: WasmLinkInfo[] | null, images?: WasmImageInfo[] | null, feeds?: WasmFeedInfo[] | null, jsonLd?: WasmJsonLdEntry[] | null, isAllowed?: boolean | null, noindexDetected?: boolean | null, nofollowDetected?: boolean | null, isPdf?: boolean | null, wasSkipped?: boolean | null, authHeaderSent?: boolean | null, assets?: WasmDownloadedAsset[] | null, jsRenderHint?: boolean | null, browserUsed?: boolean | null, crawlDelay?: bigint | null, xRobotsTag?: string | null, detectedCharset?: string | null, responseMeta?: WasmResponseMeta | null, markdown?: WasmMarkdownResult | null, extractedData?: any | null, extractionMeta?: WasmExtractionMeta | null, screenshotBase64?: string | null, downloadedDocument?: WasmDownloadedDocument | null, browser?: WasmBrowserExtras | null);
948
+ constructor(statusCode?: number | null, finalUrl?: string | null, contentType?: string | null, html?: string | null, bodySize?: number | null, metadata?: WasmPageMetadata | null, links?: WasmLinkInfo[] | null, images?: WasmImageInfo[] | null, feeds?: WasmFeedInfo[] | null, jsonLd?: WasmJsonLdEntry[] | null, isAllowed?: boolean | null, noindexDetected?: boolean | null, nofollowDetected?: boolean | null, isPdf?: boolean | null, wasSkipped?: boolean | null, authHeaderSent?: boolean | null, assets?: WasmDownloadedAsset[] | null, jsRenderHint?: boolean | null, browserUsed?: boolean | null, ssrfRefusedUrls?: string[] | null, crawlDelay?: bigint | null, xRobotsTag?: string | null, detectedCharset?: string | null, responseMeta?: WasmResponseMeta | null, markdown?: WasmMarkdownResult | null, extractedData?: any | null, extractionMeta?: WasmExtractionMeta | null, screenshotBase64?: string | null, downloadedDocument?: WasmDownloadedDocument | null, browser?: WasmBrowserExtras | null);
927
949
  assets: WasmDownloadedAsset[];
928
950
  authHeaderSent: boolean;
929
951
  bodySize: number;
@@ -959,6 +981,7 @@ export class WasmScrapeResult {
959
981
  set responseMeta(value: WasmResponseMeta | null | undefined);
960
982
  get screenshotBase64(): string | undefined;
961
983
  set screenshotBase64(value: string | null | undefined);
984
+ ssrfRefusedUrls: string[];
962
985
  statusCode: number;
963
986
  wasSkipped: boolean;
964
987
  get xRobotsTag(): string | undefined;
@@ -1199,28 +1222,36 @@ export interface InitOutput {
1199
1222
  readonly wasmbrowserconfig_backend: (a: number) => [number, number];
1200
1223
  readonly wasmbrowserconfig_blockUrlPatterns: (a: number) => [number, number];
1201
1224
  readonly wasmbrowserconfig_captureNetworkEvents: (a: number) => number;
1225
+ readonly wasmbrowserconfig_chromeArgs: (a: number) => [number, number];
1226
+ readonly wasmbrowserconfig_chromePath: (a: number) => [number, number];
1202
1227
  readonly wasmbrowserconfig_default: () => number;
1203
1228
  readonly wasmbrowserconfig_endpoint: (a: number) => [number, number];
1204
1229
  readonly wasmbrowserconfig_evalScript: (a: number) => [number, number];
1205
1230
  readonly wasmbrowserconfig_extraWait: (a: number) => [number, bigint];
1206
1231
  readonly wasmbrowserconfig_mode: (a: number) => [number, number];
1207
- readonly wasmbrowserconfig_new: (a: number, b: number, c: number, d: bigint, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number, n: number, o: bigint, p: number, q: number, r: number, s: number, t: number) => number;
1232
+ readonly wasmbrowserconfig_new: (a: number, b: number, c: number, d: bigint, e: number, f: bigint, g: number, h: bigint, i: number, j: number, k: number, l: number, m: number, n: number, o: number, p: number, q: number, r: number, s: number, t: number, u: bigint, v: number, w: number, x: number, y: number, z: number, a1: number, b1: number) => number;
1233
+ readonly wasmbrowserconfig_overallTimeout: (a: number) => [number, bigint];
1208
1234
  readonly wasmbrowserconfig_proxy: (a: number) => number;
1209
1235
  readonly wasmbrowserconfig_robotsUserAgent: (a: number) => [number, number];
1210
1236
  readonly wasmbrowserconfig_sessionAffinity: (a: number) => number;
1211
1237
  readonly wasmbrowserconfig_set_backend: (a: number, b: number) => void;
1212
1238
  readonly wasmbrowserconfig_set_blockUrlPatterns: (a: number, b: number, c: number) => void;
1213
1239
  readonly wasmbrowserconfig_set_captureNetworkEvents: (a: number, b: number) => void;
1240
+ readonly wasmbrowserconfig_set_chromeArgs: (a: number, b: number, c: number) => void;
1241
+ readonly wasmbrowserconfig_set_chromePath: (a: number, b: number, c: number) => void;
1214
1242
  readonly wasmbrowserconfig_set_endpoint: (a: number, b: number, c: number) => void;
1215
1243
  readonly wasmbrowserconfig_set_evalScript: (a: number, b: number, c: number) => void;
1216
1244
  readonly wasmbrowserconfig_set_extraWait: (a: number, b: number, c: bigint) => void;
1217
1245
  readonly wasmbrowserconfig_set_mode: (a: number, b: number) => void;
1246
+ readonly wasmbrowserconfig_set_overallTimeout: (a: number, b: number, c: bigint) => void;
1218
1247
  readonly wasmbrowserconfig_set_proxy: (a: number, b: number) => void;
1219
1248
  readonly wasmbrowserconfig_set_robotsUserAgent: (a: number, b: number, c: number) => void;
1220
1249
  readonly wasmbrowserconfig_set_sessionAffinity: (a: number, b: number) => void;
1250
+ readonly wasmbrowserconfig_set_shutdownTimeout: (a: number, b: number, c: bigint) => void;
1221
1251
  readonly wasmbrowserconfig_set_timeout: (a: number, b: number, c: bigint) => void;
1222
1252
  readonly wasmbrowserconfig_set_wait: (a: number, b: number) => void;
1223
1253
  readonly wasmbrowserconfig_set_waitSelector: (a: number, b: number, c: number) => void;
1254
+ readonly wasmbrowserconfig_shutdownTimeout: (a: number) => [number, bigint];
1224
1255
  readonly wasmbrowserconfig_timeout: (a: number) => [number, bigint];
1225
1256
  readonly wasmbrowserconfig_wait: (a: number) => [number, number];
1226
1257
  readonly wasmbrowserconfig_waitSelector: (a: number) => [number, number];
@@ -1248,15 +1279,17 @@ export interface InitOutput {
1248
1279
  readonly wasmcitationresult_set_references: (a: number, b: number, c: number) => void;
1249
1280
  readonly wasmcontentconfig_default: () => number;
1250
1281
  readonly wasmcontentconfig_excludeSelectors: (a: number) => [number, number];
1282
+ readonly wasmcontentconfig_extractMetadata: (a: number) => number;
1251
1283
  readonly wasmcontentconfig_includeDocumentStructure: (a: number) => number;
1252
1284
  readonly wasmcontentconfig_maxDepth: (a: number) => number;
1253
- readonly wasmcontentconfig_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number, n: number, o: number, p: number, q: number) => number;
1285
+ readonly wasmcontentconfig_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number, n: number, o: number, p: number, q: number, r: number) => number;
1254
1286
  readonly wasmcontentconfig_outputFormat: (a: number) => [number, number];
1255
1287
  readonly wasmcontentconfig_preprocessingPreset: (a: number) => [number, number];
1256
1288
  readonly wasmcontentconfig_preserveTags: (a: number) => [number, number];
1257
1289
  readonly wasmcontentconfig_removeForms: (a: number) => number;
1258
1290
  readonly wasmcontentconfig_removeNavigation: (a: number) => number;
1259
1291
  readonly wasmcontentconfig_set_excludeSelectors: (a: number, b: number, c: number) => void;
1292
+ readonly wasmcontentconfig_set_extractMetadata: (a: number, b: number) => void;
1260
1293
  readonly wasmcontentconfig_set_includeDocumentStructure: (a: number, b: number) => void;
1261
1294
  readonly wasmcontentconfig_set_maxDepth: (a: number, b: number) => void;
1262
1295
  readonly wasmcontentconfig_set_outputFormat: (a: number, b: number, c: number) => void;
@@ -1295,6 +1328,7 @@ export interface InitOutput {
1295
1328
  readonly wasmcrawlconfig_cookiesEnabled: (a: number) => number;
1296
1329
  readonly wasmcrawlconfig_crawlStrategy: (a: number) => [number, number];
1297
1330
  readonly wasmcrawlconfig_customHeaders: (a: number) => any;
1331
+ readonly wasmcrawlconfig_dedupIncludeQuery: (a: number) => number;
1298
1332
  readonly wasmcrawlconfig_default: () => number;
1299
1333
  readonly wasmcrawlconfig_documentContentEncoding: (a: number) => [number, number];
1300
1334
  readonly wasmcrawlconfig_documentMaxSize: (a: number) => number;
@@ -1315,14 +1349,19 @@ export interface InitOutput {
1315
1349
  readonly wasmcrawlconfig_maxLinksPerPage: (a: number) => number;
1316
1350
  readonly wasmcrawlconfig_maxPages: (a: number) => number;
1317
1351
  readonly wasmcrawlconfig_maxRedirects: (a: number) => number;
1318
- readonly wasmcrawlconfig_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: bigint, m: number, n: number, o: number, p: number, q: number, r: number, s: number, t: number, u: number, v: number, w: number, x: number, y: number, z: number, a1: number, b1: number, c1: number, d1: number, e1: number, f1: number, g1: number, h1: number, i1: number, j1: number, k1: number, l1: number, m1: number, n1: number, o1: number, p1: number, q1: number, r1: number, s1: number, t1: bigint, u1: number, v1: number, w1: number, x1: number, y1: number, z1: number, a2: number, b2: number, c2: number, d2: number, e2: number, f2: number, g2: number, h2: number, i2: number, j2: number, k2: number) => number;
1352
+ readonly wasmcrawlconfig_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number, n: number, o: number, p: number, q: number, r: bigint, s: number, t: number, u: number, v: number, w: number, x: bigint, y: number, z: bigint, a1: number, b1: number, c1: number, d1: number, e1: number, f1: number, g1: number, h1: number, i1: number, j1: number, k1: number, l1: number, m1: number, n1: number, o1: number, p1: number, q1: number, r1: number, s1: number, t1: number, u1: number, v1: number, w1: number, x1: number, y1: number, z1: number, a2: number, b2: number, c2: number, d2: number, e2: number, f2: bigint, g2: number, h2: number, i2: number, j2: number, k2: number, l2: number, m2: number, n2: number, o2: number, p2: number, q2: number, r2: number, s2: number, t2: number, u2: number, v2: number, w2: number) => number;
1353
+ readonly wasmcrawlconfig_pathPatternsMatchQuery: (a: number) => number;
1354
+ readonly wasmcrawlconfig_pathPatternsMatchUrl: (a: number) => number;
1319
1355
  readonly wasmcrawlconfig_proxy: (a: number) => number;
1356
+ readonly wasmcrawlconfig_rateLimitJitterRatio: (a: number) => number;
1320
1357
  readonly wasmcrawlconfig_rateLimitMs: (a: number) => [number, bigint];
1321
1358
  readonly wasmcrawlconfig_removeTags: (a: number) => [number, number];
1322
1359
  readonly wasmcrawlconfig_requestTimeout: (a: number) => [number, bigint];
1323
1360
  readonly wasmcrawlconfig_respectRobotsTxt: (a: number) => number;
1324
1361
  readonly wasmcrawlconfig_retryCodes: (a: number) => [number, number];
1325
1362
  readonly wasmcrawlconfig_retryCount: (a: number) => number;
1363
+ readonly wasmcrawlconfig_retryInitialDelayMs: (a: number) => bigint;
1364
+ readonly wasmcrawlconfig_retryMaxDelayMs: (a: number) => bigint;
1326
1365
  readonly wasmcrawlconfig_saveBrowserProfile: (a: number) => number;
1327
1366
  readonly wasmcrawlconfig_set_allowSubdomains: (a: number, b: number) => void;
1328
1367
  readonly wasmcrawlconfig_set_assetTypes: (a: number, b: number, c: number) => void;
@@ -1337,6 +1376,7 @@ export interface InitOutput {
1337
1376
  readonly wasmcrawlconfig_set_cookiesEnabled: (a: number, b: number) => void;
1338
1377
  readonly wasmcrawlconfig_set_crawlStrategy: (a: number, b: number) => void;
1339
1378
  readonly wasmcrawlconfig_set_customHeaders: (a: number, b: any) => void;
1379
+ readonly wasmcrawlconfig_set_dedupIncludeQuery: (a: number, b: number) => void;
1340
1380
  readonly wasmcrawlconfig_set_documentContentEncoding: (a: number, b: number) => void;
1341
1381
  readonly wasmcrawlconfig_set_documentMaxSize: (a: number, b: number) => void;
1342
1382
  readonly wasmcrawlconfig_set_documentMimeTypes: (a: number, b: number, c: number) => void;
@@ -1356,18 +1396,25 @@ export interface InitOutput {
1356
1396
  readonly wasmcrawlconfig_set_maxLinksPerPage: (a: number, b: number) => void;
1357
1397
  readonly wasmcrawlconfig_set_maxPages: (a: number, b: number) => void;
1358
1398
  readonly wasmcrawlconfig_set_maxRedirects: (a: number, b: number) => void;
1399
+ readonly wasmcrawlconfig_set_pathPatternsMatchQuery: (a: number, b: number) => void;
1400
+ readonly wasmcrawlconfig_set_pathPatternsMatchUrl: (a: number, b: number) => void;
1359
1401
  readonly wasmcrawlconfig_set_proxy: (a: number, b: number) => void;
1402
+ readonly wasmcrawlconfig_set_rateLimitJitterRatio: (a: number, b: number) => void;
1360
1403
  readonly wasmcrawlconfig_set_rateLimitMs: (a: number, b: number, c: bigint) => void;
1361
1404
  readonly wasmcrawlconfig_set_removeTags: (a: number, b: number, c: number) => void;
1362
1405
  readonly wasmcrawlconfig_set_requestTimeout: (a: number, b: number, c: bigint) => void;
1363
1406
  readonly wasmcrawlconfig_set_respectRobotsTxt: (a: number, b: number) => void;
1364
1407
  readonly wasmcrawlconfig_set_retryCodes: (a: number, b: number, c: number) => void;
1365
1408
  readonly wasmcrawlconfig_set_retryCount: (a: number, b: number) => void;
1409
+ readonly wasmcrawlconfig_set_retryInitialDelayMs: (a: number, b: bigint) => void;
1410
+ readonly wasmcrawlconfig_set_retryMaxDelayMs: (a: number, b: bigint) => void;
1366
1411
  readonly wasmcrawlconfig_set_saveBrowserProfile: (a: number, b: number) => void;
1367
1412
  readonly wasmcrawlconfig_set_softHttpErrors: (a: number, b: number) => void;
1368
1413
  readonly wasmcrawlconfig_set_ssrf: (a: number, b: number) => void;
1369
1414
  readonly wasmcrawlconfig_set_ssrfDenyPrivateExplicit: (a: number, b: number) => void;
1370
1415
  readonly wasmcrawlconfig_set_stayOnDomain: (a: number, b: number) => void;
1416
+ readonly wasmcrawlconfig_set_stripTrackingParams: (a: number, b: number) => void;
1417
+ readonly wasmcrawlconfig_set_trackingParams: (a: number, b: number, c: number) => void;
1371
1418
  readonly wasmcrawlconfig_set_userAgent: (a: number, b: number, c: number) => void;
1372
1419
  readonly wasmcrawlconfig_set_userAgents: (a: number, b: number, c: number) => void;
1373
1420
  readonly wasmcrawlconfig_set_warcOutput: (a: number, b: number, c: number) => void;
@@ -1375,6 +1422,8 @@ export interface InitOutput {
1375
1422
  readonly wasmcrawlconfig_ssrf: (a: number) => number;
1376
1423
  readonly wasmcrawlconfig_ssrfDenyPrivateExplicit: (a: number) => number;
1377
1424
  readonly wasmcrawlconfig_stayOnDomain: (a: number) => number;
1425
+ readonly wasmcrawlconfig_stripTrackingParams: (a: number) => number;
1426
+ readonly wasmcrawlconfig_trackingParams: (a: number) => [number, number];
1378
1427
  readonly wasmcrawlconfig_userAgent: (a: number) => [number, number];
1379
1428
  readonly wasmcrawlconfig_userAgents: (a: number) => [number, number];
1380
1429
  readonly wasmcrawlconfig_validate: (a: number) => [number, number];
@@ -1389,6 +1438,7 @@ export interface InitOutput {
1389
1438
  readonly wasmcrawlpageresult_extractedData: (a: number) => any;
1390
1439
  readonly wasmcrawlpageresult_extractionMeta: (a: number) => number;
1391
1440
  readonly wasmcrawlpageresult_feeds: (a: number) => [number, number];
1441
+ readonly wasmcrawlpageresult_finalUrl: (a: number) => [number, number];
1392
1442
  readonly wasmcrawlpageresult_html: (a: number) => [number, number];
1393
1443
  readonly wasmcrawlpageresult_images: (a: number) => [number, number];
1394
1444
  readonly wasmcrawlpageresult_isPdf: (a: number) => number;
@@ -1396,8 +1446,11 @@ export interface InitOutput {
1396
1446
  readonly wasmcrawlpageresult_links: (a: number) => [number, number];
1397
1447
  readonly wasmcrawlpageresult_markdown: (a: number) => number;
1398
1448
  readonly wasmcrawlpageresult_metadata: (a: number) => number;
1399
- readonly wasmcrawlpageresult_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number, n: number, o: number, p: number, q: number, r: number, s: number, t: number, u: number, v: number, w: number, x: number, y: number, z: number, a1: number, b1: number, c1: number, d1: number) => number;
1449
+ readonly wasmcrawlpageresult_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number, n: number, o: number, p: number, q: number, r: number, s: number, t: number, u: number, v: number, w: number, x: number, y: number, z: number, a1: number, b1: number, c1: number, d1: number, e1: number, f1: number, g1: number, h1: number, i1: number, j1: number, k1: number) => number;
1450
+ readonly wasmcrawlpageresult_nofollowDetected: (a: number) => number;
1451
+ readonly wasmcrawlpageresult_noindexDetected: (a: number) => number;
1400
1452
  readonly wasmcrawlpageresult_normalizedUrl: (a: number) => [number, number];
1453
+ readonly wasmcrawlpageresult_redirectCount: (a: number) => number;
1401
1454
  readonly wasmcrawlpageresult_set_bodySize: (a: number, b: number) => void;
1402
1455
  readonly wasmcrawlpageresult_set_browserUsed: (a: number, b: number) => void;
1403
1456
  readonly wasmcrawlpageresult_set_contentType: (a: number, b: number, c: number) => void;
@@ -1407,6 +1460,7 @@ export interface InitOutput {
1407
1460
  readonly wasmcrawlpageresult_set_extractedData: (a: number, b: number) => void;
1408
1461
  readonly wasmcrawlpageresult_set_extractionMeta: (a: number, b: number) => void;
1409
1462
  readonly wasmcrawlpageresult_set_feeds: (a: number, b: number, c: number) => void;
1463
+ readonly wasmcrawlpageresult_set_finalUrl: (a: number, b: number, c: number) => void;
1410
1464
  readonly wasmcrawlpageresult_set_html: (a: number, b: number, c: number) => void;
1411
1465
  readonly wasmcrawlpageresult_set_images: (a: number, b: number, c: number) => void;
1412
1466
  readonly wasmcrawlpageresult_set_isPdf: (a: number, b: number) => void;
@@ -1414,11 +1468,16 @@ export interface InitOutput {
1414
1468
  readonly wasmcrawlpageresult_set_links: (a: number, b: number, c: number) => void;
1415
1469
  readonly wasmcrawlpageresult_set_markdown: (a: number, b: number) => void;
1416
1470
  readonly wasmcrawlpageresult_set_metadata: (a: number, b: number) => void;
1471
+ readonly wasmcrawlpageresult_set_nofollowDetected: (a: number, b: number) => void;
1472
+ readonly wasmcrawlpageresult_set_noindexDetected: (a: number, b: number) => void;
1417
1473
  readonly wasmcrawlpageresult_set_normalizedUrl: (a: number, b: number, c: number) => void;
1474
+ readonly wasmcrawlpageresult_set_redirectCount: (a: number, b: number) => void;
1475
+ readonly wasmcrawlpageresult_set_ssrfRefusedUrls: (a: number, b: number, c: number) => void;
1418
1476
  readonly wasmcrawlpageresult_set_statusCode: (a: number, b: number) => void;
1419
1477
  readonly wasmcrawlpageresult_set_stayedOnDomain: (a: number, b: number) => void;
1420
1478
  readonly wasmcrawlpageresult_set_url: (a: number, b: number, c: number) => void;
1421
1479
  readonly wasmcrawlpageresult_set_wasSkipped: (a: number, b: number) => void;
1480
+ readonly wasmcrawlpageresult_ssrfRefusedUrls: (a: number) => [number, number];
1422
1481
  readonly wasmcrawlpageresult_statusCode: (a: number) => number;
1423
1482
  readonly wasmcrawlpageresult_stayedOnDomain: (a: number) => number;
1424
1483
  readonly wasmcrawlpageresult_url: (a: number) => [number, number];
@@ -1540,12 +1599,14 @@ export interface InitOutput {
1540
1599
  readonly wasminteractionresult_default: () => number;
1541
1600
  readonly wasminteractionresult_finalHtml: (a: number) => [number, number];
1542
1601
  readonly wasminteractionresult_finalUrl: (a: number) => [number, number];
1543
- readonly wasminteractionresult_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number) => number;
1602
+ readonly wasminteractionresult_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number) => number;
1544
1603
  readonly wasminteractionresult_screenshotBase64: (a: number) => [number, number];
1545
1604
  readonly wasminteractionresult_set_actionResults: (a: number, b: number, c: number) => void;
1546
1605
  readonly wasminteractionresult_set_finalHtml: (a: number, b: number, c: number) => void;
1547
1606
  readonly wasminteractionresult_set_finalUrl: (a: number, b: number, c: number) => void;
1548
1607
  readonly wasminteractionresult_set_screenshotBase64: (a: number, b: number, c: number) => void;
1608
+ readonly wasminteractionresult_set_ssrfRefusedUrls: (a: number, b: number, c: number) => void;
1609
+ readonly wasminteractionresult_ssrfRefusedUrls: (a: number) => [number, number];
1549
1610
  readonly wasmjsonldentry_default: () => number;
1550
1611
  readonly wasmjsonldentry_name: (a: number) => [number, number];
1551
1612
  readonly wasmjsonldentry_new: (a: number, b: number, c: number, d: number, e: number, f: number) => number;
@@ -1739,7 +1800,7 @@ export interface InitOutput {
1739
1800
  readonly wasmscraperesult_links: (a: number) => [number, number];
1740
1801
  readonly wasmscraperesult_markdown: (a: number) => number;
1741
1802
  readonly wasmscraperesult_metadata: (a: number) => number;
1742
- readonly wasmscraperesult_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number, n: number, o: number, p: number, q: number, r: number, s: number, t: number, u: number, v: number, w: number, x: number, y: number, z: number, a1: number, b1: number, c1: bigint, d1: number, e1: number, f1: number, g1: number, h1: number, i1: number, j1: number, k1: number, l1: number, m1: number, n1: number, o1: number) => number;
1803
+ readonly wasmscraperesult_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number, n: number, o: number, p: number, q: number, r: number, s: number, t: number, u: number, v: number, w: number, x: number, y: number, z: number, a1: number, b1: number, c1: number, d1: number, e1: bigint, f1: number, g1: number, h1: number, i1: number, j1: number, k1: number, l1: number, m1: number, n1: number, o1: number, p1: number, q1: number) => number;
1743
1804
  readonly wasmscraperesult_nofollowDetected: (a: number) => number;
1744
1805
  readonly wasmscraperesult_noindexDetected: (a: number) => number;
1745
1806
  readonly wasmscraperesult_responseMeta: (a: number) => number;
@@ -1770,9 +1831,11 @@ export interface InitOutput {
1770
1831
  readonly wasmscraperesult_set_noindexDetected: (a: number, b: number) => void;
1771
1832
  readonly wasmscraperesult_set_responseMeta: (a: number, b: number) => void;
1772
1833
  readonly wasmscraperesult_set_screenshotBase64: (a: number, b: number, c: number) => void;
1834
+ readonly wasmscraperesult_set_ssrfRefusedUrls: (a: number, b: number, c: number) => void;
1773
1835
  readonly wasmscraperesult_set_statusCode: (a: number, b: number) => void;
1774
1836
  readonly wasmscraperesult_set_wasSkipped: (a: number, b: number) => void;
1775
1837
  readonly wasmscraperesult_set_xRobotsTag: (a: number, b: number, c: number) => void;
1838
+ readonly wasmscraperesult_ssrfRefusedUrls: (a: number) => [number, number];
1776
1839
  readonly wasmscraperesult_statusCode: (a: number) => number;
1777
1840
  readonly wasmscraperesult_wasSkipped: (a: number) => number;
1778
1841
  readonly wasmscraperesult_xRobotsTag: (a: number) => [number, number];
@@ -1797,9 +1860,9 @@ export interface InitOutput {
1797
1860
  readonly wasmssrfpolicy_set_denyPrivate: (a: number, b: number) => void;
1798
1861
  readonly wasmssrfpolicy_set_maxRedirects: (a: number, b: number) => void;
1799
1862
  readonly wasmssrfpolicy_set_schemeAllowlist: (a: number, b: number, c: number) => void;
1800
- readonly wasm_bindgen__convert__closures_____invoke__h5567083e9d00da74: (a: number, b: number, c: any) => [number, number];
1801
- readonly wasm_bindgen__convert__closures_____invoke__h9cb213bb50b5f7d0: (a: number, b: number, c: any, d: any) => void;
1802
- readonly wasm_bindgen__convert__closures_____invoke__h00e67b488e823f1f: (a: number, b: number) => void;
1863
+ readonly wasm_bindgen__convert__closures_____invoke__h2eedacf566f3748c: (a: number, b: number, c: any) => [number, number];
1864
+ readonly wasm_bindgen__convert__closures_____invoke__h8ad11faaa2905452: (a: number, b: number, c: any, d: any) => void;
1865
+ readonly wasm_bindgen__convert__closures_____invoke__h3a7ccac3e94bbb5c: (a: number, b: number) => void;
1803
1866
  readonly __wbindgen_malloc: (a: number, b: number) => number;
1804
1867
  readonly __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number;
1805
1868
  readonly __wbindgen_exn_store: (a: number) => void;