@xberg-io/crawlberg-wasm 1.7.2 → 1.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/pkg/bundler/crawlberg_wasm.d.ts +29 -6
- package/pkg/bundler/crawlberg_wasm_bg.js +494 -163
- package/pkg/bundler/crawlberg_wasm_bg.wasm +0 -0
- package/pkg/bundler/crawlberg_wasm_bg.wasm.d.ts +49 -9
- package/pkg/bundler/package.json +1 -1
- package/pkg/deno/crawlberg_wasm.d.ts +29 -6
- package/pkg/deno/crawlberg_wasm.js +494 -163
- package/pkg/deno/crawlberg_wasm_bg.wasm +0 -0
- package/pkg/deno/crawlberg_wasm_bg.wasm.d.ts +49 -9
- package/pkg/nodejs/crawlberg_wasm.d.ts +29 -6
- package/pkg/nodejs/crawlberg_wasm.js +494 -163
- package/pkg/nodejs/crawlberg_wasm_bg.wasm +0 -0
- package/pkg/nodejs/crawlberg_wasm_bg.wasm.d.ts +49 -9
- package/pkg/nodejs/package.json +1 -1
- package/pkg/web/crawlberg_wasm.d.ts +78 -15
- package/pkg/web/crawlberg_wasm.js +494 -163
- package/pkg/web/crawlberg_wasm_bg.wasm +0 -0
- package/pkg/web/crawlberg_wasm_bg.wasm.d.ts +49 -9
- package/pkg/web/package.json +1 -1
|
Binary file
|
|
@@ -124,28 +124,36 @@ export const wasmbatchscraperesults_totalCount: (a: number) => number;
|
|
|
124
124
|
export const wasmbrowserconfig_backend: (a: number) => [number, number];
|
|
125
125
|
export const wasmbrowserconfig_blockUrlPatterns: (a: number) => [number, number];
|
|
126
126
|
export const wasmbrowserconfig_captureNetworkEvents: (a: number) => number;
|
|
127
|
+
export const wasmbrowserconfig_chromeArgs: (a: number) => [number, number];
|
|
128
|
+
export const wasmbrowserconfig_chromePath: (a: number) => [number, number];
|
|
127
129
|
export const wasmbrowserconfig_default: () => number;
|
|
128
130
|
export const wasmbrowserconfig_endpoint: (a: number) => [number, number];
|
|
129
131
|
export const wasmbrowserconfig_evalScript: (a: number) => [number, number];
|
|
130
132
|
export const wasmbrowserconfig_extraWait: (a: number) => [number, bigint];
|
|
131
133
|
export const wasmbrowserconfig_mode: (a: number) => [number, number];
|
|
132
|
-
export const wasmbrowserconfig_new: (a: number, b: number, c: number, d: bigint, e: number, f:
|
|
134
|
+
export const wasmbrowserconfig_new: (a: number, b: number, c: number, d: bigint, e: number, f: bigint, g: number, h: bigint, i: number, j: number, k: number, l: number, m: number, n: number, o: number, p: number, q: number, r: number, s: number, t: number, u: bigint, v: number, w: number, x: number, y: number, z: number, a1: number, b1: number) => number;
|
|
135
|
+
export const wasmbrowserconfig_overallTimeout: (a: number) => [number, bigint];
|
|
133
136
|
export const wasmbrowserconfig_proxy: (a: number) => number;
|
|
134
137
|
export const wasmbrowserconfig_robotsUserAgent: (a: number) => [number, number];
|
|
135
138
|
export const wasmbrowserconfig_sessionAffinity: (a: number) => number;
|
|
136
139
|
export const wasmbrowserconfig_set_backend: (a: number, b: number) => void;
|
|
137
140
|
export const wasmbrowserconfig_set_blockUrlPatterns: (a: number, b: number, c: number) => void;
|
|
138
141
|
export const wasmbrowserconfig_set_captureNetworkEvents: (a: number, b: number) => void;
|
|
142
|
+
export const wasmbrowserconfig_set_chromeArgs: (a: number, b: number, c: number) => void;
|
|
143
|
+
export const wasmbrowserconfig_set_chromePath: (a: number, b: number, c: number) => void;
|
|
139
144
|
export const wasmbrowserconfig_set_endpoint: (a: number, b: number, c: number) => void;
|
|
140
145
|
export const wasmbrowserconfig_set_evalScript: (a: number, b: number, c: number) => void;
|
|
141
146
|
export const wasmbrowserconfig_set_extraWait: (a: number, b: number, c: bigint) => void;
|
|
142
147
|
export const wasmbrowserconfig_set_mode: (a: number, b: number) => void;
|
|
148
|
+
export const wasmbrowserconfig_set_overallTimeout: (a: number, b: number, c: bigint) => void;
|
|
143
149
|
export const wasmbrowserconfig_set_proxy: (a: number, b: number) => void;
|
|
144
150
|
export const wasmbrowserconfig_set_robotsUserAgent: (a: number, b: number, c: number) => void;
|
|
145
151
|
export const wasmbrowserconfig_set_sessionAffinity: (a: number, b: number) => void;
|
|
152
|
+
export const wasmbrowserconfig_set_shutdownTimeout: (a: number, b: number, c: bigint) => void;
|
|
146
153
|
export const wasmbrowserconfig_set_timeout: (a: number, b: number, c: bigint) => void;
|
|
147
154
|
export const wasmbrowserconfig_set_wait: (a: number, b: number) => void;
|
|
148
155
|
export const wasmbrowserconfig_set_waitSelector: (a: number, b: number, c: number) => void;
|
|
156
|
+
export const wasmbrowserconfig_shutdownTimeout: (a: number) => [number, bigint];
|
|
149
157
|
export const wasmbrowserconfig_timeout: (a: number) => [number, bigint];
|
|
150
158
|
export const wasmbrowserconfig_wait: (a: number) => [number, number];
|
|
151
159
|
export const wasmbrowserconfig_waitSelector: (a: number) => [number, number];
|
|
@@ -173,15 +181,17 @@ export const wasmcitationresult_set_content: (a: number, b: number, c: number) =
|
|
|
173
181
|
export const wasmcitationresult_set_references: (a: number, b: number, c: number) => void;
|
|
174
182
|
export const wasmcontentconfig_default: () => number;
|
|
175
183
|
export const wasmcontentconfig_excludeSelectors: (a: number) => [number, number];
|
|
184
|
+
export const wasmcontentconfig_extractMetadata: (a: number) => number;
|
|
176
185
|
export const wasmcontentconfig_includeDocumentStructure: (a: number) => number;
|
|
177
186
|
export const wasmcontentconfig_maxDepth: (a: number) => number;
|
|
178
|
-
export const wasmcontentconfig_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number, n: number, o: number, p: number, q: number) => number;
|
|
187
|
+
export const wasmcontentconfig_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number, n: number, o: number, p: number, q: number, r: number) => number;
|
|
179
188
|
export const wasmcontentconfig_outputFormat: (a: number) => [number, number];
|
|
180
189
|
export const wasmcontentconfig_preprocessingPreset: (a: number) => [number, number];
|
|
181
190
|
export const wasmcontentconfig_preserveTags: (a: number) => [number, number];
|
|
182
191
|
export const wasmcontentconfig_removeForms: (a: number) => number;
|
|
183
192
|
export const wasmcontentconfig_removeNavigation: (a: number) => number;
|
|
184
193
|
export const wasmcontentconfig_set_excludeSelectors: (a: number, b: number, c: number) => void;
|
|
194
|
+
export const wasmcontentconfig_set_extractMetadata: (a: number, b: number) => void;
|
|
185
195
|
export const wasmcontentconfig_set_includeDocumentStructure: (a: number, b: number) => void;
|
|
186
196
|
export const wasmcontentconfig_set_maxDepth: (a: number, b: number) => void;
|
|
187
197
|
export const wasmcontentconfig_set_outputFormat: (a: number, b: number, c: number) => void;
|
|
@@ -220,6 +230,7 @@ export const wasmcrawlconfig_contentFilter: (a: number) => [number, number];
|
|
|
220
230
|
export const wasmcrawlconfig_cookiesEnabled: (a: number) => number;
|
|
221
231
|
export const wasmcrawlconfig_crawlStrategy: (a: number) => [number, number];
|
|
222
232
|
export const wasmcrawlconfig_customHeaders: (a: number) => any;
|
|
233
|
+
export const wasmcrawlconfig_dedupIncludeQuery: (a: number) => number;
|
|
223
234
|
export const wasmcrawlconfig_default: () => number;
|
|
224
235
|
export const wasmcrawlconfig_documentContentEncoding: (a: number) => [number, number];
|
|
225
236
|
export const wasmcrawlconfig_documentMaxSize: (a: number) => number;
|
|
@@ -240,14 +251,19 @@ export const wasmcrawlconfig_maxDepth: (a: number) => number;
|
|
|
240
251
|
export const wasmcrawlconfig_maxLinksPerPage: (a: number) => number;
|
|
241
252
|
export const wasmcrawlconfig_maxPages: (a: number) => number;
|
|
242
253
|
export const wasmcrawlconfig_maxRedirects: (a: number) => number;
|
|
243
|
-
export const wasmcrawlconfig_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l:
|
|
254
|
+
export const wasmcrawlconfig_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number, n: number, o: number, p: number, q: number, r: bigint, s: number, t: number, u: number, v: number, w: number, x: bigint, y: number, z: bigint, a1: number, b1: number, c1: number, d1: number, e1: number, f1: number, g1: number, h1: number, i1: number, j1: number, k1: number, l1: number, m1: number, n1: number, o1: number, p1: number, q1: number, r1: number, s1: number, t1: number, u1: number, v1: number, w1: number, x1: number, y1: number, z1: number, a2: number, b2: number, c2: number, d2: number, e2: number, f2: bigint, g2: number, h2: number, i2: number, j2: number, k2: number, l2: number, m2: number, n2: number, o2: number, p2: number, q2: number, r2: number, s2: number, t2: number, u2: number, v2: number, w2: number) => number;
|
|
255
|
+
export const wasmcrawlconfig_pathPatternsMatchQuery: (a: number) => number;
|
|
256
|
+
export const wasmcrawlconfig_pathPatternsMatchUrl: (a: number) => number;
|
|
244
257
|
export const wasmcrawlconfig_proxy: (a: number) => number;
|
|
258
|
+
export const wasmcrawlconfig_rateLimitJitterRatio: (a: number) => number;
|
|
245
259
|
export const wasmcrawlconfig_rateLimitMs: (a: number) => [number, bigint];
|
|
246
260
|
export const wasmcrawlconfig_removeTags: (a: number) => [number, number];
|
|
247
261
|
export const wasmcrawlconfig_requestTimeout: (a: number) => [number, bigint];
|
|
248
262
|
export const wasmcrawlconfig_respectRobotsTxt: (a: number) => number;
|
|
249
263
|
export const wasmcrawlconfig_retryCodes: (a: number) => [number, number];
|
|
250
264
|
export const wasmcrawlconfig_retryCount: (a: number) => number;
|
|
265
|
+
export const wasmcrawlconfig_retryInitialDelayMs: (a: number) => bigint;
|
|
266
|
+
export const wasmcrawlconfig_retryMaxDelayMs: (a: number) => bigint;
|
|
251
267
|
export const wasmcrawlconfig_saveBrowserProfile: (a: number) => number;
|
|
252
268
|
export const wasmcrawlconfig_set_allowSubdomains: (a: number, b: number) => void;
|
|
253
269
|
export const wasmcrawlconfig_set_assetTypes: (a: number, b: number, c: number) => void;
|
|
@@ -262,6 +278,7 @@ export const wasmcrawlconfig_set_contentFilter: (a: number, b: number) => void;
|
|
|
262
278
|
export const wasmcrawlconfig_set_cookiesEnabled: (a: number, b: number) => void;
|
|
263
279
|
export const wasmcrawlconfig_set_crawlStrategy: (a: number, b: number) => void;
|
|
264
280
|
export const wasmcrawlconfig_set_customHeaders: (a: number, b: any) => void;
|
|
281
|
+
export const wasmcrawlconfig_set_dedupIncludeQuery: (a: number, b: number) => void;
|
|
265
282
|
export const wasmcrawlconfig_set_documentContentEncoding: (a: number, b: number) => void;
|
|
266
283
|
export const wasmcrawlconfig_set_documentMaxSize: (a: number, b: number) => void;
|
|
267
284
|
export const wasmcrawlconfig_set_documentMimeTypes: (a: number, b: number, c: number) => void;
|
|
@@ -281,18 +298,25 @@ export const wasmcrawlconfig_set_maxDepth: (a: number, b: number) => void;
|
|
|
281
298
|
export const wasmcrawlconfig_set_maxLinksPerPage: (a: number, b: number) => void;
|
|
282
299
|
export const wasmcrawlconfig_set_maxPages: (a: number, b: number) => void;
|
|
283
300
|
export const wasmcrawlconfig_set_maxRedirects: (a: number, b: number) => void;
|
|
301
|
+
export const wasmcrawlconfig_set_pathPatternsMatchQuery: (a: number, b: number) => void;
|
|
302
|
+
export const wasmcrawlconfig_set_pathPatternsMatchUrl: (a: number, b: number) => void;
|
|
284
303
|
export const wasmcrawlconfig_set_proxy: (a: number, b: number) => void;
|
|
304
|
+
export const wasmcrawlconfig_set_rateLimitJitterRatio: (a: number, b: number) => void;
|
|
285
305
|
export const wasmcrawlconfig_set_rateLimitMs: (a: number, b: number, c: bigint) => void;
|
|
286
306
|
export const wasmcrawlconfig_set_removeTags: (a: number, b: number, c: number) => void;
|
|
287
307
|
export const wasmcrawlconfig_set_requestTimeout: (a: number, b: number, c: bigint) => void;
|
|
288
308
|
export const wasmcrawlconfig_set_respectRobotsTxt: (a: number, b: number) => void;
|
|
289
309
|
export const wasmcrawlconfig_set_retryCodes: (a: number, b: number, c: number) => void;
|
|
290
310
|
export const wasmcrawlconfig_set_retryCount: (a: number, b: number) => void;
|
|
311
|
+
export const wasmcrawlconfig_set_retryInitialDelayMs: (a: number, b: bigint) => void;
|
|
312
|
+
export const wasmcrawlconfig_set_retryMaxDelayMs: (a: number, b: bigint) => void;
|
|
291
313
|
export const wasmcrawlconfig_set_saveBrowserProfile: (a: number, b: number) => void;
|
|
292
314
|
export const wasmcrawlconfig_set_softHttpErrors: (a: number, b: number) => void;
|
|
293
315
|
export const wasmcrawlconfig_set_ssrf: (a: number, b: number) => void;
|
|
294
316
|
export const wasmcrawlconfig_set_ssrfDenyPrivateExplicit: (a: number, b: number) => void;
|
|
295
317
|
export const wasmcrawlconfig_set_stayOnDomain: (a: number, b: number) => void;
|
|
318
|
+
export const wasmcrawlconfig_set_stripTrackingParams: (a: number, b: number) => void;
|
|
319
|
+
export const wasmcrawlconfig_set_trackingParams: (a: number, b: number, c: number) => void;
|
|
296
320
|
export const wasmcrawlconfig_set_userAgent: (a: number, b: number, c: number) => void;
|
|
297
321
|
export const wasmcrawlconfig_set_userAgents: (a: number, b: number, c: number) => void;
|
|
298
322
|
export const wasmcrawlconfig_set_warcOutput: (a: number, b: number, c: number) => void;
|
|
@@ -300,6 +324,8 @@ export const wasmcrawlconfig_softHttpErrors: (a: number) => number;
|
|
|
300
324
|
export const wasmcrawlconfig_ssrf: (a: number) => number;
|
|
301
325
|
export const wasmcrawlconfig_ssrfDenyPrivateExplicit: (a: number) => number;
|
|
302
326
|
export const wasmcrawlconfig_stayOnDomain: (a: number) => number;
|
|
327
|
+
export const wasmcrawlconfig_stripTrackingParams: (a: number) => number;
|
|
328
|
+
export const wasmcrawlconfig_trackingParams: (a: number) => [number, number];
|
|
303
329
|
export const wasmcrawlconfig_userAgent: (a: number) => [number, number];
|
|
304
330
|
export const wasmcrawlconfig_userAgents: (a: number) => [number, number];
|
|
305
331
|
export const wasmcrawlconfig_validate: (a: number) => [number, number];
|
|
@@ -314,6 +340,7 @@ export const wasmcrawlpageresult_downloadedDocument: (a: number) => number;
|
|
|
314
340
|
export const wasmcrawlpageresult_extractedData: (a: number) => any;
|
|
315
341
|
export const wasmcrawlpageresult_extractionMeta: (a: number) => number;
|
|
316
342
|
export const wasmcrawlpageresult_feeds: (a: number) => [number, number];
|
|
343
|
+
export const wasmcrawlpageresult_finalUrl: (a: number) => [number, number];
|
|
317
344
|
export const wasmcrawlpageresult_html: (a: number) => [number, number];
|
|
318
345
|
export const wasmcrawlpageresult_images: (a: number) => [number, number];
|
|
319
346
|
export const wasmcrawlpageresult_isPdf: (a: number) => number;
|
|
@@ -321,8 +348,11 @@ export const wasmcrawlpageresult_jsonLd: (a: number) => [number, number];
|
|
|
321
348
|
export const wasmcrawlpageresult_links: (a: number) => [number, number];
|
|
322
349
|
export const wasmcrawlpageresult_markdown: (a: number) => number;
|
|
323
350
|
export const wasmcrawlpageresult_metadata: (a: number) => number;
|
|
324
|
-
export const wasmcrawlpageresult_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number, n: number, o: number, p: number, q: number, r: number, s: number, t: number, u: number, v: number, w: number, x: number, y: number, z: number, a1: number, b1: number, c1: number, d1: number) => number;
|
|
351
|
+
export const wasmcrawlpageresult_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number, n: number, o: number, p: number, q: number, r: number, s: number, t: number, u: number, v: number, w: number, x: number, y: number, z: number, a1: number, b1: number, c1: number, d1: number, e1: number, f1: number, g1: number, h1: number, i1: number, j1: number, k1: number) => number;
|
|
352
|
+
export const wasmcrawlpageresult_nofollowDetected: (a: number) => number;
|
|
353
|
+
export const wasmcrawlpageresult_noindexDetected: (a: number) => number;
|
|
325
354
|
export const wasmcrawlpageresult_normalizedUrl: (a: number) => [number, number];
|
|
355
|
+
export const wasmcrawlpageresult_redirectCount: (a: number) => number;
|
|
326
356
|
export const wasmcrawlpageresult_set_bodySize: (a: number, b: number) => void;
|
|
327
357
|
export const wasmcrawlpageresult_set_browserUsed: (a: number, b: number) => void;
|
|
328
358
|
export const wasmcrawlpageresult_set_contentType: (a: number, b: number, c: number) => void;
|
|
@@ -332,6 +362,7 @@ export const wasmcrawlpageresult_set_downloadedDocument: (a: number, b: number)
|
|
|
332
362
|
export const wasmcrawlpageresult_set_extractedData: (a: number, b: number) => void;
|
|
333
363
|
export const wasmcrawlpageresult_set_extractionMeta: (a: number, b: number) => void;
|
|
334
364
|
export const wasmcrawlpageresult_set_feeds: (a: number, b: number, c: number) => void;
|
|
365
|
+
export const wasmcrawlpageresult_set_finalUrl: (a: number, b: number, c: number) => void;
|
|
335
366
|
export const wasmcrawlpageresult_set_html: (a: number, b: number, c: number) => void;
|
|
336
367
|
export const wasmcrawlpageresult_set_images: (a: number, b: number, c: number) => void;
|
|
337
368
|
export const wasmcrawlpageresult_set_isPdf: (a: number, b: number) => void;
|
|
@@ -339,11 +370,16 @@ export const wasmcrawlpageresult_set_jsonLd: (a: number, b: number, c: number) =
|
|
|
339
370
|
export const wasmcrawlpageresult_set_links: (a: number, b: number, c: number) => void;
|
|
340
371
|
export const wasmcrawlpageresult_set_markdown: (a: number, b: number) => void;
|
|
341
372
|
export const wasmcrawlpageresult_set_metadata: (a: number, b: number) => void;
|
|
373
|
+
export const wasmcrawlpageresult_set_nofollowDetected: (a: number, b: number) => void;
|
|
374
|
+
export const wasmcrawlpageresult_set_noindexDetected: (a: number, b: number) => void;
|
|
342
375
|
export const wasmcrawlpageresult_set_normalizedUrl: (a: number, b: number, c: number) => void;
|
|
376
|
+
export const wasmcrawlpageresult_set_redirectCount: (a: number, b: number) => void;
|
|
377
|
+
export const wasmcrawlpageresult_set_ssrfRefusedUrls: (a: number, b: number, c: number) => void;
|
|
343
378
|
export const wasmcrawlpageresult_set_statusCode: (a: number, b: number) => void;
|
|
344
379
|
export const wasmcrawlpageresult_set_stayedOnDomain: (a: number, b: number) => void;
|
|
345
380
|
export const wasmcrawlpageresult_set_url: (a: number, b: number, c: number) => void;
|
|
346
381
|
export const wasmcrawlpageresult_set_wasSkipped: (a: number, b: number) => void;
|
|
382
|
+
export const wasmcrawlpageresult_ssrfRefusedUrls: (a: number) => [number, number];
|
|
347
383
|
export const wasmcrawlpageresult_statusCode: (a: number) => number;
|
|
348
384
|
export const wasmcrawlpageresult_stayedOnDomain: (a: number) => number;
|
|
349
385
|
export const wasmcrawlpageresult_url: (a: number) => [number, number];
|
|
@@ -465,12 +501,14 @@ export const wasminteractionresult_actionResults: (a: number) => [number, number
|
|
|
465
501
|
export const wasminteractionresult_default: () => number;
|
|
466
502
|
export const wasminteractionresult_finalHtml: (a: number) => [number, number];
|
|
467
503
|
export const wasminteractionresult_finalUrl: (a: number) => [number, number];
|
|
468
|
-
export const wasminteractionresult_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number) => number;
|
|
504
|
+
export const wasminteractionresult_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number) => number;
|
|
469
505
|
export const wasminteractionresult_screenshotBase64: (a: number) => [number, number];
|
|
470
506
|
export const wasminteractionresult_set_actionResults: (a: number, b: number, c: number) => void;
|
|
471
507
|
export const wasminteractionresult_set_finalHtml: (a: number, b: number, c: number) => void;
|
|
472
508
|
export const wasminteractionresult_set_finalUrl: (a: number, b: number, c: number) => void;
|
|
473
509
|
export const wasminteractionresult_set_screenshotBase64: (a: number, b: number, c: number) => void;
|
|
510
|
+
export const wasminteractionresult_set_ssrfRefusedUrls: (a: number, b: number, c: number) => void;
|
|
511
|
+
export const wasminteractionresult_ssrfRefusedUrls: (a: number) => [number, number];
|
|
474
512
|
export const wasmjsonldentry_default: () => number;
|
|
475
513
|
export const wasmjsonldentry_name: (a: number) => [number, number];
|
|
476
514
|
export const wasmjsonldentry_new: (a: number, b: number, c: number, d: number, e: number, f: number) => number;
|
|
@@ -664,7 +702,7 @@ export const wasmscraperesult_jsonLd: (a: number) => [number, number];
|
|
|
664
702
|
export const wasmscraperesult_links: (a: number) => [number, number];
|
|
665
703
|
export const wasmscraperesult_markdown: (a: number) => number;
|
|
666
704
|
export const wasmscraperesult_metadata: (a: number) => number;
|
|
667
|
-
export const wasmscraperesult_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number, n: number, o: number, p: number, q: number, r: number, s: number, t: number, u: number, v: number, w: number, x: number, y: number, z: number, a1: number, b1: number, c1:
|
|
705
|
+
export const wasmscraperesult_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number, n: number, o: number, p: number, q: number, r: number, s: number, t: number, u: number, v: number, w: number, x: number, y: number, z: number, a1: number, b1: number, c1: number, d1: number, e1: bigint, f1: number, g1: number, h1: number, i1: number, j1: number, k1: number, l1: number, m1: number, n1: number, o1: number, p1: number, q1: number) => number;
|
|
668
706
|
export const wasmscraperesult_nofollowDetected: (a: number) => number;
|
|
669
707
|
export const wasmscraperesult_noindexDetected: (a: number) => number;
|
|
670
708
|
export const wasmscraperesult_responseMeta: (a: number) => number;
|
|
@@ -695,9 +733,11 @@ export const wasmscraperesult_set_nofollowDetected: (a: number, b: number) => vo
|
|
|
695
733
|
export const wasmscraperesult_set_noindexDetected: (a: number, b: number) => void;
|
|
696
734
|
export const wasmscraperesult_set_responseMeta: (a: number, b: number) => void;
|
|
697
735
|
export const wasmscraperesult_set_screenshotBase64: (a: number, b: number, c: number) => void;
|
|
736
|
+
export const wasmscraperesult_set_ssrfRefusedUrls: (a: number, b: number, c: number) => void;
|
|
698
737
|
export const wasmscraperesult_set_statusCode: (a: number, b: number) => void;
|
|
699
738
|
export const wasmscraperesult_set_wasSkipped: (a: number, b: number) => void;
|
|
700
739
|
export const wasmscraperesult_set_xRobotsTag: (a: number, b: number, c: number) => void;
|
|
740
|
+
export const wasmscraperesult_ssrfRefusedUrls: (a: number) => [number, number];
|
|
701
741
|
export const wasmscraperesult_statusCode: (a: number) => number;
|
|
702
742
|
export const wasmscraperesult_wasSkipped: (a: number) => number;
|
|
703
743
|
export const wasmscraperesult_xRobotsTag: (a: number) => [number, number];
|
|
@@ -722,9 +762,9 @@ export const wasmssrfpolicy_set_allowlist: (a: number, b: any) => void;
|
|
|
722
762
|
export const wasmssrfpolicy_set_denyPrivate: (a: number, b: number) => void;
|
|
723
763
|
export const wasmssrfpolicy_set_maxRedirects: (a: number, b: number) => void;
|
|
724
764
|
export const wasmssrfpolicy_set_schemeAllowlist: (a: number, b: number, c: number) => void;
|
|
725
|
-
export const
|
|
726
|
-
export const
|
|
727
|
-
export const
|
|
765
|
+
export const wasm_bindgen__convert__closures_____invoke__h2eedacf566f3748c: (a: number, b: number, c: any) => [number, number];
|
|
766
|
+
export const wasm_bindgen__convert__closures_____invoke__h8ad11faaa2905452: (a: number, b: number, c: any, d: any) => void;
|
|
767
|
+
export const wasm_bindgen__convert__closures_____invoke__h3a7ccac3e94bbb5c: (a: number, b: number) => void;
|
|
728
768
|
export const __wbindgen_malloc: (a: number, b: number) => number;
|
|
729
769
|
export const __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number;
|
|
730
770
|
export const __wbindgen_exn_store: (a: number) => void;
|
|
@@ -153,11 +153,14 @@ export class WasmBrowserConfig {
|
|
|
153
153
|
free(): void;
|
|
154
154
|
[Symbol.dispose](): void;
|
|
155
155
|
static default(): WasmBrowserConfig;
|
|
156
|
-
constructor(mode?: WasmBrowserMode | null, backend?: WasmBrowserBackend | null, timeout?: bigint | null, wait?: WasmBrowserWait | null, blockUrlPatterns?: string[] | null, captureNetworkEvents?: boolean | null, sessionAffinity?: boolean | null, endpoint?: string | null, waitSelector?: string | null, extraWait?: bigint | null, proxy?: WasmProxyConfig | null, evalScript?: string | null, robotsUserAgent?: string | null);
|
|
156
|
+
constructor(mode?: WasmBrowserMode | null, backend?: WasmBrowserBackend | null, timeout?: bigint | null, overallTimeout?: bigint | null, shutdownTimeout?: bigint | null, wait?: WasmBrowserWait | null, blockUrlPatterns?: string[] | null, captureNetworkEvents?: boolean | null, sessionAffinity?: boolean | null, chromeArgs?: string[] | null, endpoint?: string | null, waitSelector?: string | null, extraWait?: bigint | null, proxy?: WasmProxyConfig | null, evalScript?: string | null, robotsUserAgent?: string | null, chromePath?: string | null);
|
|
157
157
|
get backend(): string;
|
|
158
158
|
set backend(value: WasmBrowserBackend);
|
|
159
159
|
blockUrlPatterns: string[];
|
|
160
160
|
captureNetworkEvents: boolean;
|
|
161
|
+
chromeArgs: string[];
|
|
162
|
+
get chromePath(): string | undefined;
|
|
163
|
+
set chromePath(value: string | null | undefined);
|
|
161
164
|
get endpoint(): string | undefined;
|
|
162
165
|
set endpoint(value: string | null | undefined);
|
|
163
166
|
get evalScript(): string | undefined;
|
|
@@ -166,11 +169,15 @@ export class WasmBrowserConfig {
|
|
|
166
169
|
set extraWait(value: bigint | null | undefined);
|
|
167
170
|
get mode(): string;
|
|
168
171
|
set mode(value: WasmBrowserMode);
|
|
172
|
+
get overallTimeout(): bigint | undefined;
|
|
173
|
+
set overallTimeout(value: bigint | null | undefined);
|
|
169
174
|
get proxy(): WasmProxyConfig | undefined;
|
|
170
175
|
set proxy(value: WasmProxyConfig | null | undefined);
|
|
171
176
|
get robotsUserAgent(): string | undefined;
|
|
172
177
|
set robotsUserAgent(value: string | null | undefined);
|
|
173
178
|
sessionAffinity: boolean;
|
|
179
|
+
get shutdownTimeout(): bigint | undefined;
|
|
180
|
+
set shutdownTimeout(value: bigint | null | undefined);
|
|
174
181
|
get timeout(): bigint | undefined;
|
|
175
182
|
set timeout(value: bigint | null | undefined);
|
|
176
183
|
get waitSelector(): string | undefined;
|
|
@@ -251,8 +258,9 @@ export class WasmContentConfig {
|
|
|
251
258
|
free(): void;
|
|
252
259
|
[Symbol.dispose](): void;
|
|
253
260
|
static default(): WasmContentConfig;
|
|
254
|
-
constructor(outputFormat?: string | null, preprocessingPreset?: string | null, removeNavigation?: boolean | null, removeForms?: boolean | null, stripTags?: string[] | null, preserveTags?: string[] | null, excludeSelectors?: string[] | null, skipImages?: boolean | null, wrap?: boolean | null, wrapWidth?: number | null, includeDocumentStructure?: boolean | null, maxDepth?: number | null);
|
|
261
|
+
constructor(outputFormat?: string | null, preprocessingPreset?: string | null, removeNavigation?: boolean | null, removeForms?: boolean | null, stripTags?: string[] | null, preserveTags?: string[] | null, excludeSelectors?: string[] | null, skipImages?: boolean | null, wrap?: boolean | null, wrapWidth?: number | null, includeDocumentStructure?: boolean | null, extractMetadata?: boolean | null, maxDepth?: number | null);
|
|
255
262
|
excludeSelectors: string[];
|
|
263
|
+
extractMetadata: boolean;
|
|
256
264
|
includeDocumentStructure: boolean;
|
|
257
265
|
get maxDepth(): number | undefined;
|
|
258
266
|
set maxDepth(value: number | null | undefined);
|
|
@@ -297,7 +305,7 @@ export class WasmCrawlConfig {
|
|
|
297
305
|
free(): void;
|
|
298
306
|
[Symbol.dispose](): void;
|
|
299
307
|
static default(): WasmCrawlConfig;
|
|
300
|
-
constructor(crawlStrategy?: WasmCrawlStrategyKind | null, respectRobotsTxt?: boolean | null, softHttpErrors?: boolean | null, stayOnDomain?: boolean | null, allowSubdomains?: boolean | null, includePaths?: string[] | null, excludePaths?: string[] | null, customHeaders?: any | null, requestTimeout?: bigint | null, maxRedirects?: number | null, retryCount?: number | null, retryCodes?: Uint16Array | null, cookiesEnabled?: boolean | null, removeTags?: string[] | null, content?: WasmContentConfig | null, downloadAssets?: boolean | null, assetTypes?: WasmAssetCategory[] | null, browser?: WasmBrowserConfig | null, userAgents?: string[] | null, captureScreenshot?: boolean | null, followDocumentUrls?: boolean | null, downloadDocuments?: boolean | null, documentMimeTypes?: string[] | null, saveBrowserProfile?: boolean | null, ssrf?: WasmSsrfPolicy | null, maxDepth?: number | null, maxPages?: number | null, maxLinksPerPage?: number | null, maxConcurrent?: number | null, contentFilter?: WasmContentFilterKind | null, bm25Query?: string | null, bm25Threshold?: number | null, userAgent?: string | null, rateLimitMs?: bigint | null, auth?: any | null, maxBodySize?: number | null, mapLimit?: number | null, mapSearch?: string | null, maxAssetSize?: number | null, proxy?: WasmProxyConfig | null, documentUrlDepth?: number | null, documentMaxSize?: number | null, documentOutputDir?: string | null, documentContentEncoding?: WasmDocumentContentEncoding | null, warcOutput?: string | null, browserProfile?: string | null, ssrfDenyPrivateExplicit?: boolean | null);
|
|
308
|
+
constructor(crawlStrategy?: WasmCrawlStrategyKind | null, respectRobotsTxt?: boolean | null, softHttpErrors?: boolean | null, stayOnDomain?: boolean | null, allowSubdomains?: boolean | null, includePaths?: string[] | null, excludePaths?: string[] | null, pathPatternsMatchQuery?: boolean | null, pathPatternsMatchUrl?: boolean | null, dedupIncludeQuery?: boolean | null, stripTrackingParams?: boolean | null, trackingParams?: string[] | null, customHeaders?: any | null, requestTimeout?: bigint | null, maxRedirects?: number | null, retryCount?: number | null, retryCodes?: Uint16Array | null, retryInitialDelayMs?: bigint | null, retryMaxDelayMs?: bigint | null, rateLimitJitterRatio?: number | null, cookiesEnabled?: boolean | null, removeTags?: string[] | null, content?: WasmContentConfig | null, downloadAssets?: boolean | null, assetTypes?: WasmAssetCategory[] | null, browser?: WasmBrowserConfig | null, userAgents?: string[] | null, captureScreenshot?: boolean | null, followDocumentUrls?: boolean | null, downloadDocuments?: boolean | null, documentMimeTypes?: string[] | null, saveBrowserProfile?: boolean | null, ssrf?: WasmSsrfPolicy | null, maxDepth?: number | null, maxPages?: number | null, maxLinksPerPage?: number | null, maxConcurrent?: number | null, contentFilter?: WasmContentFilterKind | null, bm25Query?: string | null, bm25Threshold?: number | null, userAgent?: string | null, rateLimitMs?: bigint | null, auth?: any | null, maxBodySize?: number | null, mapLimit?: number | null, mapSearch?: string | null, maxAssetSize?: number | null, proxy?: WasmProxyConfig | null, documentUrlDepth?: number | null, documentMaxSize?: number | null, documentOutputDir?: string | null, documentContentEncoding?: WasmDocumentContentEncoding | null, warcOutput?: string | null, browserProfile?: string | null, ssrfDenyPrivateExplicit?: boolean | null);
|
|
301
309
|
/**
|
|
302
310
|
* Validate the configuration, returning an error if any values are invalid.
|
|
303
311
|
*
|
|
@@ -324,6 +332,7 @@ export class WasmCrawlConfig {
|
|
|
324
332
|
get crawlStrategy(): string;
|
|
325
333
|
set crawlStrategy(value: WasmCrawlStrategyKind);
|
|
326
334
|
customHeaders: any;
|
|
335
|
+
dedupIncludeQuery: boolean;
|
|
327
336
|
get documentContentEncoding(): string | undefined;
|
|
328
337
|
set documentContentEncoding(value: WasmDocumentContentEncoding | null | undefined);
|
|
329
338
|
get documentMaxSize(): number | undefined;
|
|
@@ -355,8 +364,11 @@ export class WasmCrawlConfig {
|
|
|
355
364
|
get maxPages(): number | undefined;
|
|
356
365
|
set maxPages(value: number | null | undefined);
|
|
357
366
|
maxRedirects: number;
|
|
367
|
+
pathPatternsMatchQuery: boolean;
|
|
368
|
+
pathPatternsMatchUrl: boolean;
|
|
358
369
|
get proxy(): WasmProxyConfig | undefined;
|
|
359
370
|
set proxy(value: WasmProxyConfig | null | undefined);
|
|
371
|
+
rateLimitJitterRatio: number;
|
|
360
372
|
get rateLimitMs(): bigint | undefined;
|
|
361
373
|
set rateLimitMs(value: bigint | null | undefined);
|
|
362
374
|
removeTags: string[];
|
|
@@ -365,12 +377,16 @@ export class WasmCrawlConfig {
|
|
|
365
377
|
respectRobotsTxt: boolean;
|
|
366
378
|
retryCodes: Uint16Array;
|
|
367
379
|
retryCount: number;
|
|
380
|
+
retryInitialDelayMs: bigint;
|
|
381
|
+
retryMaxDelayMs: bigint;
|
|
368
382
|
saveBrowserProfile: boolean;
|
|
369
383
|
softHttpErrors: boolean;
|
|
370
384
|
get ssrfDenyPrivateExplicit(): boolean | undefined;
|
|
371
385
|
set ssrfDenyPrivateExplicit(value: boolean | null | undefined);
|
|
372
386
|
ssrf: WasmSsrfPolicy;
|
|
373
387
|
stayOnDomain: boolean;
|
|
388
|
+
stripTrackingParams: boolean;
|
|
389
|
+
trackingParams: string[];
|
|
374
390
|
get userAgent(): string | undefined;
|
|
375
391
|
set userAgent(value: string | null | undefined);
|
|
376
392
|
userAgents: string[];
|
|
@@ -397,7 +413,7 @@ export class WasmCrawlPageResult {
|
|
|
397
413
|
free(): void;
|
|
398
414
|
[Symbol.dispose](): void;
|
|
399
415
|
static default(): WasmCrawlPageResult;
|
|
400
|
-
constructor(url?: string | null, normalizedUrl?: string | null, statusCode?: number | null, contentType?: string | null, html?: string | null, bodySize?: number | null, metadata?: WasmPageMetadata | null, links?: WasmLinkInfo[] | null, images?: WasmImageInfo[] | null, feeds?: WasmFeedInfo[] | null, jsonLd?: WasmJsonLdEntry[] | null, depth?: number | null, stayedOnDomain?: boolean | null, wasSkipped?: boolean | null, isPdf?: boolean | null, browserUsed?: boolean | null, detectedCharset?: string | null, markdown?: WasmMarkdownResult | null, extractedData?: any | null, extractionMeta?: WasmExtractionMeta | null, downloadedDocument?: WasmDownloadedDocument | null);
|
|
416
|
+
constructor(url?: string | null, normalizedUrl?: string | null, statusCode?: number | null, contentType?: string | null, html?: string | null, bodySize?: number | null, metadata?: WasmPageMetadata | null, links?: WasmLinkInfo[] | null, images?: WasmImageInfo[] | null, feeds?: WasmFeedInfo[] | null, jsonLd?: WasmJsonLdEntry[] | null, depth?: number | null, stayedOnDomain?: boolean | null, wasSkipped?: boolean | null, isPdf?: boolean | null, browserUsed?: boolean | null, finalUrl?: string | null, redirectCount?: number | null, noindexDetected?: boolean | null, nofollowDetected?: boolean | null, ssrfRefusedUrls?: string[] | null, detectedCharset?: string | null, markdown?: WasmMarkdownResult | null, extractedData?: any | null, extractionMeta?: WasmExtractionMeta | null, downloadedDocument?: WasmDownloadedDocument | null);
|
|
401
417
|
bodySize: number;
|
|
402
418
|
browserUsed: boolean;
|
|
403
419
|
contentType: string;
|
|
@@ -411,6 +427,7 @@ export class WasmCrawlPageResult {
|
|
|
411
427
|
get extractionMeta(): WasmExtractionMeta | undefined;
|
|
412
428
|
set extractionMeta(value: WasmExtractionMeta | null | undefined);
|
|
413
429
|
feeds: WasmFeedInfo[];
|
|
430
|
+
finalUrl: string;
|
|
414
431
|
html: string;
|
|
415
432
|
images: WasmImageInfo[];
|
|
416
433
|
isPdf: boolean;
|
|
@@ -419,7 +436,11 @@ export class WasmCrawlPageResult {
|
|
|
419
436
|
get markdown(): WasmMarkdownResult | undefined;
|
|
420
437
|
set markdown(value: WasmMarkdownResult | null | undefined);
|
|
421
438
|
metadata: WasmPageMetadata;
|
|
439
|
+
nofollowDetected: boolean;
|
|
440
|
+
noindexDetected: boolean;
|
|
422
441
|
normalizedUrl: string;
|
|
442
|
+
redirectCount: number;
|
|
443
|
+
ssrfRefusedUrls: string[];
|
|
423
444
|
statusCode: number;
|
|
424
445
|
stayedOnDomain: boolean;
|
|
425
446
|
url: string;
|
|
@@ -673,12 +694,13 @@ export class WasmInteractionResult {
|
|
|
673
694
|
free(): void;
|
|
674
695
|
[Symbol.dispose](): void;
|
|
675
696
|
static default(): WasmInteractionResult;
|
|
676
|
-
constructor(actionResults?: WasmActionResult[] | null, finalHtml?: string | null, finalUrl?: string | null, screenshotBase64?: string | null);
|
|
697
|
+
constructor(actionResults?: WasmActionResult[] | null, finalHtml?: string | null, finalUrl?: string | null, ssrfRefusedUrls?: string[] | null, screenshotBase64?: string | null);
|
|
677
698
|
actionResults: WasmActionResult[];
|
|
678
699
|
finalHtml: string;
|
|
679
700
|
finalUrl: string;
|
|
680
701
|
get screenshotBase64(): string | undefined;
|
|
681
702
|
set screenshotBase64(value: string | null | undefined);
|
|
703
|
+
ssrfRefusedUrls: string[];
|
|
682
704
|
}
|
|
683
705
|
|
|
684
706
|
/**
|
|
@@ -923,7 +945,7 @@ export class WasmScrapeResult {
|
|
|
923
945
|
free(): void;
|
|
924
946
|
[Symbol.dispose](): void;
|
|
925
947
|
static default(): WasmScrapeResult;
|
|
926
|
-
constructor(statusCode?: number | null, finalUrl?: string | null, contentType?: string | null, html?: string | null, bodySize?: number | null, metadata?: WasmPageMetadata | null, links?: WasmLinkInfo[] | null, images?: WasmImageInfo[] | null, feeds?: WasmFeedInfo[] | null, jsonLd?: WasmJsonLdEntry[] | null, isAllowed?: boolean | null, noindexDetected?: boolean | null, nofollowDetected?: boolean | null, isPdf?: boolean | null, wasSkipped?: boolean | null, authHeaderSent?: boolean | null, assets?: WasmDownloadedAsset[] | null, jsRenderHint?: boolean | null, browserUsed?: boolean | null, crawlDelay?: bigint | null, xRobotsTag?: string | null, detectedCharset?: string | null, responseMeta?: WasmResponseMeta | null, markdown?: WasmMarkdownResult | null, extractedData?: any | null, extractionMeta?: WasmExtractionMeta | null, screenshotBase64?: string | null, downloadedDocument?: WasmDownloadedDocument | null, browser?: WasmBrowserExtras | null);
|
|
948
|
+
constructor(statusCode?: number | null, finalUrl?: string | null, contentType?: string | null, html?: string | null, bodySize?: number | null, metadata?: WasmPageMetadata | null, links?: WasmLinkInfo[] | null, images?: WasmImageInfo[] | null, feeds?: WasmFeedInfo[] | null, jsonLd?: WasmJsonLdEntry[] | null, isAllowed?: boolean | null, noindexDetected?: boolean | null, nofollowDetected?: boolean | null, isPdf?: boolean | null, wasSkipped?: boolean | null, authHeaderSent?: boolean | null, assets?: WasmDownloadedAsset[] | null, jsRenderHint?: boolean | null, browserUsed?: boolean | null, ssrfRefusedUrls?: string[] | null, crawlDelay?: bigint | null, xRobotsTag?: string | null, detectedCharset?: string | null, responseMeta?: WasmResponseMeta | null, markdown?: WasmMarkdownResult | null, extractedData?: any | null, extractionMeta?: WasmExtractionMeta | null, screenshotBase64?: string | null, downloadedDocument?: WasmDownloadedDocument | null, browser?: WasmBrowserExtras | null);
|
|
927
949
|
assets: WasmDownloadedAsset[];
|
|
928
950
|
authHeaderSent: boolean;
|
|
929
951
|
bodySize: number;
|
|
@@ -959,6 +981,7 @@ export class WasmScrapeResult {
|
|
|
959
981
|
set responseMeta(value: WasmResponseMeta | null | undefined);
|
|
960
982
|
get screenshotBase64(): string | undefined;
|
|
961
983
|
set screenshotBase64(value: string | null | undefined);
|
|
984
|
+
ssrfRefusedUrls: string[];
|
|
962
985
|
statusCode: number;
|
|
963
986
|
wasSkipped: boolean;
|
|
964
987
|
get xRobotsTag(): string | undefined;
|