@xberg-io/crawlberg-wasm 1.8.0 → 1.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/pkg/bundler/crawlberg_wasm.d.ts +634 -26
- package/pkg/bundler/crawlberg_wasm_bg.js +936 -154
- package/pkg/bundler/crawlberg_wasm_bg.wasm +0 -0
- package/pkg/bundler/crawlberg_wasm_bg.wasm.d.ts +69 -7
- package/pkg/bundler/package.json +1 -1
- package/pkg/deno/crawlberg_wasm.d.ts +634 -26
- package/pkg/deno/crawlberg_wasm.js +936 -154
- package/pkg/deno/crawlberg_wasm_bg.wasm +0 -0
- package/pkg/deno/crawlberg_wasm_bg.wasm.d.ts +69 -7
- package/pkg/nodejs/crawlberg_wasm.d.ts +634 -26
- package/pkg/nodejs/crawlberg_wasm.js +936 -154
- package/pkg/nodejs/crawlberg_wasm_bg.wasm +0 -0
- package/pkg/nodejs/crawlberg_wasm_bg.wasm.d.ts +69 -7
- package/pkg/nodejs/package.json +1 -1
- package/pkg/web/crawlberg_wasm.d.ts +703 -33
- package/pkg/web/crawlberg_wasm.js +936 -154
- package/pkg/web/crawlberg_wasm_bg.wasm +0 -0
- package/pkg/web/crawlberg_wasm_bg.wasm.d.ts +69 -7
- package/pkg/web/package.json +1 -1
|
@@ -7,6 +7,13 @@
|
|
|
7
7
|
export class WasmActionResult {
|
|
8
8
|
free(): void;
|
|
9
9
|
[Symbol.dispose](): void;
|
|
10
|
+
/**
|
|
11
|
+
* Return an independent handle for an ownership-taking API.
|
|
12
|
+
*
|
|
13
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
14
|
+
* handle must remain usable.
|
|
15
|
+
*/
|
|
16
|
+
copyForTransfer(): WasmActionResult;
|
|
10
17
|
static default(): WasmActionResult;
|
|
11
18
|
constructor(actionIndex?: number | null, actionType?: string | null, success?: boolean | null, data?: any | null, error?: string | null);
|
|
12
19
|
actionIndex: number;
|
|
@@ -24,6 +31,13 @@ export class WasmActionResult {
|
|
|
24
31
|
export class WasmArticleMetadata {
|
|
25
32
|
free(): void;
|
|
26
33
|
[Symbol.dispose](): void;
|
|
34
|
+
/**
|
|
35
|
+
* Return an independent handle for an ownership-taking API.
|
|
36
|
+
*
|
|
37
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
38
|
+
* handle must remain usable.
|
|
39
|
+
*/
|
|
40
|
+
copyForTransfer(): WasmArticleMetadata;
|
|
27
41
|
static default(): WasmArticleMetadata;
|
|
28
42
|
constructor(tags?: string[] | null, publishedTime?: string | null, modifiedTime?: string | null, author?: string | null, section?: string | null);
|
|
29
43
|
get author(): string | undefined;
|
|
@@ -80,12 +94,33 @@ export class WasmAuthConfig {
|
|
|
80
94
|
export class WasmBatchCrawlResult {
|
|
81
95
|
free(): void;
|
|
82
96
|
[Symbol.dispose](): void;
|
|
97
|
+
clearResult(): void;
|
|
98
|
+
/**
|
|
99
|
+
* Return an independent handle for an ownership-taking API.
|
|
100
|
+
*
|
|
101
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
102
|
+
* handle must remain usable.
|
|
103
|
+
*/
|
|
104
|
+
copyForTransfer(): WasmBatchCrawlResult;
|
|
83
105
|
static default(): WasmBatchCrawlResult;
|
|
106
|
+
/**
|
|
107
|
+
* Takes ownership of `result`.
|
|
108
|
+
* wasm-bindgen lowers a by-value class argument through `__destroy_into_raw()`, so each
|
|
109
|
+
* of those handles is dead once this returns. Passing a freshly built value is fine. To
|
|
110
|
+
* keep a handle you already hold, build with `default()` and assign the property instead:
|
|
111
|
+
* the generated setter borrows and leaves your handle alive.
|
|
112
|
+
*/
|
|
84
113
|
constructor(url?: string | null, result?: WasmCrawlResult | null, error?: string | null);
|
|
85
114
|
get error(): string | undefined;
|
|
86
115
|
set error(value: string | null | undefined);
|
|
116
|
+
/**
|
|
117
|
+
* Returns a detached copy of `result`.
|
|
118
|
+
*
|
|
119
|
+
* Changes to the returned value do not update this object. To update `result`,
|
|
120
|
+
* read it, modify the copy, then assign the copy back to `result`.
|
|
121
|
+
*/
|
|
87
122
|
get result(): WasmCrawlResult | undefined;
|
|
88
|
-
set result(value: WasmCrawlResult
|
|
123
|
+
set result(value: WasmCrawlResult);
|
|
89
124
|
url: string;
|
|
90
125
|
}
|
|
91
126
|
|
|
@@ -98,10 +133,28 @@ export class WasmBatchCrawlResult {
|
|
|
98
133
|
export class WasmBatchCrawlResults {
|
|
99
134
|
free(): void;
|
|
100
135
|
[Symbol.dispose](): void;
|
|
136
|
+
/**
|
|
137
|
+
* Return an independent handle for an ownership-taking API.
|
|
138
|
+
*
|
|
139
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
140
|
+
* handle must remain usable.
|
|
141
|
+
*/
|
|
142
|
+
copyForTransfer(): WasmBatchCrawlResults;
|
|
101
143
|
static default(): WasmBatchCrawlResults;
|
|
144
|
+
/**
|
|
145
|
+
* Takes ownership of every element in `results`.
|
|
146
|
+
* wasm-bindgen cannot borrow class values nested in an array. To keep those handles,
|
|
147
|
+
* copy each retained element with `copyForTransfer()` before passing the array.
|
|
148
|
+
*/
|
|
102
149
|
constructor(results?: WasmBatchCrawlResult[] | null, totalCount?: number | null, completedCount?: number | null, failedCount?: number | null);
|
|
103
150
|
completedCount: number;
|
|
104
151
|
failedCount: number;
|
|
152
|
+
/**
|
|
153
|
+
* Returns detached copies of the elements in `results`.
|
|
154
|
+
*
|
|
155
|
+
* Changes to the returned values do not update this object. Modify the copies,
|
|
156
|
+
* then assign the array back to `results`.
|
|
157
|
+
*/
|
|
105
158
|
results: WasmBatchCrawlResult[];
|
|
106
159
|
totalCount: number;
|
|
107
160
|
}
|
|
@@ -112,12 +165,33 @@ export class WasmBatchCrawlResults {
|
|
|
112
165
|
export class WasmBatchScrapeResult {
|
|
113
166
|
free(): void;
|
|
114
167
|
[Symbol.dispose](): void;
|
|
168
|
+
clearResult(): void;
|
|
169
|
+
/**
|
|
170
|
+
* Return an independent handle for an ownership-taking API.
|
|
171
|
+
*
|
|
172
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
173
|
+
* handle must remain usable.
|
|
174
|
+
*/
|
|
175
|
+
copyForTransfer(): WasmBatchScrapeResult;
|
|
115
176
|
static default(): WasmBatchScrapeResult;
|
|
177
|
+
/**
|
|
178
|
+
* Takes ownership of `result`.
|
|
179
|
+
* wasm-bindgen lowers a by-value class argument through `__destroy_into_raw()`, so each
|
|
180
|
+
* of those handles is dead once this returns. Passing a freshly built value is fine. To
|
|
181
|
+
* keep a handle you already hold, build with `default()` and assign the property instead:
|
|
182
|
+
* the generated setter borrows and leaves your handle alive.
|
|
183
|
+
*/
|
|
116
184
|
constructor(url?: string | null, result?: WasmScrapeResult | null, error?: string | null);
|
|
117
185
|
get error(): string | undefined;
|
|
118
186
|
set error(value: string | null | undefined);
|
|
187
|
+
/**
|
|
188
|
+
* Returns a detached copy of `result`.
|
|
189
|
+
*
|
|
190
|
+
* Changes to the returned value do not update this object. To update `result`,
|
|
191
|
+
* read it, modify the copy, then assign the copy back to `result`.
|
|
192
|
+
*/
|
|
119
193
|
get result(): WasmScrapeResult | undefined;
|
|
120
|
-
set result(value: WasmScrapeResult
|
|
194
|
+
set result(value: WasmScrapeResult);
|
|
121
195
|
url: string;
|
|
122
196
|
}
|
|
123
197
|
|
|
@@ -130,10 +204,28 @@ export class WasmBatchScrapeResult {
|
|
|
130
204
|
export class WasmBatchScrapeResults {
|
|
131
205
|
free(): void;
|
|
132
206
|
[Symbol.dispose](): void;
|
|
207
|
+
/**
|
|
208
|
+
* Return an independent handle for an ownership-taking API.
|
|
209
|
+
*
|
|
210
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
211
|
+
* handle must remain usable.
|
|
212
|
+
*/
|
|
213
|
+
copyForTransfer(): WasmBatchScrapeResults;
|
|
133
214
|
static default(): WasmBatchScrapeResults;
|
|
215
|
+
/**
|
|
216
|
+
* Takes ownership of every element in `results`.
|
|
217
|
+
* wasm-bindgen cannot borrow class values nested in an array. To keep those handles,
|
|
218
|
+
* copy each retained element with `copyForTransfer()` before passing the array.
|
|
219
|
+
*/
|
|
134
220
|
constructor(results?: WasmBatchScrapeResult[] | null, totalCount?: number | null, completedCount?: number | null, failedCount?: number | null);
|
|
135
221
|
completedCount: number;
|
|
136
222
|
failedCount: number;
|
|
223
|
+
/**
|
|
224
|
+
* Returns detached copies of the elements in `results`.
|
|
225
|
+
*
|
|
226
|
+
* Changes to the returned values do not update this object. Modify the copies,
|
|
227
|
+
* then assign the array back to `results`.
|
|
228
|
+
*/
|
|
137
229
|
results: WasmBatchScrapeResult[];
|
|
138
230
|
totalCount: number;
|
|
139
231
|
}
|
|
@@ -152,12 +244,30 @@ export enum WasmBrowserBackend {
|
|
|
152
244
|
export class WasmBrowserConfig {
|
|
153
245
|
free(): void;
|
|
154
246
|
[Symbol.dispose](): void;
|
|
247
|
+
clearProxy(): void;
|
|
248
|
+
/**
|
|
249
|
+
* Return an independent handle for an ownership-taking API.
|
|
250
|
+
*
|
|
251
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
252
|
+
* handle must remain usable.
|
|
253
|
+
*/
|
|
254
|
+
copyForTransfer(): WasmBrowserConfig;
|
|
155
255
|
static default(): WasmBrowserConfig;
|
|
156
|
-
|
|
256
|
+
/**
|
|
257
|
+
* Takes ownership of `proxy`.
|
|
258
|
+
* wasm-bindgen lowers a by-value class argument through `__destroy_into_raw()`, so each
|
|
259
|
+
* of those handles is dead once this returns. Passing a freshly built value is fine. To
|
|
260
|
+
* keep a handle you already hold, build with `default()` and assign the property instead:
|
|
261
|
+
* the generated setter borrows and leaves your handle alive.
|
|
262
|
+
*/
|
|
263
|
+
constructor(mode?: WasmBrowserMode | null, backend?: WasmBrowserBackend | null, timeout?: bigint | null, overallTimeout?: bigint | null, shutdownTimeout?: bigint | null, wait?: WasmBrowserWait | null, blockUrlPatterns?: string[] | null, captureNetworkEvents?: boolean | null, sessionAffinity?: boolean | null, chromeArgs?: string[] | null, endpoint?: string | null, waitSelector?: string | null, extraWait?: bigint | null, proxy?: WasmProxyConfig | null, evalScript?: string | null, robotsUserAgent?: string | null, chromePath?: string | null);
|
|
157
264
|
get backend(): string;
|
|
158
265
|
set backend(value: WasmBrowserBackend);
|
|
159
266
|
blockUrlPatterns: string[];
|
|
160
267
|
captureNetworkEvents: boolean;
|
|
268
|
+
chromeArgs: string[];
|
|
269
|
+
get chromePath(): string | undefined;
|
|
270
|
+
set chromePath(value: string | null | undefined);
|
|
161
271
|
get endpoint(): string | undefined;
|
|
162
272
|
set endpoint(value: string | null | undefined);
|
|
163
273
|
get evalScript(): string | undefined;
|
|
@@ -168,8 +278,14 @@ export class WasmBrowserConfig {
|
|
|
168
278
|
set mode(value: WasmBrowserMode);
|
|
169
279
|
get overallTimeout(): bigint | undefined;
|
|
170
280
|
set overallTimeout(value: bigint | null | undefined);
|
|
281
|
+
/**
|
|
282
|
+
* Returns a detached copy of `proxy`.
|
|
283
|
+
*
|
|
284
|
+
* Changes to the returned value do not update this object. To update `proxy`,
|
|
285
|
+
* read it, modify the copy, then assign the copy back to `proxy`.
|
|
286
|
+
*/
|
|
171
287
|
get proxy(): WasmProxyConfig | undefined;
|
|
172
|
-
set proxy(value: WasmProxyConfig
|
|
288
|
+
set proxy(value: WasmProxyConfig);
|
|
173
289
|
get robotsUserAgent(): string | undefined;
|
|
174
290
|
set robotsUserAgent(value: string | null | undefined);
|
|
175
291
|
sessionAffinity: boolean;
|
|
@@ -191,11 +307,35 @@ export class WasmBrowserConfig {
|
|
|
191
307
|
export class WasmBrowserExtras {
|
|
192
308
|
free(): void;
|
|
193
309
|
[Symbol.dispose](): void;
|
|
310
|
+
/**
|
|
311
|
+
* Return an independent handle for an ownership-taking API.
|
|
312
|
+
*
|
|
313
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
314
|
+
* handle must remain usable.
|
|
315
|
+
*/
|
|
316
|
+
copyForTransfer(): WasmBrowserExtras;
|
|
194
317
|
static default(): WasmBrowserExtras;
|
|
318
|
+
/**
|
|
319
|
+
* Takes ownership of every element in `networkEvents`, `cookies`.
|
|
320
|
+
* wasm-bindgen cannot borrow class values nested in an array. To keep those handles,
|
|
321
|
+
* copy each retained element with `copyForTransfer()` before passing the array.
|
|
322
|
+
*/
|
|
195
323
|
constructor(networkEvents?: WasmResponseMeta[] | null, cookies?: WasmCookieInfo[] | null, evalResult?: any | null);
|
|
324
|
+
/**
|
|
325
|
+
* Returns detached copies of the elements in `cookies`.
|
|
326
|
+
*
|
|
327
|
+
* Changes to the returned values do not update this object. Modify the copies,
|
|
328
|
+
* then assign the array back to `cookies`.
|
|
329
|
+
*/
|
|
196
330
|
cookies: WasmCookieInfo[];
|
|
197
331
|
get evalResult(): any | undefined;
|
|
198
332
|
set evalResult(value: any | null | undefined);
|
|
333
|
+
/**
|
|
334
|
+
* Returns detached copies of the elements in `networkEvents`.
|
|
335
|
+
*
|
|
336
|
+
* Changes to the returned values do not update this object. Modify the copies,
|
|
337
|
+
* then assign the array back to `networkEvents`.
|
|
338
|
+
*/
|
|
199
339
|
networkEvents: WasmResponseMeta[];
|
|
200
340
|
}
|
|
201
341
|
|
|
@@ -225,6 +365,13 @@ export enum WasmBrowserWait {
|
|
|
225
365
|
export class WasmCitationReference {
|
|
226
366
|
free(): void;
|
|
227
367
|
[Symbol.dispose](): void;
|
|
368
|
+
/**
|
|
369
|
+
* Return an independent handle for an ownership-taking API.
|
|
370
|
+
*
|
|
371
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
372
|
+
* handle must remain usable.
|
|
373
|
+
*/
|
|
374
|
+
copyForTransfer(): WasmCitationReference;
|
|
228
375
|
static default(): WasmCitationReference;
|
|
229
376
|
constructor(index?: number | null, url?: string | null, text?: string | null);
|
|
230
377
|
index: number;
|
|
@@ -238,9 +385,27 @@ export class WasmCitationReference {
|
|
|
238
385
|
export class WasmCitationResult {
|
|
239
386
|
free(): void;
|
|
240
387
|
[Symbol.dispose](): void;
|
|
388
|
+
/**
|
|
389
|
+
* Return an independent handle for an ownership-taking API.
|
|
390
|
+
*
|
|
391
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
392
|
+
* handle must remain usable.
|
|
393
|
+
*/
|
|
394
|
+
copyForTransfer(): WasmCitationResult;
|
|
241
395
|
static default(): WasmCitationResult;
|
|
396
|
+
/**
|
|
397
|
+
* Takes ownership of every element in `references`.
|
|
398
|
+
* wasm-bindgen cannot borrow class values nested in an array. To keep those handles,
|
|
399
|
+
* copy each retained element with `copyForTransfer()` before passing the array.
|
|
400
|
+
*/
|
|
242
401
|
constructor(content?: string | null, references?: WasmCitationReference[] | null);
|
|
243
402
|
content: string;
|
|
403
|
+
/**
|
|
404
|
+
* Returns detached copies of the elements in `references`.
|
|
405
|
+
*
|
|
406
|
+
* Changes to the returned values do not update this object. Modify the copies,
|
|
407
|
+
* then assign the array back to `references`.
|
|
408
|
+
*/
|
|
244
409
|
references: WasmCitationReference[];
|
|
245
410
|
}
|
|
246
411
|
|
|
@@ -254,6 +419,13 @@ export class WasmCitationResult {
|
|
|
254
419
|
export class WasmContentConfig {
|
|
255
420
|
free(): void;
|
|
256
421
|
[Symbol.dispose](): void;
|
|
422
|
+
/**
|
|
423
|
+
* Return an independent handle for an ownership-taking API.
|
|
424
|
+
*
|
|
425
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
426
|
+
* handle must remain usable.
|
|
427
|
+
*/
|
|
428
|
+
copyForTransfer(): WasmContentConfig;
|
|
257
429
|
static default(): WasmContentConfig;
|
|
258
430
|
constructor(outputFormat?: string | null, preprocessingPreset?: string | null, removeNavigation?: boolean | null, removeForms?: boolean | null, stripTags?: string[] | null, preserveTags?: string[] | null, excludeSelectors?: string[] | null, skipImages?: boolean | null, wrap?: boolean | null, wrapWidth?: number | null, includeDocumentStructure?: boolean | null, extractMetadata?: boolean | null, maxDepth?: number | null);
|
|
259
431
|
excludeSelectors: string[];
|
|
@@ -285,6 +457,13 @@ export enum WasmContentFilterKind {
|
|
|
285
457
|
export class WasmCookieInfo {
|
|
286
458
|
free(): void;
|
|
287
459
|
[Symbol.dispose](): void;
|
|
460
|
+
/**
|
|
461
|
+
* Return an independent handle for an ownership-taking API.
|
|
462
|
+
*
|
|
463
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
464
|
+
* handle must remain usable.
|
|
465
|
+
*/
|
|
466
|
+
copyForTransfer(): WasmCookieInfo;
|
|
288
467
|
static default(): WasmCookieInfo;
|
|
289
468
|
constructor(name?: string | null, value?: string | null, domain?: string | null, path?: string | null);
|
|
290
469
|
get domain(): string | undefined;
|
|
@@ -301,8 +480,23 @@ export class WasmCookieInfo {
|
|
|
301
480
|
export class WasmCrawlConfig {
|
|
302
481
|
free(): void;
|
|
303
482
|
[Symbol.dispose](): void;
|
|
483
|
+
clearProxy(): void;
|
|
484
|
+
/**
|
|
485
|
+
* Return an independent handle for an ownership-taking API.
|
|
486
|
+
*
|
|
487
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
488
|
+
* handle must remain usable.
|
|
489
|
+
*/
|
|
490
|
+
copyForTransfer(): WasmCrawlConfig;
|
|
304
491
|
static default(): WasmCrawlConfig;
|
|
305
|
-
|
|
492
|
+
/**
|
|
493
|
+
* Takes ownership of `content`, `browser`, `proxy`, `ssrf`.
|
|
494
|
+
* wasm-bindgen lowers a by-value class argument through `__destroy_into_raw()`, so each
|
|
495
|
+
* of those handles is dead once this returns. Passing a freshly built value is fine. To
|
|
496
|
+
* keep a handle you already hold, build with `default()` and assign the property instead:
|
|
497
|
+
* the generated setter borrows and leaves your handle alive.
|
|
498
|
+
*/
|
|
499
|
+
constructor(crawlStrategy?: WasmCrawlStrategyKind | null, respectRobotsTxt?: boolean | null, softHttpErrors?: boolean | null, stayOnDomain?: boolean | null, allowSubdomains?: boolean | null, includePaths?: string[] | null, excludePaths?: string[] | null, pathPatternsMatchQuery?: boolean | null, pathPatternsMatchUrl?: boolean | null, dedupIncludeQuery?: boolean | null, stripTrackingParams?: boolean | null, trackingParams?: string[] | null, customHeaders?: any | null, requestTimeout?: bigint | null, maxRedirects?: number | null, retryCount?: number | null, retryCodes?: Uint16Array | null, retryInitialDelayMs?: bigint | null, retryMaxDelayMs?: bigint | null, rateLimitJitterRatio?: number | null, cookiesEnabled?: boolean | null, removeTags?: string[] | null, content?: WasmContentConfig | null, downloadAssets?: boolean | null, assetTypes?: WasmAssetCategory[] | null, browser?: WasmBrowserConfig | null, userAgents?: string[] | null, captureScreenshot?: boolean | null, followDocumentUrls?: boolean | null, downloadDocuments?: boolean | null, documentMimeTypes?: string[] | null, saveBrowserProfile?: boolean | null, ssrf?: WasmSsrfPolicy | null, maxDepth?: number | null, maxPages?: number | null, maxLinksPerPage?: number | null, maxConcurrent?: number | null, contentFilter?: WasmContentFilterKind | null, bm25Query?: string | null, bm25Threshold?: number | null, userAgent?: string | null, rateLimitMs?: bigint | null, auth?: any | null, maxBodySize?: number | null, mapLimit?: number | null, mapSearch?: string | null, maxAssetSize?: number | null, proxy?: WasmProxyConfig | null, documentUrlDepth?: number | null, documentMaxSize?: number | null, documentOutputDir?: string | null, documentContentEncoding?: WasmDocumentContentEncoding | null, warcOutput?: string | null, browserProfile?: string | null, ssrfDenyPrivateExplicit?: boolean | null);
|
|
306
500
|
/**
|
|
307
501
|
* Validate the configuration, returning an error if any values are invalid.
|
|
308
502
|
*
|
|
@@ -320,10 +514,22 @@ export class WasmCrawlConfig {
|
|
|
320
514
|
set bm25Threshold(value: number | null | undefined);
|
|
321
515
|
get browserProfile(): string | undefined;
|
|
322
516
|
set browserProfile(value: string | null | undefined);
|
|
517
|
+
/**
|
|
518
|
+
* Returns a detached copy of `browser`.
|
|
519
|
+
*
|
|
520
|
+
* Changes to the returned value do not update this object. To update `browser`,
|
|
521
|
+
* read it, modify the copy, then assign the copy back to `browser`.
|
|
522
|
+
*/
|
|
323
523
|
browser: WasmBrowserConfig;
|
|
324
524
|
captureScreenshot: boolean;
|
|
325
525
|
get contentFilter(): string | undefined;
|
|
326
526
|
set contentFilter(value: WasmContentFilterKind | null | undefined);
|
|
527
|
+
/**
|
|
528
|
+
* Returns a detached copy of `content`.
|
|
529
|
+
*
|
|
530
|
+
* Changes to the returned value do not update this object. To update `content`,
|
|
531
|
+
* read it, modify the copy, then assign the copy back to `content`.
|
|
532
|
+
*/
|
|
327
533
|
content: WasmContentConfig;
|
|
328
534
|
cookiesEnabled: boolean;
|
|
329
535
|
get crawlStrategy(): string;
|
|
@@ -362,8 +568,15 @@ export class WasmCrawlConfig {
|
|
|
362
568
|
set maxPages(value: number | null | undefined);
|
|
363
569
|
maxRedirects: number;
|
|
364
570
|
pathPatternsMatchQuery: boolean;
|
|
571
|
+
pathPatternsMatchUrl: boolean;
|
|
572
|
+
/**
|
|
573
|
+
* Returns a detached copy of `proxy`.
|
|
574
|
+
*
|
|
575
|
+
* Changes to the returned value do not update this object. To update `proxy`,
|
|
576
|
+
* read it, modify the copy, then assign the copy back to `proxy`.
|
|
577
|
+
*/
|
|
365
578
|
get proxy(): WasmProxyConfig | undefined;
|
|
366
|
-
set proxy(value: WasmProxyConfig
|
|
579
|
+
set proxy(value: WasmProxyConfig);
|
|
367
580
|
rateLimitJitterRatio: number;
|
|
368
581
|
get rateLimitMs(): bigint | undefined;
|
|
369
582
|
set rateLimitMs(value: bigint | null | undefined);
|
|
@@ -379,6 +592,12 @@ export class WasmCrawlConfig {
|
|
|
379
592
|
softHttpErrors: boolean;
|
|
380
593
|
get ssrfDenyPrivateExplicit(): boolean | undefined;
|
|
381
594
|
set ssrfDenyPrivateExplicit(value: boolean | null | undefined);
|
|
595
|
+
/**
|
|
596
|
+
* Returns a detached copy of `ssrf`.
|
|
597
|
+
*
|
|
598
|
+
* Changes to the returned value do not update this object. To update `ssrf`,
|
|
599
|
+
* read it, modify the copy, then assign the copy back to `ssrf`.
|
|
600
|
+
*/
|
|
382
601
|
ssrf: WasmSsrfPolicy;
|
|
383
602
|
stayOnDomain: boolean;
|
|
384
603
|
stripTrackingParams: boolean;
|
|
@@ -400,6 +619,13 @@ export class WasmCrawlEngineHandle {
|
|
|
400
619
|
private constructor();
|
|
401
620
|
free(): void;
|
|
402
621
|
[Symbol.dispose](): void;
|
|
622
|
+
/**
|
|
623
|
+
* Return an independent handle for an ownership-taking API.
|
|
624
|
+
*
|
|
625
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
626
|
+
* handle must remain usable.
|
|
627
|
+
*/
|
|
628
|
+
copyForTransfer(): WasmCrawlEngineHandle;
|
|
403
629
|
}
|
|
404
630
|
|
|
405
631
|
/**
|
|
@@ -408,34 +634,103 @@ export class WasmCrawlEngineHandle {
|
|
|
408
634
|
export class WasmCrawlPageResult {
|
|
409
635
|
free(): void;
|
|
410
636
|
[Symbol.dispose](): void;
|
|
637
|
+
clearDownloadedDocument(): void;
|
|
638
|
+
clearExtractionMeta(): void;
|
|
639
|
+
clearMarkdown(): void;
|
|
640
|
+
/**
|
|
641
|
+
* Return an independent handle for an ownership-taking API.
|
|
642
|
+
*
|
|
643
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
644
|
+
* handle must remain usable.
|
|
645
|
+
*/
|
|
646
|
+
copyForTransfer(): WasmCrawlPageResult;
|
|
411
647
|
static default(): WasmCrawlPageResult;
|
|
412
|
-
|
|
648
|
+
/**
|
|
649
|
+
* Takes ownership of `metadata`, `markdown`, `extractionMeta`, `downloadedDocument`.
|
|
650
|
+
* wasm-bindgen lowers a by-value class argument through `__destroy_into_raw()`, so each
|
|
651
|
+
* of those handles is dead once this returns. Passing a freshly built value is fine. To
|
|
652
|
+
* keep a handle you already hold, build with `default()` and assign the property instead:
|
|
653
|
+
* the generated setter borrows and leaves your handle alive.
|
|
654
|
+
* Takes ownership of every element in `links`, `images`, `feeds`, `jsonLd`.
|
|
655
|
+
* wasm-bindgen cannot borrow class values nested in an array. To keep those handles,
|
|
656
|
+
* copy each retained element with `copyForTransfer()` before passing the array.
|
|
657
|
+
*/
|
|
658
|
+
constructor(url?: string | null, normalizedUrl?: string | null, statusCode?: number | null, contentType?: string | null, html?: string | null, bodySize?: number | null, metadata?: WasmPageMetadata | null, links?: WasmLinkInfo[] | null, images?: WasmImageInfo[] | null, feeds?: WasmFeedInfo[] | null, jsonLd?: WasmJsonLdEntry[] | null, depth?: number | null, stayedOnDomain?: boolean | null, wasSkipped?: boolean | null, isPdf?: boolean | null, browserUsed?: boolean | null, finalUrl?: string | null, redirectCount?: number | null, noindexDetected?: boolean | null, nofollowDetected?: boolean | null, ssrfRefusedUrls?: string[] | null, detectedCharset?: string | null, markdown?: WasmMarkdownResult | null, extractedData?: any | null, extractionMeta?: WasmExtractionMeta | null, downloadedDocument?: WasmDownloadedDocument | null);
|
|
413
659
|
bodySize: number;
|
|
414
660
|
browserUsed: boolean;
|
|
415
661
|
contentType: string;
|
|
416
662
|
depth: number;
|
|
417
663
|
get detectedCharset(): string | undefined;
|
|
418
664
|
set detectedCharset(value: string | null | undefined);
|
|
665
|
+
/**
|
|
666
|
+
* Returns a detached copy of `downloadedDocument`.
|
|
667
|
+
*
|
|
668
|
+
* Changes to the returned value do not update this object. To update `downloadedDocument`,
|
|
669
|
+
* read it, modify the copy, then assign the copy back to `downloadedDocument`.
|
|
670
|
+
*/
|
|
419
671
|
get downloadedDocument(): WasmDownloadedDocument | undefined;
|
|
420
|
-
set downloadedDocument(value: WasmDownloadedDocument
|
|
672
|
+
set downloadedDocument(value: WasmDownloadedDocument);
|
|
421
673
|
get extractedData(): any | undefined;
|
|
422
674
|
set extractedData(value: any | null | undefined);
|
|
675
|
+
/**
|
|
676
|
+
* Returns a detached copy of `extractionMeta`.
|
|
677
|
+
*
|
|
678
|
+
* Changes to the returned value do not update this object. To update `extractionMeta`,
|
|
679
|
+
* read it, modify the copy, then assign the copy back to `extractionMeta`.
|
|
680
|
+
*/
|
|
423
681
|
get extractionMeta(): WasmExtractionMeta | undefined;
|
|
424
|
-
set extractionMeta(value: WasmExtractionMeta
|
|
682
|
+
set extractionMeta(value: WasmExtractionMeta);
|
|
683
|
+
/**
|
|
684
|
+
* Returns detached copies of the elements in `feeds`.
|
|
685
|
+
*
|
|
686
|
+
* Changes to the returned values do not update this object. Modify the copies,
|
|
687
|
+
* then assign the array back to `feeds`.
|
|
688
|
+
*/
|
|
425
689
|
feeds: WasmFeedInfo[];
|
|
426
690
|
finalUrl: string;
|
|
427
691
|
html: string;
|
|
692
|
+
/**
|
|
693
|
+
* Returns detached copies of the elements in `images`.
|
|
694
|
+
*
|
|
695
|
+
* Changes to the returned values do not update this object. Modify the copies,
|
|
696
|
+
* then assign the array back to `images`.
|
|
697
|
+
*/
|
|
428
698
|
images: WasmImageInfo[];
|
|
429
699
|
isPdf: boolean;
|
|
700
|
+
/**
|
|
701
|
+
* Returns detached copies of the elements in `jsonLd`.
|
|
702
|
+
*
|
|
703
|
+
* Changes to the returned values do not update this object. Modify the copies,
|
|
704
|
+
* then assign the array back to `jsonLd`.
|
|
705
|
+
*/
|
|
430
706
|
jsonLd: WasmJsonLdEntry[];
|
|
707
|
+
/**
|
|
708
|
+
* Returns detached copies of the elements in `links`.
|
|
709
|
+
*
|
|
710
|
+
* Changes to the returned values do not update this object. Modify the copies,
|
|
711
|
+
* then assign the array back to `links`.
|
|
712
|
+
*/
|
|
431
713
|
links: WasmLinkInfo[];
|
|
714
|
+
/**
|
|
715
|
+
* Returns a detached copy of `markdown`.
|
|
716
|
+
*
|
|
717
|
+
* Changes to the returned value do not update this object. To update `markdown`,
|
|
718
|
+
* read it, modify the copy, then assign the copy back to `markdown`.
|
|
719
|
+
*/
|
|
432
720
|
get markdown(): WasmMarkdownResult | undefined;
|
|
433
|
-
set markdown(value: WasmMarkdownResult
|
|
721
|
+
set markdown(value: WasmMarkdownResult);
|
|
722
|
+
/**
|
|
723
|
+
* Returns a detached copy of `metadata`.
|
|
724
|
+
*
|
|
725
|
+
* Changes to the returned value do not update this object. To update `metadata`,
|
|
726
|
+
* read it, modify the copy, then assign the copy back to `metadata`.
|
|
727
|
+
*/
|
|
434
728
|
metadata: WasmPageMetadata;
|
|
435
729
|
nofollowDetected: boolean;
|
|
436
730
|
noindexDetected: boolean;
|
|
437
731
|
normalizedUrl: string;
|
|
438
732
|
redirectCount: number;
|
|
733
|
+
ssrfRefusedUrls: string[];
|
|
439
734
|
statusCode: number;
|
|
440
735
|
stayedOnDomain: boolean;
|
|
441
736
|
url: string;
|
|
@@ -448,7 +743,19 @@ export class WasmCrawlPageResult {
|
|
|
448
743
|
export class WasmCrawlResult {
|
|
449
744
|
free(): void;
|
|
450
745
|
[Symbol.dispose](): void;
|
|
746
|
+
/**
|
|
747
|
+
* Return an independent handle for an ownership-taking API.
|
|
748
|
+
*
|
|
749
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
750
|
+
* handle must remain usable.
|
|
751
|
+
*/
|
|
752
|
+
copyForTransfer(): WasmCrawlResult;
|
|
451
753
|
static default(): WasmCrawlResult;
|
|
754
|
+
/**
|
|
755
|
+
* Takes ownership of every element in `pages`, `cookies`.
|
|
756
|
+
* wasm-bindgen cannot borrow class values nested in an array. To keep those handles,
|
|
757
|
+
* copy each retained element with `copyForTransfer()` before passing the array.
|
|
758
|
+
*/
|
|
452
759
|
constructor(pages?: WasmCrawlPageResult[] | null, finalUrl?: string | null, redirectCount?: number | null, wasSkipped?: boolean | null, cookies?: WasmCookieInfo[] | null, stayedOnDomain?: boolean | null, browserUsed?: boolean | null, error?: string | null);
|
|
453
760
|
/**
|
|
454
761
|
* Returns the count of unique normalized URLs encountered during crawling.
|
|
@@ -462,10 +769,22 @@ export class WasmCrawlResult {
|
|
|
462
769
|
*/
|
|
463
770
|
uniqueNormalizedUrls(): number;
|
|
464
771
|
browserUsed: boolean;
|
|
772
|
+
/**
|
|
773
|
+
* Returns detached copies of the elements in `cookies`.
|
|
774
|
+
*
|
|
775
|
+
* Changes to the returned values do not update this object. Modify the copies,
|
|
776
|
+
* then assign the array back to `cookies`.
|
|
777
|
+
*/
|
|
465
778
|
cookies: WasmCookieInfo[];
|
|
466
779
|
get error(): string | undefined;
|
|
467
780
|
set error(value: string | null | undefined);
|
|
468
781
|
finalUrl: string;
|
|
782
|
+
/**
|
|
783
|
+
* Returns detached copies of the elements in `pages`.
|
|
784
|
+
*
|
|
785
|
+
* Changes to the returned values do not update this object. Modify the copies,
|
|
786
|
+
* then assign the array back to `pages`.
|
|
787
|
+
*/
|
|
469
788
|
pages: WasmCrawlPageResult[];
|
|
470
789
|
redirectCount: number;
|
|
471
790
|
stayedOnDomain: boolean;
|
|
@@ -504,6 +823,13 @@ export enum WasmDocumentContentEncoding {
|
|
|
504
823
|
export class WasmDownloadedAsset {
|
|
505
824
|
free(): void;
|
|
506
825
|
[Symbol.dispose](): void;
|
|
826
|
+
/**
|
|
827
|
+
* Return an independent handle for an ownership-taking API.
|
|
828
|
+
*
|
|
829
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
830
|
+
* handle must remain usable.
|
|
831
|
+
*/
|
|
832
|
+
copyForTransfer(): WasmDownloadedAsset;
|
|
507
833
|
static default(): WasmDownloadedAsset;
|
|
508
834
|
constructor(url?: string | null, contentHash?: string | null, size?: number | null, assetCategory?: WasmAssetCategory | null, mimeType?: string | null, htmlTag?: string | null);
|
|
509
835
|
get assetCategory(): string;
|
|
@@ -527,6 +853,13 @@ export class WasmDownloadedAsset {
|
|
|
527
853
|
export class WasmDownloadedDocument {
|
|
528
854
|
free(): void;
|
|
529
855
|
[Symbol.dispose](): void;
|
|
856
|
+
/**
|
|
857
|
+
* Return an independent handle for an ownership-taking API.
|
|
858
|
+
*
|
|
859
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
860
|
+
* handle must remain usable.
|
|
861
|
+
*/
|
|
862
|
+
copyForTransfer(): WasmDownloadedDocument;
|
|
530
863
|
static default(): WasmDownloadedDocument;
|
|
531
864
|
constructor(url?: string | null, mimeType?: string | null, size?: number | null, contentHash?: string | null, headers?: any | null, truncated?: boolean | null, filename?: string | null, contentPath?: string | null, contentBase64?: string | null);
|
|
532
865
|
get contentBase64(): string | undefined;
|
|
@@ -549,6 +882,13 @@ export class WasmDownloadedDocument {
|
|
|
549
882
|
export class WasmExtractionMeta {
|
|
550
883
|
free(): void;
|
|
551
884
|
[Symbol.dispose](): void;
|
|
885
|
+
/**
|
|
886
|
+
* Return an independent handle for an ownership-taking API.
|
|
887
|
+
*
|
|
888
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
889
|
+
* handle must remain usable.
|
|
890
|
+
*/
|
|
891
|
+
copyForTransfer(): WasmExtractionMeta;
|
|
552
892
|
static default(): WasmExtractionMeta;
|
|
553
893
|
constructor(chunksProcessed?: number | null, cost?: number | null, promptTokens?: bigint | null, completionTokens?: bigint | null, model?: string | null);
|
|
554
894
|
chunksProcessed: number;
|
|
@@ -568,6 +908,13 @@ export class WasmExtractionMeta {
|
|
|
568
908
|
export class WasmFaviconInfo {
|
|
569
909
|
free(): void;
|
|
570
910
|
[Symbol.dispose](): void;
|
|
911
|
+
/**
|
|
912
|
+
* Return an independent handle for an ownership-taking API.
|
|
913
|
+
*
|
|
914
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
915
|
+
* handle must remain usable.
|
|
916
|
+
*/
|
|
917
|
+
copyForTransfer(): WasmFaviconInfo;
|
|
571
918
|
static default(): WasmFaviconInfo;
|
|
572
919
|
constructor(url?: string | null, rel?: string | null, sizes?: string | null, mimeType?: string | null);
|
|
573
920
|
get mimeType(): string | undefined;
|
|
@@ -584,6 +931,13 @@ export class WasmFaviconInfo {
|
|
|
584
931
|
export class WasmFeedInfo {
|
|
585
932
|
free(): void;
|
|
586
933
|
[Symbol.dispose](): void;
|
|
934
|
+
/**
|
|
935
|
+
* Return an independent handle for an ownership-taking API.
|
|
936
|
+
*
|
|
937
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
938
|
+
* handle must remain usable.
|
|
939
|
+
*/
|
|
940
|
+
copyForTransfer(): WasmFeedInfo;
|
|
587
941
|
static default(): WasmFeedInfo;
|
|
588
942
|
constructor(url?: string | null, feedType?: WasmFeedType | null, title?: string | null);
|
|
589
943
|
get feedType(): string;
|
|
@@ -608,6 +962,13 @@ export enum WasmFeedType {
|
|
|
608
962
|
export class WasmHeadingInfo {
|
|
609
963
|
free(): void;
|
|
610
964
|
[Symbol.dispose](): void;
|
|
965
|
+
/**
|
|
966
|
+
* Return an independent handle for an ownership-taking API.
|
|
967
|
+
*
|
|
968
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
969
|
+
* handle must remain usable.
|
|
970
|
+
*/
|
|
971
|
+
copyForTransfer(): WasmHeadingInfo;
|
|
611
972
|
static default(): WasmHeadingInfo;
|
|
612
973
|
constructor(level?: number | null, text?: string | null);
|
|
613
974
|
level: number;
|
|
@@ -647,6 +1008,13 @@ export class WasmHostMatcher {
|
|
|
647
1008
|
export class WasmHreflangEntry {
|
|
648
1009
|
free(): void;
|
|
649
1010
|
[Symbol.dispose](): void;
|
|
1011
|
+
/**
|
|
1012
|
+
* Return an independent handle for an ownership-taking API.
|
|
1013
|
+
*
|
|
1014
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
1015
|
+
* handle must remain usable.
|
|
1016
|
+
*/
|
|
1017
|
+
copyForTransfer(): WasmHreflangEntry;
|
|
650
1018
|
static default(): WasmHreflangEntry;
|
|
651
1019
|
constructor(lang?: string | null, url?: string | null);
|
|
652
1020
|
lang: string;
|
|
@@ -659,6 +1027,13 @@ export class WasmHreflangEntry {
|
|
|
659
1027
|
export class WasmImageInfo {
|
|
660
1028
|
free(): void;
|
|
661
1029
|
[Symbol.dispose](): void;
|
|
1030
|
+
/**
|
|
1031
|
+
* Return an independent handle for an ownership-taking API.
|
|
1032
|
+
*
|
|
1033
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
1034
|
+
* handle must remain usable.
|
|
1035
|
+
*/
|
|
1036
|
+
copyForTransfer(): WasmImageInfo;
|
|
662
1037
|
static default(): WasmImageInfo;
|
|
663
1038
|
constructor(url?: string | null, source?: WasmImageSource | null, alt?: string | null, width?: number | null, height?: number | null);
|
|
664
1039
|
get alt(): string | undefined;
|
|
@@ -688,13 +1063,32 @@ export enum WasmImageSource {
|
|
|
688
1063
|
export class WasmInteractionResult {
|
|
689
1064
|
free(): void;
|
|
690
1065
|
[Symbol.dispose](): void;
|
|
1066
|
+
/**
|
|
1067
|
+
* Return an independent handle for an ownership-taking API.
|
|
1068
|
+
*
|
|
1069
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
1070
|
+
* handle must remain usable.
|
|
1071
|
+
*/
|
|
1072
|
+
copyForTransfer(): WasmInteractionResult;
|
|
691
1073
|
static default(): WasmInteractionResult;
|
|
692
|
-
|
|
1074
|
+
/**
|
|
1075
|
+
* Takes ownership of every element in `actionResults`.
|
|
1076
|
+
* wasm-bindgen cannot borrow class values nested in an array. To keep those handles,
|
|
1077
|
+
* copy each retained element with `copyForTransfer()` before passing the array.
|
|
1078
|
+
*/
|
|
1079
|
+
constructor(actionResults?: WasmActionResult[] | null, finalHtml?: string | null, finalUrl?: string | null, ssrfRefusedUrls?: string[] | null, screenshotBase64?: string | null);
|
|
1080
|
+
/**
|
|
1081
|
+
* Returns detached copies of the elements in `actionResults`.
|
|
1082
|
+
*
|
|
1083
|
+
* Changes to the returned values do not update this object. Modify the copies,
|
|
1084
|
+
* then assign the array back to `actionResults`.
|
|
1085
|
+
*/
|
|
693
1086
|
actionResults: WasmActionResult[];
|
|
694
1087
|
finalHtml: string;
|
|
695
1088
|
finalUrl: string;
|
|
696
1089
|
get screenshotBase64(): string | undefined;
|
|
697
1090
|
set screenshotBase64(value: string | null | undefined);
|
|
1091
|
+
ssrfRefusedUrls: string[];
|
|
698
1092
|
}
|
|
699
1093
|
|
|
700
1094
|
/**
|
|
@@ -703,6 +1097,13 @@ export class WasmInteractionResult {
|
|
|
703
1097
|
export class WasmJsonLdEntry {
|
|
704
1098
|
free(): void;
|
|
705
1099
|
[Symbol.dispose](): void;
|
|
1100
|
+
/**
|
|
1101
|
+
* Return an independent handle for an ownership-taking API.
|
|
1102
|
+
*
|
|
1103
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
1104
|
+
* handle must remain usable.
|
|
1105
|
+
*/
|
|
1106
|
+
copyForTransfer(): WasmJsonLdEntry;
|
|
706
1107
|
static default(): WasmJsonLdEntry;
|
|
707
1108
|
constructor(schemaType?: string | null, raw?: string | null, name?: string | null);
|
|
708
1109
|
get name(): string | undefined;
|
|
@@ -717,6 +1118,13 @@ export class WasmJsonLdEntry {
|
|
|
717
1118
|
export class WasmLinkInfo {
|
|
718
1119
|
free(): void;
|
|
719
1120
|
[Symbol.dispose](): void;
|
|
1121
|
+
/**
|
|
1122
|
+
* Return an independent handle for an ownership-taking API.
|
|
1123
|
+
*
|
|
1124
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
1125
|
+
* handle must remain usable.
|
|
1126
|
+
*/
|
|
1127
|
+
copyForTransfer(): WasmLinkInfo;
|
|
720
1128
|
static default(): WasmLinkInfo;
|
|
721
1129
|
constructor(url?: string | null, text?: string | null, linkType?: WasmLinkType | null, nofollow?: boolean | null, rel?: string | null);
|
|
722
1130
|
get linkType(): string;
|
|
@@ -744,8 +1152,26 @@ export enum WasmLinkType {
|
|
|
744
1152
|
export class WasmMapResult {
|
|
745
1153
|
free(): void;
|
|
746
1154
|
[Symbol.dispose](): void;
|
|
1155
|
+
/**
|
|
1156
|
+
* Return an independent handle for an ownership-taking API.
|
|
1157
|
+
*
|
|
1158
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
1159
|
+
* handle must remain usable.
|
|
1160
|
+
*/
|
|
1161
|
+
copyForTransfer(): WasmMapResult;
|
|
747
1162
|
static default(): WasmMapResult;
|
|
1163
|
+
/**
|
|
1164
|
+
* Takes ownership of every element in `urls`.
|
|
1165
|
+
* wasm-bindgen cannot borrow class values nested in an array. To keep those handles,
|
|
1166
|
+
* copy each retained element with `copyForTransfer()` before passing the array.
|
|
1167
|
+
*/
|
|
748
1168
|
constructor(urls?: WasmSitemapUrl[] | null);
|
|
1169
|
+
/**
|
|
1170
|
+
* Returns detached copies of the elements in `urls`.
|
|
1171
|
+
*
|
|
1172
|
+
* Changes to the returned values do not update this object. Modify the copies,
|
|
1173
|
+
* then assign the array back to `urls`.
|
|
1174
|
+
*/
|
|
749
1175
|
urls: WasmSitemapUrl[];
|
|
750
1176
|
}
|
|
751
1177
|
|
|
@@ -755,6 +1181,13 @@ export class WasmMapResult {
|
|
|
755
1181
|
export class WasmMarkdownResult {
|
|
756
1182
|
free(): void;
|
|
757
1183
|
[Symbol.dispose](): void;
|
|
1184
|
+
/**
|
|
1185
|
+
* Return an independent handle for an ownership-taking API.
|
|
1186
|
+
*
|
|
1187
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
1188
|
+
* handle must remain usable.
|
|
1189
|
+
*/
|
|
1190
|
+
copyForTransfer(): WasmMarkdownResult;
|
|
758
1191
|
static default(): WasmMarkdownResult;
|
|
759
1192
|
constructor(content?: string | null, tables?: any | null, warnings?: string[] | null, citations?: boolean | null, documentStructure?: any | null, fitContent?: string | null);
|
|
760
1193
|
citations: boolean;
|
|
@@ -803,10 +1236,34 @@ export class WasmPageAction {
|
|
|
803
1236
|
export class WasmPageMetadata {
|
|
804
1237
|
free(): void;
|
|
805
1238
|
[Symbol.dispose](): void;
|
|
1239
|
+
clearArticle(): void;
|
|
1240
|
+
/**
|
|
1241
|
+
* Return an independent handle for an ownership-taking API.
|
|
1242
|
+
*
|
|
1243
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
1244
|
+
* handle must remain usable.
|
|
1245
|
+
*/
|
|
1246
|
+
copyForTransfer(): WasmPageMetadata;
|
|
806
1247
|
static default(): WasmPageMetadata;
|
|
1248
|
+
/**
|
|
1249
|
+
* Takes ownership of `article`.
|
|
1250
|
+
* wasm-bindgen lowers a by-value class argument through `__destroy_into_raw()`, so each
|
|
1251
|
+
* of those handles is dead once this returns. Passing a freshly built value is fine. To
|
|
1252
|
+
* keep a handle you already hold, build with `default()` and assign the property instead:
|
|
1253
|
+
* the generated setter borrows and leaves your handle alive.
|
|
1254
|
+
* Takes ownership of every element in `hreflangs`, `favicons`, `headings`.
|
|
1255
|
+
* wasm-bindgen cannot borrow class values nested in an array. To keep those handles,
|
|
1256
|
+
* copy each retained element with `copyForTransfer()` before passing the array.
|
|
1257
|
+
*/
|
|
807
1258
|
constructor(title?: string | null, description?: string | null, canonicalUrl?: string | null, keywords?: string | null, author?: string | null, viewport?: string | null, themeColor?: string | null, generator?: string | null, robots?: string | null, htmlLang?: string | null, htmlDir?: string | null, ogTitle?: string | null, ogType?: string | null, ogImage?: string | null, ogDescription?: string | null, ogUrl?: string | null, ogSiteName?: string | null, ogLocale?: string | null, ogVideo?: string | null, ogAudio?: string | null, ogLocaleAlternates?: string[] | null, twitterCard?: string | null, twitterTitle?: string | null, twitterDescription?: string | null, twitterImage?: string | null, twitterSite?: string | null, twitterCreator?: string | null, dcTitle?: string | null, dcCreator?: string | null, dcSubject?: string | null, dcDescription?: string | null, dcPublisher?: string | null, dcDate?: string | null, dcType?: string | null, dcFormat?: string | null, dcIdentifier?: string | null, dcLanguage?: string | null, dcRights?: string | null, article?: WasmArticleMetadata | null, hreflangs?: WasmHreflangEntry[] | null, favicons?: WasmFaviconInfo[] | null, headings?: WasmHeadingInfo[] | null, wordCount?: number | null);
|
|
1259
|
+
/**
|
|
1260
|
+
* Returns a detached copy of `article`.
|
|
1261
|
+
*
|
|
1262
|
+
* Changes to the returned value do not update this object. To update `article`,
|
|
1263
|
+
* read it, modify the copy, then assign the copy back to `article`.
|
|
1264
|
+
*/
|
|
808
1265
|
get article(): WasmArticleMetadata | undefined;
|
|
809
|
-
set article(value: WasmArticleMetadata
|
|
1266
|
+
set article(value: WasmArticleMetadata);
|
|
810
1267
|
get author(): string | undefined;
|
|
811
1268
|
set author(value: string | null | undefined);
|
|
812
1269
|
get canonicalUrl(): string | undefined;
|
|
@@ -835,13 +1292,46 @@ export class WasmPageMetadata {
|
|
|
835
1292
|
set dcType(value: string | null | undefined);
|
|
836
1293
|
get description(): string | undefined;
|
|
837
1294
|
set description(value: string | null | undefined);
|
|
1295
|
+
/**
|
|
1296
|
+
* Returns detached copies of the elements in `favicons`.
|
|
1297
|
+
*
|
|
1298
|
+
* Changes to the returned values do not update this object. Modify the copies,
|
|
1299
|
+
* then assign the array back to `favicons`.
|
|
1300
|
+
*/
|
|
838
1301
|
get favicons(): Array<any> | undefined;
|
|
1302
|
+
/**
|
|
1303
|
+
* Takes ownership of every element in `favicons`.
|
|
1304
|
+
* wasm-bindgen cannot borrow class values nested in an array; copy each retained
|
|
1305
|
+
* element with `copyForTransfer()` before passing the array.
|
|
1306
|
+
*/
|
|
839
1307
|
set favicons(value: WasmFaviconInfo[] | null | undefined);
|
|
840
1308
|
get generator(): string | undefined;
|
|
841
1309
|
set generator(value: string | null | undefined);
|
|
1310
|
+
/**
|
|
1311
|
+
* Returns detached copies of the elements in `headings`.
|
|
1312
|
+
*
|
|
1313
|
+
* Changes to the returned values do not update this object. Modify the copies,
|
|
1314
|
+
* then assign the array back to `headings`.
|
|
1315
|
+
*/
|
|
842
1316
|
get headings(): Array<any> | undefined;
|
|
1317
|
+
/**
|
|
1318
|
+
* Takes ownership of every element in `headings`.
|
|
1319
|
+
* wasm-bindgen cannot borrow class values nested in an array; copy each retained
|
|
1320
|
+
* element with `copyForTransfer()` before passing the array.
|
|
1321
|
+
*/
|
|
843
1322
|
set headings(value: WasmHeadingInfo[] | null | undefined);
|
|
1323
|
+
/**
|
|
1324
|
+
* Returns detached copies of the elements in `hreflangs`.
|
|
1325
|
+
*
|
|
1326
|
+
* Changes to the returned values do not update this object. Modify the copies,
|
|
1327
|
+
* then assign the array back to `hreflangs`.
|
|
1328
|
+
*/
|
|
844
1329
|
get hreflangs(): Array<any> | undefined;
|
|
1330
|
+
/**
|
|
1331
|
+
* Takes ownership of every element in `hreflangs`.
|
|
1332
|
+
* wasm-bindgen cannot borrow class values nested in an array; copy each retained
|
|
1333
|
+
* element with `copyForTransfer()` before passing the array.
|
|
1334
|
+
*/
|
|
845
1335
|
set hreflangs(value: WasmHreflangEntry[] | null | undefined);
|
|
846
1336
|
get htmlDir(): string | undefined;
|
|
847
1337
|
set htmlDir(value: string | null | undefined);
|
|
@@ -899,6 +1389,13 @@ export class WasmPageMetadata {
|
|
|
899
1389
|
export class WasmProxyConfig {
|
|
900
1390
|
free(): void;
|
|
901
1391
|
[Symbol.dispose](): void;
|
|
1392
|
+
/**
|
|
1393
|
+
* Return an independent handle for an ownership-taking API.
|
|
1394
|
+
*
|
|
1395
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
1396
|
+
* handle must remain usable.
|
|
1397
|
+
*/
|
|
1398
|
+
copyForTransfer(): WasmProxyConfig;
|
|
902
1399
|
static default(): WasmProxyConfig;
|
|
903
1400
|
constructor(url?: string | null, username?: string | null, password?: string | null);
|
|
904
1401
|
get password(): string | undefined;
|
|
@@ -914,6 +1411,13 @@ export class WasmProxyConfig {
|
|
|
914
1411
|
export class WasmResponseMeta {
|
|
915
1412
|
free(): void;
|
|
916
1413
|
[Symbol.dispose](): void;
|
|
1414
|
+
/**
|
|
1415
|
+
* Return an independent handle for an ownership-taking API.
|
|
1416
|
+
*
|
|
1417
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
1418
|
+
* handle must remain usable.
|
|
1419
|
+
*/
|
|
1420
|
+
copyForTransfer(): WasmResponseMeta;
|
|
917
1421
|
static default(): WasmResponseMeta;
|
|
918
1422
|
constructor(etag?: string | null, lastModified?: string | null, cacheControl?: string | null, server?: string | null, xPoweredBy?: string | null, contentLanguage?: string | null, contentEncoding?: string | null);
|
|
919
1423
|
get cacheControl(): string | undefined;
|
|
@@ -938,43 +1442,132 @@ export class WasmResponseMeta {
|
|
|
938
1442
|
export class WasmScrapeResult {
|
|
939
1443
|
free(): void;
|
|
940
1444
|
[Symbol.dispose](): void;
|
|
1445
|
+
clearBrowser(): void;
|
|
1446
|
+
clearDownloadedDocument(): void;
|
|
1447
|
+
clearExtractionMeta(): void;
|
|
1448
|
+
clearMarkdown(): void;
|
|
1449
|
+
clearResponseMeta(): void;
|
|
1450
|
+
/**
|
|
1451
|
+
* Return an independent handle for an ownership-taking API.
|
|
1452
|
+
*
|
|
1453
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
1454
|
+
* handle must remain usable.
|
|
1455
|
+
*/
|
|
1456
|
+
copyForTransfer(): WasmScrapeResult;
|
|
941
1457
|
static default(): WasmScrapeResult;
|
|
942
|
-
|
|
1458
|
+
/**
|
|
1459
|
+
* Takes ownership of `metadata`, `responseMeta`, `markdown`, `extractionMeta`, `downloadedDocument`, `browser`.
|
|
1460
|
+
* wasm-bindgen lowers a by-value class argument through `__destroy_into_raw()`, so each
|
|
1461
|
+
* of those handles is dead once this returns. Passing a freshly built value is fine. To
|
|
1462
|
+
* keep a handle you already hold, build with `default()` and assign the property instead:
|
|
1463
|
+
* the generated setter borrows and leaves your handle alive.
|
|
1464
|
+
* Takes ownership of every element in `links`, `images`, `feeds`, `jsonLd`, `assets`.
|
|
1465
|
+
* wasm-bindgen cannot borrow class values nested in an array. To keep those handles,
|
|
1466
|
+
* copy each retained element with `copyForTransfer()` before passing the array.
|
|
1467
|
+
*/
|
|
1468
|
+
constructor(statusCode?: number | null, finalUrl?: string | null, contentType?: string | null, html?: string | null, bodySize?: number | null, metadata?: WasmPageMetadata | null, links?: WasmLinkInfo[] | null, images?: WasmImageInfo[] | null, feeds?: WasmFeedInfo[] | null, jsonLd?: WasmJsonLdEntry[] | null, isAllowed?: boolean | null, noindexDetected?: boolean | null, nofollowDetected?: boolean | null, isPdf?: boolean | null, wasSkipped?: boolean | null, authHeaderSent?: boolean | null, assets?: WasmDownloadedAsset[] | null, jsRenderHint?: boolean | null, browserUsed?: boolean | null, ssrfRefusedUrls?: string[] | null, crawlDelay?: bigint | null, xRobotsTag?: string | null, detectedCharset?: string | null, responseMeta?: WasmResponseMeta | null, markdown?: WasmMarkdownResult | null, extractedData?: any | null, extractionMeta?: WasmExtractionMeta | null, screenshotBase64?: string | null, downloadedDocument?: WasmDownloadedDocument | null, browser?: WasmBrowserExtras | null);
|
|
1469
|
+
/**
|
|
1470
|
+
* Returns detached copies of the elements in `assets`.
|
|
1471
|
+
*
|
|
1472
|
+
* Changes to the returned values do not update this object. Modify the copies,
|
|
1473
|
+
* then assign the array back to `assets`.
|
|
1474
|
+
*/
|
|
943
1475
|
assets: WasmDownloadedAsset[];
|
|
944
1476
|
authHeaderSent: boolean;
|
|
945
1477
|
bodySize: number;
|
|
946
1478
|
browserUsed: boolean;
|
|
1479
|
+
/**
|
|
1480
|
+
* Returns a detached copy of `browser`.
|
|
1481
|
+
*
|
|
1482
|
+
* Changes to the returned value do not update this object. To update `browser`,
|
|
1483
|
+
* read it, modify the copy, then assign the copy back to `browser`.
|
|
1484
|
+
*/
|
|
947
1485
|
get browser(): WasmBrowserExtras | undefined;
|
|
948
|
-
set browser(value: WasmBrowserExtras
|
|
1486
|
+
set browser(value: WasmBrowserExtras);
|
|
949
1487
|
contentType: string;
|
|
950
1488
|
get crawlDelay(): bigint | undefined;
|
|
951
1489
|
set crawlDelay(value: bigint | null | undefined);
|
|
952
1490
|
get detectedCharset(): string | undefined;
|
|
953
1491
|
set detectedCharset(value: string | null | undefined);
|
|
1492
|
+
/**
|
|
1493
|
+
* Returns a detached copy of `downloadedDocument`.
|
|
1494
|
+
*
|
|
1495
|
+
* Changes to the returned value do not update this object. To update `downloadedDocument`,
|
|
1496
|
+
* read it, modify the copy, then assign the copy back to `downloadedDocument`.
|
|
1497
|
+
*/
|
|
954
1498
|
get downloadedDocument(): WasmDownloadedDocument | undefined;
|
|
955
|
-
set downloadedDocument(value: WasmDownloadedDocument
|
|
1499
|
+
set downloadedDocument(value: WasmDownloadedDocument);
|
|
956
1500
|
get extractedData(): any | undefined;
|
|
957
1501
|
set extractedData(value: any | null | undefined);
|
|
1502
|
+
/**
|
|
1503
|
+
* Returns a detached copy of `extractionMeta`.
|
|
1504
|
+
*
|
|
1505
|
+
* Changes to the returned value do not update this object. To update `extractionMeta`,
|
|
1506
|
+
* read it, modify the copy, then assign the copy back to `extractionMeta`.
|
|
1507
|
+
*/
|
|
958
1508
|
get extractionMeta(): WasmExtractionMeta | undefined;
|
|
959
|
-
set extractionMeta(value: WasmExtractionMeta
|
|
1509
|
+
set extractionMeta(value: WasmExtractionMeta);
|
|
1510
|
+
/**
|
|
1511
|
+
* Returns detached copies of the elements in `feeds`.
|
|
1512
|
+
*
|
|
1513
|
+
* Changes to the returned values do not update this object. Modify the copies,
|
|
1514
|
+
* then assign the array back to `feeds`.
|
|
1515
|
+
*/
|
|
960
1516
|
feeds: WasmFeedInfo[];
|
|
961
1517
|
finalUrl: string;
|
|
962
1518
|
html: string;
|
|
1519
|
+
/**
|
|
1520
|
+
* Returns detached copies of the elements in `images`.
|
|
1521
|
+
*
|
|
1522
|
+
* Changes to the returned values do not update this object. Modify the copies,
|
|
1523
|
+
* then assign the array back to `images`.
|
|
1524
|
+
*/
|
|
963
1525
|
images: WasmImageInfo[];
|
|
964
1526
|
isAllowed: boolean;
|
|
965
1527
|
isPdf: boolean;
|
|
966
1528
|
jsRenderHint: boolean;
|
|
1529
|
+
/**
|
|
1530
|
+
* Returns detached copies of the elements in `jsonLd`.
|
|
1531
|
+
*
|
|
1532
|
+
* Changes to the returned values do not update this object. Modify the copies,
|
|
1533
|
+
* then assign the array back to `jsonLd`.
|
|
1534
|
+
*/
|
|
967
1535
|
jsonLd: WasmJsonLdEntry[];
|
|
1536
|
+
/**
|
|
1537
|
+
* Returns detached copies of the elements in `links`.
|
|
1538
|
+
*
|
|
1539
|
+
* Changes to the returned values do not update this object. Modify the copies,
|
|
1540
|
+
* then assign the array back to `links`.
|
|
1541
|
+
*/
|
|
968
1542
|
links: WasmLinkInfo[];
|
|
1543
|
+
/**
|
|
1544
|
+
* Returns a detached copy of `markdown`.
|
|
1545
|
+
*
|
|
1546
|
+
* Changes to the returned value do not update this object. To update `markdown`,
|
|
1547
|
+
* read it, modify the copy, then assign the copy back to `markdown`.
|
|
1548
|
+
*/
|
|
969
1549
|
get markdown(): WasmMarkdownResult | undefined;
|
|
970
|
-
set markdown(value: WasmMarkdownResult
|
|
1550
|
+
set markdown(value: WasmMarkdownResult);
|
|
1551
|
+
/**
|
|
1552
|
+
* Returns a detached copy of `metadata`.
|
|
1553
|
+
*
|
|
1554
|
+
* Changes to the returned value do not update this object. To update `metadata`,
|
|
1555
|
+
* read it, modify the copy, then assign the copy back to `metadata`.
|
|
1556
|
+
*/
|
|
971
1557
|
metadata: WasmPageMetadata;
|
|
972
1558
|
nofollowDetected: boolean;
|
|
973
1559
|
noindexDetected: boolean;
|
|
1560
|
+
/**
|
|
1561
|
+
* Returns a detached copy of `responseMeta`.
|
|
1562
|
+
*
|
|
1563
|
+
* Changes to the returned value do not update this object. To update `responseMeta`,
|
|
1564
|
+
* read it, modify the copy, then assign the copy back to `responseMeta`.
|
|
1565
|
+
*/
|
|
974
1566
|
get responseMeta(): WasmResponseMeta | undefined;
|
|
975
|
-
set responseMeta(value: WasmResponseMeta
|
|
1567
|
+
set responseMeta(value: WasmResponseMeta);
|
|
976
1568
|
get screenshotBase64(): string | undefined;
|
|
977
1569
|
set screenshotBase64(value: string | null | undefined);
|
|
1570
|
+
ssrfRefusedUrls: string[];
|
|
978
1571
|
statusCode: number;
|
|
979
1572
|
wasSkipped: boolean;
|
|
980
1573
|
get xRobotsTag(): string | undefined;
|
|
@@ -995,6 +1588,13 @@ export enum WasmScrollDirection {
|
|
|
995
1588
|
export class WasmSitemapUrl {
|
|
996
1589
|
free(): void;
|
|
997
1590
|
[Symbol.dispose](): void;
|
|
1591
|
+
/**
|
|
1592
|
+
* Return an independent handle for an ownership-taking API.
|
|
1593
|
+
*
|
|
1594
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
1595
|
+
* handle must remain usable.
|
|
1596
|
+
*/
|
|
1597
|
+
copyForTransfer(): WasmSitemapUrl;
|
|
998
1598
|
static default(): WasmSitemapUrl;
|
|
999
1599
|
constructor(url?: string | null, lastmod?: string | null, changefreq?: string | null, priority?: string | null);
|
|
1000
1600
|
get changefreq(): string | undefined;
|
|
@@ -1012,6 +1612,13 @@ export class WasmSitemapUrl {
|
|
|
1012
1612
|
export class WasmSsrfPolicy {
|
|
1013
1613
|
free(): void;
|
|
1014
1614
|
[Symbol.dispose](): void;
|
|
1615
|
+
/**
|
|
1616
|
+
* Return an independent handle for an ownership-taking API.
|
|
1617
|
+
*
|
|
1618
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
1619
|
+
* handle must remain usable.
|
|
1620
|
+
*/
|
|
1621
|
+
copyForTransfer(): WasmSsrfPolicy;
|
|
1015
1622
|
static default(): WasmSsrfPolicy;
|
|
1016
1623
|
/**
|
|
1017
1624
|
* Create a policy from environment variables.
|
|
@@ -1025,18 +1632,19 @@ export class WasmSsrfPolicy {
|
|
|
1025
1632
|
* - Rust-side SSRF checking is unenforceable and redundant in a wasm32 context.
|
|
1026
1633
|
* - For testing and localhost access, the host's network sandbox is the enforcing boundary.
|
|
1027
1634
|
*
|
|
1028
|
-
* **Node.js caveat:** `deny_private`
|
|
1029
|
-
*
|
|
1030
|
-
*
|
|
1031
|
-
*
|
|
1032
|
-
*
|
|
1033
|
-
*
|
|
1034
|
-
*
|
|
1635
|
+
* **Node.js caveat:** `deny_private` and `denylist` have no effect on hostname-based requests
|
|
1636
|
+
* under `wasm32`. There is no DNS resolution on this target, so `validate_url` only ever
|
|
1637
|
+
* checks a literal IP host; a domain name falls straight through to `Ok(())`. In a browser
|
|
1638
|
+
* this is covered by same-origin/CORS. Node's `fetch` enforces no CORS, so a Node service
|
|
1639
|
+
* embedding this wasm module can be driven to internal hosts by domain name despite the
|
|
1640
|
+
* policy. Enforce egress restrictions (network policy, firewall, proxy allowlist) outside
|
|
1641
|
+
* the process.
|
|
1035
1642
|
*/
|
|
1036
1643
|
static fromEnv(): WasmSsrfPolicy;
|
|
1037
|
-
constructor(denyPrivate?: boolean | null, allowlist?: any | null, maxRedirects?: number | null, schemeAllowlist?: string[] | null);
|
|
1644
|
+
constructor(denyPrivate?: boolean | null, allowlist?: any | null, denylist?: any | null, maxRedirects?: number | null, schemeAllowlist?: string[] | null);
|
|
1038
1645
|
allowlist: any;
|
|
1039
1646
|
denyPrivate: boolean;
|
|
1647
|
+
denylist: any;
|
|
1040
1648
|
maxRedirects: number;
|
|
1041
1649
|
schemeAllowlist: string[];
|
|
1042
1650
|
}
|