@xberg-io/crawlberg-wasm 1.9.0 → 1.10.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/pkg/bundler/crawlberg_wasm.d.ts +622 -21
- package/pkg/bundler/crawlberg_wasm_bg.js +777 -112
- package/pkg/bundler/crawlberg_wasm_bg.wasm +0 -0
- package/pkg/bundler/crawlberg_wasm_bg.wasm.d.ts +51 -1
- package/pkg/bundler/package.json +1 -1
- package/pkg/deno/crawlberg_wasm.d.ts +622 -21
- package/pkg/deno/crawlberg_wasm.js +777 -112
- package/pkg/deno/crawlberg_wasm_bg.wasm +0 -0
- package/pkg/deno/crawlberg_wasm_bg.wasm.d.ts +51 -1
- package/pkg/nodejs/crawlberg_wasm.d.ts +622 -21
- package/pkg/nodejs/crawlberg_wasm.js +777 -112
- package/pkg/nodejs/crawlberg_wasm_bg.wasm +0 -0
- package/pkg/nodejs/crawlberg_wasm_bg.wasm.d.ts +51 -1
- package/pkg/nodejs/package.json +1 -1
- package/pkg/web/crawlberg_wasm.d.ts +673 -22
- package/pkg/web/crawlberg_wasm.js +777 -112
- package/pkg/web/crawlberg_wasm_bg.wasm +0 -0
- package/pkg/web/crawlberg_wasm_bg.wasm.d.ts +51 -1
- package/pkg/web/package.json +1 -1
|
@@ -7,6 +7,13 @@
|
|
|
7
7
|
export class WasmActionResult {
|
|
8
8
|
free(): void;
|
|
9
9
|
[Symbol.dispose](): void;
|
|
10
|
+
/**
|
|
11
|
+
* Return an independent handle for an ownership-taking API.
|
|
12
|
+
*
|
|
13
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
14
|
+
* handle must remain usable.
|
|
15
|
+
*/
|
|
16
|
+
copyForTransfer(): WasmActionResult;
|
|
10
17
|
static default(): WasmActionResult;
|
|
11
18
|
constructor(actionIndex?: number | null, actionType?: string | null, success?: boolean | null, data?: any | null, error?: string | null);
|
|
12
19
|
actionIndex: number;
|
|
@@ -24,6 +31,13 @@ export class WasmActionResult {
|
|
|
24
31
|
export class WasmArticleMetadata {
|
|
25
32
|
free(): void;
|
|
26
33
|
[Symbol.dispose](): void;
|
|
34
|
+
/**
|
|
35
|
+
* Return an independent handle for an ownership-taking API.
|
|
36
|
+
*
|
|
37
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
38
|
+
* handle must remain usable.
|
|
39
|
+
*/
|
|
40
|
+
copyForTransfer(): WasmArticleMetadata;
|
|
27
41
|
static default(): WasmArticleMetadata;
|
|
28
42
|
constructor(tags?: string[] | null, publishedTime?: string | null, modifiedTime?: string | null, author?: string | null, section?: string | null);
|
|
29
43
|
get author(): string | undefined;
|
|
@@ -80,12 +94,33 @@ export class WasmAuthConfig {
|
|
|
80
94
|
export class WasmBatchCrawlResult {
|
|
81
95
|
free(): void;
|
|
82
96
|
[Symbol.dispose](): void;
|
|
97
|
+
clearResult(): void;
|
|
98
|
+
/**
|
|
99
|
+
* Return an independent handle for an ownership-taking API.
|
|
100
|
+
*
|
|
101
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
102
|
+
* handle must remain usable.
|
|
103
|
+
*/
|
|
104
|
+
copyForTransfer(): WasmBatchCrawlResult;
|
|
83
105
|
static default(): WasmBatchCrawlResult;
|
|
106
|
+
/**
|
|
107
|
+
* Takes ownership of `result`.
|
|
108
|
+
* wasm-bindgen lowers a by-value class argument through `__destroy_into_raw()`, so each
|
|
109
|
+
* of those handles is dead once this returns. Passing a freshly built value is fine. To
|
|
110
|
+
* keep a handle you already hold, build with `default()` and assign the property instead:
|
|
111
|
+
* the generated setter borrows and leaves your handle alive.
|
|
112
|
+
*/
|
|
84
113
|
constructor(url?: string | null, result?: WasmCrawlResult | null, error?: string | null);
|
|
85
114
|
get error(): string | undefined;
|
|
86
115
|
set error(value: string | null | undefined);
|
|
116
|
+
/**
|
|
117
|
+
* Returns a detached copy of `result`.
|
|
118
|
+
*
|
|
119
|
+
* Changes to the returned value do not update this object. To update `result`,
|
|
120
|
+
* read it, modify the copy, then assign the copy back to `result`.
|
|
121
|
+
*/
|
|
87
122
|
get result(): WasmCrawlResult | undefined;
|
|
88
|
-
set result(value: WasmCrawlResult
|
|
123
|
+
set result(value: WasmCrawlResult);
|
|
89
124
|
url: string;
|
|
90
125
|
}
|
|
91
126
|
|
|
@@ -98,10 +133,28 @@ export class WasmBatchCrawlResult {
|
|
|
98
133
|
export class WasmBatchCrawlResults {
|
|
99
134
|
free(): void;
|
|
100
135
|
[Symbol.dispose](): void;
|
|
136
|
+
/**
|
|
137
|
+
* Return an independent handle for an ownership-taking API.
|
|
138
|
+
*
|
|
139
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
140
|
+
* handle must remain usable.
|
|
141
|
+
*/
|
|
142
|
+
copyForTransfer(): WasmBatchCrawlResults;
|
|
101
143
|
static default(): WasmBatchCrawlResults;
|
|
144
|
+
/**
|
|
145
|
+
* Takes ownership of every element in `results`.
|
|
146
|
+
* wasm-bindgen cannot borrow class values nested in an array. To keep those handles,
|
|
147
|
+
* copy each retained element with `copyForTransfer()` before passing the array.
|
|
148
|
+
*/
|
|
102
149
|
constructor(results?: WasmBatchCrawlResult[] | null, totalCount?: number | null, completedCount?: number | null, failedCount?: number | null);
|
|
103
150
|
completedCount: number;
|
|
104
151
|
failedCount: number;
|
|
152
|
+
/**
|
|
153
|
+
* Returns detached copies of the elements in `results`.
|
|
154
|
+
*
|
|
155
|
+
* Changes to the returned values do not update this object. Modify the copies,
|
|
156
|
+
* then assign the array back to `results`.
|
|
157
|
+
*/
|
|
105
158
|
results: WasmBatchCrawlResult[];
|
|
106
159
|
totalCount: number;
|
|
107
160
|
}
|
|
@@ -112,12 +165,33 @@ export class WasmBatchCrawlResults {
|
|
|
112
165
|
export class WasmBatchScrapeResult {
|
|
113
166
|
free(): void;
|
|
114
167
|
[Symbol.dispose](): void;
|
|
168
|
+
clearResult(): void;
|
|
169
|
+
/**
|
|
170
|
+
* Return an independent handle for an ownership-taking API.
|
|
171
|
+
*
|
|
172
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
173
|
+
* handle must remain usable.
|
|
174
|
+
*/
|
|
175
|
+
copyForTransfer(): WasmBatchScrapeResult;
|
|
115
176
|
static default(): WasmBatchScrapeResult;
|
|
177
|
+
/**
|
|
178
|
+
* Takes ownership of `result`.
|
|
179
|
+
* wasm-bindgen lowers a by-value class argument through `__destroy_into_raw()`, so each
|
|
180
|
+
* of those handles is dead once this returns. Passing a freshly built value is fine. To
|
|
181
|
+
* keep a handle you already hold, build with `default()` and assign the property instead:
|
|
182
|
+
* the generated setter borrows and leaves your handle alive.
|
|
183
|
+
*/
|
|
116
184
|
constructor(url?: string | null, result?: WasmScrapeResult | null, error?: string | null);
|
|
117
185
|
get error(): string | undefined;
|
|
118
186
|
set error(value: string | null | undefined);
|
|
187
|
+
/**
|
|
188
|
+
* Returns a detached copy of `result`.
|
|
189
|
+
*
|
|
190
|
+
* Changes to the returned value do not update this object. To update `result`,
|
|
191
|
+
* read it, modify the copy, then assign the copy back to `result`.
|
|
192
|
+
*/
|
|
119
193
|
get result(): WasmScrapeResult | undefined;
|
|
120
|
-
set result(value: WasmScrapeResult
|
|
194
|
+
set result(value: WasmScrapeResult);
|
|
121
195
|
url: string;
|
|
122
196
|
}
|
|
123
197
|
|
|
@@ -130,10 +204,28 @@ export class WasmBatchScrapeResult {
|
|
|
130
204
|
export class WasmBatchScrapeResults {
|
|
131
205
|
free(): void;
|
|
132
206
|
[Symbol.dispose](): void;
|
|
207
|
+
/**
|
|
208
|
+
* Return an independent handle for an ownership-taking API.
|
|
209
|
+
*
|
|
210
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
211
|
+
* handle must remain usable.
|
|
212
|
+
*/
|
|
213
|
+
copyForTransfer(): WasmBatchScrapeResults;
|
|
133
214
|
static default(): WasmBatchScrapeResults;
|
|
215
|
+
/**
|
|
216
|
+
* Takes ownership of every element in `results`.
|
|
217
|
+
* wasm-bindgen cannot borrow class values nested in an array. To keep those handles,
|
|
218
|
+
* copy each retained element with `copyForTransfer()` before passing the array.
|
|
219
|
+
*/
|
|
134
220
|
constructor(results?: WasmBatchScrapeResult[] | null, totalCount?: number | null, completedCount?: number | null, failedCount?: number | null);
|
|
135
221
|
completedCount: number;
|
|
136
222
|
failedCount: number;
|
|
223
|
+
/**
|
|
224
|
+
* Returns detached copies of the elements in `results`.
|
|
225
|
+
*
|
|
226
|
+
* Changes to the returned values do not update this object. Modify the copies,
|
|
227
|
+
* then assign the array back to `results`.
|
|
228
|
+
*/
|
|
137
229
|
results: WasmBatchScrapeResult[];
|
|
138
230
|
totalCount: number;
|
|
139
231
|
}
|
|
@@ -152,7 +244,22 @@ export enum WasmBrowserBackend {
|
|
|
152
244
|
export class WasmBrowserConfig {
|
|
153
245
|
free(): void;
|
|
154
246
|
[Symbol.dispose](): void;
|
|
247
|
+
clearProxy(): void;
|
|
248
|
+
/**
|
|
249
|
+
* Return an independent handle for an ownership-taking API.
|
|
250
|
+
*
|
|
251
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
252
|
+
* handle must remain usable.
|
|
253
|
+
*/
|
|
254
|
+
copyForTransfer(): WasmBrowserConfig;
|
|
155
255
|
static default(): WasmBrowserConfig;
|
|
256
|
+
/**
|
|
257
|
+
* Takes ownership of `proxy`.
|
|
258
|
+
* wasm-bindgen lowers a by-value class argument through `__destroy_into_raw()`, so each
|
|
259
|
+
* of those handles is dead once this returns. Passing a freshly built value is fine. To
|
|
260
|
+
* keep a handle you already hold, build with `default()` and assign the property instead:
|
|
261
|
+
* the generated setter borrows and leaves your handle alive.
|
|
262
|
+
*/
|
|
156
263
|
constructor(mode?: WasmBrowserMode | null, backend?: WasmBrowserBackend | null, timeout?: bigint | null, overallTimeout?: bigint | null, shutdownTimeout?: bigint | null, wait?: WasmBrowserWait | null, blockUrlPatterns?: string[] | null, captureNetworkEvents?: boolean | null, sessionAffinity?: boolean | null, chromeArgs?: string[] | null, endpoint?: string | null, waitSelector?: string | null, extraWait?: bigint | null, proxy?: WasmProxyConfig | null, evalScript?: string | null, robotsUserAgent?: string | null, chromePath?: string | null);
|
|
157
264
|
get backend(): string;
|
|
158
265
|
set backend(value: WasmBrowserBackend);
|
|
@@ -171,8 +278,14 @@ export class WasmBrowserConfig {
|
|
|
171
278
|
set mode(value: WasmBrowserMode);
|
|
172
279
|
get overallTimeout(): bigint | undefined;
|
|
173
280
|
set overallTimeout(value: bigint | null | undefined);
|
|
281
|
+
/**
|
|
282
|
+
* Returns a detached copy of `proxy`.
|
|
283
|
+
*
|
|
284
|
+
* Changes to the returned value do not update this object. To update `proxy`,
|
|
285
|
+
* read it, modify the copy, then assign the copy back to `proxy`.
|
|
286
|
+
*/
|
|
174
287
|
get proxy(): WasmProxyConfig | undefined;
|
|
175
|
-
set proxy(value: WasmProxyConfig
|
|
288
|
+
set proxy(value: WasmProxyConfig);
|
|
176
289
|
get robotsUserAgent(): string | undefined;
|
|
177
290
|
set robotsUserAgent(value: string | null | undefined);
|
|
178
291
|
sessionAffinity: boolean;
|
|
@@ -194,11 +307,35 @@ export class WasmBrowserConfig {
|
|
|
194
307
|
export class WasmBrowserExtras {
|
|
195
308
|
free(): void;
|
|
196
309
|
[Symbol.dispose](): void;
|
|
310
|
+
/**
|
|
311
|
+
* Return an independent handle for an ownership-taking API.
|
|
312
|
+
*
|
|
313
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
314
|
+
* handle must remain usable.
|
|
315
|
+
*/
|
|
316
|
+
copyForTransfer(): WasmBrowserExtras;
|
|
197
317
|
static default(): WasmBrowserExtras;
|
|
318
|
+
/**
|
|
319
|
+
* Takes ownership of every element in `networkEvents`, `cookies`.
|
|
320
|
+
* wasm-bindgen cannot borrow class values nested in an array. To keep those handles,
|
|
321
|
+
* copy each retained element with `copyForTransfer()` before passing the array.
|
|
322
|
+
*/
|
|
198
323
|
constructor(networkEvents?: WasmResponseMeta[] | null, cookies?: WasmCookieInfo[] | null, evalResult?: any | null);
|
|
324
|
+
/**
|
|
325
|
+
* Returns detached copies of the elements in `cookies`.
|
|
326
|
+
*
|
|
327
|
+
* Changes to the returned values do not update this object. Modify the copies,
|
|
328
|
+
* then assign the array back to `cookies`.
|
|
329
|
+
*/
|
|
199
330
|
cookies: WasmCookieInfo[];
|
|
200
331
|
get evalResult(): any | undefined;
|
|
201
332
|
set evalResult(value: any | null | undefined);
|
|
333
|
+
/**
|
|
334
|
+
* Returns detached copies of the elements in `networkEvents`.
|
|
335
|
+
*
|
|
336
|
+
* Changes to the returned values do not update this object. Modify the copies,
|
|
337
|
+
* then assign the array back to `networkEvents`.
|
|
338
|
+
*/
|
|
202
339
|
networkEvents: WasmResponseMeta[];
|
|
203
340
|
}
|
|
204
341
|
|
|
@@ -228,6 +365,13 @@ export enum WasmBrowserWait {
|
|
|
228
365
|
export class WasmCitationReference {
|
|
229
366
|
free(): void;
|
|
230
367
|
[Symbol.dispose](): void;
|
|
368
|
+
/**
|
|
369
|
+
* Return an independent handle for an ownership-taking API.
|
|
370
|
+
*
|
|
371
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
372
|
+
* handle must remain usable.
|
|
373
|
+
*/
|
|
374
|
+
copyForTransfer(): WasmCitationReference;
|
|
231
375
|
static default(): WasmCitationReference;
|
|
232
376
|
constructor(index?: number | null, url?: string | null, text?: string | null);
|
|
233
377
|
index: number;
|
|
@@ -241,9 +385,27 @@ export class WasmCitationReference {
|
|
|
241
385
|
export class WasmCitationResult {
|
|
242
386
|
free(): void;
|
|
243
387
|
[Symbol.dispose](): void;
|
|
388
|
+
/**
|
|
389
|
+
* Return an independent handle for an ownership-taking API.
|
|
390
|
+
*
|
|
391
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
392
|
+
* handle must remain usable.
|
|
393
|
+
*/
|
|
394
|
+
copyForTransfer(): WasmCitationResult;
|
|
244
395
|
static default(): WasmCitationResult;
|
|
396
|
+
/**
|
|
397
|
+
* Takes ownership of every element in `references`.
|
|
398
|
+
* wasm-bindgen cannot borrow class values nested in an array. To keep those handles,
|
|
399
|
+
* copy each retained element with `copyForTransfer()` before passing the array.
|
|
400
|
+
*/
|
|
245
401
|
constructor(content?: string | null, references?: WasmCitationReference[] | null);
|
|
246
402
|
content: string;
|
|
403
|
+
/**
|
|
404
|
+
* Returns detached copies of the elements in `references`.
|
|
405
|
+
*
|
|
406
|
+
* Changes to the returned values do not update this object. Modify the copies,
|
|
407
|
+
* then assign the array back to `references`.
|
|
408
|
+
*/
|
|
247
409
|
references: WasmCitationReference[];
|
|
248
410
|
}
|
|
249
411
|
|
|
@@ -257,6 +419,13 @@ export class WasmCitationResult {
|
|
|
257
419
|
export class WasmContentConfig {
|
|
258
420
|
free(): void;
|
|
259
421
|
[Symbol.dispose](): void;
|
|
422
|
+
/**
|
|
423
|
+
* Return an independent handle for an ownership-taking API.
|
|
424
|
+
*
|
|
425
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
426
|
+
* handle must remain usable.
|
|
427
|
+
*/
|
|
428
|
+
copyForTransfer(): WasmContentConfig;
|
|
260
429
|
static default(): WasmContentConfig;
|
|
261
430
|
constructor(outputFormat?: string | null, preprocessingPreset?: string | null, removeNavigation?: boolean | null, removeForms?: boolean | null, stripTags?: string[] | null, preserveTags?: string[] | null, excludeSelectors?: string[] | null, skipImages?: boolean | null, wrap?: boolean | null, wrapWidth?: number | null, includeDocumentStructure?: boolean | null, extractMetadata?: boolean | null, maxDepth?: number | null);
|
|
262
431
|
excludeSelectors: string[];
|
|
@@ -288,6 +457,13 @@ export enum WasmContentFilterKind {
|
|
|
288
457
|
export class WasmCookieInfo {
|
|
289
458
|
free(): void;
|
|
290
459
|
[Symbol.dispose](): void;
|
|
460
|
+
/**
|
|
461
|
+
* Return an independent handle for an ownership-taking API.
|
|
462
|
+
*
|
|
463
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
464
|
+
* handle must remain usable.
|
|
465
|
+
*/
|
|
466
|
+
copyForTransfer(): WasmCookieInfo;
|
|
291
467
|
static default(): WasmCookieInfo;
|
|
292
468
|
constructor(name?: string | null, value?: string | null, domain?: string | null, path?: string | null);
|
|
293
469
|
get domain(): string | undefined;
|
|
@@ -304,7 +480,22 @@ export class WasmCookieInfo {
|
|
|
304
480
|
export class WasmCrawlConfig {
|
|
305
481
|
free(): void;
|
|
306
482
|
[Symbol.dispose](): void;
|
|
483
|
+
clearProxy(): void;
|
|
484
|
+
/**
|
|
485
|
+
* Return an independent handle for an ownership-taking API.
|
|
486
|
+
*
|
|
487
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
488
|
+
* handle must remain usable.
|
|
489
|
+
*/
|
|
490
|
+
copyForTransfer(): WasmCrawlConfig;
|
|
307
491
|
static default(): WasmCrawlConfig;
|
|
492
|
+
/**
|
|
493
|
+
* Takes ownership of `content`, `browser`, `proxy`, `ssrf`.
|
|
494
|
+
* wasm-bindgen lowers a by-value class argument through `__destroy_into_raw()`, so each
|
|
495
|
+
* of those handles is dead once this returns. Passing a freshly built value is fine. To
|
|
496
|
+
* keep a handle you already hold, build with `default()` and assign the property instead:
|
|
497
|
+
* the generated setter borrows and leaves your handle alive.
|
|
498
|
+
*/
|
|
308
499
|
constructor(crawlStrategy?: WasmCrawlStrategyKind | null, respectRobotsTxt?: boolean | null, softHttpErrors?: boolean | null, stayOnDomain?: boolean | null, allowSubdomains?: boolean | null, includePaths?: string[] | null, excludePaths?: string[] | null, pathPatternsMatchQuery?: boolean | null, pathPatternsMatchUrl?: boolean | null, dedupIncludeQuery?: boolean | null, stripTrackingParams?: boolean | null, trackingParams?: string[] | null, customHeaders?: any | null, requestTimeout?: bigint | null, maxRedirects?: number | null, retryCount?: number | null, retryCodes?: Uint16Array | null, retryInitialDelayMs?: bigint | null, retryMaxDelayMs?: bigint | null, rateLimitJitterRatio?: number | null, cookiesEnabled?: boolean | null, removeTags?: string[] | null, content?: WasmContentConfig | null, downloadAssets?: boolean | null, assetTypes?: WasmAssetCategory[] | null, browser?: WasmBrowserConfig | null, userAgents?: string[] | null, captureScreenshot?: boolean | null, followDocumentUrls?: boolean | null, downloadDocuments?: boolean | null, documentMimeTypes?: string[] | null, saveBrowserProfile?: boolean | null, ssrf?: WasmSsrfPolicy | null, maxDepth?: number | null, maxPages?: number | null, maxLinksPerPage?: number | null, maxConcurrent?: number | null, contentFilter?: WasmContentFilterKind | null, bm25Query?: string | null, bm25Threshold?: number | null, userAgent?: string | null, rateLimitMs?: bigint | null, auth?: any | null, maxBodySize?: number | null, mapLimit?: number | null, mapSearch?: string | null, maxAssetSize?: number | null, proxy?: WasmProxyConfig | null, documentUrlDepth?: number | null, documentMaxSize?: number | null, documentOutputDir?: string | null, documentContentEncoding?: WasmDocumentContentEncoding | null, warcOutput?: string | null, browserProfile?: string | null, ssrfDenyPrivateExplicit?: boolean | null);
|
|
309
500
|
/**
|
|
310
501
|
* Validate the configuration, returning an error if any values are invalid.
|
|
@@ -323,10 +514,22 @@ export class WasmCrawlConfig {
|
|
|
323
514
|
set bm25Threshold(value: number | null | undefined);
|
|
324
515
|
get browserProfile(): string | undefined;
|
|
325
516
|
set browserProfile(value: string | null | undefined);
|
|
517
|
+
/**
|
|
518
|
+
* Returns a detached copy of `browser`.
|
|
519
|
+
*
|
|
520
|
+
* Changes to the returned value do not update this object. To update `browser`,
|
|
521
|
+
* read it, modify the copy, then assign the copy back to `browser`.
|
|
522
|
+
*/
|
|
326
523
|
browser: WasmBrowserConfig;
|
|
327
524
|
captureScreenshot: boolean;
|
|
328
525
|
get contentFilter(): string | undefined;
|
|
329
526
|
set contentFilter(value: WasmContentFilterKind | null | undefined);
|
|
527
|
+
/**
|
|
528
|
+
* Returns a detached copy of `content`.
|
|
529
|
+
*
|
|
530
|
+
* Changes to the returned value do not update this object. To update `content`,
|
|
531
|
+
* read it, modify the copy, then assign the copy back to `content`.
|
|
532
|
+
*/
|
|
330
533
|
content: WasmContentConfig;
|
|
331
534
|
cookiesEnabled: boolean;
|
|
332
535
|
get crawlStrategy(): string;
|
|
@@ -366,8 +569,14 @@ export class WasmCrawlConfig {
|
|
|
366
569
|
maxRedirects: number;
|
|
367
570
|
pathPatternsMatchQuery: boolean;
|
|
368
571
|
pathPatternsMatchUrl: boolean;
|
|
572
|
+
/**
|
|
573
|
+
* Returns a detached copy of `proxy`.
|
|
574
|
+
*
|
|
575
|
+
* Changes to the returned value do not update this object. To update `proxy`,
|
|
576
|
+
* read it, modify the copy, then assign the copy back to `proxy`.
|
|
577
|
+
*/
|
|
369
578
|
get proxy(): WasmProxyConfig | undefined;
|
|
370
|
-
set proxy(value: WasmProxyConfig
|
|
579
|
+
set proxy(value: WasmProxyConfig);
|
|
371
580
|
rateLimitJitterRatio: number;
|
|
372
581
|
get rateLimitMs(): bigint | undefined;
|
|
373
582
|
set rateLimitMs(value: bigint | null | undefined);
|
|
@@ -383,6 +592,12 @@ export class WasmCrawlConfig {
|
|
|
383
592
|
softHttpErrors: boolean;
|
|
384
593
|
get ssrfDenyPrivateExplicit(): boolean | undefined;
|
|
385
594
|
set ssrfDenyPrivateExplicit(value: boolean | null | undefined);
|
|
595
|
+
/**
|
|
596
|
+
* Returns a detached copy of `ssrf`.
|
|
597
|
+
*
|
|
598
|
+
* Changes to the returned value do not update this object. To update `ssrf`,
|
|
599
|
+
* read it, modify the copy, then assign the copy back to `ssrf`.
|
|
600
|
+
*/
|
|
386
601
|
ssrf: WasmSsrfPolicy;
|
|
387
602
|
stayOnDomain: boolean;
|
|
388
603
|
stripTrackingParams: boolean;
|
|
@@ -404,6 +619,13 @@ export class WasmCrawlEngineHandle {
|
|
|
404
619
|
private constructor();
|
|
405
620
|
free(): void;
|
|
406
621
|
[Symbol.dispose](): void;
|
|
622
|
+
/**
|
|
623
|
+
* Return an independent handle for an ownership-taking API.
|
|
624
|
+
*
|
|
625
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
626
|
+
* handle must remain usable.
|
|
627
|
+
*/
|
|
628
|
+
copyForTransfer(): WasmCrawlEngineHandle;
|
|
407
629
|
}
|
|
408
630
|
|
|
409
631
|
/**
|
|
@@ -412,7 +634,27 @@ export class WasmCrawlEngineHandle {
|
|
|
412
634
|
export class WasmCrawlPageResult {
|
|
413
635
|
free(): void;
|
|
414
636
|
[Symbol.dispose](): void;
|
|
637
|
+
clearDownloadedDocument(): void;
|
|
638
|
+
clearExtractionMeta(): void;
|
|
639
|
+
clearMarkdown(): void;
|
|
640
|
+
/**
|
|
641
|
+
* Return an independent handle for an ownership-taking API.
|
|
642
|
+
*
|
|
643
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
644
|
+
* handle must remain usable.
|
|
645
|
+
*/
|
|
646
|
+
copyForTransfer(): WasmCrawlPageResult;
|
|
415
647
|
static default(): WasmCrawlPageResult;
|
|
648
|
+
/**
|
|
649
|
+
* Takes ownership of `metadata`, `markdown`, `extractionMeta`, `downloadedDocument`.
|
|
650
|
+
* wasm-bindgen lowers a by-value class argument through `__destroy_into_raw()`, so each
|
|
651
|
+
* of those handles is dead once this returns. Passing a freshly built value is fine. To
|
|
652
|
+
* keep a handle you already hold, build with `default()` and assign the property instead:
|
|
653
|
+
* the generated setter borrows and leaves your handle alive.
|
|
654
|
+
* Takes ownership of every element in `links`, `images`, `feeds`, `jsonLd`.
|
|
655
|
+
* wasm-bindgen cannot borrow class values nested in an array. To keep those handles,
|
|
656
|
+
* copy each retained element with `copyForTransfer()` before passing the array.
|
|
657
|
+
*/
|
|
416
658
|
constructor(url?: string | null, normalizedUrl?: string | null, statusCode?: number | null, contentType?: string | null, html?: string | null, bodySize?: number | null, metadata?: WasmPageMetadata | null, links?: WasmLinkInfo[] | null, images?: WasmImageInfo[] | null, feeds?: WasmFeedInfo[] | null, jsonLd?: WasmJsonLdEntry[] | null, depth?: number | null, stayedOnDomain?: boolean | null, wasSkipped?: boolean | null, isPdf?: boolean | null, browserUsed?: boolean | null, finalUrl?: string | null, redirectCount?: number | null, noindexDetected?: boolean | null, nofollowDetected?: boolean | null, ssrfRefusedUrls?: string[] | null, detectedCharset?: string | null, markdown?: WasmMarkdownResult | null, extractedData?: any | null, extractionMeta?: WasmExtractionMeta | null, downloadedDocument?: WasmDownloadedDocument | null);
|
|
417
659
|
bodySize: number;
|
|
418
660
|
browserUsed: boolean;
|
|
@@ -420,21 +662,69 @@ export class WasmCrawlPageResult {
|
|
|
420
662
|
depth: number;
|
|
421
663
|
get detectedCharset(): string | undefined;
|
|
422
664
|
set detectedCharset(value: string | null | undefined);
|
|
665
|
+
/**
|
|
666
|
+
* Returns a detached copy of `downloadedDocument`.
|
|
667
|
+
*
|
|
668
|
+
* Changes to the returned value do not update this object. To update `downloadedDocument`,
|
|
669
|
+
* read it, modify the copy, then assign the copy back to `downloadedDocument`.
|
|
670
|
+
*/
|
|
423
671
|
get downloadedDocument(): WasmDownloadedDocument | undefined;
|
|
424
|
-
set downloadedDocument(value: WasmDownloadedDocument
|
|
672
|
+
set downloadedDocument(value: WasmDownloadedDocument);
|
|
425
673
|
get extractedData(): any | undefined;
|
|
426
674
|
set extractedData(value: any | null | undefined);
|
|
675
|
+
/**
|
|
676
|
+
* Returns a detached copy of `extractionMeta`.
|
|
677
|
+
*
|
|
678
|
+
* Changes to the returned value do not update this object. To update `extractionMeta`,
|
|
679
|
+
* read it, modify the copy, then assign the copy back to `extractionMeta`.
|
|
680
|
+
*/
|
|
427
681
|
get extractionMeta(): WasmExtractionMeta | undefined;
|
|
428
|
-
set extractionMeta(value: WasmExtractionMeta
|
|
682
|
+
set extractionMeta(value: WasmExtractionMeta);
|
|
683
|
+
/**
|
|
684
|
+
* Returns detached copies of the elements in `feeds`.
|
|
685
|
+
*
|
|
686
|
+
* Changes to the returned values do not update this object. Modify the copies,
|
|
687
|
+
* then assign the array back to `feeds`.
|
|
688
|
+
*/
|
|
429
689
|
feeds: WasmFeedInfo[];
|
|
430
690
|
finalUrl: string;
|
|
431
691
|
html: string;
|
|
692
|
+
/**
|
|
693
|
+
* Returns detached copies of the elements in `images`.
|
|
694
|
+
*
|
|
695
|
+
* Changes to the returned values do not update this object. Modify the copies,
|
|
696
|
+
* then assign the array back to `images`.
|
|
697
|
+
*/
|
|
432
698
|
images: WasmImageInfo[];
|
|
433
699
|
isPdf: boolean;
|
|
700
|
+
/**
|
|
701
|
+
* Returns detached copies of the elements in `jsonLd`.
|
|
702
|
+
*
|
|
703
|
+
* Changes to the returned values do not update this object. Modify the copies,
|
|
704
|
+
* then assign the array back to `jsonLd`.
|
|
705
|
+
*/
|
|
434
706
|
jsonLd: WasmJsonLdEntry[];
|
|
707
|
+
/**
|
|
708
|
+
* Returns detached copies of the elements in `links`.
|
|
709
|
+
*
|
|
710
|
+
* Changes to the returned values do not update this object. Modify the copies,
|
|
711
|
+
* then assign the array back to `links`.
|
|
712
|
+
*/
|
|
435
713
|
links: WasmLinkInfo[];
|
|
714
|
+
/**
|
|
715
|
+
* Returns a detached copy of `markdown`.
|
|
716
|
+
*
|
|
717
|
+
* Changes to the returned value do not update this object. To update `markdown`,
|
|
718
|
+
* read it, modify the copy, then assign the copy back to `markdown`.
|
|
719
|
+
*/
|
|
436
720
|
get markdown(): WasmMarkdownResult | undefined;
|
|
437
|
-
set markdown(value: WasmMarkdownResult
|
|
721
|
+
set markdown(value: WasmMarkdownResult);
|
|
722
|
+
/**
|
|
723
|
+
* Returns a detached copy of `metadata`.
|
|
724
|
+
*
|
|
725
|
+
* Changes to the returned value do not update this object. To update `metadata`,
|
|
726
|
+
* read it, modify the copy, then assign the copy back to `metadata`.
|
|
727
|
+
*/
|
|
438
728
|
metadata: WasmPageMetadata;
|
|
439
729
|
nofollowDetected: boolean;
|
|
440
730
|
noindexDetected: boolean;
|
|
@@ -453,7 +743,19 @@ export class WasmCrawlPageResult {
|
|
|
453
743
|
export class WasmCrawlResult {
|
|
454
744
|
free(): void;
|
|
455
745
|
[Symbol.dispose](): void;
|
|
746
|
+
/**
|
|
747
|
+
* Return an independent handle for an ownership-taking API.
|
|
748
|
+
*
|
|
749
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
750
|
+
* handle must remain usable.
|
|
751
|
+
*/
|
|
752
|
+
copyForTransfer(): WasmCrawlResult;
|
|
456
753
|
static default(): WasmCrawlResult;
|
|
754
|
+
/**
|
|
755
|
+
* Takes ownership of every element in `pages`, `cookies`.
|
|
756
|
+
* wasm-bindgen cannot borrow class values nested in an array. To keep those handles,
|
|
757
|
+
* copy each retained element with `copyForTransfer()` before passing the array.
|
|
758
|
+
*/
|
|
457
759
|
constructor(pages?: WasmCrawlPageResult[] | null, finalUrl?: string | null, redirectCount?: number | null, wasSkipped?: boolean | null, cookies?: WasmCookieInfo[] | null, stayedOnDomain?: boolean | null, browserUsed?: boolean | null, error?: string | null);
|
|
458
760
|
/**
|
|
459
761
|
* Returns the count of unique normalized URLs encountered during crawling.
|
|
@@ -467,10 +769,22 @@ export class WasmCrawlResult {
|
|
|
467
769
|
*/
|
|
468
770
|
uniqueNormalizedUrls(): number;
|
|
469
771
|
browserUsed: boolean;
|
|
772
|
+
/**
|
|
773
|
+
* Returns detached copies of the elements in `cookies`.
|
|
774
|
+
*
|
|
775
|
+
* Changes to the returned values do not update this object. Modify the copies,
|
|
776
|
+
* then assign the array back to `cookies`.
|
|
777
|
+
*/
|
|
470
778
|
cookies: WasmCookieInfo[];
|
|
471
779
|
get error(): string | undefined;
|
|
472
780
|
set error(value: string | null | undefined);
|
|
473
781
|
finalUrl: string;
|
|
782
|
+
/**
|
|
783
|
+
* Returns detached copies of the elements in `pages`.
|
|
784
|
+
*
|
|
785
|
+
* Changes to the returned values do not update this object. Modify the copies,
|
|
786
|
+
* then assign the array back to `pages`.
|
|
787
|
+
*/
|
|
474
788
|
pages: WasmCrawlPageResult[];
|
|
475
789
|
redirectCount: number;
|
|
476
790
|
stayedOnDomain: boolean;
|
|
@@ -509,6 +823,13 @@ export enum WasmDocumentContentEncoding {
|
|
|
509
823
|
export class WasmDownloadedAsset {
|
|
510
824
|
free(): void;
|
|
511
825
|
[Symbol.dispose](): void;
|
|
826
|
+
/**
|
|
827
|
+
* Return an independent handle for an ownership-taking API.
|
|
828
|
+
*
|
|
829
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
830
|
+
* handle must remain usable.
|
|
831
|
+
*/
|
|
832
|
+
copyForTransfer(): WasmDownloadedAsset;
|
|
512
833
|
static default(): WasmDownloadedAsset;
|
|
513
834
|
constructor(url?: string | null, contentHash?: string | null, size?: number | null, assetCategory?: WasmAssetCategory | null, mimeType?: string | null, htmlTag?: string | null);
|
|
514
835
|
get assetCategory(): string;
|
|
@@ -532,6 +853,13 @@ export class WasmDownloadedAsset {
|
|
|
532
853
|
export class WasmDownloadedDocument {
|
|
533
854
|
free(): void;
|
|
534
855
|
[Symbol.dispose](): void;
|
|
856
|
+
/**
|
|
857
|
+
* Return an independent handle for an ownership-taking API.
|
|
858
|
+
*
|
|
859
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
860
|
+
* handle must remain usable.
|
|
861
|
+
*/
|
|
862
|
+
copyForTransfer(): WasmDownloadedDocument;
|
|
535
863
|
static default(): WasmDownloadedDocument;
|
|
536
864
|
constructor(url?: string | null, mimeType?: string | null, size?: number | null, contentHash?: string | null, headers?: any | null, truncated?: boolean | null, filename?: string | null, contentPath?: string | null, contentBase64?: string | null);
|
|
537
865
|
get contentBase64(): string | undefined;
|
|
@@ -554,6 +882,13 @@ export class WasmDownloadedDocument {
|
|
|
554
882
|
export class WasmExtractionMeta {
|
|
555
883
|
free(): void;
|
|
556
884
|
[Symbol.dispose](): void;
|
|
885
|
+
/**
|
|
886
|
+
* Return an independent handle for an ownership-taking API.
|
|
887
|
+
*
|
|
888
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
889
|
+
* handle must remain usable.
|
|
890
|
+
*/
|
|
891
|
+
copyForTransfer(): WasmExtractionMeta;
|
|
557
892
|
static default(): WasmExtractionMeta;
|
|
558
893
|
constructor(chunksProcessed?: number | null, cost?: number | null, promptTokens?: bigint | null, completionTokens?: bigint | null, model?: string | null);
|
|
559
894
|
chunksProcessed: number;
|
|
@@ -573,6 +908,13 @@ export class WasmExtractionMeta {
|
|
|
573
908
|
export class WasmFaviconInfo {
|
|
574
909
|
free(): void;
|
|
575
910
|
[Symbol.dispose](): void;
|
|
911
|
+
/**
|
|
912
|
+
* Return an independent handle for an ownership-taking API.
|
|
913
|
+
*
|
|
914
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
915
|
+
* handle must remain usable.
|
|
916
|
+
*/
|
|
917
|
+
copyForTransfer(): WasmFaviconInfo;
|
|
576
918
|
static default(): WasmFaviconInfo;
|
|
577
919
|
constructor(url?: string | null, rel?: string | null, sizes?: string | null, mimeType?: string | null);
|
|
578
920
|
get mimeType(): string | undefined;
|
|
@@ -589,6 +931,13 @@ export class WasmFaviconInfo {
|
|
|
589
931
|
export class WasmFeedInfo {
|
|
590
932
|
free(): void;
|
|
591
933
|
[Symbol.dispose](): void;
|
|
934
|
+
/**
|
|
935
|
+
* Return an independent handle for an ownership-taking API.
|
|
936
|
+
*
|
|
937
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
938
|
+
* handle must remain usable.
|
|
939
|
+
*/
|
|
940
|
+
copyForTransfer(): WasmFeedInfo;
|
|
592
941
|
static default(): WasmFeedInfo;
|
|
593
942
|
constructor(url?: string | null, feedType?: WasmFeedType | null, title?: string | null);
|
|
594
943
|
get feedType(): string;
|
|
@@ -613,6 +962,13 @@ export enum WasmFeedType {
|
|
|
613
962
|
export class WasmHeadingInfo {
|
|
614
963
|
free(): void;
|
|
615
964
|
[Symbol.dispose](): void;
|
|
965
|
+
/**
|
|
966
|
+
* Return an independent handle for an ownership-taking API.
|
|
967
|
+
*
|
|
968
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
969
|
+
* handle must remain usable.
|
|
970
|
+
*/
|
|
971
|
+
copyForTransfer(): WasmHeadingInfo;
|
|
616
972
|
static default(): WasmHeadingInfo;
|
|
617
973
|
constructor(level?: number | null, text?: string | null);
|
|
618
974
|
level: number;
|
|
@@ -652,6 +1008,13 @@ export class WasmHostMatcher {
|
|
|
652
1008
|
export class WasmHreflangEntry {
|
|
653
1009
|
free(): void;
|
|
654
1010
|
[Symbol.dispose](): void;
|
|
1011
|
+
/**
|
|
1012
|
+
* Return an independent handle for an ownership-taking API.
|
|
1013
|
+
*
|
|
1014
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
1015
|
+
* handle must remain usable.
|
|
1016
|
+
*/
|
|
1017
|
+
copyForTransfer(): WasmHreflangEntry;
|
|
655
1018
|
static default(): WasmHreflangEntry;
|
|
656
1019
|
constructor(lang?: string | null, url?: string | null);
|
|
657
1020
|
lang: string;
|
|
@@ -664,6 +1027,13 @@ export class WasmHreflangEntry {
|
|
|
664
1027
|
export class WasmImageInfo {
|
|
665
1028
|
free(): void;
|
|
666
1029
|
[Symbol.dispose](): void;
|
|
1030
|
+
/**
|
|
1031
|
+
* Return an independent handle for an ownership-taking API.
|
|
1032
|
+
*
|
|
1033
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
1034
|
+
* handle must remain usable.
|
|
1035
|
+
*/
|
|
1036
|
+
copyForTransfer(): WasmImageInfo;
|
|
667
1037
|
static default(): WasmImageInfo;
|
|
668
1038
|
constructor(url?: string | null, source?: WasmImageSource | null, alt?: string | null, width?: number | null, height?: number | null);
|
|
669
1039
|
get alt(): string | undefined;
|
|
@@ -693,8 +1063,26 @@ export enum WasmImageSource {
|
|
|
693
1063
|
export class WasmInteractionResult {
|
|
694
1064
|
free(): void;
|
|
695
1065
|
[Symbol.dispose](): void;
|
|
1066
|
+
/**
|
|
1067
|
+
* Return an independent handle for an ownership-taking API.
|
|
1068
|
+
*
|
|
1069
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
1070
|
+
* handle must remain usable.
|
|
1071
|
+
*/
|
|
1072
|
+
copyForTransfer(): WasmInteractionResult;
|
|
696
1073
|
static default(): WasmInteractionResult;
|
|
1074
|
+
/**
|
|
1075
|
+
* Takes ownership of every element in `actionResults`.
|
|
1076
|
+
* wasm-bindgen cannot borrow class values nested in an array. To keep those handles,
|
|
1077
|
+
* copy each retained element with `copyForTransfer()` before passing the array.
|
|
1078
|
+
*/
|
|
697
1079
|
constructor(actionResults?: WasmActionResult[] | null, finalHtml?: string | null, finalUrl?: string | null, ssrfRefusedUrls?: string[] | null, screenshotBase64?: string | null);
|
|
1080
|
+
/**
|
|
1081
|
+
* Returns detached copies of the elements in `actionResults`.
|
|
1082
|
+
*
|
|
1083
|
+
* Changes to the returned values do not update this object. Modify the copies,
|
|
1084
|
+
* then assign the array back to `actionResults`.
|
|
1085
|
+
*/
|
|
698
1086
|
actionResults: WasmActionResult[];
|
|
699
1087
|
finalHtml: string;
|
|
700
1088
|
finalUrl: string;
|
|
@@ -709,6 +1097,13 @@ export class WasmInteractionResult {
|
|
|
709
1097
|
export class WasmJsonLdEntry {
|
|
710
1098
|
free(): void;
|
|
711
1099
|
[Symbol.dispose](): void;
|
|
1100
|
+
/**
|
|
1101
|
+
* Return an independent handle for an ownership-taking API.
|
|
1102
|
+
*
|
|
1103
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
1104
|
+
* handle must remain usable.
|
|
1105
|
+
*/
|
|
1106
|
+
copyForTransfer(): WasmJsonLdEntry;
|
|
712
1107
|
static default(): WasmJsonLdEntry;
|
|
713
1108
|
constructor(schemaType?: string | null, raw?: string | null, name?: string | null);
|
|
714
1109
|
get name(): string | undefined;
|
|
@@ -723,6 +1118,13 @@ export class WasmJsonLdEntry {
|
|
|
723
1118
|
export class WasmLinkInfo {
|
|
724
1119
|
free(): void;
|
|
725
1120
|
[Symbol.dispose](): void;
|
|
1121
|
+
/**
|
|
1122
|
+
* Return an independent handle for an ownership-taking API.
|
|
1123
|
+
*
|
|
1124
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
1125
|
+
* handle must remain usable.
|
|
1126
|
+
*/
|
|
1127
|
+
copyForTransfer(): WasmLinkInfo;
|
|
726
1128
|
static default(): WasmLinkInfo;
|
|
727
1129
|
constructor(url?: string | null, text?: string | null, linkType?: WasmLinkType | null, nofollow?: boolean | null, rel?: string | null);
|
|
728
1130
|
get linkType(): string;
|
|
@@ -750,8 +1152,26 @@ export enum WasmLinkType {
|
|
|
750
1152
|
export class WasmMapResult {
|
|
751
1153
|
free(): void;
|
|
752
1154
|
[Symbol.dispose](): void;
|
|
1155
|
+
/**
|
|
1156
|
+
* Return an independent handle for an ownership-taking API.
|
|
1157
|
+
*
|
|
1158
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
1159
|
+
* handle must remain usable.
|
|
1160
|
+
*/
|
|
1161
|
+
copyForTransfer(): WasmMapResult;
|
|
753
1162
|
static default(): WasmMapResult;
|
|
1163
|
+
/**
|
|
1164
|
+
* Takes ownership of every element in `urls`.
|
|
1165
|
+
* wasm-bindgen cannot borrow class values nested in an array. To keep those handles,
|
|
1166
|
+
* copy each retained element with `copyForTransfer()` before passing the array.
|
|
1167
|
+
*/
|
|
754
1168
|
constructor(urls?: WasmSitemapUrl[] | null);
|
|
1169
|
+
/**
|
|
1170
|
+
* Returns detached copies of the elements in `urls`.
|
|
1171
|
+
*
|
|
1172
|
+
* Changes to the returned values do not update this object. Modify the copies,
|
|
1173
|
+
* then assign the array back to `urls`.
|
|
1174
|
+
*/
|
|
755
1175
|
urls: WasmSitemapUrl[];
|
|
756
1176
|
}
|
|
757
1177
|
|
|
@@ -761,6 +1181,13 @@ export class WasmMapResult {
|
|
|
761
1181
|
export class WasmMarkdownResult {
|
|
762
1182
|
free(): void;
|
|
763
1183
|
[Symbol.dispose](): void;
|
|
1184
|
+
/**
|
|
1185
|
+
* Return an independent handle for an ownership-taking API.
|
|
1186
|
+
*
|
|
1187
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
1188
|
+
* handle must remain usable.
|
|
1189
|
+
*/
|
|
1190
|
+
copyForTransfer(): WasmMarkdownResult;
|
|
764
1191
|
static default(): WasmMarkdownResult;
|
|
765
1192
|
constructor(content?: string | null, tables?: any | null, warnings?: string[] | null, citations?: boolean | null, documentStructure?: any | null, fitContent?: string | null);
|
|
766
1193
|
citations: boolean;
|
|
@@ -809,10 +1236,34 @@ export class WasmPageAction {
|
|
|
809
1236
|
export class WasmPageMetadata {
|
|
810
1237
|
free(): void;
|
|
811
1238
|
[Symbol.dispose](): void;
|
|
1239
|
+
clearArticle(): void;
|
|
1240
|
+
/**
|
|
1241
|
+
* Return an independent handle for an ownership-taking API.
|
|
1242
|
+
*
|
|
1243
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
1244
|
+
* handle must remain usable.
|
|
1245
|
+
*/
|
|
1246
|
+
copyForTransfer(): WasmPageMetadata;
|
|
812
1247
|
static default(): WasmPageMetadata;
|
|
1248
|
+
/**
|
|
1249
|
+
* Takes ownership of `article`.
|
|
1250
|
+
* wasm-bindgen lowers a by-value class argument through `__destroy_into_raw()`, so each
|
|
1251
|
+
* of those handles is dead once this returns. Passing a freshly built value is fine. To
|
|
1252
|
+
* keep a handle you already hold, build with `default()` and assign the property instead:
|
|
1253
|
+
* the generated setter borrows and leaves your handle alive.
|
|
1254
|
+
* Takes ownership of every element in `hreflangs`, `favicons`, `headings`.
|
|
1255
|
+
* wasm-bindgen cannot borrow class values nested in an array. To keep those handles,
|
|
1256
|
+
* copy each retained element with `copyForTransfer()` before passing the array.
|
|
1257
|
+
*/
|
|
813
1258
|
constructor(title?: string | null, description?: string | null, canonicalUrl?: string | null, keywords?: string | null, author?: string | null, viewport?: string | null, themeColor?: string | null, generator?: string | null, robots?: string | null, htmlLang?: string | null, htmlDir?: string | null, ogTitle?: string | null, ogType?: string | null, ogImage?: string | null, ogDescription?: string | null, ogUrl?: string | null, ogSiteName?: string | null, ogLocale?: string | null, ogVideo?: string | null, ogAudio?: string | null, ogLocaleAlternates?: string[] | null, twitterCard?: string | null, twitterTitle?: string | null, twitterDescription?: string | null, twitterImage?: string | null, twitterSite?: string | null, twitterCreator?: string | null, dcTitle?: string | null, dcCreator?: string | null, dcSubject?: string | null, dcDescription?: string | null, dcPublisher?: string | null, dcDate?: string | null, dcType?: string | null, dcFormat?: string | null, dcIdentifier?: string | null, dcLanguage?: string | null, dcRights?: string | null, article?: WasmArticleMetadata | null, hreflangs?: WasmHreflangEntry[] | null, favicons?: WasmFaviconInfo[] | null, headings?: WasmHeadingInfo[] | null, wordCount?: number | null);
|
|
1259
|
+
/**
|
|
1260
|
+
* Returns a detached copy of `article`.
|
|
1261
|
+
*
|
|
1262
|
+
* Changes to the returned value do not update this object. To update `article`,
|
|
1263
|
+
* read it, modify the copy, then assign the copy back to `article`.
|
|
1264
|
+
*/
|
|
814
1265
|
get article(): WasmArticleMetadata | undefined;
|
|
815
|
-
set article(value: WasmArticleMetadata
|
|
1266
|
+
set article(value: WasmArticleMetadata);
|
|
816
1267
|
get author(): string | undefined;
|
|
817
1268
|
set author(value: string | null | undefined);
|
|
818
1269
|
get canonicalUrl(): string | undefined;
|
|
@@ -841,13 +1292,46 @@ export class WasmPageMetadata {
|
|
|
841
1292
|
set dcType(value: string | null | undefined);
|
|
842
1293
|
get description(): string | undefined;
|
|
843
1294
|
set description(value: string | null | undefined);
|
|
1295
|
+
/**
|
|
1296
|
+
* Returns detached copies of the elements in `favicons`.
|
|
1297
|
+
*
|
|
1298
|
+
* Changes to the returned values do not update this object. Modify the copies,
|
|
1299
|
+
* then assign the array back to `favicons`.
|
|
1300
|
+
*/
|
|
844
1301
|
get favicons(): Array<any> | undefined;
|
|
1302
|
+
/**
|
|
1303
|
+
* Takes ownership of every element in `favicons`.
|
|
1304
|
+
* wasm-bindgen cannot borrow class values nested in an array; copy each retained
|
|
1305
|
+
* element with `copyForTransfer()` before passing the array.
|
|
1306
|
+
*/
|
|
845
1307
|
set favicons(value: WasmFaviconInfo[] | null | undefined);
|
|
846
1308
|
get generator(): string | undefined;
|
|
847
1309
|
set generator(value: string | null | undefined);
|
|
1310
|
+
/**
|
|
1311
|
+
* Returns detached copies of the elements in `headings`.
|
|
1312
|
+
*
|
|
1313
|
+
* Changes to the returned values do not update this object. Modify the copies,
|
|
1314
|
+
* then assign the array back to `headings`.
|
|
1315
|
+
*/
|
|
848
1316
|
get headings(): Array<any> | undefined;
|
|
1317
|
+
/**
|
|
1318
|
+
* Takes ownership of every element in `headings`.
|
|
1319
|
+
* wasm-bindgen cannot borrow class values nested in an array; copy each retained
|
|
1320
|
+
* element with `copyForTransfer()` before passing the array.
|
|
1321
|
+
*/
|
|
849
1322
|
set headings(value: WasmHeadingInfo[] | null | undefined);
|
|
1323
|
+
/**
|
|
1324
|
+
* Returns detached copies of the elements in `hreflangs`.
|
|
1325
|
+
*
|
|
1326
|
+
* Changes to the returned values do not update this object. Modify the copies,
|
|
1327
|
+
* then assign the array back to `hreflangs`.
|
|
1328
|
+
*/
|
|
850
1329
|
get hreflangs(): Array<any> | undefined;
|
|
1330
|
+
/**
|
|
1331
|
+
* Takes ownership of every element in `hreflangs`.
|
|
1332
|
+
* wasm-bindgen cannot borrow class values nested in an array; copy each retained
|
|
1333
|
+
* element with `copyForTransfer()` before passing the array.
|
|
1334
|
+
*/
|
|
851
1335
|
set hreflangs(value: WasmHreflangEntry[] | null | undefined);
|
|
852
1336
|
get htmlDir(): string | undefined;
|
|
853
1337
|
set htmlDir(value: string | null | undefined);
|
|
@@ -905,6 +1389,13 @@ export class WasmPageMetadata {
|
|
|
905
1389
|
export class WasmProxyConfig {
|
|
906
1390
|
free(): void;
|
|
907
1391
|
[Symbol.dispose](): void;
|
|
1392
|
+
/**
|
|
1393
|
+
* Return an independent handle for an ownership-taking API.
|
|
1394
|
+
*
|
|
1395
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
1396
|
+
* handle must remain usable.
|
|
1397
|
+
*/
|
|
1398
|
+
copyForTransfer(): WasmProxyConfig;
|
|
908
1399
|
static default(): WasmProxyConfig;
|
|
909
1400
|
constructor(url?: string | null, username?: string | null, password?: string | null);
|
|
910
1401
|
get password(): string | undefined;
|
|
@@ -920,6 +1411,13 @@ export class WasmProxyConfig {
|
|
|
920
1411
|
export class WasmResponseMeta {
|
|
921
1412
|
free(): void;
|
|
922
1413
|
[Symbol.dispose](): void;
|
|
1414
|
+
/**
|
|
1415
|
+
* Return an independent handle for an ownership-taking API.
|
|
1416
|
+
*
|
|
1417
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
1418
|
+
* handle must remain usable.
|
|
1419
|
+
*/
|
|
1420
|
+
copyForTransfer(): WasmResponseMeta;
|
|
923
1421
|
static default(): WasmResponseMeta;
|
|
924
1422
|
constructor(etag?: string | null, lastModified?: string | null, cacheControl?: string | null, server?: string | null, xPoweredBy?: string | null, contentLanguage?: string | null, contentEncoding?: string | null);
|
|
925
1423
|
get cacheControl(): string | undefined;
|
|
@@ -944,41 +1442,129 @@ export class WasmResponseMeta {
|
|
|
944
1442
|
export class WasmScrapeResult {
|
|
945
1443
|
free(): void;
|
|
946
1444
|
[Symbol.dispose](): void;
|
|
1445
|
+
clearBrowser(): void;
|
|
1446
|
+
clearDownloadedDocument(): void;
|
|
1447
|
+
clearExtractionMeta(): void;
|
|
1448
|
+
clearMarkdown(): void;
|
|
1449
|
+
clearResponseMeta(): void;
|
|
1450
|
+
/**
|
|
1451
|
+
* Return an independent handle for an ownership-taking API.
|
|
1452
|
+
*
|
|
1453
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
1454
|
+
* handle must remain usable.
|
|
1455
|
+
*/
|
|
1456
|
+
copyForTransfer(): WasmScrapeResult;
|
|
947
1457
|
static default(): WasmScrapeResult;
|
|
1458
|
+
/**
|
|
1459
|
+
* Takes ownership of `metadata`, `responseMeta`, `markdown`, `extractionMeta`, `downloadedDocument`, `browser`.
|
|
1460
|
+
* wasm-bindgen lowers a by-value class argument through `__destroy_into_raw()`, so each
|
|
1461
|
+
* of those handles is dead once this returns. Passing a freshly built value is fine. To
|
|
1462
|
+
* keep a handle you already hold, build with `default()` and assign the property instead:
|
|
1463
|
+
* the generated setter borrows and leaves your handle alive.
|
|
1464
|
+
* Takes ownership of every element in `links`, `images`, `feeds`, `jsonLd`, `assets`.
|
|
1465
|
+
* wasm-bindgen cannot borrow class values nested in an array. To keep those handles,
|
|
1466
|
+
* copy each retained element with `copyForTransfer()` before passing the array.
|
|
1467
|
+
*/
|
|
948
1468
|
constructor(statusCode?: number | null, finalUrl?: string | null, contentType?: string | null, html?: string | null, bodySize?: number | null, metadata?: WasmPageMetadata | null, links?: WasmLinkInfo[] | null, images?: WasmImageInfo[] | null, feeds?: WasmFeedInfo[] | null, jsonLd?: WasmJsonLdEntry[] | null, isAllowed?: boolean | null, noindexDetected?: boolean | null, nofollowDetected?: boolean | null, isPdf?: boolean | null, wasSkipped?: boolean | null, authHeaderSent?: boolean | null, assets?: WasmDownloadedAsset[] | null, jsRenderHint?: boolean | null, browserUsed?: boolean | null, ssrfRefusedUrls?: string[] | null, crawlDelay?: bigint | null, xRobotsTag?: string | null, detectedCharset?: string | null, responseMeta?: WasmResponseMeta | null, markdown?: WasmMarkdownResult | null, extractedData?: any | null, extractionMeta?: WasmExtractionMeta | null, screenshotBase64?: string | null, downloadedDocument?: WasmDownloadedDocument | null, browser?: WasmBrowserExtras | null);
|
|
1469
|
+
/**
|
|
1470
|
+
* Returns detached copies of the elements in `assets`.
|
|
1471
|
+
*
|
|
1472
|
+
* Changes to the returned values do not update this object. Modify the copies,
|
|
1473
|
+
* then assign the array back to `assets`.
|
|
1474
|
+
*/
|
|
949
1475
|
assets: WasmDownloadedAsset[];
|
|
950
1476
|
authHeaderSent: boolean;
|
|
951
1477
|
bodySize: number;
|
|
952
1478
|
browserUsed: boolean;
|
|
1479
|
+
/**
|
|
1480
|
+
* Returns a detached copy of `browser`.
|
|
1481
|
+
*
|
|
1482
|
+
* Changes to the returned value do not update this object. To update `browser`,
|
|
1483
|
+
* read it, modify the copy, then assign the copy back to `browser`.
|
|
1484
|
+
*/
|
|
953
1485
|
get browser(): WasmBrowserExtras | undefined;
|
|
954
|
-
set browser(value: WasmBrowserExtras
|
|
1486
|
+
set browser(value: WasmBrowserExtras);
|
|
955
1487
|
contentType: string;
|
|
956
1488
|
get crawlDelay(): bigint | undefined;
|
|
957
1489
|
set crawlDelay(value: bigint | null | undefined);
|
|
958
1490
|
get detectedCharset(): string | undefined;
|
|
959
1491
|
set detectedCharset(value: string | null | undefined);
|
|
1492
|
+
/**
|
|
1493
|
+
* Returns a detached copy of `downloadedDocument`.
|
|
1494
|
+
*
|
|
1495
|
+
* Changes to the returned value do not update this object. To update `downloadedDocument`,
|
|
1496
|
+
* read it, modify the copy, then assign the copy back to `downloadedDocument`.
|
|
1497
|
+
*/
|
|
960
1498
|
get downloadedDocument(): WasmDownloadedDocument | undefined;
|
|
961
|
-
set downloadedDocument(value: WasmDownloadedDocument
|
|
1499
|
+
set downloadedDocument(value: WasmDownloadedDocument);
|
|
962
1500
|
get extractedData(): any | undefined;
|
|
963
1501
|
set extractedData(value: any | null | undefined);
|
|
1502
|
+
/**
|
|
1503
|
+
* Returns a detached copy of `extractionMeta`.
|
|
1504
|
+
*
|
|
1505
|
+
* Changes to the returned value do not update this object. To update `extractionMeta`,
|
|
1506
|
+
* read it, modify the copy, then assign the copy back to `extractionMeta`.
|
|
1507
|
+
*/
|
|
964
1508
|
get extractionMeta(): WasmExtractionMeta | undefined;
|
|
965
|
-
set extractionMeta(value: WasmExtractionMeta
|
|
1509
|
+
set extractionMeta(value: WasmExtractionMeta);
|
|
1510
|
+
/**
|
|
1511
|
+
* Returns detached copies of the elements in `feeds`.
|
|
1512
|
+
*
|
|
1513
|
+
* Changes to the returned values do not update this object. Modify the copies,
|
|
1514
|
+
* then assign the array back to `feeds`.
|
|
1515
|
+
*/
|
|
966
1516
|
feeds: WasmFeedInfo[];
|
|
967
1517
|
finalUrl: string;
|
|
968
1518
|
html: string;
|
|
1519
|
+
/**
|
|
1520
|
+
* Returns detached copies of the elements in `images`.
|
|
1521
|
+
*
|
|
1522
|
+
* Changes to the returned values do not update this object. Modify the copies,
|
|
1523
|
+
* then assign the array back to `images`.
|
|
1524
|
+
*/
|
|
969
1525
|
images: WasmImageInfo[];
|
|
970
1526
|
isAllowed: boolean;
|
|
971
1527
|
isPdf: boolean;
|
|
972
1528
|
jsRenderHint: boolean;
|
|
1529
|
+
/**
|
|
1530
|
+
* Returns detached copies of the elements in `jsonLd`.
|
|
1531
|
+
*
|
|
1532
|
+
* Changes to the returned values do not update this object. Modify the copies,
|
|
1533
|
+
* then assign the array back to `jsonLd`.
|
|
1534
|
+
*/
|
|
973
1535
|
jsonLd: WasmJsonLdEntry[];
|
|
1536
|
+
/**
|
|
1537
|
+
* Returns detached copies of the elements in `links`.
|
|
1538
|
+
*
|
|
1539
|
+
* Changes to the returned values do not update this object. Modify the copies,
|
|
1540
|
+
* then assign the array back to `links`.
|
|
1541
|
+
*/
|
|
974
1542
|
links: WasmLinkInfo[];
|
|
1543
|
+
/**
|
|
1544
|
+
* Returns a detached copy of `markdown`.
|
|
1545
|
+
*
|
|
1546
|
+
* Changes to the returned value do not update this object. To update `markdown`,
|
|
1547
|
+
* read it, modify the copy, then assign the copy back to `markdown`.
|
|
1548
|
+
*/
|
|
975
1549
|
get markdown(): WasmMarkdownResult | undefined;
|
|
976
|
-
set markdown(value: WasmMarkdownResult
|
|
1550
|
+
set markdown(value: WasmMarkdownResult);
|
|
1551
|
+
/**
|
|
1552
|
+
* Returns a detached copy of `metadata`.
|
|
1553
|
+
*
|
|
1554
|
+
* Changes to the returned value do not update this object. To update `metadata`,
|
|
1555
|
+
* read it, modify the copy, then assign the copy back to `metadata`.
|
|
1556
|
+
*/
|
|
977
1557
|
metadata: WasmPageMetadata;
|
|
978
1558
|
nofollowDetected: boolean;
|
|
979
1559
|
noindexDetected: boolean;
|
|
1560
|
+
/**
|
|
1561
|
+
* Returns a detached copy of `responseMeta`.
|
|
1562
|
+
*
|
|
1563
|
+
* Changes to the returned value do not update this object. To update `responseMeta`,
|
|
1564
|
+
* read it, modify the copy, then assign the copy back to `responseMeta`.
|
|
1565
|
+
*/
|
|
980
1566
|
get responseMeta(): WasmResponseMeta | undefined;
|
|
981
|
-
set responseMeta(value: WasmResponseMeta
|
|
1567
|
+
set responseMeta(value: WasmResponseMeta);
|
|
982
1568
|
get screenshotBase64(): string | undefined;
|
|
983
1569
|
set screenshotBase64(value: string | null | undefined);
|
|
984
1570
|
ssrfRefusedUrls: string[];
|
|
@@ -1002,6 +1588,13 @@ export enum WasmScrollDirection {
|
|
|
1002
1588
|
export class WasmSitemapUrl {
|
|
1003
1589
|
free(): void;
|
|
1004
1590
|
[Symbol.dispose](): void;
|
|
1591
|
+
/**
|
|
1592
|
+
* Return an independent handle for an ownership-taking API.
|
|
1593
|
+
*
|
|
1594
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
1595
|
+
* handle must remain usable.
|
|
1596
|
+
*/
|
|
1597
|
+
copyForTransfer(): WasmSitemapUrl;
|
|
1005
1598
|
static default(): WasmSitemapUrl;
|
|
1006
1599
|
constructor(url?: string | null, lastmod?: string | null, changefreq?: string | null, priority?: string | null);
|
|
1007
1600
|
get changefreq(): string | undefined;
|
|
@@ -1019,6 +1612,13 @@ export class WasmSitemapUrl {
|
|
|
1019
1612
|
export class WasmSsrfPolicy {
|
|
1020
1613
|
free(): void;
|
|
1021
1614
|
[Symbol.dispose](): void;
|
|
1615
|
+
/**
|
|
1616
|
+
* Return an independent handle for an ownership-taking API.
|
|
1617
|
+
*
|
|
1618
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
1619
|
+
* handle must remain usable.
|
|
1620
|
+
*/
|
|
1621
|
+
copyForTransfer(): WasmSsrfPolicy;
|
|
1022
1622
|
static default(): WasmSsrfPolicy;
|
|
1023
1623
|
/**
|
|
1024
1624
|
* Create a policy from environment variables.
|
|
@@ -1032,18 +1632,19 @@ export class WasmSsrfPolicy {
|
|
|
1032
1632
|
* - Rust-side SSRF checking is unenforceable and redundant in a wasm32 context.
|
|
1033
1633
|
* - For testing and localhost access, the host's network sandbox is the enforcing boundary.
|
|
1034
1634
|
*
|
|
1035
|
-
* **Node.js caveat:** `deny_private`
|
|
1036
|
-
*
|
|
1037
|
-
*
|
|
1038
|
-
*
|
|
1039
|
-
*
|
|
1040
|
-
*
|
|
1041
|
-
*
|
|
1635
|
+
* **Node.js caveat:** `deny_private` and `denylist` have no effect on hostname-based requests
|
|
1636
|
+
* under `wasm32`. There is no DNS resolution on this target, so `validate_url` only ever
|
|
1637
|
+
* checks a literal IP host; a domain name falls straight through to `Ok(())`. In a browser
|
|
1638
|
+
* this is covered by same-origin/CORS. Node's `fetch` enforces no CORS, so a Node service
|
|
1639
|
+
* embedding this wasm module can be driven to internal hosts by domain name despite the
|
|
1640
|
+
* policy. Enforce egress restrictions (network policy, firewall, proxy allowlist) outside
|
|
1641
|
+
* the process.
|
|
1042
1642
|
*/
|
|
1043
1643
|
static fromEnv(): WasmSsrfPolicy;
|
|
1044
|
-
constructor(denyPrivate?: boolean | null, allowlist?: any | null, maxRedirects?: number | null, schemeAllowlist?: string[] | null);
|
|
1644
|
+
constructor(denyPrivate?: boolean | null, allowlist?: any | null, denylist?: any | null, maxRedirects?: number | null, schemeAllowlist?: string[] | null);
|
|
1045
1645
|
allowlist: any;
|
|
1046
1646
|
denyPrivate: boolean;
|
|
1647
|
+
denylist: any;
|
|
1047
1648
|
maxRedirects: number;
|
|
1048
1649
|
schemeAllowlist: string[];
|
|
1049
1650
|
}
|