@xberg-io/crawlberg-wasm 1.11.0 → 1.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +3 -3
- package/pkg/bundler/crawlberg_wasm.d.ts +43 -3
- package/pkg/bundler/crawlberg_wasm.js +1 -1
- package/pkg/bundler/crawlberg_wasm_bg.js +214 -17
- package/pkg/bundler/crawlberg_wasm_bg.wasm +0 -0
- package/pkg/bundler/crawlberg_wasm_bg.wasm.d.ts +25 -6
- package/pkg/bundler/package.json +1 -1
- package/pkg/deno/crawlberg_wasm.d.ts +43 -3
- package/pkg/deno/crawlberg_wasm.js +214 -17
- package/pkg/deno/crawlberg_wasm_bg.wasm +0 -0
- package/pkg/deno/crawlberg_wasm_bg.wasm.d.ts +25 -6
- package/pkg/nodejs/crawlberg_wasm.d.ts +43 -3
- package/pkg/nodejs/crawlberg_wasm.js +215 -17
- package/pkg/nodejs/crawlberg_wasm_bg.wasm +0 -0
- package/pkg/nodejs/crawlberg_wasm_bg.wasm.d.ts +25 -6
- package/pkg/nodejs/package.json +1 -1
- package/pkg/web/crawlberg_wasm.d.ts +68 -9
- package/pkg/web/crawlberg_wasm.js +214 -17
- package/pkg/web/crawlberg_wasm_bg.wasm +0 -0
- package/pkg/web/crawlberg_wasm_bg.wasm.d.ts +25 -6
- package/pkg/web/package.json +1 -1
|
@@ -260,7 +260,7 @@ export class WasmBrowserConfig {
|
|
|
260
260
|
* keep a handle you already hold, build with `default()` and assign the property instead:
|
|
261
261
|
* the generated setter borrows and leaves your handle alive.
|
|
262
262
|
*/
|
|
263
|
-
constructor(mode?: WasmBrowserMode | null, backend?: WasmBrowserBackend | null, timeout?: bigint | null, overallTimeout?: bigint | null, shutdownTimeout?: bigint | null, wait?: WasmBrowserWait | null, blockUrlPatterns?: string[] | null, captureNetworkEvents?: boolean | null, sessionAffinity?: boolean | null, chromeArgs?: string[] | null, endpoint?: string | null, waitSelector?: string | null, extraWait?: bigint | null, proxy?: WasmProxyConfig | null, evalScript?: string | null, robotsUserAgent?: string | null, chromePath?: string | null);
|
|
263
|
+
constructor(mode?: WasmBrowserMode | null, backend?: WasmBrowserBackend | null, timeout?: bigint | null, overallTimeout?: bigint | null, shutdownTimeout?: bigint | null, wait?: WasmBrowserWait | null, blockUrlPatterns?: string[] | null, captureNetworkEvents?: boolean | null, sessionAffinity?: boolean | null, chromeArgs?: string[] | null, endpoint?: string | null, waitSelector?: string | null, extraWait?: bigint | null, proxy?: WasmProxyConfig | null, evalScript?: string | null, robotsUserAgent?: string | null, chromePath?: string | null, scrollContainers?: boolean | null, domQuietWindow?: bigint | null);
|
|
264
264
|
get backend(): string;
|
|
265
265
|
set backend(value: WasmBrowserBackend);
|
|
266
266
|
blockUrlPatterns: string[];
|
|
@@ -268,6 +268,8 @@ export class WasmBrowserConfig {
|
|
|
268
268
|
chromeArgs: string[];
|
|
269
269
|
get chromePath(): string | undefined;
|
|
270
270
|
set chromePath(value: string | null | undefined);
|
|
271
|
+
get domQuietWindow(): bigint | undefined;
|
|
272
|
+
set domQuietWindow(value: bigint | null | undefined);
|
|
271
273
|
get endpoint(): string | undefined;
|
|
272
274
|
set endpoint(value: string | null | undefined);
|
|
273
275
|
get evalScript(): string | undefined;
|
|
@@ -288,6 +290,7 @@ export class WasmBrowserConfig {
|
|
|
288
290
|
set proxy(value: WasmProxyConfig);
|
|
289
291
|
get robotsUserAgent(): string | undefined;
|
|
290
292
|
set robotsUserAgent(value: string | null | undefined);
|
|
293
|
+
scrollContainers: boolean;
|
|
291
294
|
sessionAffinity: boolean;
|
|
292
295
|
get shutdownTimeout(): bigint | undefined;
|
|
293
296
|
set shutdownTimeout(value: bigint | null | undefined);
|
|
@@ -506,7 +509,7 @@ export class WasmCrawlConfig {
|
|
|
506
509
|
* keep a handle you already hold, build with `default()` and assign the property instead:
|
|
507
510
|
* the generated setter borrows and leaves your handle alive.
|
|
508
511
|
*/
|
|
509
|
-
constructor(crawlStrategy?: WasmCrawlStrategyKind | null, respectRobotsTxt?: boolean | null, softHttpErrors?: boolean | null, stayOnDomain?: boolean | null, allowSubdomains?: boolean | null, includePaths?: string[] | null, excludePaths?: string[] | null, pathPatternsMatchQuery?: boolean | null, pathPatternsMatchUrl?: boolean | null, dedupIncludeQuery?: boolean | null, stripTrackingParams?: boolean | null, trackingParams?: string[] | null, customHeaders?: any | null, requestTimeout?: bigint | null, maxRedirects?: number | null, retryCount?: number | null, retryCodes?: Uint16Array | null, retryInitialDelayMs?: bigint | null, retryMaxDelayMs?: bigint | null, rateLimitJitterRatio?: number | null, cookiesEnabled?: boolean | null, removeTags?: string[] | null, content?: WasmContentConfig | null, downloadAssets?: boolean | null, assetTypes?: WasmAssetCategory[] | null, browser?: WasmBrowserConfig | null, userAgents?: string[] | null, captureScreenshot?: boolean | null, followDocumentUrls?: boolean | null, downloadDocuments?: boolean | null, documentMimeTypes?: string[] | null, saveBrowserProfile?: boolean | null, ssrf?: WasmSsrfPolicy | null, maxDepth?: number | null, maxPages?: number | null, maxLinksPerPage?: number | null, maxConcurrent?: number | null, contentFilter?: WasmContentFilterKind | null, bm25Query?: string | null, bm25Threshold?: number | null, userAgent?: string | null, rateLimitMs?: bigint | null, auth?: any | null, maxBodySize?: number | null, mapLimit?: number | null, mapSearch?: string | null, maxAssetSize?: number | null, proxy?: WasmProxyConfig | null, documentUrlDepth?: number | null, documentMaxSize?: number | null, documentOutputDir?: string | null, documentContentEncoding?: WasmDocumentContentEncoding | null, warcOutput?: string | null, browserProfile?: string | null, ssrfDenyPrivateExplicit?: boolean | null);
|
|
512
|
+
constructor(crawlStrategy?: WasmCrawlStrategyKind | null, respectRobotsTxt?: boolean | null, softHttpErrors?: boolean | null, stayOnDomain?: boolean | null, allowSubdomains?: boolean | null, includePaths?: string[] | null, excludePaths?: string[] | null, pathPatternsMatchQuery?: boolean | null, pathPatternsMatchUrl?: boolean | null, dedupIncludeQuery?: boolean | null, stripTrackingParams?: boolean | null, trackingParams?: string[] | null, customHeaders?: any | null, requestTimeout?: bigint | null, maxRedirects?: number | null, retryCount?: number | null, retryCodes?: Uint16Array | null, retryInitialDelayMs?: bigint | null, retryMaxDelayMs?: bigint | null, rateLimitJitterRatio?: number | null, cookiesEnabled?: boolean | null, removeTags?: string[] | null, content?: WasmContentConfig | null, downloadAssets?: boolean | null, assetTypes?: WasmAssetCategory[] | null, browser?: WasmBrowserConfig | null, userAgents?: string[] | null, captureScreenshot?: boolean | null, followDocumentUrls?: boolean | null, downloadDocuments?: boolean | null, documentMimeTypes?: string[] | null, saveBrowserProfile?: boolean | null, ssrf?: WasmSsrfPolicy | null, maxDepth?: number | null, maxPages?: number | null, maxLinksPerPage?: number | null, maxConcurrent?: number | null, contentFilter?: WasmContentFilterKind | null, bm25Query?: string | null, bm25Threshold?: number | null, userAgent?: string | null, rateLimitMs?: bigint | null, auth?: any | null, maxBodySize?: number | null, mapLimit?: number | null, mapSearch?: string | null, maxAssetSize?: number | null, proxy?: WasmProxyConfig | null, documentUrlDepth?: number | null, documentMaxSize?: number | null, documentOutputDir?: string | null, documentContentEncoding?: WasmDocumentContentEncoding | null, warcOutput?: string | null, browserProfile?: string | null, ssrfDenyPrivateExplicit?: boolean | null, dangerAcceptInvalidCerts?: boolean | null);
|
|
510
513
|
/**
|
|
511
514
|
* Validate the configuration, returning an error if any values are invalid.
|
|
512
515
|
*
|
|
@@ -545,6 +548,7 @@ export class WasmCrawlConfig {
|
|
|
545
548
|
get crawlStrategy(): string;
|
|
546
549
|
set crawlStrategy(value: WasmCrawlStrategyKind);
|
|
547
550
|
customHeaders: any;
|
|
551
|
+
dangerAcceptInvalidCerts: boolean;
|
|
548
552
|
dedupIncludeQuery: boolean;
|
|
549
553
|
get documentContentEncoding(): string | undefined;
|
|
550
554
|
set documentContentEncoding(value: WasmDocumentContentEncoding | null | undefined);
|
|
@@ -754,6 +758,7 @@ export class WasmCrawlPageResult {
|
|
|
754
758
|
export class WasmCrawlResult {
|
|
755
759
|
free(): void;
|
|
756
760
|
[Symbol.dispose](): void;
|
|
761
|
+
clearWafBlock(): void;
|
|
757
762
|
/**
|
|
758
763
|
* Return an independent handle for an ownership-taking API.
|
|
759
764
|
*
|
|
@@ -763,11 +768,16 @@ export class WasmCrawlResult {
|
|
|
763
768
|
copyForTransfer(): WasmCrawlResult;
|
|
764
769
|
static default(): WasmCrawlResult;
|
|
765
770
|
/**
|
|
771
|
+
* Takes ownership of `wafBlock`.
|
|
772
|
+
* wasm-bindgen lowers a by-value class argument through `__destroy_into_raw()`, so each
|
|
773
|
+
* of those handles is dead once this returns. Passing a freshly built value is fine. To
|
|
774
|
+
* keep a handle you already hold, build with `default()` and assign the property instead:
|
|
775
|
+
* the generated setter borrows and leaves your handle alive.
|
|
766
776
|
* Takes ownership of every element in `pages`, `cookies`.
|
|
767
777
|
* wasm-bindgen cannot borrow class values nested in an array. To keep those handles,
|
|
768
778
|
* copy each retained element with `copyForTransfer()` before passing the array.
|
|
769
779
|
*/
|
|
770
|
-
constructor(pages?: WasmCrawlPageResult[] | null, finalUrl?: string | null, redirectCount?: number | null, wasSkipped?: boolean | null, cookies?: WasmCookieInfo[] | null, stayedOnDomain?: boolean | null, browserUsed?: boolean | null, error?: string | null);
|
|
780
|
+
constructor(pages?: WasmCrawlPageResult[] | null, finalUrl?: string | null, redirectCount?: number | null, wasSkipped?: boolean | null, cookies?: WasmCookieInfo[] | null, stayedOnDomain?: boolean | null, browserUsed?: boolean | null, error?: string | null, wafBlock?: WasmWafBlockDetails | null);
|
|
771
781
|
/**
|
|
772
782
|
* Returns the count of unique normalized URLs encountered during crawling.
|
|
773
783
|
*
|
|
@@ -799,6 +809,14 @@ export class WasmCrawlResult {
|
|
|
799
809
|
pages: WasmCrawlPageResult[];
|
|
800
810
|
redirectCount: number;
|
|
801
811
|
stayedOnDomain: boolean;
|
|
812
|
+
/**
|
|
813
|
+
* Returns a detached copy of `wafBlock`.
|
|
814
|
+
*
|
|
815
|
+
* Changes to the returned value do not update this object. To update `wafBlock`,
|
|
816
|
+
* read it, modify the copy, then assign the copy back to `wafBlock`.
|
|
817
|
+
*/
|
|
818
|
+
get wafBlock(): WasmWafBlockDetails | undefined;
|
|
819
|
+
set wafBlock(value: WasmWafBlockDetails);
|
|
802
820
|
wasSkipped: boolean;
|
|
803
821
|
}
|
|
804
822
|
|
|
@@ -1661,6 +1679,28 @@ export class WasmSsrfPolicy {
|
|
|
1661
1679
|
schemeAllowlist: string[];
|
|
1662
1680
|
}
|
|
1663
1681
|
|
|
1682
|
+
/**
|
|
1683
|
+
* Structured context for a WAF-blocked response. <!-- -->
|
|
1684
|
+
*/
|
|
1685
|
+
export class WasmWafBlockDetails {
|
|
1686
|
+
free(): void;
|
|
1687
|
+
[Symbol.dispose](): void;
|
|
1688
|
+
/**
|
|
1689
|
+
* Return an independent handle for an ownership-taking API.
|
|
1690
|
+
*
|
|
1691
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
1692
|
+
* handle must remain usable.
|
|
1693
|
+
*/
|
|
1694
|
+
copyForTransfer(): WasmWafBlockDetails;
|
|
1695
|
+
static default(): WasmWafBlockDetails;
|
|
1696
|
+
constructor(vendor?: string | null, status?: number | null, bodyPrefix?: string | null);
|
|
1697
|
+
get bodyPrefix(): string | undefined;
|
|
1698
|
+
set bodyPrefix(value: string | null | undefined);
|
|
1699
|
+
get status(): number | undefined;
|
|
1700
|
+
set status(value: number | null | undefined);
|
|
1701
|
+
vendor: string;
|
|
1702
|
+
}
|
|
1703
|
+
|
|
1664
1704
|
/**
|
|
1665
1705
|
* Crawl multiple seed URLs concurrently, each following links to configured depth.
|
|
1666
1706
|
*/
|
|
@@ -1750,6 +1790,7 @@ export interface InitOutput {
|
|
|
1750
1790
|
readonly __wbg_wasmscraperesult_free: (a: number, b: number) => void;
|
|
1751
1791
|
readonly __wbg_wasmsitemapurl_free: (a: number, b: number) => void;
|
|
1752
1792
|
readonly __wbg_wasmssrfpolicy_free: (a: number, b: number) => void;
|
|
1793
|
+
readonly __wbg_wasmwafblockdetails_free: (a: number, b: number) => void;
|
|
1753
1794
|
readonly batchCrawl: (a: number, b: number, c: number) => any;
|
|
1754
1795
|
readonly batchScrape: (a: number, b: number, c: number) => any;
|
|
1755
1796
|
readonly crawl: (a: number, b: number, c: number) => any;
|
|
@@ -1848,20 +1889,23 @@ export interface InitOutput {
|
|
|
1848
1889
|
readonly wasmbrowserconfig_clearProxy: (a: number) => void;
|
|
1849
1890
|
readonly wasmbrowserconfig_copyForTransfer: (a: number) => number;
|
|
1850
1891
|
readonly wasmbrowserconfig_default: () => number;
|
|
1892
|
+
readonly wasmbrowserconfig_domQuietWindow: (a: number) => [number, bigint];
|
|
1851
1893
|
readonly wasmbrowserconfig_endpoint: (a: number) => [number, number];
|
|
1852
1894
|
readonly wasmbrowserconfig_evalScript: (a: number) => [number, number];
|
|
1853
1895
|
readonly wasmbrowserconfig_extraWait: (a: number) => [number, bigint];
|
|
1854
1896
|
readonly wasmbrowserconfig_mode: (a: number) => [number, number];
|
|
1855
|
-
readonly wasmbrowserconfig_new: (a: number, b: number, c: number, d: bigint, e: number, f: bigint, g: number, h: bigint, i: number, j: number, k: number, l: number, m: number, n: number, o: number, p: number, q: number, r: number, s: number, t: number, u: bigint, v: number, w: number, x: number, y: number, z: number, a1: number, b1: number) => number;
|
|
1897
|
+
readonly wasmbrowserconfig_new: (a: number, b: number, c: number, d: bigint, e: number, f: bigint, g: number, h: bigint, i: number, j: number, k: number, l: number, m: number, n: number, o: number, p: number, q: number, r: number, s: number, t: number, u: bigint, v: number, w: number, x: number, y: number, z: number, a1: number, b1: number, c1: number, d1: number, e1: bigint) => number;
|
|
1856
1898
|
readonly wasmbrowserconfig_overallTimeout: (a: number) => [number, bigint];
|
|
1857
1899
|
readonly wasmbrowserconfig_proxy: (a: number) => number;
|
|
1858
1900
|
readonly wasmbrowserconfig_robotsUserAgent: (a: number) => [number, number];
|
|
1901
|
+
readonly wasmbrowserconfig_scrollContainers: (a: number) => number;
|
|
1859
1902
|
readonly wasmbrowserconfig_sessionAffinity: (a: number) => number;
|
|
1860
1903
|
readonly wasmbrowserconfig_set_backend: (a: number, b: number) => void;
|
|
1861
1904
|
readonly wasmbrowserconfig_set_blockUrlPatterns: (a: number, b: number, c: number) => void;
|
|
1862
1905
|
readonly wasmbrowserconfig_set_captureNetworkEvents: (a: number, b: number) => void;
|
|
1863
1906
|
readonly wasmbrowserconfig_set_chromeArgs: (a: number, b: number, c: number) => void;
|
|
1864
1907
|
readonly wasmbrowserconfig_set_chromePath: (a: number, b: number, c: number) => void;
|
|
1908
|
+
readonly wasmbrowserconfig_set_domQuietWindow: (a: number, b: number, c: bigint) => void;
|
|
1865
1909
|
readonly wasmbrowserconfig_set_endpoint: (a: number, b: number, c: number) => void;
|
|
1866
1910
|
readonly wasmbrowserconfig_set_evalScript: (a: number, b: number, c: number) => void;
|
|
1867
1911
|
readonly wasmbrowserconfig_set_extraWait: (a: number, b: number, c: bigint) => void;
|
|
@@ -1869,6 +1913,7 @@ export interface InitOutput {
|
|
|
1869
1913
|
readonly wasmbrowserconfig_set_overallTimeout: (a: number, b: number, c: bigint) => void;
|
|
1870
1914
|
readonly wasmbrowserconfig_set_proxy: (a: number, b: number) => void;
|
|
1871
1915
|
readonly wasmbrowserconfig_set_robotsUserAgent: (a: number, b: number, c: number) => void;
|
|
1916
|
+
readonly wasmbrowserconfig_set_scrollContainers: (a: number, b: number) => void;
|
|
1872
1917
|
readonly wasmbrowserconfig_set_sessionAffinity: (a: number, b: number) => void;
|
|
1873
1918
|
readonly wasmbrowserconfig_set_shutdownTimeout: (a: number, b: number, c: bigint) => void;
|
|
1874
1919
|
readonly wasmbrowserconfig_set_timeout: (a: number, b: number, c: bigint) => void;
|
|
@@ -1972,6 +2017,7 @@ export interface InitOutput {
|
|
|
1972
2017
|
readonly wasmcrawlconfig_copyForTransfer: (a: number) => number;
|
|
1973
2018
|
readonly wasmcrawlconfig_crawlStrategy: (a: number) => [number, number];
|
|
1974
2019
|
readonly wasmcrawlconfig_customHeaders: (a: number) => any;
|
|
2020
|
+
readonly wasmcrawlconfig_dangerAcceptInvalidCerts: (a: number) => number;
|
|
1975
2021
|
readonly wasmcrawlconfig_dedupIncludeQuery: (a: number) => number;
|
|
1976
2022
|
readonly wasmcrawlconfig_default: () => number;
|
|
1977
2023
|
readonly wasmcrawlconfig_documentContentEncoding: (a: number) => [number, number];
|
|
@@ -1993,7 +2039,7 @@ export interface InitOutput {
|
|
|
1993
2039
|
readonly wasmcrawlconfig_maxLinksPerPage: (a: number) => number;
|
|
1994
2040
|
readonly wasmcrawlconfig_maxPages: (a: number) => number;
|
|
1995
2041
|
readonly wasmcrawlconfig_maxRedirects: (a: number) => number;
|
|
1996
|
-
readonly wasmcrawlconfig_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number, n: number, o: number, p: number, q: number, r: bigint, s: number, t: number, u: number, v: number, w: number, x: bigint, y: number, z: bigint, a1: number, b1: number, c1: number, d1: number, e1: number, f1: number, g1: number, h1: number, i1: number, j1: number, k1: number, l1: number, m1: number, n1: number, o1: number, p1: number, q1: number, r1: number, s1: number, t1: number, u1: number, v1: number, w1: number, x1: number, y1: number, z1: number, a2: number, b2: number, c2: number, d2: number, e2: number, f2: bigint, g2: number, h2: number, i2: number, j2: number, k2: number, l2: number, m2: number, n2: number, o2: number, p2: number, q2: number, r2: number, s2: number, t2: number, u2: number, v2: number, w2: number) => number;
|
|
2042
|
+
readonly wasmcrawlconfig_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number, n: number, o: number, p: number, q: number, r: bigint, s: number, t: number, u: number, v: number, w: number, x: bigint, y: number, z: bigint, a1: number, b1: number, c1: number, d1: number, e1: number, f1: number, g1: number, h1: number, i1: number, j1: number, k1: number, l1: number, m1: number, n1: number, o1: number, p1: number, q1: number, r1: number, s1: number, t1: number, u1: number, v1: number, w1: number, x1: number, y1: number, z1: number, a2: number, b2: number, c2: number, d2: number, e2: number, f2: bigint, g2: number, h2: number, i2: number, j2: number, k2: number, l2: number, m2: number, n2: number, o2: number, p2: number, q2: number, r2: number, s2: number, t2: number, u2: number, v2: number, w2: number, x2: number) => number;
|
|
1997
2043
|
readonly wasmcrawlconfig_pathPatternsMatchQuery: (a: number) => number;
|
|
1998
2044
|
readonly wasmcrawlconfig_pathPatternsMatchUrl: (a: number) => number;
|
|
1999
2045
|
readonly wasmcrawlconfig_proxy: (a: number) => number;
|
|
@@ -2020,6 +2066,7 @@ export interface InitOutput {
|
|
|
2020
2066
|
readonly wasmcrawlconfig_set_cookiesEnabled: (a: number, b: number) => void;
|
|
2021
2067
|
readonly wasmcrawlconfig_set_crawlStrategy: (a: number, b: number) => void;
|
|
2022
2068
|
readonly wasmcrawlconfig_set_customHeaders: (a: number, b: any) => void;
|
|
2069
|
+
readonly wasmcrawlconfig_set_dangerAcceptInvalidCerts: (a: number, b: number) => void;
|
|
2023
2070
|
readonly wasmcrawlconfig_set_dedupIncludeQuery: (a: number, b: number) => void;
|
|
2024
2071
|
readonly wasmcrawlconfig_set_documentContentEncoding: (a: number, b: number) => void;
|
|
2025
2072
|
readonly wasmcrawlconfig_set_documentMaxSize: (a: number, b: number) => void;
|
|
@@ -2134,12 +2181,13 @@ export interface InitOutput {
|
|
|
2134
2181
|
readonly wasmcrawlpageresult_url: (a: number) => [number, number];
|
|
2135
2182
|
readonly wasmcrawlpageresult_wasSkipped: (a: number) => number;
|
|
2136
2183
|
readonly wasmcrawlresult_browserUsed: (a: number) => number;
|
|
2184
|
+
readonly wasmcrawlresult_clearWafBlock: (a: number) => void;
|
|
2137
2185
|
readonly wasmcrawlresult_cookies: (a: number) => [number, number];
|
|
2138
2186
|
readonly wasmcrawlresult_copyForTransfer: (a: number) => number;
|
|
2139
2187
|
readonly wasmcrawlresult_default: () => number;
|
|
2140
2188
|
readonly wasmcrawlresult_error: (a: number) => [number, number];
|
|
2141
2189
|
readonly wasmcrawlresult_finalUrl: (a: number) => [number, number];
|
|
2142
|
-
readonly wasmcrawlresult_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number) => number;
|
|
2190
|
+
readonly wasmcrawlresult_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number) => number;
|
|
2143
2191
|
readonly wasmcrawlresult_pages: (a: number) => [number, number];
|
|
2144
2192
|
readonly wasmcrawlresult_redirectCount: (a: number) => number;
|
|
2145
2193
|
readonly wasmcrawlresult_set_browserUsed: (a: number, b: number) => void;
|
|
@@ -2149,9 +2197,11 @@ export interface InitOutput {
|
|
|
2149
2197
|
readonly wasmcrawlresult_set_pages: (a: number, b: number, c: number) => void;
|
|
2150
2198
|
readonly wasmcrawlresult_set_redirectCount: (a: number, b: number) => void;
|
|
2151
2199
|
readonly wasmcrawlresult_set_stayedOnDomain: (a: number, b: number) => void;
|
|
2200
|
+
readonly wasmcrawlresult_set_wafBlock: (a: number, b: number) => void;
|
|
2152
2201
|
readonly wasmcrawlresult_set_wasSkipped: (a: number, b: number) => void;
|
|
2153
2202
|
readonly wasmcrawlresult_stayedOnDomain: (a: number) => number;
|
|
2154
2203
|
readonly wasmcrawlresult_uniqueNormalizedUrls: (a: number) => number;
|
|
2204
|
+
readonly wasmcrawlresult_wafBlock: (a: number) => number;
|
|
2155
2205
|
readonly wasmcrawlresult_wasSkipped: (a: number) => number;
|
|
2156
2206
|
readonly wasmdownloadedasset_assetCategory: (a: number) => [number, number];
|
|
2157
2207
|
readonly wasmdownloadedasset_contentHash: (a: number) => [number, number];
|
|
@@ -2541,9 +2591,18 @@ export interface InitOutput {
|
|
|
2541
2591
|
readonly wasmssrfpolicy_set_denylist: (a: number, b: any) => void;
|
|
2542
2592
|
readonly wasmssrfpolicy_set_maxRedirects: (a: number, b: number) => void;
|
|
2543
2593
|
readonly wasmssrfpolicy_set_schemeAllowlist: (a: number, b: number, c: number) => void;
|
|
2544
|
-
readonly
|
|
2545
|
-
readonly
|
|
2546
|
-
readonly
|
|
2594
|
+
readonly wasmwafblockdetails_bodyPrefix: (a: number) => [number, number];
|
|
2595
|
+
readonly wasmwafblockdetails_copyForTransfer: (a: number) => number;
|
|
2596
|
+
readonly wasmwafblockdetails_default: () => number;
|
|
2597
|
+
readonly wasmwafblockdetails_new: (a: number, b: number, c: number, d: number, e: number) => number;
|
|
2598
|
+
readonly wasmwafblockdetails_set_bodyPrefix: (a: number, b: number, c: number) => void;
|
|
2599
|
+
readonly wasmwafblockdetails_set_status: (a: number, b: number) => void;
|
|
2600
|
+
readonly wasmwafblockdetails_set_vendor: (a: number, b: number, c: number) => void;
|
|
2601
|
+
readonly wasmwafblockdetails_status: (a: number) => number;
|
|
2602
|
+
readonly wasmwafblockdetails_vendor: (a: number) => [number, number];
|
|
2603
|
+
readonly wasm_bindgen_a37d5a31ee71bc5f___convert__closures_____invoke___js_sys_a6a58716f022baea___Function_fn_wasm_bindgen_a37d5a31ee71bc5f___JsValue_____wasm_bindgen_a37d5a31ee71bc5f___sys__Undefined___js_sys_a6a58716f022baea___Function_fn_wasm_bindgen_a37d5a31ee71bc5f___JsValue_____wasm_bindgen_a37d5a31ee71bc5f___sys__Undefined_______true_: (a: number, b: number, c: any, d: any) => void;
|
|
2604
|
+
readonly wasm_bindgen_a37d5a31ee71bc5f___convert__closures_____invoke___wasm_bindgen_a37d5a31ee71bc5f___JsValue__core_37ee0ce0c4cd57d5___result__Result_____wasm_bindgen_a37d5a31ee71bc5f___JsError___true_: (a: number, b: number, c: any) => [number, number];
|
|
2605
|
+
readonly wasm_bindgen_a37d5a31ee71bc5f___convert__closures_____invoke_______true_: (a: number, b: number) => void;
|
|
2547
2606
|
readonly __wbindgen_malloc: (a: number, b: number) => number;
|
|
2548
2607
|
readonly __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number;
|
|
2549
2608
|
readonly __wbindgen_exn_store: (a: number) => void;
|
|
@@ -1105,6 +1105,13 @@ export class WasmBrowserConfig {
|
|
|
1105
1105
|
const ret = wasm.wasmbrowserconfig_default();
|
|
1106
1106
|
return WasmBrowserConfig.__wrap(ret);
|
|
1107
1107
|
}
|
|
1108
|
+
/**
|
|
1109
|
+
* @returns {bigint | undefined}
|
|
1110
|
+
*/
|
|
1111
|
+
get domQuietWindow() {
|
|
1112
|
+
const ret = wasm.wasmbrowserconfig_domQuietWindow(this.__wbg_ptr);
|
|
1113
|
+
return ret[0] === 0 ? undefined : BigInt.asUintN(64, ret[1]);
|
|
1114
|
+
}
|
|
1108
1115
|
/**
|
|
1109
1116
|
* @returns {string | undefined}
|
|
1110
1117
|
*/
|
|
@@ -1174,8 +1181,10 @@ export class WasmBrowserConfig {
|
|
|
1174
1181
|
* @param {string | null} [evalScript]
|
|
1175
1182
|
* @param {string | null} [robotsUserAgent]
|
|
1176
1183
|
* @param {string | null} [chromePath]
|
|
1184
|
+
* @param {boolean | null} [scrollContainers]
|
|
1185
|
+
* @param {bigint | null} [domQuietWindow]
|
|
1177
1186
|
*/
|
|
1178
|
-
constructor(mode, backend, timeout, overallTimeout, shutdownTimeout, wait, blockUrlPatterns, captureNetworkEvents, sessionAffinity, chromeArgs, endpoint, waitSelector, extraWait, proxy, evalScript, robotsUserAgent, chromePath) {
|
|
1187
|
+
constructor(mode, backend, timeout, overallTimeout, shutdownTimeout, wait, blockUrlPatterns, captureNetworkEvents, sessionAffinity, chromeArgs, endpoint, waitSelector, extraWait, proxy, evalScript, robotsUserAgent, chromePath, scrollContainers, domQuietWindow) {
|
|
1179
1188
|
var ptr0 = isLikeNone(blockUrlPatterns) ? 0 : passArrayJsValueToWasm0(blockUrlPatterns, wasm.__wbindgen_malloc);
|
|
1180
1189
|
var len0 = WASM_VECTOR_LEN;
|
|
1181
1190
|
var ptr1 = isLikeNone(chromeArgs) ? 0 : passArrayJsValueToWasm0(chromeArgs, wasm.__wbindgen_malloc);
|
|
@@ -1195,7 +1204,7 @@ export class WasmBrowserConfig {
|
|
|
1195
1204
|
var len6 = WASM_VECTOR_LEN;
|
|
1196
1205
|
var ptr7 = isLikeNone(chromePath) ? 0 : passStringToWasm0(chromePath, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
|
|
1197
1206
|
var len7 = WASM_VECTOR_LEN;
|
|
1198
|
-
const ret = wasm.wasmbrowserconfig_new(isLikeNone(mode) ? 4 : mode, isLikeNone(backend) ? 2 : backend, !isLikeNone(timeout), isLikeNone(timeout) ? BigInt(0) : timeout, !isLikeNone(overallTimeout), isLikeNone(overallTimeout) ? BigInt(0) : overallTimeout, !isLikeNone(shutdownTimeout), isLikeNone(shutdownTimeout) ? BigInt(0) : shutdownTimeout, isLikeNone(wait) ? 3 : wait, ptr0, len0, isLikeNone(captureNetworkEvents) ? 0xFFFFFF : captureNetworkEvents ? 1 : 0, isLikeNone(sessionAffinity) ? 0xFFFFFF : sessionAffinity ? 1 : 0, ptr1, len1, ptr2, len2, ptr3, len3, !isLikeNone(extraWait), isLikeNone(extraWait) ? BigInt(0) : extraWait, ptr4, ptr5, len5, ptr6, len6, ptr7, len7);
|
|
1207
|
+
const ret = wasm.wasmbrowserconfig_new(isLikeNone(mode) ? 4 : mode, isLikeNone(backend) ? 2 : backend, !isLikeNone(timeout), isLikeNone(timeout) ? BigInt(0) : timeout, !isLikeNone(overallTimeout), isLikeNone(overallTimeout) ? BigInt(0) : overallTimeout, !isLikeNone(shutdownTimeout), isLikeNone(shutdownTimeout) ? BigInt(0) : shutdownTimeout, isLikeNone(wait) ? 3 : wait, ptr0, len0, isLikeNone(captureNetworkEvents) ? 0xFFFFFF : captureNetworkEvents ? 1 : 0, isLikeNone(sessionAffinity) ? 0xFFFFFF : sessionAffinity ? 1 : 0, ptr1, len1, ptr2, len2, ptr3, len3, !isLikeNone(extraWait), isLikeNone(extraWait) ? BigInt(0) : extraWait, ptr4, ptr5, len5, ptr6, len6, ptr7, len7, isLikeNone(scrollContainers) ? 0xFFFFFF : scrollContainers ? 1 : 0, !isLikeNone(domQuietWindow), isLikeNone(domQuietWindow) ? BigInt(0) : domQuietWindow);
|
|
1199
1208
|
this.__wbg_ptr = ret;
|
|
1200
1209
|
WasmBrowserConfigFinalization.register(this, this.__wbg_ptr, this);
|
|
1201
1210
|
return this;
|
|
@@ -1230,6 +1239,13 @@ export class WasmBrowserConfig {
|
|
|
1230
1239
|
}
|
|
1231
1240
|
return v1;
|
|
1232
1241
|
}
|
|
1242
|
+
/**
|
|
1243
|
+
* @returns {boolean}
|
|
1244
|
+
*/
|
|
1245
|
+
get scrollContainers() {
|
|
1246
|
+
const ret = wasm.wasmbrowserconfig_scrollContainers(this.__wbg_ptr);
|
|
1247
|
+
return ret !== 0;
|
|
1248
|
+
}
|
|
1233
1249
|
/**
|
|
1234
1250
|
* @returns {boolean}
|
|
1235
1251
|
*/
|
|
@@ -1273,6 +1289,12 @@ export class WasmBrowserConfig {
|
|
|
1273
1289
|
var len0 = WASM_VECTOR_LEN;
|
|
1274
1290
|
wasm.wasmbrowserconfig_set_chromePath(this.__wbg_ptr, ptr0, len0);
|
|
1275
1291
|
}
|
|
1292
|
+
/**
|
|
1293
|
+
* @param {bigint | null} [value]
|
|
1294
|
+
*/
|
|
1295
|
+
set domQuietWindow(value) {
|
|
1296
|
+
wasm.wasmbrowserconfig_set_domQuietWindow(this.__wbg_ptr, !isLikeNone(value), isLikeNone(value) ? BigInt(0) : value);
|
|
1297
|
+
}
|
|
1276
1298
|
/**
|
|
1277
1299
|
* @param {string | null} [value]
|
|
1278
1300
|
*/
|
|
@@ -1322,6 +1344,12 @@ export class WasmBrowserConfig {
|
|
|
1322
1344
|
var len0 = WASM_VECTOR_LEN;
|
|
1323
1345
|
wasm.wasmbrowserconfig_set_robotsUserAgent(this.__wbg_ptr, ptr0, len0);
|
|
1324
1346
|
}
|
|
1347
|
+
/**
|
|
1348
|
+
* @param {boolean} value
|
|
1349
|
+
*/
|
|
1350
|
+
set scrollContainers(value) {
|
|
1351
|
+
wasm.wasmbrowserconfig_set_scrollContainers(this.__wbg_ptr, value);
|
|
1352
|
+
}
|
|
1325
1353
|
/**
|
|
1326
1354
|
* @param {boolean} value
|
|
1327
1355
|
*/
|
|
@@ -2495,6 +2523,13 @@ export class WasmCrawlConfig {
|
|
|
2495
2523
|
const ret = wasm.wasmcrawlconfig_customHeaders(this.__wbg_ptr);
|
|
2496
2524
|
return ret;
|
|
2497
2525
|
}
|
|
2526
|
+
/**
|
|
2527
|
+
* @returns {boolean}
|
|
2528
|
+
*/
|
|
2529
|
+
get dangerAcceptInvalidCerts() {
|
|
2530
|
+
const ret = wasm.wasmcrawlconfig_dangerAcceptInvalidCerts(this.__wbg_ptr);
|
|
2531
|
+
return ret !== 0;
|
|
2532
|
+
}
|
|
2498
2533
|
/**
|
|
2499
2534
|
* @returns {boolean}
|
|
2500
2535
|
*/
|
|
@@ -2724,8 +2759,9 @@ export class WasmCrawlConfig {
|
|
|
2724
2759
|
* @param {string | null} [warcOutput]
|
|
2725
2760
|
* @param {string | null} [browserProfile]
|
|
2726
2761
|
* @param {boolean | null} [ssrfDenyPrivateExplicit]
|
|
2762
|
+
* @param {boolean | null} [dangerAcceptInvalidCerts]
|
|
2727
2763
|
*/
|
|
2728
|
-
constructor(crawlStrategy, respectRobotsTxt, softHttpErrors, stayOnDomain, allowSubdomains, includePaths, excludePaths, pathPatternsMatchQuery, pathPatternsMatchUrl, dedupIncludeQuery, stripTrackingParams, trackingParams, customHeaders, requestTimeout, maxRedirects, retryCount, retryCodes, retryInitialDelayMs, retryMaxDelayMs, rateLimitJitterRatio, cookiesEnabled, removeTags, content, downloadAssets, assetTypes, browser, userAgents, captureScreenshot, followDocumentUrls, downloadDocuments, documentMimeTypes, saveBrowserProfile, ssrf, maxDepth, maxPages, maxLinksPerPage, maxConcurrent, contentFilter, bm25Query, bm25Threshold, userAgent, rateLimitMs, auth, maxBodySize, mapLimit, mapSearch, maxAssetSize, proxy, documentUrlDepth, documentMaxSize, documentOutputDir, documentContentEncoding, warcOutput, browserProfile, ssrfDenyPrivateExplicit) {
|
|
2764
|
+
constructor(crawlStrategy, respectRobotsTxt, softHttpErrors, stayOnDomain, allowSubdomains, includePaths, excludePaths, pathPatternsMatchQuery, pathPatternsMatchUrl, dedupIncludeQuery, stripTrackingParams, trackingParams, customHeaders, requestTimeout, maxRedirects, retryCount, retryCodes, retryInitialDelayMs, retryMaxDelayMs, rateLimitJitterRatio, cookiesEnabled, removeTags, content, downloadAssets, assetTypes, browser, userAgents, captureScreenshot, followDocumentUrls, downloadDocuments, documentMimeTypes, saveBrowserProfile, ssrf, maxDepth, maxPages, maxLinksPerPage, maxConcurrent, contentFilter, bm25Query, bm25Threshold, userAgent, rateLimitMs, auth, maxBodySize, mapLimit, mapSearch, maxAssetSize, proxy, documentUrlDepth, documentMaxSize, documentOutputDir, documentContentEncoding, warcOutput, browserProfile, ssrfDenyPrivateExplicit, dangerAcceptInvalidCerts) {
|
|
2729
2765
|
var ptr0 = isLikeNone(includePaths) ? 0 : passArrayJsValueToWasm0(includePaths, wasm.__wbindgen_malloc);
|
|
2730
2766
|
var len0 = WASM_VECTOR_LEN;
|
|
2731
2767
|
var ptr1 = isLikeNone(excludePaths) ? 0 : passArrayJsValueToWasm0(excludePaths, wasm.__wbindgen_malloc);
|
|
@@ -2774,7 +2810,7 @@ export class WasmCrawlConfig {
|
|
|
2774
2810
|
var len16 = WASM_VECTOR_LEN;
|
|
2775
2811
|
var ptr17 = isLikeNone(browserProfile) ? 0 : passStringToWasm0(browserProfile, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
|
|
2776
2812
|
var len17 = WASM_VECTOR_LEN;
|
|
2777
|
-
const ret = wasm.wasmcrawlconfig_new(isLikeNone(crawlStrategy) ? 4 : crawlStrategy, isLikeNone(respectRobotsTxt) ? 0xFFFFFF : respectRobotsTxt ? 1 : 0, isLikeNone(softHttpErrors) ? 0xFFFFFF : softHttpErrors ? 1 : 0, isLikeNone(stayOnDomain) ? 0xFFFFFF : stayOnDomain ? 1 : 0, isLikeNone(allowSubdomains) ? 0xFFFFFF : allowSubdomains ? 1 : 0, ptr0, len0, ptr1, len1, isLikeNone(pathPatternsMatchQuery) ? 0xFFFFFF : pathPatternsMatchQuery ? 1 : 0, isLikeNone(pathPatternsMatchUrl) ? 0xFFFFFF : pathPatternsMatchUrl ? 1 : 0, isLikeNone(dedupIncludeQuery) ? 0xFFFFFF : dedupIncludeQuery ? 1 : 0, isLikeNone(stripTrackingParams) ? 0xFFFFFF : stripTrackingParams ? 1 : 0, ptr2, len2, isLikeNone(customHeaders) ? 0 : addToExternrefTable0(customHeaders), !isLikeNone(requestTimeout), isLikeNone(requestTimeout) ? BigInt(0) : requestTimeout, isLikeNone(maxRedirects) ? Number.MAX_SAFE_INTEGER : (maxRedirects) >>> 0, isLikeNone(retryCount) ? Number.MAX_SAFE_INTEGER : (retryCount) >>> 0, ptr3, len3, !isLikeNone(retryInitialDelayMs), isLikeNone(retryInitialDelayMs) ? BigInt(0) : retryInitialDelayMs, !isLikeNone(retryMaxDelayMs), isLikeNone(retryMaxDelayMs) ? BigInt(0) : retryMaxDelayMs, !isLikeNone(rateLimitJitterRatio), isLikeNone(rateLimitJitterRatio) ? 0 : rateLimitJitterRatio, isLikeNone(cookiesEnabled) ? 0xFFFFFF : cookiesEnabled ? 1 : 0, ptr4, len4, ptr5, isLikeNone(downloadAssets) ? 0xFFFFFF : downloadAssets ? 1 : 0, ptr6, len6, ptr7, ptr8, len8, isLikeNone(captureScreenshot) ? 0xFFFFFF : captureScreenshot ? 1 : 0, isLikeNone(followDocumentUrls) ? 0xFFFFFF : followDocumentUrls ? 1 : 0, isLikeNone(downloadDocuments) ? 0xFFFFFF : downloadDocuments ? 1 : 0, ptr9, len9, isLikeNone(saveBrowserProfile) ? 0xFFFFFF : saveBrowserProfile ? 1 : 0, ptr10, isLikeNone(maxDepth) ? Number.MAX_SAFE_INTEGER : (maxDepth) >>> 0, isLikeNone(maxPages) ? Number.MAX_SAFE_INTEGER : (maxPages) >>> 0, isLikeNone(maxLinksPerPage) ? Number.MAX_SAFE_INTEGER : (maxLinksPerPage) >>> 0, isLikeNone(maxConcurrent) ? Number.MAX_SAFE_INTEGER : (maxConcurrent) >>> 0, isLikeNone(contentFilter) ? 1 : contentFilter, ptr11, len11, !isLikeNone(bm25Threshold), isLikeNone(bm25Threshold) ? 0 : bm25Threshold, ptr12, len12, !isLikeNone(rateLimitMs), isLikeNone(rateLimitMs) ? BigInt(0) : rateLimitMs, isLikeNone(auth) ? 0 : addToExternrefTable0(auth), isLikeNone(maxBodySize) ? Number.MAX_SAFE_INTEGER : (maxBodySize) >>> 0, isLikeNone(mapLimit) ? Number.MAX_SAFE_INTEGER : (mapLimit) >>> 0, ptr13, len13, isLikeNone(maxAssetSize) ? Number.MAX_SAFE_INTEGER : (maxAssetSize) >>> 0, ptr14, isLikeNone(documentUrlDepth) ? Number.MAX_SAFE_INTEGER : (documentUrlDepth) >>> 0, isLikeNone(documentMaxSize) ? Number.MAX_SAFE_INTEGER : (documentMaxSize) >>> 0, ptr15, len15, isLikeNone(documentContentEncoding) ? 1 : documentContentEncoding, ptr16, len16, ptr17, len17, isLikeNone(ssrfDenyPrivateExplicit) ? 0xFFFFFF : ssrfDenyPrivateExplicit ? 1 : 0);
|
|
2813
|
+
const ret = wasm.wasmcrawlconfig_new(isLikeNone(crawlStrategy) ? 4 : crawlStrategy, isLikeNone(respectRobotsTxt) ? 0xFFFFFF : respectRobotsTxt ? 1 : 0, isLikeNone(softHttpErrors) ? 0xFFFFFF : softHttpErrors ? 1 : 0, isLikeNone(stayOnDomain) ? 0xFFFFFF : stayOnDomain ? 1 : 0, isLikeNone(allowSubdomains) ? 0xFFFFFF : allowSubdomains ? 1 : 0, ptr0, len0, ptr1, len1, isLikeNone(pathPatternsMatchQuery) ? 0xFFFFFF : pathPatternsMatchQuery ? 1 : 0, isLikeNone(pathPatternsMatchUrl) ? 0xFFFFFF : pathPatternsMatchUrl ? 1 : 0, isLikeNone(dedupIncludeQuery) ? 0xFFFFFF : dedupIncludeQuery ? 1 : 0, isLikeNone(stripTrackingParams) ? 0xFFFFFF : stripTrackingParams ? 1 : 0, ptr2, len2, isLikeNone(customHeaders) ? 0 : addToExternrefTable0(customHeaders), !isLikeNone(requestTimeout), isLikeNone(requestTimeout) ? BigInt(0) : requestTimeout, isLikeNone(maxRedirects) ? Number.MAX_SAFE_INTEGER : (maxRedirects) >>> 0, isLikeNone(retryCount) ? Number.MAX_SAFE_INTEGER : (retryCount) >>> 0, ptr3, len3, !isLikeNone(retryInitialDelayMs), isLikeNone(retryInitialDelayMs) ? BigInt(0) : retryInitialDelayMs, !isLikeNone(retryMaxDelayMs), isLikeNone(retryMaxDelayMs) ? BigInt(0) : retryMaxDelayMs, !isLikeNone(rateLimitJitterRatio), isLikeNone(rateLimitJitterRatio) ? 0 : rateLimitJitterRatio, isLikeNone(cookiesEnabled) ? 0xFFFFFF : cookiesEnabled ? 1 : 0, ptr4, len4, ptr5, isLikeNone(downloadAssets) ? 0xFFFFFF : downloadAssets ? 1 : 0, ptr6, len6, ptr7, ptr8, len8, isLikeNone(captureScreenshot) ? 0xFFFFFF : captureScreenshot ? 1 : 0, isLikeNone(followDocumentUrls) ? 0xFFFFFF : followDocumentUrls ? 1 : 0, isLikeNone(downloadDocuments) ? 0xFFFFFF : downloadDocuments ? 1 : 0, ptr9, len9, isLikeNone(saveBrowserProfile) ? 0xFFFFFF : saveBrowserProfile ? 1 : 0, ptr10, isLikeNone(maxDepth) ? Number.MAX_SAFE_INTEGER : (maxDepth) >>> 0, isLikeNone(maxPages) ? Number.MAX_SAFE_INTEGER : (maxPages) >>> 0, isLikeNone(maxLinksPerPage) ? Number.MAX_SAFE_INTEGER : (maxLinksPerPage) >>> 0, isLikeNone(maxConcurrent) ? Number.MAX_SAFE_INTEGER : (maxConcurrent) >>> 0, isLikeNone(contentFilter) ? 1 : contentFilter, ptr11, len11, !isLikeNone(bm25Threshold), isLikeNone(bm25Threshold) ? 0 : bm25Threshold, ptr12, len12, !isLikeNone(rateLimitMs), isLikeNone(rateLimitMs) ? BigInt(0) : rateLimitMs, isLikeNone(auth) ? 0 : addToExternrefTable0(auth), isLikeNone(maxBodySize) ? Number.MAX_SAFE_INTEGER : (maxBodySize) >>> 0, isLikeNone(mapLimit) ? Number.MAX_SAFE_INTEGER : (mapLimit) >>> 0, ptr13, len13, isLikeNone(maxAssetSize) ? Number.MAX_SAFE_INTEGER : (maxAssetSize) >>> 0, ptr14, isLikeNone(documentUrlDepth) ? Number.MAX_SAFE_INTEGER : (documentUrlDepth) >>> 0, isLikeNone(documentMaxSize) ? Number.MAX_SAFE_INTEGER : (documentMaxSize) >>> 0, ptr15, len15, isLikeNone(documentContentEncoding) ? 1 : documentContentEncoding, ptr16, len16, ptr17, len17, isLikeNone(ssrfDenyPrivateExplicit) ? 0xFFFFFF : ssrfDenyPrivateExplicit ? 1 : 0, isLikeNone(dangerAcceptInvalidCerts) ? 0xFFFFFF : dangerAcceptInvalidCerts ? 1 : 0);
|
|
2778
2814
|
this.__wbg_ptr = ret;
|
|
2779
2815
|
WasmCrawlConfigFinalization.register(this, this.__wbg_ptr, this);
|
|
2780
2816
|
return this;
|
|
@@ -2964,6 +3000,12 @@ export class WasmCrawlConfig {
|
|
|
2964
3000
|
set customHeaders(value) {
|
|
2965
3001
|
wasm.wasmcrawlconfig_set_customHeaders(this.__wbg_ptr, value);
|
|
2966
3002
|
}
|
|
3003
|
+
/**
|
|
3004
|
+
* @param {boolean} value
|
|
3005
|
+
*/
|
|
3006
|
+
set dangerAcceptInvalidCerts(value) {
|
|
3007
|
+
wasm.wasmcrawlconfig_set_dangerAcceptInvalidCerts(this.__wbg_ptr, value);
|
|
3008
|
+
}
|
|
2967
3009
|
/**
|
|
2968
3010
|
* @param {boolean} value
|
|
2969
3011
|
*/
|
|
@@ -4029,6 +4071,9 @@ export class WasmCrawlResult {
|
|
|
4029
4071
|
const ret = wasm.wasmcrawlresult_browserUsed(this.__wbg_ptr);
|
|
4030
4072
|
return ret !== 0;
|
|
4031
4073
|
}
|
|
4074
|
+
clearWafBlock() {
|
|
4075
|
+
wasm.wasmcrawlresult_clearWafBlock(this.__wbg_ptr);
|
|
4076
|
+
}
|
|
4032
4077
|
/**
|
|
4033
4078
|
* Returns detached copies of the elements in `cookies`.
|
|
4034
4079
|
*
|
|
@@ -4088,6 +4133,11 @@ export class WasmCrawlResult {
|
|
|
4088
4133
|
}
|
|
4089
4134
|
}
|
|
4090
4135
|
/**
|
|
4136
|
+
* Takes ownership of `wafBlock`.
|
|
4137
|
+
* wasm-bindgen lowers a by-value class argument through `__destroy_into_raw()`, so each
|
|
4138
|
+
* of those handles is dead once this returns. Passing a freshly built value is fine. To
|
|
4139
|
+
* keep a handle you already hold, build with `default()` and assign the property instead:
|
|
4140
|
+
* the generated setter borrows and leaves your handle alive.
|
|
4091
4141
|
* Takes ownership of every element in `pages`, `cookies`.
|
|
4092
4142
|
* wasm-bindgen cannot borrow class values nested in an array. To keep those handles,
|
|
4093
4143
|
* copy each retained element with `copyForTransfer()` before passing the array.
|
|
@@ -4099,8 +4149,9 @@ export class WasmCrawlResult {
|
|
|
4099
4149
|
* @param {boolean | null} [stayedOnDomain]
|
|
4100
4150
|
* @param {boolean | null} [browserUsed]
|
|
4101
4151
|
* @param {string | null} [error]
|
|
4152
|
+
* @param {WasmWafBlockDetails | null} [wafBlock]
|
|
4102
4153
|
*/
|
|
4103
|
-
constructor(pages, finalUrl, redirectCount, wasSkipped, cookies, stayedOnDomain, browserUsed, error) {
|
|
4154
|
+
constructor(pages, finalUrl, redirectCount, wasSkipped, cookies, stayedOnDomain, browserUsed, error, wafBlock) {
|
|
4104
4155
|
var ptr0 = isLikeNone(pages) ? 0 : passArrayJsValueToWasm0(pages, wasm.__wbindgen_malloc);
|
|
4105
4156
|
var len0 = WASM_VECTOR_LEN;
|
|
4106
4157
|
var ptr1 = isLikeNone(finalUrl) ? 0 : passStringToWasm0(finalUrl, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
|
|
@@ -4109,7 +4160,12 @@ export class WasmCrawlResult {
|
|
|
4109
4160
|
var len2 = WASM_VECTOR_LEN;
|
|
4110
4161
|
var ptr3 = isLikeNone(error) ? 0 : passStringToWasm0(error, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
|
|
4111
4162
|
var len3 = WASM_VECTOR_LEN;
|
|
4112
|
-
|
|
4163
|
+
let ptr4 = 0;
|
|
4164
|
+
if (!isLikeNone(wafBlock)) {
|
|
4165
|
+
_assertClass(wafBlock, WasmWafBlockDetails);
|
|
4166
|
+
ptr4 = wafBlock.__destroy_into_raw();
|
|
4167
|
+
}
|
|
4168
|
+
const ret = wasm.wasmcrawlresult_new(ptr0, len0, ptr1, len1, isLikeNone(redirectCount) ? Number.MAX_SAFE_INTEGER : (redirectCount) >>> 0, isLikeNone(wasSkipped) ? 0xFFFFFF : wasSkipped ? 1 : 0, ptr2, len2, isLikeNone(stayedOnDomain) ? 0xFFFFFF : stayedOnDomain ? 1 : 0, isLikeNone(browserUsed) ? 0xFFFFFF : browserUsed ? 1 : 0, ptr3, len3, ptr4);
|
|
4113
4169
|
this.__wbg_ptr = ret;
|
|
4114
4170
|
WasmCrawlResultFinalization.register(this, this.__wbg_ptr, this);
|
|
4115
4171
|
return this;
|
|
@@ -4190,6 +4246,13 @@ export class WasmCrawlResult {
|
|
|
4190
4246
|
set stayedOnDomain(value) {
|
|
4191
4247
|
wasm.wasmcrawlresult_set_stayedOnDomain(this.__wbg_ptr, value);
|
|
4192
4248
|
}
|
|
4249
|
+
/**
|
|
4250
|
+
* @param {WasmWafBlockDetails} value
|
|
4251
|
+
*/
|
|
4252
|
+
set wafBlock(value) {
|
|
4253
|
+
_assertClass(value, WasmWafBlockDetails);
|
|
4254
|
+
wasm.wasmcrawlresult_set_wafBlock(this.__wbg_ptr, value.__wbg_ptr);
|
|
4255
|
+
}
|
|
4193
4256
|
/**
|
|
4194
4257
|
* @param {boolean} value
|
|
4195
4258
|
*/
|
|
@@ -4218,6 +4281,17 @@ export class WasmCrawlResult {
|
|
|
4218
4281
|
const ret = wasm.wasmcrawlresult_uniqueNormalizedUrls(this.__wbg_ptr);
|
|
4219
4282
|
return ret >>> 0;
|
|
4220
4283
|
}
|
|
4284
|
+
/**
|
|
4285
|
+
* Returns a detached copy of `wafBlock`.
|
|
4286
|
+
*
|
|
4287
|
+
* Changes to the returned value do not update this object. To update `wafBlock`,
|
|
4288
|
+
* read it, modify the copy, then assign the copy back to `wafBlock`.
|
|
4289
|
+
* @returns {WasmWafBlockDetails | undefined}
|
|
4290
|
+
*/
|
|
4291
|
+
get wafBlock() {
|
|
4292
|
+
const ret = wasm.wasmcrawlresult_wafBlock(this.__wbg_ptr);
|
|
4293
|
+
return ret === 0 ? undefined : WasmWafBlockDetails.__wrap(ret);
|
|
4294
|
+
}
|
|
4221
4295
|
/**
|
|
4222
4296
|
* @returns {boolean}
|
|
4223
4297
|
*/
|
|
@@ -8889,6 +8963,118 @@ export class WasmSsrfPolicy {
|
|
|
8889
8963
|
}
|
|
8890
8964
|
if (Symbol.dispose) WasmSsrfPolicy.prototype[Symbol.dispose] = WasmSsrfPolicy.prototype.free;
|
|
8891
8965
|
|
|
8966
|
+
/**
|
|
8967
|
+
* Structured context for a WAF-blocked response. <!-- -->
|
|
8968
|
+
*/
|
|
8969
|
+
export class WasmWafBlockDetails {
|
|
8970
|
+
static __wrap(ptr) {
|
|
8971
|
+
const obj = Object.create(WasmWafBlockDetails.prototype);
|
|
8972
|
+
obj.__wbg_ptr = ptr;
|
|
8973
|
+
WasmWafBlockDetailsFinalization.register(obj, obj.__wbg_ptr, obj);
|
|
8974
|
+
return obj;
|
|
8975
|
+
}
|
|
8976
|
+
__destroy_into_raw() {
|
|
8977
|
+
const ptr = this.__wbg_ptr;
|
|
8978
|
+
this.__wbg_ptr = 0;
|
|
8979
|
+
WasmWafBlockDetailsFinalization.unregister(this);
|
|
8980
|
+
return ptr;
|
|
8981
|
+
}
|
|
8982
|
+
free() {
|
|
8983
|
+
const ptr = this.__destroy_into_raw();
|
|
8984
|
+
wasm.__wbg_wasmwafblockdetails_free(ptr, 0);
|
|
8985
|
+
}
|
|
8986
|
+
/**
|
|
8987
|
+
* @returns {string | undefined}
|
|
8988
|
+
*/
|
|
8989
|
+
get bodyPrefix() {
|
|
8990
|
+
const ret = wasm.wasmwafblockdetails_bodyPrefix(this.__wbg_ptr);
|
|
8991
|
+
let v1;
|
|
8992
|
+
if (ret[0] !== 0) {
|
|
8993
|
+
v1 = getStringFromWasm0(ret[0], ret[1]);
|
|
8994
|
+
wasm.__wbindgen_free(ret[0], ret[1] * 1, 1);
|
|
8995
|
+
}
|
|
8996
|
+
return v1;
|
|
8997
|
+
}
|
|
8998
|
+
/**
|
|
8999
|
+
* Return an independent handle for an ownership-taking API.
|
|
9000
|
+
*
|
|
9001
|
+
* Use `copyForTransfer()` for each class value placed in an array when the original
|
|
9002
|
+
* handle must remain usable.
|
|
9003
|
+
* @returns {WasmWafBlockDetails}
|
|
9004
|
+
*/
|
|
9005
|
+
copyForTransfer() {
|
|
9006
|
+
const ret = wasm.wasmwafblockdetails_copyForTransfer(this.__wbg_ptr);
|
|
9007
|
+
return WasmWafBlockDetails.__wrap(ret);
|
|
9008
|
+
}
|
|
9009
|
+
/**
|
|
9010
|
+
* @returns {WasmWafBlockDetails}
|
|
9011
|
+
*/
|
|
9012
|
+
static default() {
|
|
9013
|
+
const ret = wasm.wasmwafblockdetails_default();
|
|
9014
|
+
return WasmWafBlockDetails.__wrap(ret);
|
|
9015
|
+
}
|
|
9016
|
+
/**
|
|
9017
|
+
* @param {string | null} [vendor]
|
|
9018
|
+
* @param {number | null} [status]
|
|
9019
|
+
* @param {string | null} [bodyPrefix]
|
|
9020
|
+
*/
|
|
9021
|
+
constructor(vendor, status, bodyPrefix) {
|
|
9022
|
+
var ptr0 = isLikeNone(vendor) ? 0 : passStringToWasm0(vendor, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
|
|
9023
|
+
var len0 = WASM_VECTOR_LEN;
|
|
9024
|
+
var ptr1 = isLikeNone(bodyPrefix) ? 0 : passStringToWasm0(bodyPrefix, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
|
|
9025
|
+
var len1 = WASM_VECTOR_LEN;
|
|
9026
|
+
const ret = wasm.wasmwafblockdetails_new(ptr0, len0, isLikeNone(status) ? 0xFFFFFF : status, ptr1, len1);
|
|
9027
|
+
this.__wbg_ptr = ret;
|
|
9028
|
+
WasmWafBlockDetailsFinalization.register(this, this.__wbg_ptr, this);
|
|
9029
|
+
return this;
|
|
9030
|
+
}
|
|
9031
|
+
/**
|
|
9032
|
+
* @param {string | null} [value]
|
|
9033
|
+
*/
|
|
9034
|
+
set bodyPrefix(value) {
|
|
9035
|
+
var ptr0 = isLikeNone(value) ? 0 : passStringToWasm0(value, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
|
|
9036
|
+
var len0 = WASM_VECTOR_LEN;
|
|
9037
|
+
wasm.wasmwafblockdetails_set_bodyPrefix(this.__wbg_ptr, ptr0, len0);
|
|
9038
|
+
}
|
|
9039
|
+
/**
|
|
9040
|
+
* @param {number | null} [value]
|
|
9041
|
+
*/
|
|
9042
|
+
set status(value) {
|
|
9043
|
+
wasm.wasmwafblockdetails_set_status(this.__wbg_ptr, isLikeNone(value) ? 0xFFFFFF : value);
|
|
9044
|
+
}
|
|
9045
|
+
/**
|
|
9046
|
+
* @param {string} value
|
|
9047
|
+
*/
|
|
9048
|
+
set vendor(value) {
|
|
9049
|
+
const ptr0 = passStringToWasm0(value, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
|
|
9050
|
+
const len0 = WASM_VECTOR_LEN;
|
|
9051
|
+
wasm.wasmwafblockdetails_set_vendor(this.__wbg_ptr, ptr0, len0);
|
|
9052
|
+
}
|
|
9053
|
+
/**
|
|
9054
|
+
* @returns {number | undefined}
|
|
9055
|
+
*/
|
|
9056
|
+
get status() {
|
|
9057
|
+
const ret = wasm.wasmwafblockdetails_status(this.__wbg_ptr);
|
|
9058
|
+
return ret === 0xFFFFFF ? undefined : ret;
|
|
9059
|
+
}
|
|
9060
|
+
/**
|
|
9061
|
+
* @returns {string}
|
|
9062
|
+
*/
|
|
9063
|
+
get vendor() {
|
|
9064
|
+
let deferred1_0;
|
|
9065
|
+
let deferred1_1;
|
|
9066
|
+
try {
|
|
9067
|
+
const ret = wasm.wasmwafblockdetails_vendor(this.__wbg_ptr);
|
|
9068
|
+
deferred1_0 = ret[0];
|
|
9069
|
+
deferred1_1 = ret[1];
|
|
9070
|
+
return getStringFromWasm0(ret[0], ret[1]);
|
|
9071
|
+
} finally {
|
|
9072
|
+
wasm.__wbindgen_free(deferred1_0, deferred1_1, 1);
|
|
9073
|
+
}
|
|
9074
|
+
}
|
|
9075
|
+
}
|
|
9076
|
+
if (Symbol.dispose) WasmWafBlockDetails.prototype[Symbol.dispose] = WasmWafBlockDetails.prototype.free;
|
|
9077
|
+
|
|
8892
9078
|
/**
|
|
8893
9079
|
* Crawl multiple seed URLs concurrently, each following links to configured depth.
|
|
8894
9080
|
* @param {WasmCrawlEngineHandle} engine
|
|
@@ -9138,6 +9324,10 @@ function __wbg_get_imports() {
|
|
|
9138
9324
|
__wbg_getRandomValues_a608c4436c19407a: function() { return handleError(function (arg0, arg1) {
|
|
9139
9325
|
globalThis.crypto.getRandomValues(getArrayU8FromWasm0(arg0, arg1));
|
|
9140
9326
|
}, arguments); },
|
|
9327
|
+
__wbg_getTime_f5a55efff2585d5d: function(arg0) {
|
|
9328
|
+
const ret = arg0.getTime();
|
|
9329
|
+
return ret;
|
|
9330
|
+
},
|
|
9141
9331
|
__wbg_get_658f6698067d9515: function() { return handleError(function (arg0, arg1) {
|
|
9142
9332
|
const ret = Reflect.get(arg0, arg1);
|
|
9143
9333
|
return ret;
|
|
@@ -9218,6 +9408,10 @@ function __wbg_get_imports() {
|
|
|
9218
9408
|
const ret = arg0.length;
|
|
9219
9409
|
return ret;
|
|
9220
9410
|
},
|
|
9411
|
+
__wbg_new_0_72d020f0c63443d4: function() {
|
|
9412
|
+
const ret = new Date();
|
|
9413
|
+
return ret;
|
|
9414
|
+
},
|
|
9221
9415
|
__wbg_new_1dbf7428bba60a42: function(arg0) {
|
|
9222
9416
|
const ret = new Uint8Array(arg0);
|
|
9223
9417
|
return ret;
|
|
@@ -9253,7 +9447,7 @@ function __wbg_get_imports() {
|
|
|
9253
9447
|
const a = state0.a;
|
|
9254
9448
|
state0.a = 0;
|
|
9255
9449
|
try {
|
|
9256
|
-
return
|
|
9450
|
+
return wasm_bindgen_a37d5a31ee71bc5f___convert__closures_____invoke___js_sys_a6a58716f022baea___Function_fn_wasm_bindgen_a37d5a31ee71bc5f___JsValue_____wasm_bindgen_a37d5a31ee71bc5f___sys__Undefined___js_sys_a6a58716f022baea___Function_fn_wasm_bindgen_a37d5a31ee71bc5f___JsValue_____wasm_bindgen_a37d5a31ee71bc5f___sys__Undefined_______true_(a, state0.b, arg0, arg1);
|
|
9257
9451
|
} finally {
|
|
9258
9452
|
state0.a = a;
|
|
9259
9453
|
}
|
|
@@ -9547,13 +9741,13 @@ function __wbg_get_imports() {
|
|
|
9547
9741
|
return ret;
|
|
9548
9742
|
},
|
|
9549
9743
|
__wbindgen_generic_0000000000000001: function(arg0, arg1) {
|
|
9550
|
-
// Cast intrinsic for `Closure(Closure { owned: true, function: Function { arguments: [Externref], shim_idx:
|
|
9551
|
-
const ret = makeMutClosure(arg0, arg1,
|
|
9744
|
+
// Cast intrinsic for `Closure(Closure { owned: true, function: Function { arguments: [Externref], shim_idx: 1512, ret: Result(Unit), inner_ret: Some(Result(Unit)) }, mutable: true }) -> Externref`.
|
|
9745
|
+
const ret = makeMutClosure(arg0, arg1, wasm_bindgen_a37d5a31ee71bc5f___convert__closures_____invoke___wasm_bindgen_a37d5a31ee71bc5f___JsValue__core_37ee0ce0c4cd57d5___result__Result_____wasm_bindgen_a37d5a31ee71bc5f___JsError___true_);
|
|
9552
9746
|
return ret;
|
|
9553
9747
|
},
|
|
9554
9748
|
__wbindgen_generic_0000000000000002: function(arg0, arg1) {
|
|
9555
|
-
// Cast intrinsic for `Closure(Closure { owned: true, function: Function { arguments: [], shim_idx:
|
|
9556
|
-
const ret = makeMutClosure(arg0, arg1,
|
|
9749
|
+
// Cast intrinsic for `Closure(Closure { owned: true, function: Function { arguments: [], shim_idx: 1354, ret: Unit, inner_ret: Some(Unit) }, mutable: true }) -> Externref`.
|
|
9750
|
+
const ret = makeMutClosure(arg0, arg1, wasm_bindgen_a37d5a31ee71bc5f___convert__closures_____invoke_______true_);
|
|
9557
9751
|
return ret;
|
|
9558
9752
|
},
|
|
9559
9753
|
__wbindgen_generic_0000000000000003: function(arg0) {
|
|
@@ -9592,19 +9786,19 @@ function __wbg_get_imports() {
|
|
|
9592
9786
|
};
|
|
9593
9787
|
}
|
|
9594
9788
|
|
|
9595
|
-
function
|
|
9596
|
-
wasm.
|
|
9789
|
+
function wasm_bindgen_a37d5a31ee71bc5f___convert__closures_____invoke_______true_(arg0, arg1) {
|
|
9790
|
+
wasm.wasm_bindgen_a37d5a31ee71bc5f___convert__closures_____invoke_______true_(arg0, arg1);
|
|
9597
9791
|
}
|
|
9598
9792
|
|
|
9599
|
-
function
|
|
9600
|
-
const ret = wasm.
|
|
9793
|
+
function wasm_bindgen_a37d5a31ee71bc5f___convert__closures_____invoke___wasm_bindgen_a37d5a31ee71bc5f___JsValue__core_37ee0ce0c4cd57d5___result__Result_____wasm_bindgen_a37d5a31ee71bc5f___JsError___true_(arg0, arg1, arg2) {
|
|
9794
|
+
const ret = wasm.wasm_bindgen_a37d5a31ee71bc5f___convert__closures_____invoke___wasm_bindgen_a37d5a31ee71bc5f___JsValue__core_37ee0ce0c4cd57d5___result__Result_____wasm_bindgen_a37d5a31ee71bc5f___JsError___true_(arg0, arg1, arg2);
|
|
9601
9795
|
if (ret[1]) {
|
|
9602
9796
|
throw takeFromExternrefTable0(ret[0]);
|
|
9603
9797
|
}
|
|
9604
9798
|
}
|
|
9605
9799
|
|
|
9606
|
-
function
|
|
9607
|
-
wasm.
|
|
9800
|
+
function wasm_bindgen_a37d5a31ee71bc5f___convert__closures_____invoke___js_sys_a6a58716f022baea___Function_fn_wasm_bindgen_a37d5a31ee71bc5f___JsValue_____wasm_bindgen_a37d5a31ee71bc5f___sys__Undefined___js_sys_a6a58716f022baea___Function_fn_wasm_bindgen_a37d5a31ee71bc5f___JsValue_____wasm_bindgen_a37d5a31ee71bc5f___sys__Undefined_______true_(arg0, arg1, arg2, arg3) {
|
|
9801
|
+
wasm.wasm_bindgen_a37d5a31ee71bc5f___convert__closures_____invoke___js_sys_a6a58716f022baea___Function_fn_wasm_bindgen_a37d5a31ee71bc5f___JsValue_____wasm_bindgen_a37d5a31ee71bc5f___sys__Undefined___js_sys_a6a58716f022baea___Function_fn_wasm_bindgen_a37d5a31ee71bc5f___JsValue_____wasm_bindgen_a37d5a31ee71bc5f___sys__Undefined_______true_(arg0, arg1, arg2, arg3);
|
|
9608
9802
|
}
|
|
9609
9803
|
|
|
9610
9804
|
|
|
@@ -9732,6 +9926,9 @@ const WasmSitemapUrlFinalization = (typeof FinalizationRegistry === 'undefined')
|
|
|
9732
9926
|
const WasmSsrfPolicyFinalization = (typeof FinalizationRegistry === 'undefined')
|
|
9733
9927
|
? { register: () => {}, unregister: () => {} }
|
|
9734
9928
|
: new FinalizationRegistry(ptr => wasm.__wbg_wasmssrfpolicy_free(ptr, 1));
|
|
9929
|
+
const WasmWafBlockDetailsFinalization = (typeof FinalizationRegistry === 'undefined')
|
|
9930
|
+
? { register: () => {}, unregister: () => {} }
|
|
9931
|
+
: new FinalizationRegistry(ptr => wasm.__wbg_wasmwafblockdetails_free(ptr, 1));
|
|
9735
9932
|
|
|
9736
9933
|
function addToExternrefTable0(obj) {
|
|
9737
9934
|
const idx = wasm.__externref_table_alloc();
|
|
Binary file
|