@xberg-io/crawlberg-wasm 1.7.2 → 1.8.0

package/pkg/nodejs/crawlberg_wasm_bg.wasm.d.ts

@@ -129,7 +129,8 @@ export const wasmbrowserconfig_endpoint: (a: number) => [number, number];
 export const wasmbrowserconfig_evalScript: (a: number) => [number, number];
 export const wasmbrowserconfig_extraWait: (a: number) => [number, bigint];
 export const wasmbrowserconfig_mode: (a: number) => [number, number];
-export const wasmbrowserconfig_new: (a: number, b: number, c: number, d: bigint, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number, n: number, o: bigint, p: number, q: number, r: number, s: number, t: number) => number;
+export const wasmbrowserconfig_new: (a: number, b: number, c: number, d: bigint, e: number, f: bigint, g: number, h: bigint, i: number, j: number, k: number, l: number, m: number, n: number, o: number, p: number, q: number, r: number, s: bigint, t: number, u: number, v: number, w: number, x: number) => number;
+export const wasmbrowserconfig_overallTimeout: (a: number) => [number, bigint];
 export const wasmbrowserconfig_proxy: (a: number) => number;
 export const wasmbrowserconfig_robotsUserAgent: (a: number) => [number, number];
 export const wasmbrowserconfig_sessionAffinity: (a: number) => number;
@@ -140,12 +141,15 @@ export const wasmbrowserconfig_set_endpoint: (a: number, b: number, c: number) =
 export const wasmbrowserconfig_set_evalScript: (a: number, b: number, c: number) => void;
 export const wasmbrowserconfig_set_extraWait: (a: number, b: number, c: bigint) => void;
 export const wasmbrowserconfig_set_mode: (a: number, b: number) => void;
+export const wasmbrowserconfig_set_overallTimeout: (a: number, b: number, c: bigint) => void;
 export const wasmbrowserconfig_set_proxy: (a: number, b: number) => void;
 export const wasmbrowserconfig_set_robotsUserAgent: (a: number, b: number, c: number) => void;
 export const wasmbrowserconfig_set_sessionAffinity: (a: number, b: number) => void;
+export const wasmbrowserconfig_set_shutdownTimeout: (a: number, b: number, c: bigint) => void;
 export const wasmbrowserconfig_set_timeout: (a: number, b: number, c: bigint) => void;
 export const wasmbrowserconfig_set_wait: (a: number, b: number) => void;
 export const wasmbrowserconfig_set_waitSelector: (a: number, b: number, c: number) => void;
+export const wasmbrowserconfig_shutdownTimeout: (a: number) => [number, bigint];
 export const wasmbrowserconfig_timeout: (a: number) => [number, bigint];
 export const wasmbrowserconfig_wait: (a: number) => [number, number];
 export const wasmbrowserconfig_waitSelector: (a: number) => [number, number];
@@ -173,15 +177,17 @@ export const wasmcitationresult_set_content: (a: number, b: number, c: number) =
 export const wasmcitationresult_set_references: (a: number, b: number, c: number) => void;
 export const wasmcontentconfig_default: () => number;
 export const wasmcontentconfig_excludeSelectors: (a: number) => [number, number];
+export const wasmcontentconfig_extractMetadata: (a: number) => number;
 export const wasmcontentconfig_includeDocumentStructure: (a: number) => number;
 export const wasmcontentconfig_maxDepth: (a: number) => number;
-export const wasmcontentconfig_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number, n: number, o: number, p: number, q: number) => number;
+export const wasmcontentconfig_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number, n: number, o: number, p: number, q: number, r: number) => number;
 export const wasmcontentconfig_outputFormat: (a: number) => [number, number];
 export const wasmcontentconfig_preprocessingPreset: (a: number) => [number, number];
 export const wasmcontentconfig_preserveTags: (a: number) => [number, number];
 export const wasmcontentconfig_removeForms: (a: number) => number;
 export const wasmcontentconfig_removeNavigation: (a: number) => number;
 export const wasmcontentconfig_set_excludeSelectors: (a: number, b: number, c: number) => void;
+export const wasmcontentconfig_set_extractMetadata: (a: number, b: number) => void;
 export const wasmcontentconfig_set_includeDocumentStructure: (a: number, b: number) => void;
 export const wasmcontentconfig_set_maxDepth: (a: number, b: number) => void;
 export const wasmcontentconfig_set_outputFormat: (a: number, b: number, c: number) => void;
@@ -220,6 +226,7 @@ export const wasmcrawlconfig_contentFilter: (a: number) => [number, number];
 export const wasmcrawlconfig_cookiesEnabled: (a: number) => number;
 export const wasmcrawlconfig_crawlStrategy: (a: number) => [number, number];
 export const wasmcrawlconfig_customHeaders: (a: number) => any;
+export const wasmcrawlconfig_dedupIncludeQuery: (a: number) => number;
 export const wasmcrawlconfig_default: () => number;
 export const wasmcrawlconfig_documentContentEncoding: (a: number) => [number, number];
 export const wasmcrawlconfig_documentMaxSize: (a: number) => number;
@@ -240,14 +247,18 @@ export const wasmcrawlconfig_maxDepth: (a: number) => number;
 export const wasmcrawlconfig_maxLinksPerPage: (a: number) => number;
 export const wasmcrawlconfig_maxPages: (a: number) => number;
 export const wasmcrawlconfig_maxRedirects: (a: number) => number;
-export const wasmcrawlconfig_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: bigint, m: number, n: number, o: number, p: number, q: number, r: number, s: number, t: number, u: number, v: number, w: number, x: number, y: number, z: number, a1: number, b1: number, c1: number, d1: number, e1: number, f1: number, g1: number, h1: number, i1: number, j1: number, k1: number, l1: number, m1: number, n1: number, o1: number, p1: number, q1: number, r1: number, s1: number, t1: bigint, u1: number, v1: number, w1: number, x1: number, y1: number, z1: number, a2: number, b2: number, c2: number, d2: number, e2: number, f2: number, g2: number, h2: number, i2: number, j2: number, k2: number) => number;
+export const wasmcrawlconfig_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number, n: number, o: number, p: number, q: bigint, r: number, s: number, t: number, u: number, v: number, w: bigint, x: number, y: bigint, z: number, a1: number, b1: number, c1: number, d1: number, e1: number, f1: number, g1: number, h1: number, i1: number, j1: number, k1: number, l1: number, m1: number, n1: number, o1: number, p1: number, q1: number, r1: number, s1: number, t1: number, u1: number, v1: number, w1: number, x1: number, y1: number, z1: number, a2: number, b2: number, c2: number, d2: number, e2: bigint, f2: number, g2: number, h2: number, i2: number, j2: number, k2: number, l2: number, m2: number, n2: number, o2: number, p2: number, q2: number, r2: number, s2: number, t2: number, u2: number, v2: number) => number;
+export const wasmcrawlconfig_pathPatternsMatchQuery: (a: number) => number;
 export const wasmcrawlconfig_proxy: (a: number) => number;
+export const wasmcrawlconfig_rateLimitJitterRatio: (a: number) => number;
 export const wasmcrawlconfig_rateLimitMs: (a: number) => [number, bigint];
 export const wasmcrawlconfig_removeTags: (a: number) => [number, number];
 export const wasmcrawlconfig_requestTimeout: (a: number) => [number, bigint];
 export const wasmcrawlconfig_respectRobotsTxt: (a: number) => number;
 export const wasmcrawlconfig_retryCodes: (a: number) => [number, number];
 export const wasmcrawlconfig_retryCount: (a: number) => number;
+export const wasmcrawlconfig_retryInitialDelayMs: (a: number) => bigint;
+export const wasmcrawlconfig_retryMaxDelayMs: (a: number) => bigint;
 export const wasmcrawlconfig_saveBrowserProfile: (a: number) => number;
 export const wasmcrawlconfig_set_allowSubdomains: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_assetTypes: (a: number, b: number, c: number) => void;
@@ -262,6 +273,7 @@ export const wasmcrawlconfig_set_contentFilter: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_cookiesEnabled: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_crawlStrategy: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_customHeaders: (a: number, b: any) => void;
+export const wasmcrawlconfig_set_dedupIncludeQuery: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_documentContentEncoding: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_documentMaxSize: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_documentMimeTypes: (a: number, b: number, c: number) => void;
@@ -281,18 +293,24 @@ export const wasmcrawlconfig_set_maxDepth: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_maxLinksPerPage: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_maxPages: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_maxRedirects: (a: number, b: number) => void;
+export const wasmcrawlconfig_set_pathPatternsMatchQuery: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_proxy: (a: number, b: number) => void;
+export const wasmcrawlconfig_set_rateLimitJitterRatio: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_rateLimitMs: (a: number, b: number, c: bigint) => void;
 export const wasmcrawlconfig_set_removeTags: (a: number, b: number, c: number) => void;
 export const wasmcrawlconfig_set_requestTimeout: (a: number, b: number, c: bigint) => void;
 export const wasmcrawlconfig_set_respectRobotsTxt: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_retryCodes: (a: number, b: number, c: number) => void;
 export const wasmcrawlconfig_set_retryCount: (a: number, b: number) => void;
+export const wasmcrawlconfig_set_retryInitialDelayMs: (a: number, b: bigint) => void;
+export const wasmcrawlconfig_set_retryMaxDelayMs: (a: number, b: bigint) => void;
 export const wasmcrawlconfig_set_saveBrowserProfile: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_softHttpErrors: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_ssrf: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_ssrfDenyPrivateExplicit: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_stayOnDomain: (a: number, b: number) => void;
+export const wasmcrawlconfig_set_stripTrackingParams: (a: number, b: number) => void;
+export const wasmcrawlconfig_set_trackingParams: (a: number, b: number, c: number) => void;
 export const wasmcrawlconfig_set_userAgent: (a: number, b: number, c: number) => void;
 export const wasmcrawlconfig_set_userAgents: (a: number, b: number, c: number) => void;
 export const wasmcrawlconfig_set_warcOutput: (a: number, b: number, c: number) => void;
@@ -300,6 +318,8 @@ export const wasmcrawlconfig_softHttpErrors: (a: number) => number;
 export const wasmcrawlconfig_ssrf: (a: number) => number;
 export const wasmcrawlconfig_ssrfDenyPrivateExplicit: (a: number) => number;
 export const wasmcrawlconfig_stayOnDomain: (a: number) => number;
+export const wasmcrawlconfig_stripTrackingParams: (a: number) => number;
+export const wasmcrawlconfig_trackingParams: (a: number) => [number, number];
 export const wasmcrawlconfig_userAgent: (a: number) => [number, number];
 export const wasmcrawlconfig_userAgents: (a: number) => [number, number];
 export const wasmcrawlconfig_validate: (a: number) => [number, number];
@@ -314,6 +334,7 @@ export const wasmcrawlpageresult_downloadedDocument: (a: number) => number;
 export const wasmcrawlpageresult_extractedData: (a: number) => any;
 export const wasmcrawlpageresult_extractionMeta: (a: number) => number;
 export const wasmcrawlpageresult_feeds: (a: number) => [number, number];
+export const wasmcrawlpageresult_finalUrl: (a: number) => [number, number];
 export const wasmcrawlpageresult_html: (a: number) => [number, number];
 export const wasmcrawlpageresult_images: (a: number) => [number, number];
 export const wasmcrawlpageresult_isPdf: (a: number) => number;
@@ -321,8 +342,11 @@ export const wasmcrawlpageresult_jsonLd: (a: number) => [number, number];
 export const wasmcrawlpageresult_links: (a: number) => [number, number];
 export const wasmcrawlpageresult_markdown: (a: number) => number;
 export const wasmcrawlpageresult_metadata: (a: number) => number;
-export const wasmcrawlpageresult_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number, n: number, o: number, p: number, q: number, r: number, s: number, t: number, u: number, v: number, w: number, x: number, y: number, z: number, a1: number, b1: number, c1: number, d1: number) => number;
+export const wasmcrawlpageresult_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number, n: number, o: number, p: number, q: number, r: number, s: number, t: number, u: number, v: number, w: number, x: number, y: number, z: number, a1: number, b1: number, c1: number, d1: number, e1: number, f1: number, g1: number, h1: number, i1: number) => number;
+export const wasmcrawlpageresult_nofollowDetected: (a: number) => number;
+export const wasmcrawlpageresult_noindexDetected: (a: number) => number;
 export const wasmcrawlpageresult_normalizedUrl: (a: number) => [number, number];
+export const wasmcrawlpageresult_redirectCount: (a: number) => number;
 export const wasmcrawlpageresult_set_bodySize: (a: number, b: number) => void;
 export const wasmcrawlpageresult_set_browserUsed: (a: number, b: number) => void;
 export const wasmcrawlpageresult_set_contentType: (a: number, b: number, c: number) => void;
@@ -332,6 +356,7 @@ export const wasmcrawlpageresult_set_downloadedDocument: (a: number, b: number)
 export const wasmcrawlpageresult_set_extractedData: (a: number, b: number) => void;
 export const wasmcrawlpageresult_set_extractionMeta: (a: number, b: number) => void;
 export const wasmcrawlpageresult_set_feeds: (a: number, b: number, c: number) => void;
+export const wasmcrawlpageresult_set_finalUrl: (a: number, b: number, c: number) => void;
 export const wasmcrawlpageresult_set_html: (a: number, b: number, c: number) => void;
 export const wasmcrawlpageresult_set_images: (a: number, b: number, c: number) => void;
 export const wasmcrawlpageresult_set_isPdf: (a: number, b: number) => void;
@@ -339,7 +364,10 @@ export const wasmcrawlpageresult_set_jsonLd: (a: number, b: number, c: number) =
 export const wasmcrawlpageresult_set_links: (a: number, b: number, c: number) => void;
 export const wasmcrawlpageresult_set_markdown: (a: number, b: number) => void;
 export const wasmcrawlpageresult_set_metadata: (a: number, b: number) => void;
+export const wasmcrawlpageresult_set_nofollowDetected: (a: number, b: number) => void;
+export const wasmcrawlpageresult_set_noindexDetected: (a: number, b: number) => void;
 export const wasmcrawlpageresult_set_normalizedUrl: (a: number, b: number, c: number) => void;
+export const wasmcrawlpageresult_set_redirectCount: (a: number, b: number) => void;
 export const wasmcrawlpageresult_set_statusCode: (a: number, b: number) => void;
 export const wasmcrawlpageresult_set_stayedOnDomain: (a: number, b: number) => void;
 export const wasmcrawlpageresult_set_url: (a: number, b: number, c: number) => void;
@@ -722,9 +750,9 @@ export const wasmssrfpolicy_set_allowlist: (a: number, b: any) => void;
 export const wasmssrfpolicy_set_denyPrivate: (a: number, b: number) => void;
 export const wasmssrfpolicy_set_maxRedirects: (a: number, b: number) => void;
 export const wasmssrfpolicy_set_schemeAllowlist: (a: number, b: number, c: number) => void;
-export const wasm_bindgen__convert__closures_____invoke__h5567083e9d00da74: (a: number, b: number, c: any) => [number, number];
-export const wasm_bindgen__convert__closures_____invoke__h9cb213bb50b5f7d0: (a: number, b: number, c: any, d: any) => void;
-export const wasm_bindgen__convert__closures_____invoke__h00e67b488e823f1f: (a: number, b: number) => void;
+export const wasm_bindgen__convert__closures_____invoke__h2eedacf566f3748c: (a: number, b: number, c: any) => [number, number];
+export const wasm_bindgen__convert__closures_____invoke__h8ad11faaa2905452: (a: number, b: number, c: any, d: any) => void;
+export const wasm_bindgen__convert__closures_____invoke__h21c8118e1fe083d1: (a: number, b: number) => void;
 export const __wbindgen_malloc: (a: number, b: number) => number;
 export const __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number;
 export const __wbindgen_exn_store: (a: number) => void;

package/pkg/nodejs/package.json

@@ -1,7 +1,7 @@
 {
   "name": "crawlberg-wasm",
   "description": "High-performance web crawling engine",
-  "version": "1.7.2",
+  "version": "1.8.0",
   "license": "MIT",
   "repository": {
     "type": "git",

package/pkg/web/crawlberg_wasm.d.ts

@@ -153,7 +153,7 @@ export class WasmBrowserConfig {
     free(): void;
     [Symbol.dispose](): void;
     static default(): WasmBrowserConfig;
-    constructor(mode?: WasmBrowserMode | null, backend?: WasmBrowserBackend | null, timeout?: bigint | null, wait?: WasmBrowserWait | null, blockUrlPatterns?: string[] | null, captureNetworkEvents?: boolean | null, sessionAffinity?: boolean | null, endpoint?: string | null, waitSelector?: string | null, extraWait?: bigint | null, proxy?: WasmProxyConfig | null, evalScript?: string | null, robotsUserAgent?: string | null);
+    constructor(mode?: WasmBrowserMode | null, backend?: WasmBrowserBackend | null, timeout?: bigint | null, overallTimeout?: bigint | null, shutdownTimeout?: bigint | null, wait?: WasmBrowserWait | null, blockUrlPatterns?: string[] | null, captureNetworkEvents?: boolean | null, sessionAffinity?: boolean | null, endpoint?: string | null, waitSelector?: string | null, extraWait?: bigint | null, proxy?: WasmProxyConfig | null, evalScript?: string | null, robotsUserAgent?: string | null);
     get backend(): string;
     set backend(value: WasmBrowserBackend);
     blockUrlPatterns: string[];
@@ -166,11 +166,15 @@ export class WasmBrowserConfig {
     set extraWait(value: bigint | null | undefined);
     get mode(): string;
     set mode(value: WasmBrowserMode);
+    get overallTimeout(): bigint | undefined;
+    set overallTimeout(value: bigint | null | undefined);
     get proxy(): WasmProxyConfig | undefined;
     set proxy(value: WasmProxyConfig | null | undefined);
     get robotsUserAgent(): string | undefined;
     set robotsUserAgent(value: string | null | undefined);
     sessionAffinity: boolean;
+    get shutdownTimeout(): bigint | undefined;
+    set shutdownTimeout(value: bigint | null | undefined);
     get timeout(): bigint | undefined;
     set timeout(value: bigint | null | undefined);
     get waitSelector(): string | undefined;
@@ -251,8 +255,9 @@ export class WasmContentConfig {
     free(): void;
     [Symbol.dispose](): void;
     static default(): WasmContentConfig;
-    constructor(outputFormat?: string | null, preprocessingPreset?: string | null, removeNavigation?: boolean | null, removeForms?: boolean | null, stripTags?: string[] | null, preserveTags?: string[] | null, excludeSelectors?: string[] | null, skipImages?: boolean | null, wrap?: boolean | null, wrapWidth?: number | null, includeDocumentStructure?: boolean | null, maxDepth?: number | null);
+    constructor(outputFormat?: string | null, preprocessingPreset?: string | null, removeNavigation?: boolean | null, removeForms?: boolean | null, stripTags?: string[] | null, preserveTags?: string[] | null, excludeSelectors?: string[] | null, skipImages?: boolean | null, wrap?: boolean | null, wrapWidth?: number | null, includeDocumentStructure?: boolean | null, extractMetadata?: boolean | null, maxDepth?: number | null);
     excludeSelectors: string[];
+    extractMetadata: boolean;
     includeDocumentStructure: boolean;
     get maxDepth(): number | undefined;
     set maxDepth(value: number | null | undefined);
@@ -297,7 +302,7 @@ export class WasmCrawlConfig {
     free(): void;
     [Symbol.dispose](): void;
     static default(): WasmCrawlConfig;
-    constructor(crawlStrategy?: WasmCrawlStrategyKind | null, respectRobotsTxt?: boolean | null, softHttpErrors?: boolean | null, stayOnDomain?: boolean | null, allowSubdomains?: boolean | null, includePaths?: string[] | null, excludePaths?: string[] | null, customHeaders?: any | null, requestTimeout?: bigint | null, maxRedirects?: number | null, retryCount?: number | null, retryCodes?: Uint16Array | null, cookiesEnabled?: boolean | null, removeTags?: string[] | null, content?: WasmContentConfig | null, downloadAssets?: boolean | null, assetTypes?: WasmAssetCategory[] | null, browser?: WasmBrowserConfig | null, userAgents?: string[] | null, captureScreenshot?: boolean | null, followDocumentUrls?: boolean | null, downloadDocuments?: boolean | null, documentMimeTypes?: string[] | null, saveBrowserProfile?: boolean | null, ssrf?: WasmSsrfPolicy | null, maxDepth?: number | null, maxPages?: number | null, maxLinksPerPage?: number | null, maxConcurrent?: number | null, contentFilter?: WasmContentFilterKind | null, bm25Query?: string | null, bm25Threshold?: number | null, userAgent?: string | null, rateLimitMs?: bigint | null, auth?: any | null, maxBodySize?: number | null, mapLimit?: number | null, mapSearch?: string | null, maxAssetSize?: number | null, proxy?: WasmProxyConfig | null, documentUrlDepth?: number | null, documentMaxSize?: number | null, documentOutputDir?: string | null, documentContentEncoding?: WasmDocumentContentEncoding | null, warcOutput?: string | null, browserProfile?: string | null, ssrfDenyPrivateExplicit?: boolean | null);
+    constructor(crawlStrategy?: WasmCrawlStrategyKind | null, respectRobotsTxt?: boolean | null, softHttpErrors?: boolean | null, stayOnDomain?: boolean | null, allowSubdomains?: boolean | null, includePaths?: string[] | null, excludePaths?: string[] | null, pathPatternsMatchQuery?: boolean | null, dedupIncludeQuery?: boolean | null, stripTrackingParams?: boolean | null, trackingParams?: string[] | null, customHeaders?: any | null, requestTimeout?: bigint | null, maxRedirects?: number | null, retryCount?: number | null, retryCodes?: Uint16Array | null, retryInitialDelayMs?: bigint | null, retryMaxDelayMs?: bigint | null, rateLimitJitterRatio?: number | null, cookiesEnabled?: boolean | null, removeTags?: string[] | null, content?: WasmContentConfig | null, downloadAssets?: boolean | null, assetTypes?: WasmAssetCategory[] | null, browser?: WasmBrowserConfig | null, userAgents?: string[] | null, captureScreenshot?: boolean | null, followDocumentUrls?: boolean | null, downloadDocuments?: boolean | null, documentMimeTypes?: string[] | null, saveBrowserProfile?: boolean | null, ssrf?: WasmSsrfPolicy | null, maxDepth?: number | null, maxPages?: number | null, maxLinksPerPage?: number | null, maxConcurrent?: number | null, contentFilter?: WasmContentFilterKind | null, bm25Query?: string | null, bm25Threshold?: number | null, userAgent?: string | null, rateLimitMs?: bigint | null, auth?: any | null, maxBodySize?: number | null, mapLimit?: number | null, mapSearch?: string | null, maxAssetSize?: number | null, proxy?: WasmProxyConfig | null, documentUrlDepth?: number | null, documentMaxSize?: number | null, documentOutputDir?: string | null, documentContentEncoding?: WasmDocumentContentEncoding | null, warcOutput?: string | null, browserProfile?: string | null, ssrfDenyPrivateExplicit?: boolean | null);
     /**
      * Validate the configuration, returning an error if any values are invalid.
      *
@@ -324,6 +329,7 @@ export class WasmCrawlConfig {
     get crawlStrategy(): string;
     set crawlStrategy(value: WasmCrawlStrategyKind);
     customHeaders: any;
+    dedupIncludeQuery: boolean;
     get documentContentEncoding(): string | undefined;
     set documentContentEncoding(value: WasmDocumentContentEncoding | null | undefined);
     get documentMaxSize(): number | undefined;
@@ -355,8 +361,10 @@ export class WasmCrawlConfig {
     get maxPages(): number | undefined;
     set maxPages(value: number | null | undefined);
     maxRedirects: number;
+    pathPatternsMatchQuery: boolean;
     get proxy(): WasmProxyConfig | undefined;
     set proxy(value: WasmProxyConfig | null | undefined);
+    rateLimitJitterRatio: number;
     get rateLimitMs(): bigint | undefined;
     set rateLimitMs(value: bigint | null | undefined);
     removeTags: string[];
@@ -365,12 +373,16 @@ export class WasmCrawlConfig {
     respectRobotsTxt: boolean;
     retryCodes: Uint16Array;
     retryCount: number;
+    retryInitialDelayMs: bigint;
+    retryMaxDelayMs: bigint;
     saveBrowserProfile: boolean;
     softHttpErrors: boolean;
     get ssrfDenyPrivateExplicit(): boolean | undefined;
     set ssrfDenyPrivateExplicit(value: boolean | null | undefined);
     ssrf: WasmSsrfPolicy;
     stayOnDomain: boolean;
+    stripTrackingParams: boolean;
+    trackingParams: string[];
     get userAgent(): string | undefined;
     set userAgent(value: string | null | undefined);
     userAgents: string[];
@@ -397,7 +409,7 @@ export class WasmCrawlPageResult {
     free(): void;
     [Symbol.dispose](): void;
     static default(): WasmCrawlPageResult;
-    constructor(url?: string | null, normalizedUrl?: string | null, statusCode?: number | null, contentType?: string | null, html?: string | null, bodySize?: number | null, metadata?: WasmPageMetadata | null, links?: WasmLinkInfo[] | null, images?: WasmImageInfo[] | null, feeds?: WasmFeedInfo[] | null, jsonLd?: WasmJsonLdEntry[] | null, depth?: number | null, stayedOnDomain?: boolean | null, wasSkipped?: boolean | null, isPdf?: boolean | null, browserUsed?: boolean | null, detectedCharset?: string | null, markdown?: WasmMarkdownResult | null, extractedData?: any | null, extractionMeta?: WasmExtractionMeta | null, downloadedDocument?: WasmDownloadedDocument | null);
+    constructor(url?: string | null, normalizedUrl?: string | null, statusCode?: number | null, contentType?: string | null, html?: string | null, bodySize?: number | null, metadata?: WasmPageMetadata | null, links?: WasmLinkInfo[] | null, images?: WasmImageInfo[] | null, feeds?: WasmFeedInfo[] | null, jsonLd?: WasmJsonLdEntry[] | null, depth?: number | null, stayedOnDomain?: boolean | null, wasSkipped?: boolean | null, isPdf?: boolean | null, browserUsed?: boolean | null, finalUrl?: string | null, redirectCount?: number | null, noindexDetected?: boolean | null, nofollowDetected?: boolean | null, detectedCharset?: string | null, markdown?: WasmMarkdownResult | null, extractedData?: any | null, extractionMeta?: WasmExtractionMeta | null, downloadedDocument?: WasmDownloadedDocument | null);
     bodySize: number;
     browserUsed: boolean;
     contentType: string;
@@ -411,6 +423,7 @@ export class WasmCrawlPageResult {
     get extractionMeta(): WasmExtractionMeta | undefined;
     set extractionMeta(value: WasmExtractionMeta | null | undefined);
     feeds: WasmFeedInfo[];
+    finalUrl: string;
     html: string;
     images: WasmImageInfo[];
     isPdf: boolean;
@@ -419,7 +432,10 @@ export class WasmCrawlPageResult {
     get markdown(): WasmMarkdownResult | undefined;
     set markdown(value: WasmMarkdownResult | null | undefined);
     metadata: WasmPageMetadata;
+    nofollowDetected: boolean;
+    noindexDetected: boolean;
     normalizedUrl: string;
+    redirectCount: number;
     statusCode: number;
     stayedOnDomain: boolean;
     url: string;
@@ -1204,7 +1220,8 @@ export interface InitOutput {
     readonly wasmbrowserconfig_evalScript: (a: number) => [number, number];
     readonly wasmbrowserconfig_extraWait: (a: number) => [number, bigint];
     readonly wasmbrowserconfig_mode: (a: number) => [number, number];
-    readonly wasmbrowserconfig_new: (a: number, b: number, c: number, d: bigint, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number, n: number, o: bigint, p: number, q: number, r: number, s: number, t: number) => number;
+    readonly wasmbrowserconfig_new: (a: number, b: number, c: number, d: bigint, e: number, f: bigint, g: number, h: bigint, i: number, j: number, k: number, l: number, m: number, n: number, o: number, p: number, q: number, r: number, s: bigint, t: number, u: number, v: number, w: number, x: number) => number;
+    readonly wasmbrowserconfig_overallTimeout: (a: number) => [number, bigint];
     readonly wasmbrowserconfig_proxy: (a: number) => number;
     readonly wasmbrowserconfig_robotsUserAgent: (a: number) => [number, number];
     readonly wasmbrowserconfig_sessionAffinity: (a: number) => number;
@@ -1215,12 +1232,15 @@ export interface InitOutput {
     readonly wasmbrowserconfig_set_evalScript: (a: number, b: number, c: number) => void;
     readonly wasmbrowserconfig_set_extraWait: (a: number, b: number, c: bigint) => void;
     readonly wasmbrowserconfig_set_mode: (a: number, b: number) => void;
+    readonly wasmbrowserconfig_set_overallTimeout: (a: number, b: number, c: bigint) => void;
     readonly wasmbrowserconfig_set_proxy: (a: number, b: number) => void;
     readonly wasmbrowserconfig_set_robotsUserAgent: (a: number, b: number, c: number) => void;
     readonly wasmbrowserconfig_set_sessionAffinity: (a: number, b: number) => void;
+    readonly wasmbrowserconfig_set_shutdownTimeout: (a: number, b: number, c: bigint) => void;
     readonly wasmbrowserconfig_set_timeout: (a: number, b: number, c: bigint) => void;
     readonly wasmbrowserconfig_set_wait: (a: number, b: number) => void;
     readonly wasmbrowserconfig_set_waitSelector: (a: number, b: number, c: number) => void;
+    readonly wasmbrowserconfig_shutdownTimeout: (a: number) => [number, bigint];
     readonly wasmbrowserconfig_timeout: (a: number) => [number, bigint];
     readonly wasmbrowserconfig_wait: (a: number) => [number, number];
     readonly wasmbrowserconfig_waitSelector: (a: number) => [number, number];
@@ -1248,15 +1268,17 @@ export interface InitOutput {
     readonly wasmcitationresult_set_references: (a: number, b: number, c: number) => void;
     readonly wasmcontentconfig_default: () => number;
     readonly wasmcontentconfig_excludeSelectors: (a: number) => [number, number];
+    readonly wasmcontentconfig_extractMetadata: (a: number) => number;
     readonly wasmcontentconfig_includeDocumentStructure: (a: number) => number;
     readonly wasmcontentconfig_maxDepth: (a: number) => number;
-    readonly wasmcontentconfig_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number, n: number, o: number, p: number, q: number) => number;
+    readonly wasmcontentconfig_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number, n: number, o: number, p: number, q: number, r: number) => number;
     readonly wasmcontentconfig_outputFormat: (a: number) => [number, number];
     readonly wasmcontentconfig_preprocessingPreset: (a: number) => [number, number];
     readonly wasmcontentconfig_preserveTags: (a: number) => [number, number];
     readonly wasmcontentconfig_removeForms: (a: number) => number;
     readonly wasmcontentconfig_removeNavigation: (a: number) => number;
     readonly wasmcontentconfig_set_excludeSelectors: (a: number, b: number, c: number) => void;
+    readonly wasmcontentconfig_set_extractMetadata: (a: number, b: number) => void;
     readonly wasmcontentconfig_set_includeDocumentStructure: (a: number, b: number) => void;
     readonly wasmcontentconfig_set_maxDepth: (a: number, b: number) => void;
     readonly wasmcontentconfig_set_outputFormat: (a: number, b: number, c: number) => void;
@@ -1295,6 +1317,7 @@ export interface InitOutput {
     readonly wasmcrawlconfig_cookiesEnabled: (a: number) => number;
     readonly wasmcrawlconfig_crawlStrategy: (a: number) => [number, number];
     readonly wasmcrawlconfig_customHeaders: (a: number) => any;
+    readonly wasmcrawlconfig_dedupIncludeQuery: (a: number) => number;
     readonly wasmcrawlconfig_default: () => number;
     readonly wasmcrawlconfig_documentContentEncoding: (a: number) => [number, number];
     readonly wasmcrawlconfig_documentMaxSize: (a: number) => number;
@@ -1315,14 +1338,18 @@ export interface InitOutput {
     readonly wasmcrawlconfig_maxLinksPerPage: (a: number) => number;
     readonly wasmcrawlconfig_maxPages: (a: number) => number;
     readonly wasmcrawlconfig_maxRedirects: (a: number) => number;
-    readonly wasmcrawlconfig_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: bigint, m: number, n: number, o: number, p: number, q: number, r: number, s: number, t: number, u: number, v: number, w: number, x: number, y: number, z: number, a1: number, b1: number, c1: number, d1: number, e1: number, f1: number, g1: number, h1: number, i1: number, j1: number, k1: number, l1: number, m1: number, n1: number, o1: number, p1: number, q1: number, r1: number, s1: number, t1: bigint, u1: number, v1: number, w1: number, x1: number, y1: number, z1: number, a2: number, b2: number, c2: number, d2: number, e2: number, f2: number, g2: number, h2: number, i2: number, j2: number, k2: number) => number;
+    readonly wasmcrawlconfig_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number, n: number, o: number, p: number, q: bigint, r: number, s: number, t: number, u: number, v: number, w: bigint, x: number, y: bigint, z: number, a1: number, b1: number, c1: number, d1: number, e1: number, f1: number, g1: number, h1: number, i1: number, j1: number, k1: number, l1: number, m1: number, n1: number, o1: number, p1: number, q1: number, r1: number, s1: number, t1: number, u1: number, v1: number, w1: number, x1: number, y1: number, z1: number, a2: number, b2: number, c2: number, d2: number, e2: bigint, f2: number, g2: number, h2: number, i2: number, j2: number, k2: number, l2: number, m2: number, n2: number, o2: number, p2: number, q2: number, r2: number, s2: number, t2: number, u2: number, v2: number) => number;
+    readonly wasmcrawlconfig_pathPatternsMatchQuery: (a: number) => number;
     readonly wasmcrawlconfig_proxy: (a: number) => number;
+    readonly wasmcrawlconfig_rateLimitJitterRatio: (a: number) => number;
     readonly wasmcrawlconfig_rateLimitMs: (a: number) => [number, bigint];
     readonly wasmcrawlconfig_removeTags: (a: number) => [number, number];
     readonly wasmcrawlconfig_requestTimeout: (a: number) => [number, bigint];
     readonly wasmcrawlconfig_respectRobotsTxt: (a: number) => number;
     readonly wasmcrawlconfig_retryCodes: (a: number) => [number, number];
     readonly wasmcrawlconfig_retryCount: (a: number) => number;
+    readonly wasmcrawlconfig_retryInitialDelayMs: (a: number) => bigint;
+    readonly wasmcrawlconfig_retryMaxDelayMs: (a: number) => bigint;
     readonly wasmcrawlconfig_saveBrowserProfile: (a: number) => number;
     readonly wasmcrawlconfig_set_allowSubdomains: (a: number, b: number) => void;
     readonly wasmcrawlconfig_set_assetTypes: (a: number, b: number, c: number) => void;
@@ -1337,6 +1364,7 @@ export interface InitOutput {
     readonly wasmcrawlconfig_set_cookiesEnabled: (a: number, b: number) => void;
     readonly wasmcrawlconfig_set_crawlStrategy: (a: number, b: number) => void;
     readonly wasmcrawlconfig_set_customHeaders: (a: number, b: any) => void;
+    readonly wasmcrawlconfig_set_dedupIncludeQuery: (a: number, b: number) => void;
     readonly wasmcrawlconfig_set_documentContentEncoding: (a: number, b: number) => void;
     readonly wasmcrawlconfig_set_documentMaxSize: (a: number, b: number) => void;
     readonly wasmcrawlconfig_set_documentMimeTypes: (a: number, b: number, c: number) => void;
@@ -1356,18 +1384,24 @@ export interface InitOutput {
     readonly wasmcrawlconfig_set_maxLinksPerPage: (a: number, b: number) => void;
     readonly wasmcrawlconfig_set_maxPages: (a: number, b: number) => void;
     readonly wasmcrawlconfig_set_maxRedirects: (a: number, b: number) => void;
+    readonly wasmcrawlconfig_set_pathPatternsMatchQuery: (a: number, b: number) => void;
     readonly wasmcrawlconfig_set_proxy: (a: number, b: number) => void;
+    readonly wasmcrawlconfig_set_rateLimitJitterRatio: (a: number, b: number) => void;
     readonly wasmcrawlconfig_set_rateLimitMs: (a: number, b: number, c: bigint) => void;
     readonly wasmcrawlconfig_set_removeTags: (a: number, b: number, c: number) => void;
     readonly wasmcrawlconfig_set_requestTimeout: (a: number, b: number, c: bigint) => void;
     readonly wasmcrawlconfig_set_respectRobotsTxt: (a: number, b: number) => void;
     readonly wasmcrawlconfig_set_retryCodes: (a: number, b: number, c: number) => void;
     readonly wasmcrawlconfig_set_retryCount: (a: number, b: number) => void;
+    readonly wasmcrawlconfig_set_retryInitialDelayMs: (a: number, b: bigint) => void;
+    readonly wasmcrawlconfig_set_retryMaxDelayMs: (a: number, b: bigint) => void;
     readonly wasmcrawlconfig_set_saveBrowserProfile: (a: number, b: number) => void;
     readonly wasmcrawlconfig_set_softHttpErrors: (a: number, b: number) => void;
     readonly wasmcrawlconfig_set_ssrf: (a: number, b: number) => void;
     readonly wasmcrawlconfig_set_ssrfDenyPrivateExplicit: (a: number, b: number) => void;
     readonly wasmcrawlconfig_set_stayOnDomain: (a: number, b: number) => void;
+    readonly wasmcrawlconfig_set_stripTrackingParams: (a: number, b: number) => void;
+    readonly wasmcrawlconfig_set_trackingParams: (a: number, b: number, c: number) => void;
     readonly wasmcrawlconfig_set_userAgent: (a: number, b: number, c: number) => void;
     readonly wasmcrawlconfig_set_userAgents: (a: number, b: number, c: number) => void;
     readonly wasmcrawlconfig_set_warcOutput: (a: number, b: number, c: number) => void;
@@ -1375,6 +1409,8 @@ export interface InitOutput {
     readonly wasmcrawlconfig_ssrf: (a: number) => number;
     readonly wasmcrawlconfig_ssrfDenyPrivateExplicit: (a: number) => number;
     readonly wasmcrawlconfig_stayOnDomain: (a: number) => number;
+    readonly wasmcrawlconfig_stripTrackingParams: (a: number) => number;
+    readonly wasmcrawlconfig_trackingParams: (a: number) => [number, number];
     readonly wasmcrawlconfig_userAgent: (a: number) => [number, number];
     readonly wasmcrawlconfig_userAgents: (a: number) => [number, number];
     readonly wasmcrawlconfig_validate: (a: number) => [number, number];
@@ -1389,6 +1425,7 @@ export interface InitOutput {
     readonly wasmcrawlpageresult_extractedData: (a: number) => any;
     readonly wasmcrawlpageresult_extractionMeta: (a: number) => number;
     readonly wasmcrawlpageresult_feeds: (a: number) => [number, number];
+    readonly wasmcrawlpageresult_finalUrl: (a: number) => [number, number];
     readonly wasmcrawlpageresult_html: (a: number) => [number, number];
     readonly wasmcrawlpageresult_images: (a: number) => [number, number];
     readonly wasmcrawlpageresult_isPdf: (a: number) => number;
@@ -1396,8 +1433,11 @@ export interface InitOutput {
     readonly wasmcrawlpageresult_links: (a: number) => [number, number];
     readonly wasmcrawlpageresult_markdown: (a: number) => number;
     readonly wasmcrawlpageresult_metadata: (a: number) => number;
-    readonly wasmcrawlpageresult_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number, n: number, o: number, p: number, q: number, r: number, s: number, t: number, u: number, v: number, w: number, x: number, y: number, z: number, a1: number, b1: number, c1: number, d1: number) => number;
+    readonly wasmcrawlpageresult_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number, n: number, o: number, p: number, q: number, r: number, s: number, t: number, u: number, v: number, w: number, x: number, y: number, z: number, a1: number, b1: number, c1: number, d1: number, e1: number, f1: number, g1: number, h1: number, i1: number) => number;
+    readonly wasmcrawlpageresult_nofollowDetected: (a: number) => number;
+    readonly wasmcrawlpageresult_noindexDetected: (a: number) => number;
     readonly wasmcrawlpageresult_normalizedUrl: (a: number) => [number, number];
+    readonly wasmcrawlpageresult_redirectCount: (a: number) => number;
     readonly wasmcrawlpageresult_set_bodySize: (a: number, b: number) => void;
     readonly wasmcrawlpageresult_set_browserUsed: (a: number, b: number) => void;
     readonly wasmcrawlpageresult_set_contentType: (a: number, b: number, c: number) => void;
@@ -1407,6 +1447,7 @@ export interface InitOutput {
     readonly wasmcrawlpageresult_set_extractedData: (a: number, b: number) => void;
     readonly wasmcrawlpageresult_set_extractionMeta: (a: number, b: number) => void;
     readonly wasmcrawlpageresult_set_feeds: (a: number, b: number, c: number) => void;
+    readonly wasmcrawlpageresult_set_finalUrl: (a: number, b: number, c: number) => void;
     readonly wasmcrawlpageresult_set_html: (a: number, b: number, c: number) => void;
     readonly wasmcrawlpageresult_set_images: (a: number, b: number, c: number) => void;
     readonly wasmcrawlpageresult_set_isPdf: (a: number, b: number) => void;
@@ -1414,7 +1455,10 @@ export interface InitOutput {
     readonly wasmcrawlpageresult_set_links: (a: number, b: number, c: number) => void;
     readonly wasmcrawlpageresult_set_markdown: (a: number, b: number) => void;
     readonly wasmcrawlpageresult_set_metadata: (a: number, b: number) => void;
+    readonly wasmcrawlpageresult_set_nofollowDetected: (a: number, b: number) => void;
+    readonly wasmcrawlpageresult_set_noindexDetected: (a: number, b: number) => void;
     readonly wasmcrawlpageresult_set_normalizedUrl: (a: number, b: number, c: number) => void;
+    readonly wasmcrawlpageresult_set_redirectCount: (a: number, b: number) => void;
     readonly wasmcrawlpageresult_set_statusCode: (a: number, b: number) => void;
     readonly wasmcrawlpageresult_set_stayedOnDomain: (a: number, b: number) => void;
     readonly wasmcrawlpageresult_set_url: (a: number, b: number, c: number) => void;
@@ -1797,9 +1841,9 @@ export interface InitOutput {
     readonly wasmssrfpolicy_set_denyPrivate: (a: number, b: number) => void;
     readonly wasmssrfpolicy_set_maxRedirects: (a: number, b: number) => void;
     readonly wasmssrfpolicy_set_schemeAllowlist: (a: number, b: number, c: number) => void;
-    readonly wasm_bindgen__convert__closures_____invoke__h5567083e9d00da74: (a: number, b: number, c: any) => [number, number];
-    readonly wasm_bindgen__convert__closures_____invoke__h9cb213bb50b5f7d0: (a: number, b: number, c: any, d: any) => void;
-    readonly wasm_bindgen__convert__closures_____invoke__h00e67b488e823f1f: (a: number, b: number) => void;
+    readonly wasm_bindgen__convert__closures_____invoke__h2eedacf566f3748c: (a: number, b: number, c: any) => [number, number];
+    readonly wasm_bindgen__convert__closures_____invoke__h8ad11faaa2905452: (a: number, b: number, c: any, d: any) => void;
+    readonly wasm_bindgen__convert__closures_____invoke__h21c8118e1fe083d1: (a: number, b: number) => void;
     readonly __wbindgen_malloc: (a: number, b: number) => number;
     readonly __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number;
     readonly __wbindgen_exn_store: (a: number) => void;