@xberg-io/crawlberg-wasm 1.7.2 → 1.8.0

package/pkg/web/crawlberg_wasm_bg.wasm.d.ts

@@ -129,7 +129,8 @@ export const wasmbrowserconfig_endpoint: (a: number) => [number, number];
 export const wasmbrowserconfig_evalScript: (a: number) => [number, number];
 export const wasmbrowserconfig_extraWait: (a: number) => [number, bigint];
 export const wasmbrowserconfig_mode: (a: number) => [number, number];
-export const wasmbrowserconfig_new: (a: number, b: number, c: number, d: bigint, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number, n: number, o: bigint, p: number, q: number, r: number, s: number, t: number) => number;
+export const wasmbrowserconfig_new: (a: number, b: number, c: number, d: bigint, e: number, f: bigint, g: number, h: bigint, i: number, j: number, k: number, l: number, m: number, n: number, o: number, p: number, q: number, r: number, s: bigint, t: number, u: number, v: number, w: number, x: number) => number;
+export const wasmbrowserconfig_overallTimeout: (a: number) => [number, bigint];
 export const wasmbrowserconfig_proxy: (a: number) => number;
 export const wasmbrowserconfig_robotsUserAgent: (a: number) => [number, number];
 export const wasmbrowserconfig_sessionAffinity: (a: number) => number;
@@ -140,12 +141,15 @@ export const wasmbrowserconfig_set_endpoint: (a: number, b: number, c: number) =
 export const wasmbrowserconfig_set_evalScript: (a: number, b: number, c: number) => void;
 export const wasmbrowserconfig_set_extraWait: (a: number, b: number, c: bigint) => void;
 export const wasmbrowserconfig_set_mode: (a: number, b: number) => void;
+export const wasmbrowserconfig_set_overallTimeout: (a: number, b: number, c: bigint) => void;
 export const wasmbrowserconfig_set_proxy: (a: number, b: number) => void;
 export const wasmbrowserconfig_set_robotsUserAgent: (a: number, b: number, c: number) => void;
 export const wasmbrowserconfig_set_sessionAffinity: (a: number, b: number) => void;
+export const wasmbrowserconfig_set_shutdownTimeout: (a: number, b: number, c: bigint) => void;
 export const wasmbrowserconfig_set_timeout: (a: number, b: number, c: bigint) => void;
 export const wasmbrowserconfig_set_wait: (a: number, b: number) => void;
 export const wasmbrowserconfig_set_waitSelector: (a: number, b: number, c: number) => void;
+export const wasmbrowserconfig_shutdownTimeout: (a: number) => [number, bigint];
 export const wasmbrowserconfig_timeout: (a: number) => [number, bigint];
 export const wasmbrowserconfig_wait: (a: number) => [number, number];
 export const wasmbrowserconfig_waitSelector: (a: number) => [number, number];
@@ -173,15 +177,17 @@ export const wasmcitationresult_set_content: (a: number, b: number, c: number) =
 export const wasmcitationresult_set_references: (a: number, b: number, c: number) => void;
 export const wasmcontentconfig_default: () => number;
 export const wasmcontentconfig_excludeSelectors: (a: number) => [number, number];
+export const wasmcontentconfig_extractMetadata: (a: number) => number;
 export const wasmcontentconfig_includeDocumentStructure: (a: number) => number;
 export const wasmcontentconfig_maxDepth: (a: number) => number;
-export const wasmcontentconfig_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number, n: number, o: number, p: number, q: number) => number;
+export const wasmcontentconfig_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number, n: number, o: number, p: number, q: number, r: number) => number;
 export const wasmcontentconfig_outputFormat: (a: number) => [number, number];
 export const wasmcontentconfig_preprocessingPreset: (a: number) => [number, number];
 export const wasmcontentconfig_preserveTags: (a: number) => [number, number];
 export const wasmcontentconfig_removeForms: (a: number) => number;
 export const wasmcontentconfig_removeNavigation: (a: number) => number;
 export const wasmcontentconfig_set_excludeSelectors: (a: number, b: number, c: number) => void;
+export const wasmcontentconfig_set_extractMetadata: (a: number, b: number) => void;
 export const wasmcontentconfig_set_includeDocumentStructure: (a: number, b: number) => void;
 export const wasmcontentconfig_set_maxDepth: (a: number, b: number) => void;
 export const wasmcontentconfig_set_outputFormat: (a: number, b: number, c: number) => void;
@@ -220,6 +226,7 @@ export const wasmcrawlconfig_contentFilter: (a: number) => [number, number];
 export const wasmcrawlconfig_cookiesEnabled: (a: number) => number;
 export const wasmcrawlconfig_crawlStrategy: (a: number) => [number, number];
 export const wasmcrawlconfig_customHeaders: (a: number) => any;
+export const wasmcrawlconfig_dedupIncludeQuery: (a: number) => number;
 export const wasmcrawlconfig_default: () => number;
 export const wasmcrawlconfig_documentContentEncoding: (a: number) => [number, number];
 export const wasmcrawlconfig_documentMaxSize: (a: number) => number;
@@ -240,14 +247,18 @@ export const wasmcrawlconfig_maxDepth: (a: number) => number;
 export const wasmcrawlconfig_maxLinksPerPage: (a: number) => number;
 export const wasmcrawlconfig_maxPages: (a: number) => number;
 export const wasmcrawlconfig_maxRedirects: (a: number) => number;
-export const wasmcrawlconfig_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: bigint, m: number, n: number, o: number, p: number, q: number, r: number, s: number, t: number, u: number, v: number, w: number, x: number, y: number, z: number, a1: number, b1: number, c1: number, d1: number, e1: number, f1: number, g1: number, h1: number, i1: number, j1: number, k1: number, l1: number, m1: number, n1: number, o1: number, p1: number, q1: number, r1: number, s1: number, t1: bigint, u1: number, v1: number, w1: number, x1: number, y1: number, z1: number, a2: number, b2: number, c2: number, d2: number, e2: number, f2: number, g2: number, h2: number, i2: number, j2: number, k2: number) => number;
+export const wasmcrawlconfig_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number, n: number, o: number, p: number, q: bigint, r: number, s: number, t: number, u: number, v: number, w: bigint, x: number, y: bigint, z: number, a1: number, b1: number, c1: number, d1: number, e1: number, f1: number, g1: number, h1: number, i1: number, j1: number, k1: number, l1: number, m1: number, n1: number, o1: number, p1: number, q1: number, r1: number, s1: number, t1: number, u1: number, v1: number, w1: number, x1: number, y1: number, z1: number, a2: number, b2: number, c2: number, d2: number, e2: bigint, f2: number, g2: number, h2: number, i2: number, j2: number, k2: number, l2: number, m2: number, n2: number, o2: number, p2: number, q2: number, r2: number, s2: number, t2: number, u2: number, v2: number) => number;
+export const wasmcrawlconfig_pathPatternsMatchQuery: (a: number) => number;
 export const wasmcrawlconfig_proxy: (a: number) => number;
+export const wasmcrawlconfig_rateLimitJitterRatio: (a: number) => number;
 export const wasmcrawlconfig_rateLimitMs: (a: number) => [number, bigint];
 export const wasmcrawlconfig_removeTags: (a: number) => [number, number];
 export const wasmcrawlconfig_requestTimeout: (a: number) => [number, bigint];
 export const wasmcrawlconfig_respectRobotsTxt: (a: number) => number;
 export const wasmcrawlconfig_retryCodes: (a: number) => [number, number];
 export const wasmcrawlconfig_retryCount: (a: number) => number;
+export const wasmcrawlconfig_retryInitialDelayMs: (a: number) => bigint;
+export const wasmcrawlconfig_retryMaxDelayMs: (a: number) => bigint;
 export const wasmcrawlconfig_saveBrowserProfile: (a: number) => number;
 export const wasmcrawlconfig_set_allowSubdomains: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_assetTypes: (a: number, b: number, c: number) => void;
@@ -262,6 +273,7 @@ export const wasmcrawlconfig_set_contentFilter: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_cookiesEnabled: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_crawlStrategy: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_customHeaders: (a: number, b: any) => void;
+export const wasmcrawlconfig_set_dedupIncludeQuery: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_documentContentEncoding: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_documentMaxSize: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_documentMimeTypes: (a: number, b: number, c: number) => void;
@@ -281,18 +293,24 @@ export const wasmcrawlconfig_set_maxDepth: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_maxLinksPerPage: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_maxPages: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_maxRedirects: (a: number, b: number) => void;
+export const wasmcrawlconfig_set_pathPatternsMatchQuery: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_proxy: (a: number, b: number) => void;
+export const wasmcrawlconfig_set_rateLimitJitterRatio: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_rateLimitMs: (a: number, b: number, c: bigint) => void;
 export const wasmcrawlconfig_set_removeTags: (a: number, b: number, c: number) => void;
 export const wasmcrawlconfig_set_requestTimeout: (a: number, b: number, c: bigint) => void;
 export const wasmcrawlconfig_set_respectRobotsTxt: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_retryCodes: (a: number, b: number, c: number) => void;
 export const wasmcrawlconfig_set_retryCount: (a: number, b: number) => void;
+export const wasmcrawlconfig_set_retryInitialDelayMs: (a: number, b: bigint) => void;
+export const wasmcrawlconfig_set_retryMaxDelayMs: (a: number, b: bigint) => void;
 export const wasmcrawlconfig_set_saveBrowserProfile: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_softHttpErrors: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_ssrf: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_ssrfDenyPrivateExplicit: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_stayOnDomain: (a: number, b: number) => void;
+export const wasmcrawlconfig_set_stripTrackingParams: (a: number, b: number) => void;
+export const wasmcrawlconfig_set_trackingParams: (a: number, b: number, c: number) => void;
 export const wasmcrawlconfig_set_userAgent: (a: number, b: number, c: number) => void;
 export const wasmcrawlconfig_set_userAgents: (a: number, b: number, c: number) => void;
 export const wasmcrawlconfig_set_warcOutput: (a: number, b: number, c: number) => void;
@@ -300,6 +318,8 @@ export const wasmcrawlconfig_softHttpErrors: (a: number) => number;
 export const wasmcrawlconfig_ssrf: (a: number) => number;
 export const wasmcrawlconfig_ssrfDenyPrivateExplicit: (a: number) => number;
 export const wasmcrawlconfig_stayOnDomain: (a: number) => number;
+export const wasmcrawlconfig_stripTrackingParams: (a: number) => number;
+export const wasmcrawlconfig_trackingParams: (a: number) => [number, number];
 export const wasmcrawlconfig_userAgent: (a: number) => [number, number];
 export const wasmcrawlconfig_userAgents: (a: number) => [number, number];
 export const wasmcrawlconfig_validate: (a: number) => [number, number];
@@ -314,6 +334,7 @@ export const wasmcrawlpageresult_downloadedDocument: (a: number) => number;
 export const wasmcrawlpageresult_extractedData: (a: number) => any;
 export const wasmcrawlpageresult_extractionMeta: (a: number) => number;
 export const wasmcrawlpageresult_feeds: (a: number) => [number, number];
+export const wasmcrawlpageresult_finalUrl: (a: number) => [number, number];
 export const wasmcrawlpageresult_html: (a: number) => [number, number];
 export const wasmcrawlpageresult_images: (a: number) => [number, number];
 export const wasmcrawlpageresult_isPdf: (a: number) => number;
@@ -321,8 +342,11 @@ export const wasmcrawlpageresult_jsonLd: (a: number) => [number, number];
 export const wasmcrawlpageresult_links: (a: number) => [number, number];
 export const wasmcrawlpageresult_markdown: (a: number) => number;
 export const wasmcrawlpageresult_metadata: (a: number) => number;
-export const wasmcrawlpageresult_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number, n: number, o: number, p: number, q: number, r: number, s: number, t: number, u: number, v: number, w: number, x: number, y: number, z: number, a1: number, b1: number, c1: number, d1: number) => number;
+export const wasmcrawlpageresult_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number, n: number, o: number, p: number, q: number, r: number, s: number, t: number, u: number, v: number, w: number, x: number, y: number, z: number, a1: number, b1: number, c1: number, d1: number, e1: number, f1: number, g1: number, h1: number, i1: number) => number;
+export const wasmcrawlpageresult_nofollowDetected: (a: number) => number;
+export const wasmcrawlpageresult_noindexDetected: (a: number) => number;
 export const wasmcrawlpageresult_normalizedUrl: (a: number) => [number, number];
+export const wasmcrawlpageresult_redirectCount: (a: number) => number;
 export const wasmcrawlpageresult_set_bodySize: (a: number, b: number) => void;
 export const wasmcrawlpageresult_set_browserUsed: (a: number, b: number) => void;
 export const wasmcrawlpageresult_set_contentType: (a: number, b: number, c: number) => void;
@@ -332,6 +356,7 @@ export const wasmcrawlpageresult_set_downloadedDocument: (a: number, b: number)
 export const wasmcrawlpageresult_set_extractedData: (a: number, b: number) => void;
 export const wasmcrawlpageresult_set_extractionMeta: (a: number, b: number) => void;
 export const wasmcrawlpageresult_set_feeds: (a: number, b: number, c: number) => void;
+export const wasmcrawlpageresult_set_finalUrl: (a: number, b: number, c: number) => void;
 export const wasmcrawlpageresult_set_html: (a: number, b: number, c: number) => void;
 export const wasmcrawlpageresult_set_images: (a: number, b: number, c: number) => void;
 export const wasmcrawlpageresult_set_isPdf: (a: number, b: number) => void;
@@ -339,7 +364,10 @@ export const wasmcrawlpageresult_set_jsonLd: (a: number, b: number, c: number) =
 export const wasmcrawlpageresult_set_links: (a: number, b: number, c: number) => void;
 export const wasmcrawlpageresult_set_markdown: (a: number, b: number) => void;
 export const wasmcrawlpageresult_set_metadata: (a: number, b: number) => void;
+export const wasmcrawlpageresult_set_nofollowDetected: (a: number, b: number) => void;
+export const wasmcrawlpageresult_set_noindexDetected: (a: number, b: number) => void;
 export const wasmcrawlpageresult_set_normalizedUrl: (a: number, b: number, c: number) => void;
+export const wasmcrawlpageresult_set_redirectCount: (a: number, b: number) => void;
 export const wasmcrawlpageresult_set_statusCode: (a: number, b: number) => void;
 export const wasmcrawlpageresult_set_stayedOnDomain: (a: number, b: number) => void;
 export const wasmcrawlpageresult_set_url: (a: number, b: number, c: number) => void;
@@ -722,9 +750,9 @@ export const wasmssrfpolicy_set_allowlist: (a: number, b: any) => void;
 export const wasmssrfpolicy_set_denyPrivate: (a: number, b: number) => void;
 export const wasmssrfpolicy_set_maxRedirects: (a: number, b: number) => void;
 export const wasmssrfpolicy_set_schemeAllowlist: (a: number, b: number, c: number) => void;
-export const wasm_bindgen__convert__closures_____invoke__h5567083e9d00da74: (a: number, b: number, c: any) => [number, number];
-export const wasm_bindgen__convert__closures_____invoke__h9cb213bb50b5f7d0: (a: number, b: number, c: any, d: any) => void;
-export const wasm_bindgen__convert__closures_____invoke__h00e67b488e823f1f: (a: number, b: number) => void;
+export const wasm_bindgen__convert__closures_____invoke__h2eedacf566f3748c: (a: number, b: number, c: any) => [number, number];
+export const wasm_bindgen__convert__closures_____invoke__h8ad11faaa2905452: (a: number, b: number, c: any, d: any) => void;
+export const wasm_bindgen__convert__closures_____invoke__h21c8118e1fe083d1: (a: number, b: number) => void;
 export const __wbindgen_malloc: (a: number, b: number) => number;
 export const __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number;
 export const __wbindgen_exn_store: (a: number) => void;

package/pkg/web/package.json

@@ -2,7 +2,7 @@
   "name": "crawlberg-wasm",
   "type": "module",
   "description": "High-performance web crawling engine",
-  "version": "1.7.2",
+  "version": "1.8.0",
   "license": "MIT",
   "repository": {
     "type": "git",