@xberg-io/crawlberg-wasm 1.5.2 → 1.6.2

package/pkg/bundler/crawlberg_wasm_bg.wasm.d.ts

@@ -73,6 +73,7 @@ export const wasmarticlemetadata_set_tags: (a: number, b: number, c: number) =>
 export const wasmarticlemetadata_tags: (a: number) => [number, number];
 export const wasmauthconfig_default: () => number;
 export const wasmauthconfig_name: (a: number) => [number, number];
+export const wasmauthconfig_new: () => number;
 export const wasmauthconfig_password: (a: number) => [number, number];
 export const wasmauthconfig_set_name: (a: number, b: number, c: number) => void;
 export const wasmauthconfig_set_password: (a: number, b: number, c: number) => void;
@@ -110,10 +111,16 @@ export const wasmbatchscraperesult_set_error: (a: number, b: number, c: number)
 export const wasmbatchscraperesult_set_result: (a: number, b: number) => void;
 export const wasmbatchscraperesult_set_url: (a: number, b: number, c: number) => void;
 export const wasmbatchscraperesult_url: (a: number) => [number, number];
+export const wasmbatchscraperesults_completedCount: (a: number) => number;
 export const wasmbatchscraperesults_default: () => number;
+export const wasmbatchscraperesults_failedCount: (a: number) => number;
 export const wasmbatchscraperesults_new: (a: number, b: number, c: number, d: number, e: number) => number;
 export const wasmbatchscraperesults_results: (a: number) => [number, number];
+export const wasmbatchscraperesults_set_completedCount: (a: number, b: number) => void;
+export const wasmbatchscraperesults_set_failedCount: (a: number, b: number) => void;
 export const wasmbatchscraperesults_set_results: (a: number, b: number, c: number) => void;
+export const wasmbatchscraperesults_set_totalCount: (a: number, b: number) => void;
+export const wasmbatchscraperesults_totalCount: (a: number) => number;
 export const wasmbrowserconfig_backend: (a: number) => [number, number];
 export const wasmbrowserconfig_blockUrlPatterns: (a: number) => [number, number];
 export const wasmbrowserconfig_captureNetworkEvents: (a: number) => number;
@@ -237,6 +244,7 @@ export const wasmcrawlconfig_new: (a: number, b: number, c: number, d: number, e
 export const wasmcrawlconfig_proxy: (a: number) => number;
 export const wasmcrawlconfig_rateLimitMs: (a: number) => [number, bigint];
 export const wasmcrawlconfig_removeTags: (a: number) => [number, number];
+export const wasmcrawlconfig_requestTimeout: (a: number) => [number, bigint];
 export const wasmcrawlconfig_respectRobotsTxt: (a: number) => number;
 export const wasmcrawlconfig_retryCodes: (a: number) => [number, number];
 export const wasmcrawlconfig_retryCount: (a: number) => number;
@@ -276,6 +284,7 @@ export const wasmcrawlconfig_set_maxRedirects: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_proxy: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_rateLimitMs: (a: number, b: number, c: bigint) => void;
 export const wasmcrawlconfig_set_removeTags: (a: number, b: number, c: number) => void;
+export const wasmcrawlconfig_set_requestTimeout: (a: number, b: number, c: bigint) => void;
 export const wasmcrawlconfig_set_respectRobotsTxt: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_retryCodes: (a: number, b: number, c: number) => void;
 export const wasmcrawlconfig_set_retryCount: (a: number, b: number) => void;
@@ -368,7 +377,9 @@ export const wasmdownloadedasset_set_assetCategory: (a: number, b: number) => vo
 export const wasmdownloadedasset_set_contentHash: (a: number, b: number, c: number) => void;
 export const wasmdownloadedasset_set_htmlTag: (a: number, b: number, c: number) => void;
 export const wasmdownloadedasset_set_mimeType: (a: number, b: number, c: number) => void;
+export const wasmdownloadedasset_set_size: (a: number, b: number) => void;
 export const wasmdownloadedasset_set_url: (a: number, b: number, c: number) => void;
+export const wasmdownloadedasset_size: (a: number) => number;
 export const wasmdownloadedasset_url: (a: number) => [number, number];
 export const wasmdownloadeddocument_contentBase64: (a: number) => [number, number];
 export const wasmdownloadeddocument_contentHash: (a: number) => [number, number];
@@ -391,12 +402,20 @@ export const wasmdownloadeddocument_size: (a: number) => number;
 export const wasmdownloadeddocument_truncated: (a: number) => number;
 export const wasmdownloadeddocument_url: (a: number) => [number, number];
 export const wasmextractionmeta_chunksProcessed: (a: number) => number;
+export const wasmextractionmeta_completionTokens: (a: number) => [number, bigint];
+export const wasmextractionmeta_cost: (a: number) => [number, number];
 export const wasmextractionmeta_default: () => number;
 export const wasmextractionmeta_model: (a: number) => [number, number];
 export const wasmextractionmeta_new: (a: number, b: number, c: number, d: number, e: bigint, f: number, g: bigint, h: number, i: number) => number;
+export const wasmextractionmeta_promptTokens: (a: number) => [number, bigint];
 export const wasmextractionmeta_set_chunksProcessed: (a: number, b: number) => void;
+export const wasmextractionmeta_set_completionTokens: (a: number, b: number, c: bigint) => void;
+export const wasmextractionmeta_set_cost: (a: number, b: number, c: number) => void;
 export const wasmextractionmeta_set_model: (a: number, b: number, c: number) => void;
+export const wasmextractionmeta_set_promptTokens: (a: number, b: number, c: bigint) => void;
+export const wasmfaviconinfo_default: () => number;
 export const wasmfaviconinfo_mimeType: (a: number) => [number, number];
+export const wasmfaviconinfo_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number) => number;
 export const wasmfaviconinfo_rel: (a: number) => [number, number];
 export const wasmfaviconinfo_set_mimeType: (a: number, b: number, c: number) => void;
 export const wasmfaviconinfo_set_rel: (a: number, b: number, c: number) => void;
@@ -419,6 +438,7 @@ export const wasmheadinginfo_set_level: (a: number, b: number) => void;
 export const wasmheadinginfo_set_text: (a: number, b: number, c: number) => void;
 export const wasmheadinginfo_text: (a: number) => [number, number];
 export const wasmhostmatcher_default: () => number;
+export const wasmhostmatcher_new: () => number;
 export const wasmhostmatcher_set_type: (a: number, b: number, c: number) => void;
 export const wasmhostmatcher_set_value: (a: number, b: number, c: number) => void;
 export const wasmhostmatcher_type: (a: number) => [number, number];
@@ -437,8 +457,10 @@ export const wasmimageinfo_set_alt: (a: number, b: number, c: number) => void;
 export const wasmimageinfo_set_height: (a: number, b: number) => void;
 export const wasmimageinfo_set_source: (a: number, b: number) => void;
 export const wasmimageinfo_set_url: (a: number, b: number, c: number) => void;
+export const wasmimageinfo_set_width: (a: number, b: number) => void;
 export const wasmimageinfo_source: (a: number) => [number, number];
 export const wasmimageinfo_url: (a: number) => [number, number];
+export const wasmimageinfo_width: (a: number) => number;
 export const wasminteractionresult_actionResults: (a: number) => [number, number];
 export const wasminteractionresult_default: () => number;
 export const wasminteractionresult_finalHtml: (a: number) => [number, number];
@@ -460,8 +482,10 @@ export const wasmjsonldentry_set_schemaType: (a: number, b: number, c: number) =
 export const wasmlinkinfo_default: () => number;
 export const wasmlinkinfo_linkType: (a: number) => [number, number];
 export const wasmlinkinfo_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number) => number;
+export const wasmlinkinfo_nofollow: (a: number) => number;
 export const wasmlinkinfo_rel: (a: number) => [number, number];
 export const wasmlinkinfo_set_linkType: (a: number, b: number) => void;
+export const wasmlinkinfo_set_nofollow: (a: number, b: number) => void;
 export const wasmlinkinfo_set_rel: (a: number, b: number, c: number) => void;
 export const wasmlinkinfo_set_text: (a: number, b: number, c: number) => void;
 export const wasmlinkinfo_set_url: (a: number, b: number, c: number) => void;
@@ -485,15 +509,20 @@ export const wasmmarkdownresult_set_tables: (a: number, b: any) => void;
 export const wasmmarkdownresult_set_warnings: (a: number, b: number, c: number) => void;
 export const wasmmarkdownresult_tables: (a: number) => any;
 export const wasmmarkdownresult_warnings: (a: number) => [number, number];
+export const wasmpageaction_amount: (a: number) => [number, bigint];
 export const wasmpageaction_default: () => number;
 export const wasmpageaction_direction: (a: number) => number;
 export const wasmpageaction_fullPage: (a: number) => number;
 export const wasmpageaction_key: (a: number) => [number, number];
+export const wasmpageaction_milliseconds: (a: number) => [number, bigint];
+export const wasmpageaction_new: () => number;
 export const wasmpageaction_script: (a: number) => [number, number];
 export const wasmpageaction_selector: (a: number) => [number, number];
+export const wasmpageaction_set_amount: (a: number, b: number, c: bigint) => void;
 export const wasmpageaction_set_direction: (a: number, b: number) => void;
 export const wasmpageaction_set_fullPage: (a: number, b: number) => void;
 export const wasmpageaction_set_key: (a: number, b: number, c: number) => void;
+export const wasmpageaction_set_milliseconds: (a: number, b: number, c: bigint) => void;
 export const wasmpageaction_set_script: (a: number, b: number, c: number) => void;
 export const wasmpageaction_set_selector: (a: number, b: number, c: number) => void;
 export const wasmpageaction_set_text: (a: number, b: number, c: number) => void;
@@ -577,6 +606,7 @@ export const wasmpagemetadata_set_twitterImage: (a: number, b: number, c: number
 export const wasmpagemetadata_set_twitterSite: (a: number, b: number, c: number) => void;
 export const wasmpagemetadata_set_twitterTitle: (a: number, b: number, c: number) => void;
 export const wasmpagemetadata_set_viewport: (a: number, b: number, c: number) => void;
+export const wasmpagemetadata_set_wordCount: (a: number, b: number) => void;
 export const wasmpagemetadata_themeColor: (a: number) => [number, number];
 export const wasmpagemetadata_title: (a: number) => [number, number];
 export const wasmpagemetadata_twitterCard: (a: number) => [number, number];
@@ -586,6 +616,7 @@ export const wasmpagemetadata_twitterImage: (a: number) => [number, number];
 export const wasmpagemetadata_twitterSite: (a: number) => [number, number];
 export const wasmpagemetadata_twitterTitle: (a: number) => [number, number];
 export const wasmpagemetadata_viewport: (a: number) => [number, number];
+export const wasmpagemetadata_wordCount: (a: number) => number;
 export const wasmproxyconfig_default: () => number;
 export const wasmproxyconfig_new: (a: number, b: number, c: number, d: number, e: number, f: number) => number;
 export const wasmproxyconfig_password: (a: number) => [number, number];
@@ -616,6 +647,7 @@ export const wasmscraperesult_bodySize: (a: number) => number;
 export const wasmscraperesult_browser: (a: number) => number;
 export const wasmscraperesult_browserUsed: (a: number) => number;
 export const wasmscraperesult_contentType: (a: number) => [number, number];
+export const wasmscraperesult_crawlDelay: (a: number) => [number, bigint];
 export const wasmscraperesult_default: () => number;
 export const wasmscraperesult_detectedCharset: (a: number) => [number, number];
 export const wasmscraperesult_downloadedDocument: (a: number) => number;
@@ -643,6 +675,7 @@ export const wasmscraperesult_set_bodySize: (a: number, b: number) => void;
 export const wasmscraperesult_set_browser: (a: number, b: number) => void;
 export const wasmscraperesult_set_browserUsed: (a: number, b: number) => void;
 export const wasmscraperesult_set_contentType: (a: number, b: number, c: number) => void;
+export const wasmscraperesult_set_crawlDelay: (a: number, b: number, c: bigint) => void;
 export const wasmscraperesult_set_detectedCharset: (a: number, b: number, c: number) => void;
 export const wasmscraperesult_set_downloadedDocument: (a: number, b: number) => void;
 export const wasmscraperesult_set_extractedData: (a: number, b: number) => void;
@@ -689,42 +722,9 @@ export const wasmssrfpolicy_set_allowlist: (a: number, b: any) => void;
 export const wasmssrfpolicy_set_denyPrivate: (a: number, b: number) => void;
 export const wasmssrfpolicy_set_maxRedirects: (a: number, b: number) => void;
 export const wasmssrfpolicy_set_schemeAllowlist: (a: number, b: number, c: number) => void;
-export const wasmhostmatcher_new: () => number;
-export const wasmcrawlconfig_set_requestTimeout: (a: number, b: number, c: bigint) => void;
-export const wasmextractionmeta_set_completionTokens: (a: number, b: number, c: bigint) => void;
-export const wasmextractionmeta_set_promptTokens: (a: number, b: number, c: bigint) => void;
-export const wasmpageaction_set_milliseconds: (a: number, b: number, c: bigint) => void;
-export const wasmcrawlconfig_requestTimeout: (a: number) => [number, bigint];
-export const wasmextractionmeta_completionTokens: (a: number) => [number, bigint];
-export const wasmextractionmeta_cost: (a: number) => [number, number];
-export const wasmextractionmeta_promptTokens: (a: number) => [number, bigint];
-export const wasmpageaction_amount: (a: number) => [number, bigint];
-export const wasmpageaction_milliseconds: (a: number) => [number, bigint];
-export const wasmscraperesult_crawlDelay: (a: number) => [number, bigint];
-export const wasmfaviconinfo_default: () => number;
-export const wasmbatchscraperesults_set_completedCount: (a: number, b: number) => void;
-export const wasmbatchscraperesults_set_failedCount: (a: number, b: number) => void;
-export const wasmbatchscraperesults_set_totalCount: (a: number, b: number) => void;
-export const wasmdownloadedasset_set_size: (a: number, b: number) => void;
-export const wasmimageinfo_set_width: (a: number, b: number) => void;
-export const wasmpagemetadata_set_wordCount: (a: number, b: number) => void;
-export const wasmbatchscraperesults_completedCount: (a: number) => number;
-export const wasmbatchscraperesults_failedCount: (a: number) => number;
-export const wasmbatchscraperesults_totalCount: (a: number) => number;
-export const wasmdownloadedasset_size: (a: number) => number;
-export const wasmlinkinfo_set_nofollow: (a: number, b: number) => void;
-export const wasmfaviconinfo_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number) => number;
-export const wasmimageinfo_width: (a: number) => number;
-export const wasmpagemetadata_wordCount: (a: number) => number;
-export const wasmlinkinfo_nofollow: (a: number) => number;
-export const wasmauthconfig_new: () => number;
-export const wasmextractionmeta_set_cost: (a: number, b: number, c: number) => void;
-export const wasmpageaction_set_amount: (a: number, b: number, c: bigint) => void;
-export const wasmscraperesult_set_crawlDelay: (a: number, b: number, c: bigint) => void;
-export const wasmpageaction_new: () => number;
-export const wasm_bindgen__convert__closures_____invoke__h6e6fec37c02cb65f: (a: number, b: number, c: any) => [number, number];
-export const wasm_bindgen__convert__closures_____invoke__hbab77f1211c923b7: (a: number, b: number, c: any, d: any) => void;
-export const wasm_bindgen__convert__closures_____invoke__h1d72c46fe7a75bc3: (a: number, b: number) => void;
+export const wasm_bindgen__convert__closures_____invoke__h5567083e9d00da74: (a: number, b: number, c: any) => [number, number];
+export const wasm_bindgen__convert__closures_____invoke__h9cb213bb50b5f7d0: (a: number, b: number, c: any, d: any) => void;
+export const wasm_bindgen__convert__closures_____invoke__h00e67b488e823f1f: (a: number, b: number) => void;
 export const __wbindgen_malloc: (a: number, b: number) => number;
 export const __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number;
 export const __wbindgen_exn_store: (a: number) => void;

package/pkg/bundler/package.json

@@ -2,7 +2,7 @@
   "name": "crawlberg-wasm",
   "type": "module",
   "description": "High-performance web crawling engine",
-  "version": "1.5.2",
+  "version": "1.6.2",
   "license": "MIT",
   "repository": {
     "type": "git",

package/pkg/deno/crawlberg_wasm.d.ts

@@ -173,10 +173,10 @@ export class WasmBrowserConfig {
     sessionAffinity: boolean;
     get timeout(): bigint | undefined;
     set timeout(value: bigint | null | undefined);
-    get wait(): string;
-    set wait(value: WasmBrowserWait);
     get waitSelector(): string | undefined;
     set waitSelector(value: string | null | undefined);
+    get wait(): string;
+    set wait(value: WasmBrowserWait);
 }
 
 /**
@@ -263,8 +263,8 @@ export class WasmContentConfig {
     removeNavigation: boolean;
     skipImages: boolean;
     stripTags: string[];
-    wrap: boolean;
     wrapWidth: number;
+    wrap: boolean;
 }
 
 /**
@@ -297,7 +297,7 @@ export class WasmCrawlConfig {
     free(): void;
     [Symbol.dispose](): void;
     static default(): WasmCrawlConfig;
-    constructor(crawlStrategy?: WasmCrawlStrategyKind | null, respectRobotsTxt?: boolean | null, softHttpErrors?: boolean | null, stayOnDomain?: boolean | null, allowSubdomains?: boolean | null, includePaths?: string[] | null, excludePaths?: string[] | null, customHeaders?: any | null, requestTimeout?: bigint | null, maxRedirects?: number | null, retryCount?: number | null, retryCodes?: Uint16Array | null, cookiesEnabled?: boolean | null, removeTags?: string[] | null, content?: WasmContentConfig | null, downloadAssets?: boolean | null, assetTypes?: any[] | null, browser?: WasmBrowserConfig | null, userAgents?: string[] | null, captureScreenshot?: boolean | null, followDocumentUrls?: boolean | null, downloadDocuments?: boolean | null, documentMimeTypes?: string[] | null, saveBrowserProfile?: boolean | null, ssrf?: WasmSsrfPolicy | null, maxDepth?: number | null, maxPages?: number | null, maxLinksPerPage?: number | null, maxConcurrent?: number | null, contentFilter?: WasmContentFilterKind | null, bm25Query?: string | null, bm25Threshold?: number | null, userAgent?: string | null, rateLimitMs?: bigint | null, auth?: any | null, maxBodySize?: number | null, mapLimit?: number | null, mapSearch?: string | null, maxAssetSize?: number | null, proxy?: WasmProxyConfig | null, documentUrlDepth?: number | null, documentMaxSize?: number | null, documentOutputDir?: string | null, documentContentEncoding?: WasmDocumentContentEncoding | null, warcOutput?: string | null, browserProfile?: string | null, ssrfDenyPrivateExplicit?: boolean | null);
+    constructor(crawlStrategy?: WasmCrawlStrategyKind | null, respectRobotsTxt?: boolean | null, softHttpErrors?: boolean | null, stayOnDomain?: boolean | null, allowSubdomains?: boolean | null, includePaths?: string[] | null, excludePaths?: string[] | null, customHeaders?: any | null, requestTimeout?: bigint | null, maxRedirects?: number | null, retryCount?: number | null, retryCodes?: Uint16Array | null, cookiesEnabled?: boolean | null, removeTags?: string[] | null, content?: WasmContentConfig | null, downloadAssets?: boolean | null, assetTypes?: WasmAssetCategory[] | null, browser?: WasmBrowserConfig | null, userAgents?: string[] | null, captureScreenshot?: boolean | null, followDocumentUrls?: boolean | null, downloadDocuments?: boolean | null, documentMimeTypes?: string[] | null, saveBrowserProfile?: boolean | null, ssrf?: WasmSsrfPolicy | null, maxDepth?: number | null, maxPages?: number | null, maxLinksPerPage?: number | null, maxConcurrent?: number | null, contentFilter?: WasmContentFilterKind | null, bm25Query?: string | null, bm25Threshold?: number | null, userAgent?: string | null, rateLimitMs?: bigint | null, auth?: any | null, maxBodySize?: number | null, mapLimit?: number | null, mapSearch?: string | null, maxAssetSize?: number | null, proxy?: WasmProxyConfig | null, documentUrlDepth?: number | null, documentMaxSize?: number | null, documentOutputDir?: string | null, documentContentEncoding?: WasmDocumentContentEncoding | null, warcOutput?: string | null, browserProfile?: string | null, ssrfDenyPrivateExplicit?: boolean | null);
     /**
      * Validate the configuration, returning an error if any values are invalid.
      */
@@ -310,13 +310,13 @@ export class WasmCrawlConfig {
     set bm25Query(value: string | null | undefined);
     get bm25Threshold(): number | undefined;
     set bm25Threshold(value: number | null | undefined);
-    browser: WasmBrowserConfig;
     get browserProfile(): string | undefined;
     set browserProfile(value: string | null | undefined);
+    browser: WasmBrowserConfig;
     captureScreenshot: boolean;
-    content: WasmContentConfig;
     get contentFilter(): string | undefined;
     set contentFilter(value: WasmContentFilterKind | null | undefined);
+    content: WasmContentConfig;
     cookiesEnabled: boolean;
     get crawlStrategy(): string;
     set crawlStrategy(value: WasmCrawlStrategyKind);
@@ -364,9 +364,9 @@ export class WasmCrawlConfig {
     retryCount: number;
     saveBrowserProfile: boolean;
     softHttpErrors: boolean;
-    ssrf: WasmSsrfPolicy;
     get ssrfDenyPrivateExplicit(): boolean | undefined;
     set ssrfDenyPrivateExplicit(value: boolean | null | undefined);
+    ssrf: WasmSsrfPolicy;
     stayOnDomain: boolean;
     get userAgent(): string | undefined;
     set userAgent(value: string | null | undefined);
@@ -836,10 +836,10 @@ export class WasmPageMetadata {
     set ogDescription(value: string | null | undefined);
     get ogImage(): string | undefined;
     set ogImage(value: string | null | undefined);
-    get ogLocale(): string | undefined;
-    set ogLocale(value: string | null | undefined);
     get ogLocaleAlternates(): string[] | undefined;
     set ogLocaleAlternates(value: string[] | null | undefined);
+    get ogLocale(): string | undefined;
+    set ogLocale(value: string | null | undefined);
     get ogSiteName(): string | undefined;
     set ogSiteName(value: string | null | undefined);
     get ogTitle(): string | undefined;
@@ -924,9 +924,9 @@ export class WasmScrapeResult {
     assets: WasmDownloadedAsset[];
     authHeaderSent: boolean;
     bodySize: number;
+    browserUsed: boolean;
     get browser(): WasmBrowserExtras | undefined;
     set browser(value: WasmBrowserExtras | null | undefined);
-    browserUsed: boolean;
     contentType: string;
     get crawlDelay(): bigint | undefined;
     set crawlDelay(value: bigint | null | undefined);