@xberg-io/crawlberg-wasm 1.1.3 → 1.2.1

package/pkg/nodejs/crawlberg_wasm_bg.wasm.d.ts

@@ -24,6 +24,7 @@ export const __wbg_wasmextractionmeta_free: (a: number, b: number) => void;
 export const __wbg_wasmfaviconinfo_free: (a: number, b: number) => void;
 export const __wbg_wasmfeedinfo_free: (a: number, b: number) => void;
 export const __wbg_wasmheadinginfo_free: (a: number, b: number) => void;
+export const __wbg_wasmhostmatcher_free: (a: number, b: number) => void;
 export const __wbg_wasmhreflangentry_free: (a: number, b: number) => void;
 export const __wbg_wasmimageinfo_free: (a: number, b: number) => void;
 export const __wbg_wasminteractionresult_free: (a: number, b: number) => void;
@@ -209,8 +210,10 @@ export const wasmcrawlconfig_content: (a: number) => number;
 export const wasmcrawlconfig_cookiesEnabled: (a: number) => number;
 export const wasmcrawlconfig_customHeaders: (a: number) => any;
 export const wasmcrawlconfig_default: () => number;
+export const wasmcrawlconfig_documentContentEncoding: (a: number) => [number, number];
 export const wasmcrawlconfig_documentMaxSize: (a: number) => number;
 export const wasmcrawlconfig_documentMimeTypes: (a: number) => [number, number];
+export const wasmcrawlconfig_documentOutputDir: (a: number) => [number, number];
 export const wasmcrawlconfig_documentUrlDepth: (a: number) => number;
 export const wasmcrawlconfig_downloadAssets: (a: number) => number;
 export const wasmcrawlconfig_downloadDocuments: (a: number) => number;
@@ -223,9 +226,10 @@ export const wasmcrawlconfig_maxAssetSize: (a: number) => number;
 export const wasmcrawlconfig_maxBodySize: (a: number) => number;
 export const wasmcrawlconfig_maxConcurrent: (a: number) => number;
 export const wasmcrawlconfig_maxDepth: (a: number) => number;
+export const wasmcrawlconfig_maxLinksPerPage: (a: number) => number;
 export const wasmcrawlconfig_maxPages: (a: number) => number;
 export const wasmcrawlconfig_maxRedirects: (a: number) => number;
-export const wasmcrawlconfig_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: bigint, l: number, m: number, n: number, o: number, p: number, q: number, r: number, s: number, t: number, u: number, v: number, w: number, x: number, y: number, z: number, a1: number, b1: number, c1: number, d1: number, e1: number, f1: number, g1: number, h1: number, i1: number, j1: number, k1: number, l1: number, m1: bigint, n1: number, o1: number, p1: number, q1: number, r1: number, s1: number, t1: number, u1: number, v1: number, w1: number, x1: number, y1: number, z1: number) => number;
+export const wasmcrawlconfig_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: bigint, l: number, m: number, n: number, o: number, p: number, q: number, r: number, s: number, t: number, u: number, v: number, w: number, x: number, y: number, z: number, a1: number, b1: number, c1: number, d1: number, e1: number, f1: number, g1: number, h1: number, i1: number, j1: number, k1: number, l1: number, m1: number, n1: bigint, o1: number, p1: number, q1: number, r1: number, s1: number, t1: number, u1: number, v1: number, w1: number, x1: number, y1: number, z1: number, a2: number, b2: number, c2: number, d2: number, e2: number) => number;
 export const wasmcrawlconfig_proxy: (a: number) => number;
 export const wasmcrawlconfig_removeTags: (a: number) => [number, number];
 export const wasmcrawlconfig_respectRobotsTxt: (a: number) => number;
@@ -241,8 +245,10 @@ export const wasmcrawlconfig_set_captureScreenshot: (a: number, b: number) => vo
 export const wasmcrawlconfig_set_content: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_cookiesEnabled: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_customHeaders: (a: number, b: any) => void;
+export const wasmcrawlconfig_set_documentContentEncoding: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_documentMaxSize: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_documentMimeTypes: (a: number, b: number, c: number) => void;
+export const wasmcrawlconfig_set_documentOutputDir: (a: number, b: number, c: number) => void;
 export const wasmcrawlconfig_set_documentUrlDepth: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_downloadAssets: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_downloadDocuments: (a: number, b: number) => void;
@@ -255,6 +261,7 @@ export const wasmcrawlconfig_set_maxAssetSize: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_maxBodySize: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_maxConcurrent: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_maxDepth: (a: number, b: number) => void;
+export const wasmcrawlconfig_set_maxLinksPerPage: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_maxPages: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_maxRedirects: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_proxy: (a: number, b: number) => void;
@@ -265,12 +272,14 @@ export const wasmcrawlconfig_set_retryCount: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_saveBrowserProfile: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_softHttpErrors: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_ssrf: (a: number, b: number) => void;
+export const wasmcrawlconfig_set_ssrfDenyPrivateExplicit: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_stayOnDomain: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_userAgent: (a: number, b: number, c: number) => void;
 export const wasmcrawlconfig_set_userAgents: (a: number, b: number, c: number) => void;
 export const wasmcrawlconfig_set_warcOutput: (a: number, b: number, c: number) => void;
 export const wasmcrawlconfig_softHttpErrors: (a: number) => number;
 export const wasmcrawlconfig_ssrf: (a: number) => number;
+export const wasmcrawlconfig_ssrfDenyPrivateExplicit: (a: number) => number;
 export const wasmcrawlconfig_stayOnDomain: (a: number) => number;
 export const wasmcrawlconfig_userAgent: (a: number) => [number, number];
 export const wasmcrawlconfig_userAgents: (a: number) => [number, number];
@@ -351,17 +360,25 @@ export const wasmdownloadedasset_set_htmlTag: (a: number, b: number, c: number)
 export const wasmdownloadedasset_set_mimeType: (a: number, b: number, c: number) => void;
 export const wasmdownloadedasset_set_url: (a: number, b: number, c: number) => void;
 export const wasmdownloadedasset_url: (a: number) => [number, number];
+export const wasmdownloadeddocument_contentBase64: (a: number) => [number, number];
 export const wasmdownloadeddocument_contentHash: (a: number) => [number, number];
+export const wasmdownloadeddocument_contentPath: (a: number) => [number, number];
 export const wasmdownloadeddocument_default: () => number;
 export const wasmdownloadeddocument_filename: (a: number) => [number, number];
 export const wasmdownloadeddocument_headers: (a: number) => any;
 export const wasmdownloadeddocument_mimeType: (a: number) => [number, number];
-export const wasmdownloadeddocument_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number) => number;
+export const wasmdownloadeddocument_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number, n: number, o: number) => number;
+export const wasmdownloadeddocument_set_contentBase64: (a: number, b: number, c: number) => void;
 export const wasmdownloadeddocument_set_contentHash: (a: number, b: number, c: number) => void;
+export const wasmdownloadeddocument_set_contentPath: (a: number, b: number, c: number) => void;
 export const wasmdownloadeddocument_set_filename: (a: number, b: number, c: number) => void;
 export const wasmdownloadeddocument_set_headers: (a: number, b: any) => void;
 export const wasmdownloadeddocument_set_mimeType: (a: number, b: number, c: number) => void;
+export const wasmdownloadeddocument_set_size: (a: number, b: number) => void;
+export const wasmdownloadeddocument_set_truncated: (a: number, b: number) => void;
 export const wasmdownloadeddocument_set_url: (a: number, b: number, c: number) => void;
+export const wasmdownloadeddocument_size: (a: number) => number;
+export const wasmdownloadeddocument_truncated: (a: number) => number;
 export const wasmdownloadeddocument_url: (a: number) => [number, number];
 export const wasmextractionmeta_chunksProcessed: (a: number) => number;
 export const wasmextractionmeta_completionTokens: (a: number) => [number, bigint];
@@ -395,6 +412,11 @@ export const wasmheadinginfo_new: (a: number, b: number, c: number) => number;
 export const wasmheadinginfo_set_level: (a: number, b: number) => void;
 export const wasmheadinginfo_set_text: (a: number, b: number, c: number) => void;
 export const wasmheadinginfo_text: (a: number) => [number, number];
+export const wasmhostmatcher_default: () => number;
+export const wasmhostmatcher_set_type: (a: number, b: number, c: number) => void;
+export const wasmhostmatcher_set_value: (a: number, b: number, c: number) => void;
+export const wasmhostmatcher_type: (a: number) => [number, number];
+export const wasmhostmatcher_value: (a: number) => [number, number];
 export const wasmhreflangentry_default: () => number;
 export const wasmhreflangentry_lang: (a: number) => [number, number];
 export const wasmhreflangentry_new: (a: number, b: number, c: number, d: number) => number;
@@ -415,10 +437,12 @@ export const wasminteractionresult_actionResults: (a: number) => [number, number
 export const wasminteractionresult_default: () => number;
 export const wasminteractionresult_finalHtml: (a: number) => [number, number];
 export const wasminteractionresult_finalUrl: (a: number) => [number, number];
-export const wasminteractionresult_new: (a: number, b: number, c: number, d: number, e: number, f: number) => number;
+export const wasminteractionresult_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number) => number;
+export const wasminteractionresult_screenshotBase64: (a: number) => [number, number];
 export const wasminteractionresult_set_actionResults: (a: number, b: number, c: number) => void;
 export const wasminteractionresult_set_finalHtml: (a: number, b: number, c: number) => void;
 export const wasminteractionresult_set_finalUrl: (a: number, b: number, c: number) => void;
+export const wasminteractionresult_set_screenshotBase64: (a: number, b: number, c: number) => void;
 export const wasmjsonldentry_default: () => number;
 export const wasmjsonldentry_name: (a: number) => [number, number];
 export const wasmjsonldentry_new: (a: number, b: number, c: number, d: number, e: number, f: number) => number;
@@ -602,10 +626,11 @@ export const wasmscraperesult_jsonLd: (a: number) => [number, number];
 export const wasmscraperesult_links: (a: number) => [number, number];
 export const wasmscraperesult_markdown: (a: number) => number;
 export const wasmscraperesult_metadata: (a: number) => number;
-export const wasmscraperesult_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number, n: number, o: number, p: number, q: number, r: number, s: number, t: number, u: number, v: number, w: number, x: number, y: number, z: number, a1: number, b1: number, c1: bigint, d1: number, e1: number, f1: number, g1: number, h1: number, i1: number, j1: number, k1: number, l1: number, m1: number) => number;
+export const wasmscraperesult_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number, n: number, o: number, p: number, q: number, r: number, s: number, t: number, u: number, v: number, w: number, x: number, y: number, z: number, a1: number, b1: number, c1: bigint, d1: number, e1: number, f1: number, g1: number, h1: number, i1: number, j1: number, k1: number, l1: number, m1: number, n1: number, o1: number) => number;
 export const wasmscraperesult_nofollowDetected: (a: number) => number;
 export const wasmscraperesult_noindexDetected: (a: number) => number;
 export const wasmscraperesult_responseMeta: (a: number) => number;
+export const wasmscraperesult_screenshotBase64: (a: number) => [number, number];
 export const wasmscraperesult_set_assets: (a: number, b: number, c: number) => void;
 export const wasmscraperesult_set_authHeaderSent: (a: number, b: number) => void;
 export const wasmscraperesult_set_bodySize: (a: number, b: number) => void;
@@ -630,6 +655,7 @@ export const wasmscraperesult_set_metadata: (a: number, b: number) => void;
 export const wasmscraperesult_set_nofollowDetected: (a: number, b: number) => void;
 export const wasmscraperesult_set_noindexDetected: (a: number, b: number) => void;
 export const wasmscraperesult_set_responseMeta: (a: number, b: number) => void;
+export const wasmscraperesult_set_screenshotBase64: (a: number, b: number, c: number) => void;
 export const wasmscraperesult_set_statusCode: (a: number, b: number) => void;
 export const wasmscraperesult_set_wasSkipped: (a: number, b: number) => void;
 export const wasmscraperesult_set_xRobotsTag: (a: number, b: number, c: number) => void;
@@ -646,13 +672,16 @@ export const wasmsitemapurl_set_lastmod: (a: number, b: number, c: number) => vo
 export const wasmsitemapurl_set_priority: (a: number, b: number, c: number) => void;
 export const wasmsitemapurl_set_url: (a: number, b: number, c: number) => void;
 export const wasmsitemapurl_url: (a: number) => [number, number];
+export const wasmssrfpolicy_allowlist: (a: number) => any;
 export const wasmssrfpolicy_default: () => number;
 export const wasmssrfpolicy_denyPrivate: (a: number) => number;
 export const wasmssrfpolicy_fromEnv: () => number;
 export const wasmssrfpolicy_maxRedirects: (a: number) => number;
-export const wasmssrfpolicy_new: (a: number, b: number) => number;
+export const wasmssrfpolicy_new: (a: number, b: number, c: number) => number;
+export const wasmssrfpolicy_set_allowlist: (a: number, b: any) => void;
 export const wasmssrfpolicy_set_denyPrivate: (a: number, b: number) => void;
 export const wasmssrfpolicy_set_maxRedirects: (a: number, b: number) => void;
+export const wasmhostmatcher_new: () => number;
 export const wasmcrawlconfig_set_rateLimitMs: (a: number, b: number, c: bigint) => void;
 export const wasmextractionmeta_set_promptTokens: (a: number, b: number, c: bigint) => void;
 export const wasmpageaction_set_milliseconds: (a: number, b: number, c: bigint) => void;
@@ -667,14 +696,12 @@ export const wasmbatchscraperesults_set_completedCount: (a: number, b: number) =
 export const wasmbatchscraperesults_set_failedCount: (a: number, b: number) => void;
 export const wasmbatchscraperesults_set_totalCount: (a: number, b: number) => void;
 export const wasmdownloadedasset_set_size: (a: number, b: number) => void;
-export const wasmdownloadeddocument_set_size: (a: number, b: number) => void;
 export const wasmimageinfo_set_width: (a: number, b: number) => void;
 export const wasmpagemetadata_set_wordCount: (a: number, b: number) => void;
 export const wasmbatchscraperesults_completedCount: (a: number) => number;
 export const wasmbatchscraperesults_failedCount: (a: number) => number;
 export const wasmbatchscraperesults_totalCount: (a: number) => number;
 export const wasmdownloadedasset_size: (a: number) => number;
-export const wasmdownloadeddocument_size: (a: number) => number;
 export const wasmlinkinfo_set_nofollow: (a: number, b: number) => void;
 export const wasmfaviconinfo_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number) => number;
 export const wasmimageinfo_width: (a: number) => number;
@@ -685,9 +712,9 @@ export const wasmcrawlconfig_set_requestTimeout: (a: number, b: number, c: bigin
 export const wasmpageaction_set_amount: (a: number, b: number, c: bigint) => void;
 export const wasmscraperesult_set_crawlDelay: (a: number, b: number, c: bigint) => void;
 export const wasmpageaction_new: () => number;
-export const wasm_bindgen__convert__closures_____invoke__h96ab4f4d64e6bfe6: (a: number, b: number, c: any) => [number, number];
-export const wasm_bindgen__convert__closures_____invoke__h20406e9dd5374954: (a: number, b: number, c: any, d: any) => void;
-export const wasm_bindgen__convert__closures_____invoke__h10c3645e699af307: (a: number, b: number) => void;
+export const wasm_bindgen__convert__closures_____invoke__he29f1bb22c192c97: (a: number, b: number, c: any) => [number, number];
+export const wasm_bindgen__convert__closures_____invoke__h06ea3d672ad56c9d: (a: number, b: number, c: any, d: any) => void;
+export const wasm_bindgen__convert__closures_____invoke__hbe9f9227fe7d8a65: (a: number, b: number) => void;
 export const __wbindgen_malloc: (a: number, b: number) => number;
 export const __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number;
 export const __wbindgen_exn_store: (a: number) => void;

package/pkg/nodejs/package.json

@@ -1,7 +1,7 @@
 {
   "name": "crawlberg-wasm",
   "description": "High-performance web crawling engine",
-  "version": "1.1.3",
+  "version": "1.2.1",
   "license": "MIT",
   "repository": {
     "type": "git",

package/pkg/web/crawlberg_wasm.d.ts

@@ -290,7 +290,7 @@ export class WasmCrawlConfig {
     free(): void;
     [Symbol.dispose](): void;
     static default(): WasmCrawlConfig;
-    constructor(respectRobotsTxt?: boolean | null, softHttpErrors?: boolean | null, stayOnDomain?: boolean | null, allowSubdomains?: boolean | null, includePaths?: string[] | null, excludePaths?: string[] | null, customHeaders?: any | null, requestTimeout?: bigint | null, maxRedirects?: number | null, retryCount?: number | null, retryCodes?: Uint16Array | null, cookiesEnabled?: boolean | null, removeTags?: string[] | null, content?: WasmContentConfig | null, downloadAssets?: boolean | null, assetTypes?: any[] | null, browser?: WasmBrowserConfig | null, userAgents?: string[] | null, captureScreenshot?: boolean | null, followDocumentUrls?: boolean | null, downloadDocuments?: boolean | null, documentMimeTypes?: string[] | null, saveBrowserProfile?: boolean | null, ssrf?: WasmSsrfPolicy | null, maxDepth?: number | null, maxPages?: number | null, maxConcurrent?: number | null, userAgent?: string | null, rateLimitMs?: bigint | null, auth?: any | null, maxBodySize?: number | null, mapLimit?: number | null, mapSearch?: string | null, maxAssetSize?: number | null, proxy?: WasmProxyConfig | null, documentUrlDepth?: number | null, documentMaxSize?: number | null, warcOutput?: string | null, browserProfile?: string | null);
+    constructor(respectRobotsTxt?: boolean | null, softHttpErrors?: boolean | null, stayOnDomain?: boolean | null, allowSubdomains?: boolean | null, includePaths?: string[] | null, excludePaths?: string[] | null, customHeaders?: any | null, requestTimeout?: bigint | null, maxRedirects?: number | null, retryCount?: number | null, retryCodes?: Uint16Array | null, cookiesEnabled?: boolean | null, removeTags?: string[] | null, content?: WasmContentConfig | null, downloadAssets?: boolean | null, assetTypes?: any[] | null, browser?: WasmBrowserConfig | null, userAgents?: string[] | null, captureScreenshot?: boolean | null, followDocumentUrls?: boolean | null, downloadDocuments?: boolean | null, documentMimeTypes?: string[] | null, saveBrowserProfile?: boolean | null, ssrf?: WasmSsrfPolicy | null, maxDepth?: number | null, maxPages?: number | null, maxLinksPerPage?: number | null, maxConcurrent?: number | null, userAgent?: string | null, rateLimitMs?: bigint | null, auth?: any | null, maxBodySize?: number | null, mapLimit?: number | null, mapSearch?: string | null, maxAssetSize?: number | null, proxy?: WasmProxyConfig | null, documentUrlDepth?: number | null, documentMaxSize?: number | null, documentOutputDir?: string | null, documentContentEncoding?: WasmDocumentContentEncoding | null, warcOutput?: string | null, browserProfile?: string | null, ssrfDenyPrivateExplicit?: boolean | null);
     /**
      * Validate the configuration, returning an error if any values are invalid.
      */
@@ -306,9 +306,13 @@ export class WasmCrawlConfig {
     content: WasmContentConfig;
     cookiesEnabled: boolean;
     customHeaders: any;
+    get documentContentEncoding(): string | undefined;
+    set documentContentEncoding(value: WasmDocumentContentEncoding | null | undefined);
     get documentMaxSize(): number | undefined;
     set documentMaxSize(value: number | null | undefined);
     documentMimeTypes: string[];
+    get documentOutputDir(): string | undefined;
+    set documentOutputDir(value: string | null | undefined);
     get documentUrlDepth(): number | undefined;
     set documentUrlDepth(value: number | null | undefined);
     downloadAssets: boolean;
@@ -328,6 +332,8 @@ export class WasmCrawlConfig {
     set maxConcurrent(value: number | null | undefined);
     get maxDepth(): number | undefined;
     set maxDepth(value: number | null | undefined);
+    get maxLinksPerPage(): number | undefined;
+    set maxLinksPerPage(value: number | null | undefined);
     get maxPages(): number | undefined;
     set maxPages(value: number | null | undefined);
     maxRedirects: number;
@@ -344,6 +350,8 @@ export class WasmCrawlConfig {
     saveBrowserProfile: boolean;
     softHttpErrors: boolean;
     ssrf: WasmSsrfPolicy;
+    get ssrfDenyPrivateExplicit(): boolean | undefined;
+    set ssrfDenyPrivateExplicit(value: boolean | null | undefined);
     stayOnDomain: boolean;
     get userAgent(): string | undefined;
     set userAgent(value: string | null | undefined);
@@ -410,6 +418,13 @@ export class WasmCrawlResult {
     constructor(pages?: WasmCrawlPageResult[] | null, finalUrl?: string | null, redirectCount?: number | null, wasSkipped?: boolean | null, cookies?: WasmCookieInfo[] | null, stayedOnDomain?: boolean | null, browserUsed?: boolean | null, error?: string | null);
     /**
      * Returns the count of unique normalized URLs encountered during crawling.
+     *
+     * Computed from `pages` (not the deprecated `normalized_urls` field) so it
+     * is correct across every binding that reconstructs `CrawlResult` from
+     * `pages` alone. In streaming mode `pages` is empty, so this returns 0 on
+     * the opaque-handle (C/Go/C#/Zig/Dart) path where it previously counted
+     * streamed pages — a known, accepted cost of making the other ten binding
+     * families correct.
      */
     uniqueNormalizedUrls(): number;
     browserUsed: boolean;
@@ -423,6 +438,18 @@ export class WasmCrawlResult {
     wasSkipped: boolean;
 }
 
+/**
+ * Opt-in encoding applied to a downloaded document's bytes for callers who need the
+ * content available in a serializable field rather than reading it from disk.
+ *
+ * `None` (the `CrawlConfig.document_content_encoding` default) produces neither — unlike
+ * screenshots, base64-encoding a document by default would duplicate an already
+ * up-to-`document_max_size` buffer (50 MB default) in memory per document.
+ */
+export enum WasmDocumentContentEncoding {
+    Base64 = 0,
+}
+
 /**
  * A downloaded asset from a page.
  */
@@ -453,13 +480,18 @@ export class WasmDownloadedDocument {
     free(): void;
     [Symbol.dispose](): void;
     static default(): WasmDownloadedDocument;
-    constructor(url?: string | null, mimeType?: string | null, size?: number | null, contentHash?: string | null, headers?: any | null, filename?: string | null);
+    constructor(url?: string | null, mimeType?: string | null, size?: number | null, contentHash?: string | null, headers?: any | null, truncated?: boolean | null, filename?: string | null, contentPath?: string | null, contentBase64?: string | null);
+    get contentBase64(): string | undefined;
+    set contentBase64(value: string | null | undefined);
     contentHash: string;
+    get contentPath(): string | undefined;
+    set contentPath(value: string | null | undefined);
     get filename(): string | undefined;
     set filename(value: string | null | undefined);
     headers: any;
     mimeType: string;
     size: number;
+    truncated: boolean;
     url: string;
 }
 
@@ -534,6 +566,33 @@ export class WasmHeadingInfo {
     text: string;
 }
 
+/**
+ * Hostname/IP allowlist matcher for SSRF policy.
+ *
+ * Serializes as an internally-tagged object so each variant is distinguishable on the
+ * wire and round-trips losslessly:
+ *
+ * ```json
+ * {"type": "exact",  "value": "api.example.com"}
+ * {"type": "suffix", "value": ".example.com"}
+ * {"type": "cidr",   "value": "10.0.0.0/8"}
+ * ```
+ *
+ * A bare JSON string is still accepted on deserialization and resolves to `Exact`,
+ * preserving configs written against the previous untagged representation.
+ *
+ * `Exact`: HostMatcher.Exact
+ */
+export class WasmHostMatcher {
+    free(): void;
+    [Symbol.dispose](): void;
+    static default(): WasmHostMatcher;
+    constructor();
+    type: string;
+    get value(): string | undefined;
+    set value(value: string | null | undefined);
+}
+
 /**
  * An hreflang alternate link entry.
  */
@@ -582,10 +641,12 @@ export class WasmInteractionResult {
     free(): void;
     [Symbol.dispose](): void;
     static default(): WasmInteractionResult;
-    constructor(actionResults?: WasmActionResult[] | null, finalHtml?: string | null, finalUrl?: string | null);
+    constructor(actionResults?: WasmActionResult[] | null, finalHtml?: string | null, finalUrl?: string | null, screenshotBase64?: string | null);
     actionResults: WasmActionResult[];
     finalHtml: string;
     finalUrl: string;
+    get screenshotBase64(): string | undefined;
+    set screenshotBase64(value: string | null | undefined);
 }
 
 /**
@@ -830,7 +891,7 @@ export class WasmScrapeResult {
     free(): void;
     [Symbol.dispose](): void;
     static default(): WasmScrapeResult;
-    constructor(statusCode?: number | null, finalUrl?: string | null, contentType?: string | null, html?: string | null, bodySize?: number | null, metadata?: WasmPageMetadata | null, links?: WasmLinkInfo[] | null, images?: WasmImageInfo[] | null, feeds?: WasmFeedInfo[] | null, jsonLd?: WasmJsonLdEntry[] | null, isAllowed?: boolean | null, noindexDetected?: boolean | null, nofollowDetected?: boolean | null, isPdf?: boolean | null, wasSkipped?: boolean | null, authHeaderSent?: boolean | null, assets?: WasmDownloadedAsset[] | null, jsRenderHint?: boolean | null, browserUsed?: boolean | null, crawlDelay?: bigint | null, xRobotsTag?: string | null, detectedCharset?: string | null, responseMeta?: WasmResponseMeta | null, markdown?: WasmMarkdownResult | null, extractedData?: any | null, extractionMeta?: WasmExtractionMeta | null, downloadedDocument?: WasmDownloadedDocument | null, browser?: WasmBrowserExtras | null);
+    constructor(statusCode?: number | null, finalUrl?: string | null, contentType?: string | null, html?: string | null, bodySize?: number | null, metadata?: WasmPageMetadata | null, links?: WasmLinkInfo[] | null, images?: WasmImageInfo[] | null, feeds?: WasmFeedInfo[] | null, jsonLd?: WasmJsonLdEntry[] | null, isAllowed?: boolean | null, noindexDetected?: boolean | null, nofollowDetected?: boolean | null, isPdf?: boolean | null, wasSkipped?: boolean | null, authHeaderSent?: boolean | null, assets?: WasmDownloadedAsset[] | null, jsRenderHint?: boolean | null, browserUsed?: boolean | null, crawlDelay?: bigint | null, xRobotsTag?: string | null, detectedCharset?: string | null, responseMeta?: WasmResponseMeta | null, markdown?: WasmMarkdownResult | null, extractedData?: any | null, extractionMeta?: WasmExtractionMeta | null, screenshotBase64?: string | null, downloadedDocument?: WasmDownloadedDocument | null, browser?: WasmBrowserExtras | null);
     assets: WasmDownloadedAsset[];
     authHeaderSent: boolean;
     bodySize: number;
@@ -864,6 +925,8 @@ export class WasmScrapeResult {
     noindexDetected: boolean;
     get responseMeta(): WasmResponseMeta | undefined;
     set responseMeta(value: WasmResponseMeta | null | undefined);
+    get screenshotBase64(): string | undefined;
+    set screenshotBase64(value: string | null | undefined);
     statusCode: number;
     wasSkipped: boolean;
     get xRobotsTag(): string | undefined;
@@ -913,9 +976,18 @@ export class WasmSsrfPolicy {
      * - Outbound requests in a browser go through the fetch API, which enforces its own network policies.
      * - Rust-side SSRF checking is unenforceable and redundant in a wasm32 context.
      * - For testing and localhost access, the host's network sandbox is the enforcing boundary.
+     *
+     * **Node.js caveat:** `deny_private` (whatever its value) has no effect on hostname-based
+     * requests under `wasm32`. There is no DNS resolution on this target, so `validate_url`
+     * only ever checks a literal IP host; a domain name falls straight through to `Ok(())`. In a
+     * browser this is covered by same-origin/CORS. Node's `fetch` enforces no CORS, so a Node
+     * service embedding this wasm module can be driven to internal hosts by domain name even
+     * though `deny_private = true`. Do not rely on this policy to stop that in Node — enforce
+     * egress restrictions (network policy, firewall, proxy allowlist) outside the process.
      */
     static fromEnv(): WasmSsrfPolicy;
-    constructor(denyPrivate?: boolean | null, maxRedirects?: number | null);
+    constructor(denyPrivate?: boolean | null, allowlist?: any | null, maxRedirects?: number | null);
+    allowlist: any;
     denyPrivate: boolean;
     maxRedirects: number;
 }
@@ -994,6 +1066,7 @@ export interface InitOutput {
     readonly __wbg_wasmfaviconinfo_free: (a: number, b: number) => void;
     readonly __wbg_wasmfeedinfo_free: (a: number, b: number) => void;
     readonly __wbg_wasmheadinginfo_free: (a: number, b: number) => void;
+    readonly __wbg_wasmhostmatcher_free: (a: number, b: number) => void;
     readonly __wbg_wasmhreflangentry_free: (a: number, b: number) => void;
     readonly __wbg_wasmimageinfo_free: (a: number, b: number) => void;
     readonly __wbg_wasminteractionresult_free: (a: number, b: number) => void;
@@ -1179,8 +1252,10 @@ export interface InitOutput {
     readonly wasmcrawlconfig_cookiesEnabled: (a: number) => number;
     readonly wasmcrawlconfig_customHeaders: (a: number) => any;
     readonly wasmcrawlconfig_default: () => number;
+    readonly wasmcrawlconfig_documentContentEncoding: (a: number) => [number, number];
     readonly wasmcrawlconfig_documentMaxSize: (a: number) => number;
     readonly wasmcrawlconfig_documentMimeTypes: (a: number) => [number, number];
+    readonly wasmcrawlconfig_documentOutputDir: (a: number) => [number, number];
     readonly wasmcrawlconfig_documentUrlDepth: (a: number) => number;
     readonly wasmcrawlconfig_downloadAssets: (a: number) => number;
     readonly wasmcrawlconfig_downloadDocuments: (a: number) => number;
@@ -1193,9 +1268,10 @@ export interface InitOutput {
     readonly wasmcrawlconfig_maxBodySize: (a: number) => number;
     readonly wasmcrawlconfig_maxConcurrent: (a: number) => number;
     readonly wasmcrawlconfig_maxDepth: (a: number) => number;
+    readonly wasmcrawlconfig_maxLinksPerPage: (a: number) => number;
     readonly wasmcrawlconfig_maxPages: (a: number) => number;
     readonly wasmcrawlconfig_maxRedirects: (a: number) => number;
-    readonly wasmcrawlconfig_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: bigint, l: number, m: number, n: number, o: number, p: number, q: number, r: number, s: number, t: number, u: number, v: number, w: number, x: number, y: number, z: number, a1: number, b1: number, c1: number, d1: number, e1: number, f1: number, g1: number, h1: number, i1: number, j1: number, k1: number, l1: number, m1: bigint, n1: number, o1: number, p1: number, q1: number, r1: number, s1: number, t1: number, u1: number, v1: number, w1: number, x1: number, y1: number, z1: number) => number;
+    readonly wasmcrawlconfig_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: bigint, l: number, m: number, n: number, o: number, p: number, q: number, r: number, s: number, t: number, u: number, v: number, w: number, x: number, y: number, z: number, a1: number, b1: number, c1: number, d1: number, e1: number, f1: number, g1: number, h1: number, i1: number, j1: number, k1: number, l1: number, m1: number, n1: bigint, o1: number, p1: number, q1: number, r1: number, s1: number, t1: number, u1: number, v1: number, w1: number, x1: number, y1: number, z1: number, a2: number, b2: number, c2: number, d2: number, e2: number) => number;
     readonly wasmcrawlconfig_proxy: (a: number) => number;
     readonly wasmcrawlconfig_removeTags: (a: number) => [number, number];
     readonly wasmcrawlconfig_respectRobotsTxt: (a: number) => number;
@@ -1211,8 +1287,10 @@ export interface InitOutput {
     readonly wasmcrawlconfig_set_content: (a: number, b: number) => void;
     readonly wasmcrawlconfig_set_cookiesEnabled: (a: number, b: number) => void;
     readonly wasmcrawlconfig_set_customHeaders: (a: number, b: any) => void;
+    readonly wasmcrawlconfig_set_documentContentEncoding: (a: number, b: number) => void;
     readonly wasmcrawlconfig_set_documentMaxSize: (a: number, b: number) => void;
     readonly wasmcrawlconfig_set_documentMimeTypes: (a: number, b: number, c: number) => void;
+    readonly wasmcrawlconfig_set_documentOutputDir: (a: number, b: number, c: number) => void;
     readonly wasmcrawlconfig_set_documentUrlDepth: (a: number, b: number) => void;
     readonly wasmcrawlconfig_set_downloadAssets: (a: number, b: number) => void;
     readonly wasmcrawlconfig_set_downloadDocuments: (a: number, b: number) => void;
@@ -1225,6 +1303,7 @@ export interface InitOutput {
     readonly wasmcrawlconfig_set_maxBodySize: (a: number, b: number) => void;
     readonly wasmcrawlconfig_set_maxConcurrent: (a: number, b: number) => void;
     readonly wasmcrawlconfig_set_maxDepth: (a: number, b: number) => void;
+    readonly wasmcrawlconfig_set_maxLinksPerPage: (a: number, b: number) => void;
     readonly wasmcrawlconfig_set_maxPages: (a: number, b: number) => void;
     readonly wasmcrawlconfig_set_maxRedirects: (a: number, b: number) => void;
     readonly wasmcrawlconfig_set_proxy: (a: number, b: number) => void;
@@ -1235,12 +1314,14 @@ export interface InitOutput {
     readonly wasmcrawlconfig_set_saveBrowserProfile: (a: number, b: number) => void;
     readonly wasmcrawlconfig_set_softHttpErrors: (a: number, b: number) => void;
     readonly wasmcrawlconfig_set_ssrf: (a: number, b: number) => void;
+    readonly wasmcrawlconfig_set_ssrfDenyPrivateExplicit: (a: number, b: number) => void;
     readonly wasmcrawlconfig_set_stayOnDomain: (a: number, b: number) => void;
     readonly wasmcrawlconfig_set_userAgent: (a: number, b: number, c: number) => void;
     readonly wasmcrawlconfig_set_userAgents: (a: number, b: number, c: number) => void;
     readonly wasmcrawlconfig_set_warcOutput: (a: number, b: number, c: number) => void;
     readonly wasmcrawlconfig_softHttpErrors: (a: number) => number;
     readonly wasmcrawlconfig_ssrf: (a: number) => number;
+    readonly wasmcrawlconfig_ssrfDenyPrivateExplicit: (a: number) => number;
     readonly wasmcrawlconfig_stayOnDomain: (a: number) => number;
     readonly wasmcrawlconfig_userAgent: (a: number) => [number, number];
     readonly wasmcrawlconfig_userAgents: (a: number) => [number, number];
@@ -1321,17 +1402,25 @@ export interface InitOutput {
     readonly wasmdownloadedasset_set_mimeType: (a: number, b: number, c: number) => void;
     readonly wasmdownloadedasset_set_url: (a: number, b: number, c: number) => void;
     readonly wasmdownloadedasset_url: (a: number) => [number, number];
+    readonly wasmdownloadeddocument_contentBase64: (a: number) => [number, number];
     readonly wasmdownloadeddocument_contentHash: (a: number) => [number, number];
+    readonly wasmdownloadeddocument_contentPath: (a: number) => [number, number];
     readonly wasmdownloadeddocument_default: () => number;
     readonly wasmdownloadeddocument_filename: (a: number) => [number, number];
     readonly wasmdownloadeddocument_headers: (a: number) => any;
     readonly wasmdownloadeddocument_mimeType: (a: number) => [number, number];
-    readonly wasmdownloadeddocument_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number) => number;
+    readonly wasmdownloadeddocument_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number, n: number, o: number) => number;
+    readonly wasmdownloadeddocument_set_contentBase64: (a: number, b: number, c: number) => void;
     readonly wasmdownloadeddocument_set_contentHash: (a: number, b: number, c: number) => void;
+    readonly wasmdownloadeddocument_set_contentPath: (a: number, b: number, c: number) => void;
     readonly wasmdownloadeddocument_set_filename: (a: number, b: number, c: number) => void;
     readonly wasmdownloadeddocument_set_headers: (a: number, b: any) => void;
     readonly wasmdownloadeddocument_set_mimeType: (a: number, b: number, c: number) => void;
+    readonly wasmdownloadeddocument_set_size: (a: number, b: number) => void;
+    readonly wasmdownloadeddocument_set_truncated: (a: number, b: number) => void;
     readonly wasmdownloadeddocument_set_url: (a: number, b: number, c: number) => void;
+    readonly wasmdownloadeddocument_size: (a: number) => number;
+    readonly wasmdownloadeddocument_truncated: (a: number) => number;
     readonly wasmdownloadeddocument_url: (a: number) => [number, number];
     readonly wasmextractionmeta_chunksProcessed: (a: number) => number;
     readonly wasmextractionmeta_completionTokens: (a: number) => [number, bigint];
@@ -1365,6 +1454,11 @@ export interface InitOutput {
     readonly wasmheadinginfo_set_level: (a: number, b: number) => void;
     readonly wasmheadinginfo_set_text: (a: number, b: number, c: number) => void;
     readonly wasmheadinginfo_text: (a: number) => [number, number];
+    readonly wasmhostmatcher_default: () => number;
+    readonly wasmhostmatcher_set_type: (a: number, b: number, c: number) => void;
+    readonly wasmhostmatcher_set_value: (a: number, b: number, c: number) => void;
+    readonly wasmhostmatcher_type: (a: number) => [number, number];
+    readonly wasmhostmatcher_value: (a: number) => [number, number];
     readonly wasmhreflangentry_default: () => number;
     readonly wasmhreflangentry_lang: (a: number) => [number, number];
     readonly wasmhreflangentry_new: (a: number, b: number, c: number, d: number) => number;
@@ -1385,10 +1479,12 @@ export interface InitOutput {
     readonly wasminteractionresult_default: () => number;
     readonly wasminteractionresult_finalHtml: (a: number) => [number, number];
     readonly wasminteractionresult_finalUrl: (a: number) => [number, number];
-    readonly wasminteractionresult_new: (a: number, b: number, c: number, d: number, e: number, f: number) => number;
+    readonly wasminteractionresult_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number) => number;
+    readonly wasminteractionresult_screenshotBase64: (a: number) => [number, number];
     readonly wasminteractionresult_set_actionResults: (a: number, b: number, c: number) => void;
     readonly wasminteractionresult_set_finalHtml: (a: number, b: number, c: number) => void;
     readonly wasminteractionresult_set_finalUrl: (a: number, b: number, c: number) => void;
+    readonly wasminteractionresult_set_screenshotBase64: (a: number, b: number, c: number) => void;
     readonly wasmjsonldentry_default: () => number;
     readonly wasmjsonldentry_name: (a: number) => [number, number];
     readonly wasmjsonldentry_new: (a: number, b: number, c: number, d: number, e: number, f: number) => number;
@@ -1572,10 +1668,11 @@ export interface InitOutput {
     readonly wasmscraperesult_links: (a: number) => [number, number];
     readonly wasmscraperesult_markdown: (a: number) => number;
     readonly wasmscraperesult_metadata: (a: number) => number;
-    readonly wasmscraperesult_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number, n: number, o: number, p: number, q: number, r: number, s: number, t: number, u: number, v: number, w: number, x: number, y: number, z: number, a1: number, b1: number, c1: bigint, d1: number, e1: number, f1: number, g1: number, h1: number, i1: number, j1: number, k1: number, l1: number, m1: number) => number;
+    readonly wasmscraperesult_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number, n: number, o: number, p: number, q: number, r: number, s: number, t: number, u: number, v: number, w: number, x: number, y: number, z: number, a1: number, b1: number, c1: bigint, d1: number, e1: number, f1: number, g1: number, h1: number, i1: number, j1: number, k1: number, l1: number, m1: number, n1: number, o1: number) => number;
     readonly wasmscraperesult_nofollowDetected: (a: number) => number;
     readonly wasmscraperesult_noindexDetected: (a: number) => number;
     readonly wasmscraperesult_responseMeta: (a: number) => number;
+    readonly wasmscraperesult_screenshotBase64: (a: number) => [number, number];
     readonly wasmscraperesult_set_assets: (a: number, b: number, c: number) => void;
     readonly wasmscraperesult_set_authHeaderSent: (a: number, b: number) => void;
     readonly wasmscraperesult_set_bodySize: (a: number, b: number) => void;
@@ -1600,6 +1697,7 @@ export interface InitOutput {
     readonly wasmscraperesult_set_nofollowDetected: (a: number, b: number) => void;
     readonly wasmscraperesult_set_noindexDetected: (a: number, b: number) => void;
     readonly wasmscraperesult_set_responseMeta: (a: number, b: number) => void;
+    readonly wasmscraperesult_set_screenshotBase64: (a: number, b: number, c: number) => void;
     readonly wasmscraperesult_set_statusCode: (a: number, b: number) => void;
     readonly wasmscraperesult_set_wasSkipped: (a: number, b: number) => void;
     readonly wasmscraperesult_set_xRobotsTag: (a: number, b: number, c: number) => void;
@@ -1616,13 +1714,16 @@ export interface InitOutput {
     readonly wasmsitemapurl_set_priority: (a: number, b: number, c: number) => void;
     readonly wasmsitemapurl_set_url: (a: number, b: number, c: number) => void;
     readonly wasmsitemapurl_url: (a: number) => [number, number];
+    readonly wasmssrfpolicy_allowlist: (a: number) => any;
     readonly wasmssrfpolicy_default: () => number;
     readonly wasmssrfpolicy_denyPrivate: (a: number) => number;
     readonly wasmssrfpolicy_fromEnv: () => number;
     readonly wasmssrfpolicy_maxRedirects: (a: number) => number;
-    readonly wasmssrfpolicy_new: (a: number, b: number) => number;
+    readonly wasmssrfpolicy_new: (a: number, b: number, c: number) => number;
+    readonly wasmssrfpolicy_set_allowlist: (a: number, b: any) => void;
     readonly wasmssrfpolicy_set_denyPrivate: (a: number, b: number) => void;
     readonly wasmssrfpolicy_set_maxRedirects: (a: number, b: number) => void;
+    readonly wasmhostmatcher_new: () => number;
     readonly wasmcrawlconfig_set_rateLimitMs: (a: number, b: number, c: bigint) => void;
     readonly wasmextractionmeta_set_promptTokens: (a: number, b: number, c: bigint) => void;
     readonly wasmpageaction_set_milliseconds: (a: number, b: number, c: bigint) => void;
@@ -1637,14 +1738,12 @@ export interface InitOutput {
     readonly wasmbatchscraperesults_set_failedCount: (a: number, b: number) => void;
     readonly wasmbatchscraperesults_set_totalCount: (a: number, b: number) => void;
     readonly wasmdownloadedasset_set_size: (a: number, b: number) => void;
-    readonly wasmdownloadeddocument_set_size: (a: number, b: number) => void;
     readonly wasmimageinfo_set_width: (a: number, b: number) => void;
     readonly wasmpagemetadata_set_wordCount: (a: number, b: number) => void;
     readonly wasmbatchscraperesults_completedCount: (a: number) => number;
     readonly wasmbatchscraperesults_failedCount: (a: number) => number;
     readonly wasmbatchscraperesults_totalCount: (a: number) => number;
     readonly wasmdownloadedasset_size: (a: number) => number;
-    readonly wasmdownloadeddocument_size: (a: number) => number;
     readonly wasmlinkinfo_set_nofollow: (a: number, b: number) => void;
     readonly wasmfaviconinfo_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number) => number;
     readonly wasmimageinfo_width: (a: number) => number;
@@ -1655,9 +1754,9 @@ export interface InitOutput {
     readonly wasmpageaction_set_amount: (a: number, b: number, c: bigint) => void;
     readonly wasmscraperesult_set_crawlDelay: (a: number, b: number, c: bigint) => void;
     readonly wasmpageaction_new: () => number;
-    readonly wasm_bindgen__convert__closures_____invoke__h96ab4f4d64e6bfe6: (a: number, b: number, c: any) => [number, number];
-    readonly wasm_bindgen__convert__closures_____invoke__h20406e9dd5374954: (a: number, b: number, c: any, d: any) => void;
-    readonly wasm_bindgen__convert__closures_____invoke__h10c3645e699af307: (a: number, b: number) => void;
+    readonly wasm_bindgen__convert__closures_____invoke__he29f1bb22c192c97: (a: number, b: number, c: any) => [number, number];
+    readonly wasm_bindgen__convert__closures_____invoke__h06ea3d672ad56c9d: (a: number, b: number, c: any, d: any) => void;
+    readonly wasm_bindgen__convert__closures_____invoke__hbe9f9227fe7d8a65: (a: number, b: number) => void;
     readonly __wbindgen_malloc: (a: number, b: number) => number;
     readonly __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number;
     readonly __wbindgen_exn_store: (a: number) => void;