@xberg-io/crawlberg-wasm 1.12.0 → 1.12.3

package/package.json

@@ -10,7 +10,7 @@
     }
   ],
   "name": "@xberg-io/crawlberg-wasm",
-  "version": "1.12.0",
+  "version": "1.12.3",
   "private": false,
   "description": "High-performance web crawling engine",
   "license": "MIT",

package/pkg/bundler/crawlberg_wasm.d.ts

@@ -260,7 +260,7 @@ export class WasmBrowserConfig {
      * keep a handle you already hold, build with `default()` and assign the property instead:
      * the generated setter borrows and leaves your handle alive.
      */
-    constructor(mode?: WasmBrowserMode | null, backend?: WasmBrowserBackend | null, timeout?: bigint | null, overallTimeout?: bigint | null, shutdownTimeout?: bigint | null, wait?: WasmBrowserWait | null, blockUrlPatterns?: string[] | null, captureNetworkEvents?: boolean | null, sessionAffinity?: boolean | null, chromeArgs?: string[] | null, endpoint?: string | null, waitSelector?: string | null, extraWait?: bigint | null, proxy?: WasmProxyConfig | null, evalScript?: string | null, robotsUserAgent?: string | null, chromePath?: string | null, scrollContainers?: boolean | null, domQuietWindow?: bigint | null);
+    constructor(mode?: WasmBrowserMode | null, backend?: WasmBrowserBackend | null, timeout?: bigint | null, overallTimeout?: bigint | null, shutdownTimeout?: bigint | null, wait?: WasmBrowserWait | null, blockUrlPatterns?: string[] | null, captureNetworkEvents?: boolean | null, sessionAffinity?: boolean | null, chromeArgs?: string[] | null, endpoint?: string | null, waitSelector?: string | null, extraWait?: bigint | null, proxy?: WasmProxyConfig | null, evalScript?: string | null, robotsUserAgent?: string | null, chromePath?: string | null, scrollContainers?: boolean | null, domQuietWindow?: bigint | null, domQuietCap?: bigint | null);
     get backend(): string;
     set backend(value: WasmBrowserBackend);
     blockUrlPatterns: string[];
@@ -268,6 +268,8 @@ export class WasmBrowserConfig {
     chromeArgs: string[];
     get chromePath(): string | undefined;
     set chromePath(value: string | null | undefined);
+    get domQuietCap(): bigint | undefined;
+    set domQuietCap(value: bigint | null | undefined);
     get domQuietWindow(): bigint | undefined;
     set domQuietWindow(value: bigint | null | undefined);
     get endpoint(): string | undefined;
@@ -430,12 +432,16 @@ export class WasmContentConfig {
      */
     copyForTransfer(): WasmContentConfig;
     static default(): WasmContentConfig;
-    constructor(outputFormat?: string | null, preprocessingPreset?: string | null, removeNavigation?: boolean | null, removeForms?: boolean | null, stripTags?: string[] | null, preserveTags?: string[] | null, excludeSelectors?: string[] | null, skipImages?: boolean | null, wrap?: boolean | null, wrapWidth?: number | null, includeDocumentStructure?: boolean | null, extractMetadata?: boolean | null, maxDepth?: number | null, stripNewlines?: boolean | null, bullets?: string | null, listIndentWidth?: number | null, strongEmSymbol?: string | null, compactTables?: boolean | null, keepInlineImagesIn?: string[] | null, keepHidden?: boolean | null);
+    constructor(outputFormat?: string | null, preprocessingPreset?: string | null, removeNavigation?: boolean | null, removeForms?: boolean | null, stripTags?: string[] | null, preserveTags?: string[] | null, excludeSelectors?: string[] | null, skipImages?: boolean | null, wrap?: boolean | null, wrapWidth?: number | null, includeDocumentStructure?: boolean | null, extractMetadata?: boolean | null, maxDepth?: number | null, stripNewlines?: boolean | null, bullets?: string | null, listIndentWidth?: number | null, strongEmSymbol?: string | null, compactTables?: boolean | null, keepInlineImagesIn?: string[] | null, keepHidden?: boolean | null, expandAbbreviations?: boolean | null, escapeMisc?: boolean | null, escapeAsterisks?: boolean | null, includeIconLabels?: boolean | null);
     bullets: string;
     compactTables: boolean;
+    escapeAsterisks: boolean;
+    escapeMisc: boolean;
     excludeSelectors: string[];
+    expandAbbreviations: boolean;
     extractMetadata: boolean;
     includeDocumentStructure: boolean;
+    includeIconLabels: boolean;
     keepHidden: boolean;
     keepInlineImagesIn: string[];
     listIndentWidth: number;
@@ -509,7 +515,7 @@ export class WasmCrawlConfig {
      * keep a handle you already hold, build with `default()` and assign the property instead:
      * the generated setter borrows and leaves your handle alive.
      */
-    constructor(crawlStrategy?: WasmCrawlStrategyKind | null, respectRobotsTxt?: boolean | null, softHttpErrors?: boolean | null, stayOnDomain?: boolean | null, allowSubdomains?: boolean | null, includePaths?: string[] | null, excludePaths?: string[] | null, pathPatternsMatchQuery?: boolean | null, pathPatternsMatchUrl?: boolean | null, dedupIncludeQuery?: boolean | null, stripTrackingParams?: boolean | null, trackingParams?: string[] | null, customHeaders?: any | null, requestTimeout?: bigint | null, maxRedirects?: number | null, retryCount?: number | null, retryCodes?: Uint16Array | null, retryInitialDelayMs?: bigint | null, retryMaxDelayMs?: bigint | null, rateLimitJitterRatio?: number | null, cookiesEnabled?: boolean | null, removeTags?: string[] | null, content?: WasmContentConfig | null, downloadAssets?: boolean | null, assetTypes?: WasmAssetCategory[] | null, browser?: WasmBrowserConfig | null, userAgents?: string[] | null, captureScreenshot?: boolean | null, followDocumentUrls?: boolean | null, downloadDocuments?: boolean | null, documentMimeTypes?: string[] | null, saveBrowserProfile?: boolean | null, ssrf?: WasmSsrfPolicy | null, maxDepth?: number | null, maxPages?: number | null, maxLinksPerPage?: number | null, maxConcurrent?: number | null, contentFilter?: WasmContentFilterKind | null, bm25Query?: string | null, bm25Threshold?: number | null, userAgent?: string | null, rateLimitMs?: bigint | null, auth?: any | null, maxBodySize?: number | null, mapLimit?: number | null, mapSearch?: string | null, maxAssetSize?: number | null, proxy?: WasmProxyConfig | null, documentUrlDepth?: number | null, documentMaxSize?: number | null, documentOutputDir?: string | null, documentContentEncoding?: WasmDocumentContentEncoding | null, warcOutput?: string | null, browserProfile?: string | null, ssrfDenyPrivateExplicit?: boolean | null, dangerAcceptInvalidCerts?: boolean | null);
+    constructor(crawlStrategy?: WasmCrawlStrategyKind | null, respectRobotsTxt?: boolean | null, softHttpErrors?: boolean | null, stayOnDomain?: boolean | null, allowSubdomains?: boolean | null, includePaths?: string[] | null, excludePaths?: string[] | null, pathPatternsMatchQuery?: boolean | null, pathPatternsMatchUrl?: boolean | null, dedupIncludeQuery?: boolean | null, stripTrackingParams?: boolean | null, trackingParams?: string[] | null, customHeaders?: any | null, requestTimeout?: bigint | null, maxRedirects?: number | null, retryCount?: number | null, retryCodes?: Uint16Array | null, retryInitialDelayMs?: bigint | null, retryMaxDelayMs?: bigint | null, rateLimitJitterRatio?: number | null, cookiesEnabled?: boolean | null, removeTags?: string[] | null, content?: WasmContentConfig | null, downloadAssets?: boolean | null, assetTypes?: WasmAssetCategory[] | null, browser?: WasmBrowserConfig | null, userAgents?: string[] | null, captureScreenshot?: boolean | null, followDocumentUrls?: boolean | null, downloadDocuments?: boolean | null, documentMimeTypes?: string[] | null, saveBrowserProfile?: boolean | null, ssrf?: WasmSsrfPolicy | null, maxDepth?: number | null, maxPages?: number | null, maxLinksPerPage?: number | null, maxConcurrent?: number | null, contentFilter?: WasmContentFilterKind | null, bm25Query?: string | null, bm25Threshold?: number | null, userAgent?: string | null, rateLimitMs?: bigint | null, auth?: any | null, maxBodySize?: number | null, mapLimit?: number | null, mapSearch?: string | null, maxAssetSize?: number | null, proxy?: WasmProxyConfig | null, documentUrlDepth?: number | null, documentMaxSize?: number | null, documentOutputDir?: string | null, documentContentEncoding?: WasmDocumentContentEncoding | null, warcOutput?: string | null, browserProfile?: string | null, ssrfDenyPrivateExplicit?: boolean | null, dangerAcceptInvalidCerts?: boolean | null, subdomainHeaderAllowlist?: string[] | null);
     /**
      * Validate the configuration, returning an error if any values are invalid.
      *
@@ -615,6 +621,7 @@ export class WasmCrawlConfig {
     ssrf: WasmSsrfPolicy;
     stayOnDomain: boolean;
     stripTrackingParams: boolean;
+    subdomainHeaderAllowlist: string[];
     trackingParams: string[];
     get userAgent(): string | undefined;
     set userAgent(value: string | null | undefined);

package/pkg/bundler/crawlberg_wasm_bg.js

@@ -1103,6 +1103,13 @@ export class WasmBrowserConfig {
         const ret = wasm.wasmbrowserconfig_default();
         return WasmBrowserConfig.__wrap(ret);
     }
+    /**
+     * @returns {bigint | undefined}
+     */
+    get domQuietCap() {
+        const ret = wasm.wasmbrowserconfig_domQuietCap(this.__wbg_ptr);
+        return ret[0] === 0 ? undefined : BigInt.asUintN(64, ret[1]);
+    }
     /**
      * @returns {bigint | undefined}
      */
@@ -1181,8 +1188,9 @@ export class WasmBrowserConfig {
      * @param {string | null} [chromePath]
      * @param {boolean | null} [scrollContainers]
      * @param {bigint | null} [domQuietWindow]
+     * @param {bigint | null} [domQuietCap]
      */
-    constructor(mode, backend, timeout, overallTimeout, shutdownTimeout, wait, blockUrlPatterns, captureNetworkEvents, sessionAffinity, chromeArgs, endpoint, waitSelector, extraWait, proxy, evalScript, robotsUserAgent, chromePath, scrollContainers, domQuietWindow) {
+    constructor(mode, backend, timeout, overallTimeout, shutdownTimeout, wait, blockUrlPatterns, captureNetworkEvents, sessionAffinity, chromeArgs, endpoint, waitSelector, extraWait, proxy, evalScript, robotsUserAgent, chromePath, scrollContainers, domQuietWindow, domQuietCap) {
         var ptr0 = isLikeNone(blockUrlPatterns) ? 0 : passArrayJsValueToWasm0(blockUrlPatterns, wasm.__wbindgen_malloc);
         var len0 = WASM_VECTOR_LEN;
         var ptr1 = isLikeNone(chromeArgs) ? 0 : passArrayJsValueToWasm0(chromeArgs, wasm.__wbindgen_malloc);
@@ -1202,7 +1210,7 @@ export class WasmBrowserConfig {
         var len6 = WASM_VECTOR_LEN;
         var ptr7 = isLikeNone(chromePath) ? 0 : passStringToWasm0(chromePath, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
         var len7 = WASM_VECTOR_LEN;
-        const ret = wasm.wasmbrowserconfig_new(isLikeNone(mode) ? 4 : mode, isLikeNone(backend) ? 2 : backend, !isLikeNone(timeout), isLikeNone(timeout) ? BigInt(0) : timeout, !isLikeNone(overallTimeout), isLikeNone(overallTimeout) ? BigInt(0) : overallTimeout, !isLikeNone(shutdownTimeout), isLikeNone(shutdownTimeout) ? BigInt(0) : shutdownTimeout, isLikeNone(wait) ? 3 : wait, ptr0, len0, isLikeNone(captureNetworkEvents) ? 0xFFFFFF : captureNetworkEvents ? 1 : 0, isLikeNone(sessionAffinity) ? 0xFFFFFF : sessionAffinity ? 1 : 0, ptr1, len1, ptr2, len2, ptr3, len3, !isLikeNone(extraWait), isLikeNone(extraWait) ? BigInt(0) : extraWait, ptr4, ptr5, len5, ptr6, len6, ptr7, len7, isLikeNone(scrollContainers) ? 0xFFFFFF : scrollContainers ? 1 : 0, !isLikeNone(domQuietWindow), isLikeNone(domQuietWindow) ? BigInt(0) : domQuietWindow);
+        const ret = wasm.wasmbrowserconfig_new(isLikeNone(mode) ? 4 : mode, isLikeNone(backend) ? 2 : backend, !isLikeNone(timeout), isLikeNone(timeout) ? BigInt(0) : timeout, !isLikeNone(overallTimeout), isLikeNone(overallTimeout) ? BigInt(0) : overallTimeout, !isLikeNone(shutdownTimeout), isLikeNone(shutdownTimeout) ? BigInt(0) : shutdownTimeout, isLikeNone(wait) ? 3 : wait, ptr0, len0, isLikeNone(captureNetworkEvents) ? 0xFFFFFF : captureNetworkEvents ? 1 : 0, isLikeNone(sessionAffinity) ? 0xFFFFFF : sessionAffinity ? 1 : 0, ptr1, len1, ptr2, len2, ptr3, len3, !isLikeNone(extraWait), isLikeNone(extraWait) ? BigInt(0) : extraWait, ptr4, ptr5, len5, ptr6, len6, ptr7, len7, isLikeNone(scrollContainers) ? 0xFFFFFF : scrollContainers ? 1 : 0, !isLikeNone(domQuietWindow), isLikeNone(domQuietWindow) ? BigInt(0) : domQuietWindow, !isLikeNone(domQuietCap), isLikeNone(domQuietCap) ? BigInt(0) : domQuietCap);
         this.__wbg_ptr = ret;
         WasmBrowserConfigFinalization.register(this, this.__wbg_ptr, this);
         return this;
@@ -1287,6 +1295,12 @@ export class WasmBrowserConfig {
         var len0 = WASM_VECTOR_LEN;
         wasm.wasmbrowserconfig_set_chromePath(this.__wbg_ptr, ptr0, len0);
     }
+    /**
+     * @param {bigint | null} [value]
+     */
+    set domQuietCap(value) {
+        wasm.wasmbrowserconfig_set_domQuietCap(this.__wbg_ptr, !isLikeNone(value), isLikeNone(value) ? BigInt(0) : value);
+    }
     /**
      * @param {bigint | null} [value]
      */
@@ -1858,6 +1872,20 @@ export class WasmContentConfig {
         const ret = wasm.wasmcontentconfig_default();
         return WasmContentConfig.__wrap(ret);
     }
+    /**
+     * @returns {boolean}
+     */
+    get escapeAsterisks() {
+        const ret = wasm.wasmcontentconfig_escapeAsterisks(this.__wbg_ptr);
+        return ret !== 0;
+    }
+    /**
+     * @returns {boolean}
+     */
+    get escapeMisc() {
+        const ret = wasm.wasmcontentconfig_escapeMisc(this.__wbg_ptr);
+        return ret !== 0;
+    }
     /**
      * @returns {string[]}
      */
@@ -1867,6 +1895,13 @@ export class WasmContentConfig {
         wasm.__wbindgen_free(ret[0], ret[1] * 4, 4);
         return v1;
     }
+    /**
+     * @returns {boolean}
+     */
+    get expandAbbreviations() {
+        const ret = wasm.wasmcontentconfig_expandAbbreviations(this.__wbg_ptr);
+        return ret !== 0;
+    }
     /**
      * @returns {boolean}
      */
@@ -1881,6 +1916,13 @@ export class WasmContentConfig {
         const ret = wasm.wasmcontentconfig_includeDocumentStructure(this.__wbg_ptr);
         return ret !== 0;
     }
+    /**
+     * @returns {boolean}
+     */
+    get includeIconLabels() {
+        const ret = wasm.wasmcontentconfig_includeIconLabels(this.__wbg_ptr);
+        return ret !== 0;
+    }
     /**
      * @returns {boolean}
      */
@@ -1932,8 +1974,12 @@ export class WasmContentConfig {
      * @param {boolean | null} [compactTables]
      * @param {string[] | null} [keepInlineImagesIn]
      * @param {boolean | null} [keepHidden]
+     * @param {boolean | null} [expandAbbreviations]
+     * @param {boolean | null} [escapeMisc]
+     * @param {boolean | null} [escapeAsterisks]
+     * @param {boolean | null} [includeIconLabels]
      */
-    constructor(outputFormat, preprocessingPreset, removeNavigation, removeForms, stripTags, preserveTags, excludeSelectors, skipImages, wrap, wrapWidth, includeDocumentStructure, extractMetadata, maxDepth, stripNewlines, bullets, listIndentWidth, strongEmSymbol, compactTables, keepInlineImagesIn, keepHidden) {
+    constructor(outputFormat, preprocessingPreset, removeNavigation, removeForms, stripTags, preserveTags, excludeSelectors, skipImages, wrap, wrapWidth, includeDocumentStructure, extractMetadata, maxDepth, stripNewlines, bullets, listIndentWidth, strongEmSymbol, compactTables, keepInlineImagesIn, keepHidden, expandAbbreviations, escapeMisc, escapeAsterisks, includeIconLabels) {
         var ptr0 = isLikeNone(outputFormat) ? 0 : passStringToWasm0(outputFormat, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
         var len0 = WASM_VECTOR_LEN;
         var ptr1 = isLikeNone(preprocessingPreset) ? 0 : passStringToWasm0(preprocessingPreset, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
@@ -1950,7 +1996,7 @@ export class WasmContentConfig {
         var len6 = WASM_VECTOR_LEN;
         var ptr7 = isLikeNone(keepInlineImagesIn) ? 0 : passArrayJsValueToWasm0(keepInlineImagesIn, wasm.__wbindgen_malloc);
         var len7 = WASM_VECTOR_LEN;
-        const ret = wasm.wasmcontentconfig_new(ptr0, len0, ptr1, len1, isLikeNone(removeNavigation) ? 0xFFFFFF : removeNavigation ? 1 : 0, isLikeNone(removeForms) ? 0xFFFFFF : removeForms ? 1 : 0, ptr2, len2, ptr3, len3, ptr4, len4, isLikeNone(skipImages) ? 0xFFFFFF : skipImages ? 1 : 0, isLikeNone(wrap) ? 0xFFFFFF : wrap ? 1 : 0, isLikeNone(wrapWidth) ? Number.MAX_SAFE_INTEGER : (wrapWidth) >>> 0, isLikeNone(includeDocumentStructure) ? 0xFFFFFF : includeDocumentStructure ? 1 : 0, isLikeNone(extractMetadata) ? 0xFFFFFF : extractMetadata ? 1 : 0, isLikeNone(maxDepth) ? Number.MAX_SAFE_INTEGER : (maxDepth) >>> 0, isLikeNone(stripNewlines) ? 0xFFFFFF : stripNewlines ? 1 : 0, ptr5, len5, isLikeNone(listIndentWidth) ? Number.MAX_SAFE_INTEGER : (listIndentWidth) >>> 0, ptr6, len6, isLikeNone(compactTables) ? 0xFFFFFF : compactTables ? 1 : 0, ptr7, len7, isLikeNone(keepHidden) ? 0xFFFFFF : keepHidden ? 1 : 0);
+        const ret = wasm.wasmcontentconfig_new(ptr0, len0, ptr1, len1, isLikeNone(removeNavigation) ? 0xFFFFFF : removeNavigation ? 1 : 0, isLikeNone(removeForms) ? 0xFFFFFF : removeForms ? 1 : 0, ptr2, len2, ptr3, len3, ptr4, len4, isLikeNone(skipImages) ? 0xFFFFFF : skipImages ? 1 : 0, isLikeNone(wrap) ? 0xFFFFFF : wrap ? 1 : 0, isLikeNone(wrapWidth) ? Number.MAX_SAFE_INTEGER : (wrapWidth) >>> 0, isLikeNone(includeDocumentStructure) ? 0xFFFFFF : includeDocumentStructure ? 1 : 0, isLikeNone(extractMetadata) ? 0xFFFFFF : extractMetadata ? 1 : 0, isLikeNone(maxDepth) ? Number.MAX_SAFE_INTEGER : (maxDepth) >>> 0, isLikeNone(stripNewlines) ? 0xFFFFFF : stripNewlines ? 1 : 0, ptr5, len5, isLikeNone(listIndentWidth) ? Number.MAX_SAFE_INTEGER : (listIndentWidth) >>> 0, ptr6, len6, isLikeNone(compactTables) ? 0xFFFFFF : compactTables ? 1 : 0, ptr7, len7, isLikeNone(keepHidden) ? 0xFFFFFF : keepHidden ? 1 : 0, isLikeNone(expandAbbreviations) ? 0xFFFFFF : expandAbbreviations ? 1 : 0, isLikeNone(escapeMisc) ? 0xFFFFFF : escapeMisc ? 1 : 0, isLikeNone(escapeAsterisks) ? 0xFFFFFF : escapeAsterisks ? 1 : 0, isLikeNone(includeIconLabels) ? 0xFFFFFF : includeIconLabels ? 1 : 0);
         this.__wbg_ptr = ret;
         WasmContentConfigFinalization.register(this, this.__wbg_ptr, this);
         return this;
@@ -2022,6 +2068,18 @@ export class WasmContentConfig {
     set compactTables(value) {
         wasm.wasmcontentconfig_set_compactTables(this.__wbg_ptr, value);
     }
+    /**
+     * @param {boolean} value
+     */
+    set escapeAsterisks(value) {
+        wasm.wasmcontentconfig_set_escapeAsterisks(this.__wbg_ptr, value);
+    }
+    /**
+     * @param {boolean} value
+     */
+    set escapeMisc(value) {
+        wasm.wasmcontentconfig_set_escapeMisc(this.__wbg_ptr, value);
+    }
     /**
      * @param {string[]} value
      */
@@ -2030,6 +2088,12 @@ export class WasmContentConfig {
         const len0 = WASM_VECTOR_LEN;
         wasm.wasmcontentconfig_set_excludeSelectors(this.__wbg_ptr, ptr0, len0);
     }
+    /**
+     * @param {boolean} value
+     */
+    set expandAbbreviations(value) {
+        wasm.wasmcontentconfig_set_expandAbbreviations(this.__wbg_ptr, value);
+    }
     /**
      * @param {boolean} value
      */
@@ -2042,6 +2106,12 @@ export class WasmContentConfig {
     set includeDocumentStructure(value) {
         wasm.wasmcontentconfig_set_includeDocumentStructure(this.__wbg_ptr, value);
     }
+    /**
+     * @param {boolean} value
+     */
+    set includeIconLabels(value) {
+        wasm.wasmcontentconfig_set_includeIconLabels(this.__wbg_ptr, value);
+    }
     /**
      * @param {boolean} value
      */
@@ -2758,8 +2828,9 @@ export class WasmCrawlConfig {
      * @param {string | null} [browserProfile]
      * @param {boolean | null} [ssrfDenyPrivateExplicit]
      * @param {boolean | null} [dangerAcceptInvalidCerts]
+     * @param {string[] | null} [subdomainHeaderAllowlist]
      */
-    constructor(crawlStrategy, respectRobotsTxt, softHttpErrors, stayOnDomain, allowSubdomains, includePaths, excludePaths, pathPatternsMatchQuery, pathPatternsMatchUrl, dedupIncludeQuery, stripTrackingParams, trackingParams, customHeaders, requestTimeout, maxRedirects, retryCount, retryCodes, retryInitialDelayMs, retryMaxDelayMs, rateLimitJitterRatio, cookiesEnabled, removeTags, content, downloadAssets, assetTypes, browser, userAgents, captureScreenshot, followDocumentUrls, downloadDocuments, documentMimeTypes, saveBrowserProfile, ssrf, maxDepth, maxPages, maxLinksPerPage, maxConcurrent, contentFilter, bm25Query, bm25Threshold, userAgent, rateLimitMs, auth, maxBodySize, mapLimit, mapSearch, maxAssetSize, proxy, documentUrlDepth, documentMaxSize, documentOutputDir, documentContentEncoding, warcOutput, browserProfile, ssrfDenyPrivateExplicit, dangerAcceptInvalidCerts) {
+    constructor(crawlStrategy, respectRobotsTxt, softHttpErrors, stayOnDomain, allowSubdomains, includePaths, excludePaths, pathPatternsMatchQuery, pathPatternsMatchUrl, dedupIncludeQuery, stripTrackingParams, trackingParams, customHeaders, requestTimeout, maxRedirects, retryCount, retryCodes, retryInitialDelayMs, retryMaxDelayMs, rateLimitJitterRatio, cookiesEnabled, removeTags, content, downloadAssets, assetTypes, browser, userAgents, captureScreenshot, followDocumentUrls, downloadDocuments, documentMimeTypes, saveBrowserProfile, ssrf, maxDepth, maxPages, maxLinksPerPage, maxConcurrent, contentFilter, bm25Query, bm25Threshold, userAgent, rateLimitMs, auth, maxBodySize, mapLimit, mapSearch, maxAssetSize, proxy, documentUrlDepth, documentMaxSize, documentOutputDir, documentContentEncoding, warcOutput, browserProfile, ssrfDenyPrivateExplicit, dangerAcceptInvalidCerts, subdomainHeaderAllowlist) {
         var ptr0 = isLikeNone(includePaths) ? 0 : passArrayJsValueToWasm0(includePaths, wasm.__wbindgen_malloc);
         var len0 = WASM_VECTOR_LEN;
         var ptr1 = isLikeNone(excludePaths) ? 0 : passArrayJsValueToWasm0(excludePaths, wasm.__wbindgen_malloc);
@@ -2808,7 +2879,9 @@ export class WasmCrawlConfig {
         var len16 = WASM_VECTOR_LEN;
         var ptr17 = isLikeNone(browserProfile) ? 0 : passStringToWasm0(browserProfile, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
         var len17 = WASM_VECTOR_LEN;
-        const ret = wasm.wasmcrawlconfig_new(isLikeNone(crawlStrategy) ? 4 : crawlStrategy, isLikeNone(respectRobotsTxt) ? 0xFFFFFF : respectRobotsTxt ? 1 : 0, isLikeNone(softHttpErrors) ? 0xFFFFFF : softHttpErrors ? 1 : 0, isLikeNone(stayOnDomain) ? 0xFFFFFF : stayOnDomain ? 1 : 0, isLikeNone(allowSubdomains) ? 0xFFFFFF : allowSubdomains ? 1 : 0, ptr0, len0, ptr1, len1, isLikeNone(pathPatternsMatchQuery) ? 0xFFFFFF : pathPatternsMatchQuery ? 1 : 0, isLikeNone(pathPatternsMatchUrl) ? 0xFFFFFF : pathPatternsMatchUrl ? 1 : 0, isLikeNone(dedupIncludeQuery) ? 0xFFFFFF : dedupIncludeQuery ? 1 : 0, isLikeNone(stripTrackingParams) ? 0xFFFFFF : stripTrackingParams ? 1 : 0, ptr2, len2, isLikeNone(customHeaders) ? 0 : addToExternrefTable0(customHeaders), !isLikeNone(requestTimeout), isLikeNone(requestTimeout) ? BigInt(0) : requestTimeout, isLikeNone(maxRedirects) ? Number.MAX_SAFE_INTEGER : (maxRedirects) >>> 0, isLikeNone(retryCount) ? Number.MAX_SAFE_INTEGER : (retryCount) >>> 0, ptr3, len3, !isLikeNone(retryInitialDelayMs), isLikeNone(retryInitialDelayMs) ? BigInt(0) : retryInitialDelayMs, !isLikeNone(retryMaxDelayMs), isLikeNone(retryMaxDelayMs) ? BigInt(0) : retryMaxDelayMs, !isLikeNone(rateLimitJitterRatio), isLikeNone(rateLimitJitterRatio) ? 0 : rateLimitJitterRatio, isLikeNone(cookiesEnabled) ? 0xFFFFFF : cookiesEnabled ? 1 : 0, ptr4, len4, ptr5, isLikeNone(downloadAssets) ? 0xFFFFFF : downloadAssets ? 1 : 0, ptr6, len6, ptr7, ptr8, len8, isLikeNone(captureScreenshot) ? 0xFFFFFF : captureScreenshot ? 1 : 0, isLikeNone(followDocumentUrls) ? 0xFFFFFF : followDocumentUrls ? 1 : 0, isLikeNone(downloadDocuments) ? 0xFFFFFF : downloadDocuments ? 1 : 0, ptr9, len9, isLikeNone(saveBrowserProfile) ? 0xFFFFFF : saveBrowserProfile ? 1 : 0, ptr10, isLikeNone(maxDepth) ? Number.MAX_SAFE_INTEGER : (maxDepth) >>> 0, isLikeNone(maxPages) ? Number.MAX_SAFE_INTEGER : (maxPages) >>> 0, isLikeNone(maxLinksPerPage) ? Number.MAX_SAFE_INTEGER : (maxLinksPerPage) >>> 0, isLikeNone(maxConcurrent) ? Number.MAX_SAFE_INTEGER : (maxConcurrent) >>> 0, isLikeNone(contentFilter) ? 1 : contentFilter, ptr11, len11, !isLikeNone(bm25Threshold), isLikeNone(bm25Threshold) ? 0 : bm25Threshold, ptr12, len12, !isLikeNone(rateLimitMs), isLikeNone(rateLimitMs) ? BigInt(0) : rateLimitMs, isLikeNone(auth) ? 0 : addToExternrefTable0(auth), isLikeNone(maxBodySize) ? Number.MAX_SAFE_INTEGER : (maxBodySize) >>> 0, isLikeNone(mapLimit) ? Number.MAX_SAFE_INTEGER : (mapLimit) >>> 0, ptr13, len13, isLikeNone(maxAssetSize) ? Number.MAX_SAFE_INTEGER : (maxAssetSize) >>> 0, ptr14, isLikeNone(documentUrlDepth) ? Number.MAX_SAFE_INTEGER : (documentUrlDepth) >>> 0, isLikeNone(documentMaxSize) ? Number.MAX_SAFE_INTEGER : (documentMaxSize) >>> 0, ptr15, len15, isLikeNone(documentContentEncoding) ? 1 : documentContentEncoding, ptr16, len16, ptr17, len17, isLikeNone(ssrfDenyPrivateExplicit) ? 0xFFFFFF : ssrfDenyPrivateExplicit ? 1 : 0, isLikeNone(dangerAcceptInvalidCerts) ? 0xFFFFFF : dangerAcceptInvalidCerts ? 1 : 0);
+        var ptr18 = isLikeNone(subdomainHeaderAllowlist) ? 0 : passArrayJsValueToWasm0(subdomainHeaderAllowlist, wasm.__wbindgen_malloc);
+        var len18 = WASM_VECTOR_LEN;
+        const ret = wasm.wasmcrawlconfig_new(isLikeNone(crawlStrategy) ? 4 : crawlStrategy, isLikeNone(respectRobotsTxt) ? 0xFFFFFF : respectRobotsTxt ? 1 : 0, isLikeNone(softHttpErrors) ? 0xFFFFFF : softHttpErrors ? 1 : 0, isLikeNone(stayOnDomain) ? 0xFFFFFF : stayOnDomain ? 1 : 0, isLikeNone(allowSubdomains) ? 0xFFFFFF : allowSubdomains ? 1 : 0, ptr0, len0, ptr1, len1, isLikeNone(pathPatternsMatchQuery) ? 0xFFFFFF : pathPatternsMatchQuery ? 1 : 0, isLikeNone(pathPatternsMatchUrl) ? 0xFFFFFF : pathPatternsMatchUrl ? 1 : 0, isLikeNone(dedupIncludeQuery) ? 0xFFFFFF : dedupIncludeQuery ? 1 : 0, isLikeNone(stripTrackingParams) ? 0xFFFFFF : stripTrackingParams ? 1 : 0, ptr2, len2, isLikeNone(customHeaders) ? 0 : addToExternrefTable0(customHeaders), !isLikeNone(requestTimeout), isLikeNone(requestTimeout) ? BigInt(0) : requestTimeout, isLikeNone(maxRedirects) ? Number.MAX_SAFE_INTEGER : (maxRedirects) >>> 0, isLikeNone(retryCount) ? Number.MAX_SAFE_INTEGER : (retryCount) >>> 0, ptr3, len3, !isLikeNone(retryInitialDelayMs), isLikeNone(retryInitialDelayMs) ? BigInt(0) : retryInitialDelayMs, !isLikeNone(retryMaxDelayMs), isLikeNone(retryMaxDelayMs) ? BigInt(0) : retryMaxDelayMs, !isLikeNone(rateLimitJitterRatio), isLikeNone(rateLimitJitterRatio) ? 0 : rateLimitJitterRatio, isLikeNone(cookiesEnabled) ? 0xFFFFFF : cookiesEnabled ? 1 : 0, ptr4, len4, ptr5, isLikeNone(downloadAssets) ? 0xFFFFFF : downloadAssets ? 1 : 0, ptr6, len6, ptr7, ptr8, len8, isLikeNone(captureScreenshot) ? 0xFFFFFF : captureScreenshot ? 1 : 0, isLikeNone(followDocumentUrls) ? 0xFFFFFF : followDocumentUrls ? 1 : 0, isLikeNone(downloadDocuments) ? 0xFFFFFF : downloadDocuments ? 1 : 0, ptr9, len9, isLikeNone(saveBrowserProfile) ? 0xFFFFFF : saveBrowserProfile ? 1 : 0, ptr10, isLikeNone(maxDepth) ? Number.MAX_SAFE_INTEGER : (maxDepth) >>> 0, isLikeNone(maxPages) ? Number.MAX_SAFE_INTEGER : (maxPages) >>> 0, isLikeNone(maxLinksPerPage) ? Number.MAX_SAFE_INTEGER : (maxLinksPerPage) >>> 0, isLikeNone(maxConcurrent) ? Number.MAX_SAFE_INTEGER : (maxConcurrent) >>> 0, isLikeNone(contentFilter) ? 1 : contentFilter, ptr11, len11, !isLikeNone(bm25Threshold), isLikeNone(bm25Threshold) ? 0 : bm25Threshold, ptr12, len12, !isLikeNone(rateLimitMs), isLikeNone(rateLimitMs) ? BigInt(0) : rateLimitMs, isLikeNone(auth) ? 0 : addToExternrefTable0(auth), isLikeNone(maxBodySize) ? Number.MAX_SAFE_INTEGER : (maxBodySize) >>> 0, isLikeNone(mapLimit) ? Number.MAX_SAFE_INTEGER : (mapLimit) >>> 0, ptr13, len13, isLikeNone(maxAssetSize) ? Number.MAX_SAFE_INTEGER : (maxAssetSize) >>> 0, ptr14, isLikeNone(documentUrlDepth) ? Number.MAX_SAFE_INTEGER : (documentUrlDepth) >>> 0, isLikeNone(documentMaxSize) ? Number.MAX_SAFE_INTEGER : (documentMaxSize) >>> 0, ptr15, len15, isLikeNone(documentContentEncoding) ? 1 : documentContentEncoding, ptr16, len16, ptr17, len17, isLikeNone(ssrfDenyPrivateExplicit) ? 0xFFFFFF : ssrfDenyPrivateExplicit ? 1 : 0, isLikeNone(dangerAcceptInvalidCerts) ? 0xFFFFFF : dangerAcceptInvalidCerts ? 1 : 0, ptr18, len18);
         this.__wbg_ptr = ret;
         WasmCrawlConfigFinalization.register(this, this.__wbg_ptr, this);
         return this;
@@ -3248,6 +3321,14 @@ export class WasmCrawlConfig {
     set stripTrackingParams(value) {
         wasm.wasmcrawlconfig_set_stripTrackingParams(this.__wbg_ptr, value);
     }
+    /**
+     * @param {string[]} value
+     */
+    set subdomainHeaderAllowlist(value) {
+        const ptr0 = passArrayJsValueToWasm0(value, wasm.__wbindgen_malloc);
+        const len0 = WASM_VECTOR_LEN;
+        wasm.wasmcrawlconfig_set_subdomainHeaderAllowlist(this.__wbg_ptr, ptr0, len0);
+    }
     /**
      * @param {string[]} value
      */
@@ -3319,6 +3400,15 @@ export class WasmCrawlConfig {
         const ret = wasm.wasmcrawlconfig_stripTrackingParams(this.__wbg_ptr);
         return ret !== 0;
     }
+    /**
+     * @returns {string[]}
+     */
+    get subdomainHeaderAllowlist() {
+        const ret = wasm.wasmcrawlconfig_subdomainHeaderAllowlist(this.__wbg_ptr);
+        var v1 = getArrayJsValueFromWasm0(ret[0], ret[1]);
+        wasm.__wbindgen_free(ret[0], ret[1] * 4, 4);
+        return v1;
+    }
     /**
      * @returns {string[]}
      */
@@ -9296,6 +9386,10 @@ export function __wbg_clearTimeout_1daad5c6ee4fcd3b(arg0) {
     const ret = clearTimeout(arg0);
     return ret;
 }
+export function __wbg_create_7febfb959f8c7046(arg0) {
+    const ret = Object.create(arg0);
+    return ret;
+}
 export function __wbg_done_b41a1d26cdb37fb6(arg0) {
     const ret = arg0.done;
     return ret;
@@ -9316,6 +9410,10 @@ export function __wbg_fetch_efc64baf325b084d(arg0, arg1) {
     const ret = arg0.fetch(arg1);
     return ret;
 }
+export function __wbg_for_beb5dc4e908dd8b8(arg0, arg1) {
+    const ret = Symbol.for(getStringFromWasm0(arg0, arg1));
+    return ret;
+}
 export function __wbg_getRandomValues_a608c4436c19407a() { return handleError(function (arg0, arg1) {
     globalThis.crypto.getRandomValues(getArrayU8FromWasm0(arg0, arg1));
 }, arguments); }
@@ -9419,6 +9517,10 @@ export function __wbg_new_2e41357094e99294() { return handleError(function () {
     const ret = new AbortController();
     return ret;
 }, arguments); }
+export function __wbg_new_343a093a3c2ffb4e(arg0, arg1) {
+    const ret = new Error(getStringFromWasm0(arg0, arg1));
+    return ret;
+}
 export function __wbg_new_617a8cdb8bb1130e() {
     const ret = new Object();
     return ret;
@@ -9499,6 +9601,10 @@ export function __wbg_setTimeout_b4a9096784635514(arg0, arg1) {
     const ret = setTimeout(arg0, arg1);
     return ret;
 }
+export function __wbg_set_145a351398b48c65() { return handleError(function (arg0, arg1, arg2) {
+    const ret = Reflect.set(arg0, arg1, arg2);
+    return ret;
+}, arguments); }
 export function __wbg_set_6ae97e73113c4f0b(arg0, arg1, arg2) {
     const ret = arg0.set(arg1, arg2);
     return ret;
@@ -9736,12 +9842,12 @@ export function __wbg_wasmsitemapurl_unwrap(arg0) {
     return ret;
 }
 export function __wbindgen_generic_0000000000000001(arg0, arg1) {
-    // Cast intrinsic for `Closure(Closure { owned: true, function: Function { arguments: [Externref], shim_idx: 1512, ret: Result(Unit), inner_ret: Some(Result(Unit)) }, mutable: true }) -> Externref`.
+    // Cast intrinsic for `Closure(Closure { owned: true, function: Function { arguments: [Externref], shim_idx: 1509, ret: Result(Unit), inner_ret: Some(Result(Unit)) }, mutable: true }) -> Externref`.
     const ret = makeMutClosure(arg0, arg1, wasm_bindgen_a37d5a31ee71bc5f___convert__closures_____invoke___wasm_bindgen_a37d5a31ee71bc5f___JsValue__core_37ee0ce0c4cd57d5___result__Result_____wasm_bindgen_a37d5a31ee71bc5f___JsError___true_);
     return ret;
 }
 export function __wbindgen_generic_0000000000000002(arg0, arg1) {
-    // Cast intrinsic for `Closure(Closure { owned: true, function: Function { arguments: [], shim_idx: 1354, ret: Unit, inner_ret: Some(Unit) }, mutable: true }) -> Externref`.
+    // Cast intrinsic for `Closure(Closure { owned: true, function: Function { arguments: [], shim_idx: 1351, ret: Unit, inner_ret: Some(Unit) }, mutable: true }) -> Externref`.
     const ret = makeMutClosure(arg0, arg1, wasm_bindgen_a37d5a31ee71bc5f___convert__closures_____invoke_______true_);
     return ret;
 }

package/pkg/bundler/crawlberg_wasm_bg.wasm.d.ts

@@ -138,12 +138,13 @@ export const wasmbrowserconfig_chromePath: (a: number) => [number, number];
 export const wasmbrowserconfig_clearProxy: (a: number) => void;
 export const wasmbrowserconfig_copyForTransfer: (a: number) => number;
 export const wasmbrowserconfig_default: () => number;
+export const wasmbrowserconfig_domQuietCap: (a: number) => [number, bigint];
 export const wasmbrowserconfig_domQuietWindow: (a: number) => [number, bigint];
 export const wasmbrowserconfig_endpoint: (a: number) => [number, number];
 export const wasmbrowserconfig_evalScript: (a: number) => [number, number];
 export const wasmbrowserconfig_extraWait: (a: number) => [number, bigint];
 export const wasmbrowserconfig_mode: (a: number) => [number, number];
-export const wasmbrowserconfig_new: (a: number, b: number, c: number, d: bigint, e: number, f: bigint, g: number, h: bigint, i: number, j: number, k: number, l: number, m: number, n: number, o: number, p: number, q: number, r: number, s: number, t: number, u: bigint, v: number, w: number, x: number, y: number, z: number, a1: number, b1: number, c1: number, d1: number, e1: bigint) => number;
+export const wasmbrowserconfig_new: (a: number, b: number, c: number, d: bigint, e: number, f: bigint, g: number, h: bigint, i: number, j: number, k: number, l: number, m: number, n: number, o: number, p: number, q: number, r: number, s: number, t: number, u: bigint, v: number, w: number, x: number, y: number, z: number, a1: number, b1: number, c1: number, d1: number, e1: bigint, f1: number, g1: bigint) => number;
 export const wasmbrowserconfig_overallTimeout: (a: number) => [number, bigint];
 export const wasmbrowserconfig_proxy: (a: number) => number;
 export const wasmbrowserconfig_robotsUserAgent: (a: number) => [number, number];
@@ -154,6 +155,7 @@ export const wasmbrowserconfig_set_blockUrlPatterns: (a: number, b: number, c: n
 export const wasmbrowserconfig_set_captureNetworkEvents: (a: number, b: number) => void;
 export const wasmbrowserconfig_set_chromeArgs: (a: number, b: number, c: number) => void;
 export const wasmbrowserconfig_set_chromePath: (a: number, b: number, c: number) => void;
+export const wasmbrowserconfig_set_domQuietCap: (a: number, b: number, c: bigint) => void;
 export const wasmbrowserconfig_set_domQuietWindow: (a: number, b: number, c: bigint) => void;
 export const wasmbrowserconfig_set_endpoint: (a: number, b: number, c: number) => void;
 export const wasmbrowserconfig_set_evalScript: (a: number, b: number, c: number) => void;
@@ -201,14 +203,18 @@ export const wasmcontentconfig_bullets: (a: number) => [number, number];
 export const wasmcontentconfig_compactTables: (a: number) => number;
 export const wasmcontentconfig_copyForTransfer: (a: number) => number;
 export const wasmcontentconfig_default: () => number;
+export const wasmcontentconfig_escapeAsterisks: (a: number) => number;
+export const wasmcontentconfig_escapeMisc: (a: number) => number;
 export const wasmcontentconfig_excludeSelectors: (a: number) => [number, number];
+export const wasmcontentconfig_expandAbbreviations: (a: number) => number;
 export const wasmcontentconfig_extractMetadata: (a: number) => number;
 export const wasmcontentconfig_includeDocumentStructure: (a: number) => number;
+export const wasmcontentconfig_includeIconLabels: (a: number) => number;
 export const wasmcontentconfig_keepHidden: (a: number) => number;
 export const wasmcontentconfig_keepInlineImagesIn: (a: number) => [number, number];
 export const wasmcontentconfig_listIndentWidth: (a: number) => number;
 export const wasmcontentconfig_maxDepth: (a: number) => number;
-export const wasmcontentconfig_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number, n: number, o: number, p: number, q: number, r: number, s: number, t: number, u: number, v: number, w: number, x: number, y: number, z: number, a1: number, b1: number) => number;
+export const wasmcontentconfig_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number, n: number, o: number, p: number, q: number, r: number, s: number, t: number, u: number, v: number, w: number, x: number, y: number, z: number, a1: number, b1: number, c1: number, d1: number, e1: number, f1: number) => number;
 export const wasmcontentconfig_outputFormat: (a: number) => [number, number];
 export const wasmcontentconfig_preprocessingPreset: (a: number) => [number, number];
 export const wasmcontentconfig_preserveTags: (a: number) => [number, number];
@@ -216,9 +222,13 @@ export const wasmcontentconfig_removeForms: (a: number) => number;
 export const wasmcontentconfig_removeNavigation: (a: number) => number;
 export const wasmcontentconfig_set_bullets: (a: number, b: number, c: number) => void;
 export const wasmcontentconfig_set_compactTables: (a: number, b: number) => void;
+export const wasmcontentconfig_set_escapeAsterisks: (a: number, b: number) => void;
+export const wasmcontentconfig_set_escapeMisc: (a: number, b: number) => void;
 export const wasmcontentconfig_set_excludeSelectors: (a: number, b: number, c: number) => void;
+export const wasmcontentconfig_set_expandAbbreviations: (a: number, b: number) => void;
 export const wasmcontentconfig_set_extractMetadata: (a: number, b: number) => void;
 export const wasmcontentconfig_set_includeDocumentStructure: (a: number, b: number) => void;
+export const wasmcontentconfig_set_includeIconLabels: (a: number, b: number) => void;
 export const wasmcontentconfig_set_keepHidden: (a: number, b: number) => void;
 export const wasmcontentconfig_set_keepInlineImagesIn: (a: number, b: number, c: number) => void;
 export const wasmcontentconfig_set_listIndentWidth: (a: number, b: number) => void;
@@ -288,7 +298,7 @@ export const wasmcrawlconfig_maxDepth: (a: number) => number;
 export const wasmcrawlconfig_maxLinksPerPage: (a: number) => number;
 export const wasmcrawlconfig_maxPages: (a: number) => number;
 export const wasmcrawlconfig_maxRedirects: (a: number) => number;
-export const wasmcrawlconfig_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number, n: number, o: number, p: number, q: number, r: bigint, s: number, t: number, u: number, v: number, w: number, x: bigint, y: number, z: bigint, a1: number, b1: number, c1: number, d1: number, e1: number, f1: number, g1: number, h1: number, i1: number, j1: number, k1: number, l1: number, m1: number, n1: number, o1: number, p1: number, q1: number, r1: number, s1: number, t1: number, u1: number, v1: number, w1: number, x1: number, y1: number, z1: number, a2: number, b2: number, c2: number, d2: number, e2: number, f2: bigint, g2: number, h2: number, i2: number, j2: number, k2: number, l2: number, m2: number, n2: number, o2: number, p2: number, q2: number, r2: number, s2: number, t2: number, u2: number, v2: number, w2: number, x2: number) => number;
+export const wasmcrawlconfig_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: number, m: number, n: number, o: number, p: number, q: number, r: bigint, s: number, t: number, u: number, v: number, w: number, x: bigint, y: number, z: bigint, a1: number, b1: number, c1: number, d1: number, e1: number, f1: number, g1: number, h1: number, i1: number, j1: number, k1: number, l1: number, m1: number, n1: number, o1: number, p1: number, q1: number, r1: number, s1: number, t1: number, u1: number, v1: number, w1: number, x1: number, y1: number, z1: number, a2: number, b2: number, c2: number, d2: number, e2: number, f2: bigint, g2: number, h2: number, i2: number, j2: number, k2: number, l2: number, m2: number, n2: number, o2: number, p2: number, q2: number, r2: number, s2: number, t2: number, u2: number, v2: number, w2: number, x2: number, y2: number, z2: number) => number;
 export const wasmcrawlconfig_pathPatternsMatchQuery: (a: number) => number;
 export const wasmcrawlconfig_pathPatternsMatchUrl: (a: number) => number;
 export const wasmcrawlconfig_proxy: (a: number) => number;
@@ -354,6 +364,7 @@ export const wasmcrawlconfig_set_ssrf: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_ssrfDenyPrivateExplicit: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_stayOnDomain: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_stripTrackingParams: (a: number, b: number) => void;
+export const wasmcrawlconfig_set_subdomainHeaderAllowlist: (a: number, b: number, c: number) => void;
 export const wasmcrawlconfig_set_trackingParams: (a: number, b: number, c: number) => void;
 export const wasmcrawlconfig_set_userAgent: (a: number, b: number, c: number) => void;
 export const wasmcrawlconfig_set_userAgents: (a: number, b: number, c: number) => void;
@@ -363,6 +374,7 @@ export const wasmcrawlconfig_ssrf: (a: number) => number;
 export const wasmcrawlconfig_ssrfDenyPrivateExplicit: (a: number) => number;
 export const wasmcrawlconfig_stayOnDomain: (a: number) => number;
 export const wasmcrawlconfig_stripTrackingParams: (a: number) => number;
+export const wasmcrawlconfig_subdomainHeaderAllowlist: (a: number) => [number, number];
 export const wasmcrawlconfig_trackingParams: (a: number) => [number, number];
 export const wasmcrawlconfig_userAgent: (a: number) => [number, number];
 export const wasmcrawlconfig_userAgents: (a: number) => [number, number];

package/pkg/bundler/package.json

@@ -6,7 +6,7 @@
     "Tobias Perlstein <tobias@xberg.io>"
   ],
   "description": "High-performance web crawling engine",
-  "version": "1.12.0",
+  "version": "1.12.3",
   "license": "MIT",
   "repository": {
     "type": "git",

package/pkg/deno/crawlberg_wasm.d.ts

@@ -260,7 +260,7 @@ export class WasmBrowserConfig {
      * keep a handle you already hold, build with `default()` and assign the property instead:
      * the generated setter borrows and leaves your handle alive.
      */
-    constructor(mode?: WasmBrowserMode | null, backend?: WasmBrowserBackend | null, timeout?: bigint | null, overallTimeout?: bigint | null, shutdownTimeout?: bigint | null, wait?: WasmBrowserWait | null, blockUrlPatterns?: string[] | null, captureNetworkEvents?: boolean | null, sessionAffinity?: boolean | null, chromeArgs?: string[] | null, endpoint?: string | null, waitSelector?: string | null, extraWait?: bigint | null, proxy?: WasmProxyConfig | null, evalScript?: string | null, robotsUserAgent?: string | null, chromePath?: string | null, scrollContainers?: boolean | null, domQuietWindow?: bigint | null);
+    constructor(mode?: WasmBrowserMode | null, backend?: WasmBrowserBackend | null, timeout?: bigint | null, overallTimeout?: bigint | null, shutdownTimeout?: bigint | null, wait?: WasmBrowserWait | null, blockUrlPatterns?: string[] | null, captureNetworkEvents?: boolean | null, sessionAffinity?: boolean | null, chromeArgs?: string[] | null, endpoint?: string | null, waitSelector?: string | null, extraWait?: bigint | null, proxy?: WasmProxyConfig | null, evalScript?: string | null, robotsUserAgent?: string | null, chromePath?: string | null, scrollContainers?: boolean | null, domQuietWindow?: bigint | null, domQuietCap?: bigint | null);
     get backend(): string;
     set backend(value: WasmBrowserBackend);
     blockUrlPatterns: string[];
@@ -268,6 +268,8 @@ export class WasmBrowserConfig {
     chromeArgs: string[];
     get chromePath(): string | undefined;
     set chromePath(value: string | null | undefined);
+    get domQuietCap(): bigint | undefined;
+    set domQuietCap(value: bigint | null | undefined);
     get domQuietWindow(): bigint | undefined;
     set domQuietWindow(value: bigint | null | undefined);
     get endpoint(): string | undefined;
@@ -430,12 +432,16 @@ export class WasmContentConfig {
      */
     copyForTransfer(): WasmContentConfig;
     static default(): WasmContentConfig;
-    constructor(outputFormat?: string | null, preprocessingPreset?: string | null, removeNavigation?: boolean | null, removeForms?: boolean | null, stripTags?: string[] | null, preserveTags?: string[] | null, excludeSelectors?: string[] | null, skipImages?: boolean | null, wrap?: boolean | null, wrapWidth?: number | null, includeDocumentStructure?: boolean | null, extractMetadata?: boolean | null, maxDepth?: number | null, stripNewlines?: boolean | null, bullets?: string | null, listIndentWidth?: number | null, strongEmSymbol?: string | null, compactTables?: boolean | null, keepInlineImagesIn?: string[] | null, keepHidden?: boolean | null);
+    constructor(outputFormat?: string | null, preprocessingPreset?: string | null, removeNavigation?: boolean | null, removeForms?: boolean | null, stripTags?: string[] | null, preserveTags?: string[] | null, excludeSelectors?: string[] | null, skipImages?: boolean | null, wrap?: boolean | null, wrapWidth?: number | null, includeDocumentStructure?: boolean | null, extractMetadata?: boolean | null, maxDepth?: number | null, stripNewlines?: boolean | null, bullets?: string | null, listIndentWidth?: number | null, strongEmSymbol?: string | null, compactTables?: boolean | null, keepInlineImagesIn?: string[] | null, keepHidden?: boolean | null, expandAbbreviations?: boolean | null, escapeMisc?: boolean | null, escapeAsterisks?: boolean | null, includeIconLabels?: boolean | null);
     bullets: string;
     compactTables: boolean;
+    escapeAsterisks: boolean;
+    escapeMisc: boolean;
     excludeSelectors: string[];
+    expandAbbreviations: boolean;
     extractMetadata: boolean;
     includeDocumentStructure: boolean;
+    includeIconLabels: boolean;
     keepHidden: boolean;
     keepInlineImagesIn: string[];
     listIndentWidth: number;
@@ -509,7 +515,7 @@ export class WasmCrawlConfig {
      * keep a handle you already hold, build with `default()` and assign the property instead:
      * the generated setter borrows and leaves your handle alive.
      */
-    constructor(crawlStrategy?: WasmCrawlStrategyKind | null, respectRobotsTxt?: boolean | null, softHttpErrors?: boolean | null, stayOnDomain?: boolean | null, allowSubdomains?: boolean | null, includePaths?: string[] | null, excludePaths?: string[] | null, pathPatternsMatchQuery?: boolean | null, pathPatternsMatchUrl?: boolean | null, dedupIncludeQuery?: boolean | null, stripTrackingParams?: boolean | null, trackingParams?: string[] | null, customHeaders?: any | null, requestTimeout?: bigint | null, maxRedirects?: number | null, retryCount?: number | null, retryCodes?: Uint16Array | null, retryInitialDelayMs?: bigint | null, retryMaxDelayMs?: bigint | null, rateLimitJitterRatio?: number | null, cookiesEnabled?: boolean | null, removeTags?: string[] | null, content?: WasmContentConfig | null, downloadAssets?: boolean | null, assetTypes?: WasmAssetCategory[] | null, browser?: WasmBrowserConfig | null, userAgents?: string[] | null, captureScreenshot?: boolean | null, followDocumentUrls?: boolean | null, downloadDocuments?: boolean | null, documentMimeTypes?: string[] | null, saveBrowserProfile?: boolean | null, ssrf?: WasmSsrfPolicy | null, maxDepth?: number | null, maxPages?: number | null, maxLinksPerPage?: number | null, maxConcurrent?: number | null, contentFilter?: WasmContentFilterKind | null, bm25Query?: string | null, bm25Threshold?: number | null, userAgent?: string | null, rateLimitMs?: bigint | null, auth?: any | null, maxBodySize?: number | null, mapLimit?: number | null, mapSearch?: string | null, maxAssetSize?: number | null, proxy?: WasmProxyConfig | null, documentUrlDepth?: number | null, documentMaxSize?: number | null, documentOutputDir?: string | null, documentContentEncoding?: WasmDocumentContentEncoding | null, warcOutput?: string | null, browserProfile?: string | null, ssrfDenyPrivateExplicit?: boolean | null, dangerAcceptInvalidCerts?: boolean | null);
+    constructor(crawlStrategy?: WasmCrawlStrategyKind | null, respectRobotsTxt?: boolean | null, softHttpErrors?: boolean | null, stayOnDomain?: boolean | null, allowSubdomains?: boolean | null, includePaths?: string[] | null, excludePaths?: string[] | null, pathPatternsMatchQuery?: boolean | null, pathPatternsMatchUrl?: boolean | null, dedupIncludeQuery?: boolean | null, stripTrackingParams?: boolean | null, trackingParams?: string[] | null, customHeaders?: any | null, requestTimeout?: bigint | null, maxRedirects?: number | null, retryCount?: number | null, retryCodes?: Uint16Array | null, retryInitialDelayMs?: bigint | null, retryMaxDelayMs?: bigint | null, rateLimitJitterRatio?: number | null, cookiesEnabled?: boolean | null, removeTags?: string[] | null, content?: WasmContentConfig | null, downloadAssets?: boolean | null, assetTypes?: WasmAssetCategory[] | null, browser?: WasmBrowserConfig | null, userAgents?: string[] | null, captureScreenshot?: boolean | null, followDocumentUrls?: boolean | null, downloadDocuments?: boolean | null, documentMimeTypes?: string[] | null, saveBrowserProfile?: boolean | null, ssrf?: WasmSsrfPolicy | null, maxDepth?: number | null, maxPages?: number | null, maxLinksPerPage?: number | null, maxConcurrent?: number | null, contentFilter?: WasmContentFilterKind | null, bm25Query?: string | null, bm25Threshold?: number | null, userAgent?: string | null, rateLimitMs?: bigint | null, auth?: any | null, maxBodySize?: number | null, mapLimit?: number | null, mapSearch?: string | null, maxAssetSize?: number | null, proxy?: WasmProxyConfig | null, documentUrlDepth?: number | null, documentMaxSize?: number | null, documentOutputDir?: string | null, documentContentEncoding?: WasmDocumentContentEncoding | null, warcOutput?: string | null, browserProfile?: string | null, ssrfDenyPrivateExplicit?: boolean | null, dangerAcceptInvalidCerts?: boolean | null, subdomainHeaderAllowlist?: string[] | null);
     /**
      * Validate the configuration, returning an error if any values are invalid.
      *
@@ -615,6 +621,7 @@ export class WasmCrawlConfig {
     ssrf: WasmSsrfPolicy;
     stayOnDomain: boolean;
     stripTrackingParams: boolean;
+    subdomainHeaderAllowlist: string[];
     trackingParams: string[];
     get userAgent(): string | undefined;
     set userAgent(value: string | null | undefined);