@xberg-io/crawlberg-wasm 1.2.1 → 1.3.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xberg-io/crawlberg-wasm",
-  "version": "1.2.1",
+  "version": "1.3.2",
   "private": false,
   "description": "High-performance web crawling engine",
   "license": "MIT",
@@ -13,15 +13,19 @@
     "access": "public"
   },
   "type": "module",
-  "files": [
-    "pkg",
-    "*.wasm",
-    "*.d.ts",
-    "README.md"
-  ],
+  "files": ["pkg", "*.wasm", "*.d.ts", "README.md"],
   "main": "pkg/nodejs/crawlberg_wasm.js",
   "module": "pkg/web/crawlberg_wasm.js",
   "types": "pkg/nodejs/crawlberg_wasm.d.ts",
+  "exports": {
+    ".": {
+      "types": "./pkg/nodejs/crawlberg_wasm.d.ts",
+      "browser": "./pkg/web/crawlberg_wasm.js",
+      "import": "./pkg/nodejs/crawlberg_wasm.js",
+      "require": "./pkg/nodejs/crawlberg_wasm.js",
+      "default": "./pkg/nodejs/crawlberg_wasm.js"
+    }
+  },
   "engines": {
     "node": ">= 22"
   },

package/pkg/bundler/crawlberg_wasm.d.ts

@@ -267,6 +267,13 @@ export class WasmContentConfig {
     wrapWidth: number;
 }
 
+/**
+ * Content filter applied to each crawled page before it reaches the result.
+ */
+export enum WasmContentFilterKind {
+    Bm25 = 0,
+}
+
 /**
  * Information about an HTTP cookie received from a response.
  */
@@ -290,7 +297,7 @@ export class WasmCrawlConfig {
     free(): void;
     [Symbol.dispose](): void;
     static default(): WasmCrawlConfig;
-    constructor(respectRobotsTxt?: boolean | null, softHttpErrors?: boolean | null, stayOnDomain?: boolean | null, allowSubdomains?: boolean | null, includePaths?: string[] | null, excludePaths?: string[] | null, customHeaders?: any | null, requestTimeout?: bigint | null, maxRedirects?: number | null, retryCount?: number | null, retryCodes?: Uint16Array | null, cookiesEnabled?: boolean | null, removeTags?: string[] | null, content?: WasmContentConfig | null, downloadAssets?: boolean | null, assetTypes?: any[] | null, browser?: WasmBrowserConfig | null, userAgents?: string[] | null, captureScreenshot?: boolean | null, followDocumentUrls?: boolean | null, downloadDocuments?: boolean | null, documentMimeTypes?: string[] | null, saveBrowserProfile?: boolean | null, ssrf?: WasmSsrfPolicy | null, maxDepth?: number | null, maxPages?: number | null, maxLinksPerPage?: number | null, maxConcurrent?: number | null, userAgent?: string | null, rateLimitMs?: bigint | null, auth?: any | null, maxBodySize?: number | null, mapLimit?: number | null, mapSearch?: string | null, maxAssetSize?: number | null, proxy?: WasmProxyConfig | null, documentUrlDepth?: number | null, documentMaxSize?: number | null, documentOutputDir?: string | null, documentContentEncoding?: WasmDocumentContentEncoding | null, warcOutput?: string | null, browserProfile?: string | null, ssrfDenyPrivateExplicit?: boolean | null);
+    constructor(crawlStrategy?: WasmCrawlStrategyKind | null, respectRobotsTxt?: boolean | null, softHttpErrors?: boolean | null, stayOnDomain?: boolean | null, allowSubdomains?: boolean | null, includePaths?: string[] | null, excludePaths?: string[] | null, customHeaders?: any | null, requestTimeout?: bigint | null, maxRedirects?: number | null, retryCount?: number | null, retryCodes?: Uint16Array | null, cookiesEnabled?: boolean | null, removeTags?: string[] | null, content?: WasmContentConfig | null, downloadAssets?: boolean | null, assetTypes?: any[] | null, browser?: WasmBrowserConfig | null, userAgents?: string[] | null, captureScreenshot?: boolean | null, followDocumentUrls?: boolean | null, downloadDocuments?: boolean | null, documentMimeTypes?: string[] | null, saveBrowserProfile?: boolean | null, ssrf?: WasmSsrfPolicy | null, maxDepth?: number | null, maxPages?: number | null, maxLinksPerPage?: number | null, maxConcurrent?: number | null, contentFilter?: WasmContentFilterKind | null, bm25Query?: string | null, bm25Threshold?: number | null, userAgent?: string | null, rateLimitMs?: bigint | null, auth?: any | null, maxBodySize?: number | null, mapLimit?: number | null, mapSearch?: string | null, maxAssetSize?: number | null, proxy?: WasmProxyConfig | null, documentUrlDepth?: number | null, documentMaxSize?: number | null, documentOutputDir?: string | null, documentContentEncoding?: WasmDocumentContentEncoding | null, warcOutput?: string | null, browserProfile?: string | null, ssrfDenyPrivateExplicit?: boolean | null);
     /**
      * Validate the configuration, returning an error if any values are invalid.
      */
@@ -299,12 +306,20 @@ export class WasmCrawlConfig {
     assetTypes: string[];
     get auth(): any | undefined;
     set auth(value: any | null | undefined);
+    get bm25Query(): string | undefined;
+    set bm25Query(value: string | null | undefined);
+    get bm25Threshold(): number | undefined;
+    set bm25Threshold(value: number | null | undefined);
     browser: WasmBrowserConfig;
     get browserProfile(): string | undefined;
     set browserProfile(value: string | null | undefined);
     captureScreenshot: boolean;
     content: WasmContentConfig;
+    get contentFilter(): string | undefined;
+    set contentFilter(value: WasmContentFilterKind | null | undefined);
     cookiesEnabled: boolean;
+    get crawlStrategy(): string;
+    set crawlStrategy(value: WasmCrawlStrategyKind);
     customHeaders: any;
     get documentContentEncoding(): string | undefined;
     set documentContentEncoding(value: WasmDocumentContentEncoding | null | undefined);
@@ -438,6 +453,20 @@ export class WasmCrawlResult {
     wasSkipped: boolean;
 }
 
+/**
+ * Traversal order for a crawl.
+ *
+ * Selects both the queue discipline and the selection strategy, because global order is a
+ * property of the frontier: the engine hands its bounded selection window to the strategy, so
+ * a strategy alone can only reorder URLs that have already been dequeued.
+ */
+export enum WasmCrawlStrategyKind {
+    Bfs = 0,
+    Dfs = 1,
+    BestFirst = 2,
+    Adaptive = 3,
+}
+
 /**
  * Opt-in encoding applied to a downloaded document's bytes for callers who need the
  * content available in a serializable field rather than reading it from disk.

package/pkg/bundler/crawlberg_wasm.js

@@ -5,5 +5,5 @@ import { __wbg_set_wasm } from "./crawlberg_wasm_bg.js";
 __wbg_set_wasm(wasm);
 wasm.__wbindgen_start();
 export {
-    WasmActionResult, WasmArticleMetadata, WasmAssetCategory, WasmAuthConfig, WasmBatchCrawlResult, WasmBatchCrawlResults, WasmBatchScrapeResult, WasmBatchScrapeResults, WasmBrowserBackend, WasmBrowserConfig, WasmBrowserExtras, WasmBrowserMode, WasmBrowserWait, WasmCitationReference, WasmCitationResult, WasmContentConfig, WasmCookieInfo, WasmCrawlConfig, WasmCrawlEngineHandle, WasmCrawlPageResult, WasmCrawlResult, WasmDocumentContentEncoding, WasmDownloadedAsset, WasmDownloadedDocument, WasmExtractionMeta, WasmFaviconInfo, WasmFeedInfo, WasmFeedType, WasmHeadingInfo, WasmHostMatcher, WasmHreflangEntry, WasmImageInfo, WasmImageSource, WasmInteractionResult, WasmJsonLdEntry, WasmLinkInfo, WasmLinkType, WasmMapResult, WasmMarkdownResult, WasmPageAction, WasmPageMetadata, WasmProxyConfig, WasmResponseMeta, WasmScrapeResult, WasmScrollDirection, WasmSitemapUrl, WasmSsrfPolicy, batchCrawl, batchScrape, crawl, createEngine, generateCitations, interact, mapUrls, scrape
+    WasmActionResult, WasmArticleMetadata, WasmAssetCategory, WasmAuthConfig, WasmBatchCrawlResult, WasmBatchCrawlResults, WasmBatchScrapeResult, WasmBatchScrapeResults, WasmBrowserBackend, WasmBrowserConfig, WasmBrowserExtras, WasmBrowserMode, WasmBrowserWait, WasmCitationReference, WasmCitationResult, WasmContentConfig, WasmContentFilterKind, WasmCookieInfo, WasmCrawlConfig, WasmCrawlEngineHandle, WasmCrawlPageResult, WasmCrawlResult, WasmCrawlStrategyKind, WasmDocumentContentEncoding, WasmDownloadedAsset, WasmDownloadedDocument, WasmExtractionMeta, WasmFaviconInfo, WasmFeedInfo, WasmFeedType, WasmHeadingInfo, WasmHostMatcher, WasmHreflangEntry, WasmImageInfo, WasmImageSource, WasmInteractionResult, WasmJsonLdEntry, WasmLinkInfo, WasmLinkType, WasmMapResult, WasmMarkdownResult, WasmPageAction, WasmPageMetadata, WasmProxyConfig, WasmResponseMeta, WasmScrapeResult, WasmScrollDirection, WasmSitemapUrl, WasmSsrfPolicy, batchCrawl, batchScrape, crawl, createEngine, generateCitations, interact, mapUrls, scrape
 } from "./crawlberg_wasm_bg.js";

package/pkg/bundler/crawlberg_wasm_bg.js

@@ -1766,6 +1766,14 @@ export class WasmContentConfig {
 }
 if (Symbol.dispose) WasmContentConfig.prototype[Symbol.dispose] = WasmContentConfig.prototype.free;
 
+/**
+ * Content filter applied to each crawled page before it reaches the result.
+ * @enum {0}
+ */
+export const WasmContentFilterKind = Object.freeze({
+    Bm25: 0, "0": "Bm25",
+});
+
 /**
  * Information about an HTTP cookie received from a response.
  */
@@ -1951,6 +1959,25 @@ export class WasmCrawlConfig {
         const ret = wasm.wasmcrawlconfig_auth(this.__wbg_ptr);
         return ret;
     }
+    /**
+     * @returns {string | undefined}
+     */
+    get bm25Query() {
+        const ret = wasm.wasmcrawlconfig_bm25Query(this.__wbg_ptr);
+        let v1;
+        if (ret[0] !== 0) {
+            v1 = getStringFromWasm0(ret[0], ret[1]);
+            wasm.__wbindgen_free(ret[0], ret[1] * 1, 1);
+        }
+        return v1;
+    }
+    /**
+     * @returns {number | undefined}
+     */
+    get bm25Threshold() {
+        const ret = wasm.wasmcrawlconfig_bm25Threshold(this.__wbg_ptr);
+        return ret[0] === 0 ? undefined : ret[1];
+    }
     /**
      * @returns {WasmBrowserConfig}
      */
@@ -1984,6 +2011,18 @@ export class WasmCrawlConfig {
         const ret = wasm.wasmcrawlconfig_content(this.__wbg_ptr);
         return WasmContentConfig.__wrap(ret);
     }
+    /**
+     * @returns {string | undefined}
+     */
+    get contentFilter() {
+        const ret = wasm.wasmcrawlconfig_contentFilter(this.__wbg_ptr);
+        let v1;
+        if (ret[0] !== 0) {
+            v1 = getStringFromWasm0(ret[0], ret[1]);
+            wasm.__wbindgen_free(ret[0], ret[1] * 1, 1);
+        }
+        return v1;
+    }
     /**
      * @returns {boolean}
      */
@@ -1991,6 +2030,21 @@ export class WasmCrawlConfig {
         const ret = wasm.wasmcrawlconfig_cookiesEnabled(this.__wbg_ptr);
         return ret !== 0;
     }
+    /**
+     * @returns {string}
+     */
+    get crawlStrategy() {
+        let deferred1_0;
+        let deferred1_1;
+        try {
+            const ret = wasm.wasmcrawlconfig_crawlStrategy(this.__wbg_ptr);
+            deferred1_0 = ret[0];
+            deferred1_1 = ret[1];
+            return getStringFromWasm0(ret[0], ret[1]);
+        } finally {
+            wasm.__wbindgen_free(deferred1_0, deferred1_1, 1);
+        }
+    }
     /**
      * @returns {any}
      */
@@ -2160,6 +2214,7 @@ export class WasmCrawlConfig {
         return ret >>> 0;
     }
     /**
+     * @param {WasmCrawlStrategyKind | null} [crawlStrategy]
      * @param {boolean | null} [respectRobotsTxt]
      * @param {boolean | null} [softHttpErrors]
      * @param {boolean | null} [stayOnDomain]
@@ -2188,6 +2243,9 @@ export class WasmCrawlConfig {
      * @param {number | null} [maxPages]
      * @param {number | null} [maxLinksPerPage]
      * @param {number | null} [maxConcurrent]
+     * @param {WasmContentFilterKind | null} [contentFilter]
+     * @param {string | null} [bm25Query]
+     * @param {number | null} [bm25Threshold]
      * @param {string | null} [userAgent]
      * @param {bigint | null} [rateLimitMs]
      * @param {any | null} [auth]
@@ -2204,7 +2262,7 @@ export class WasmCrawlConfig {
      * @param {string | null} [browserProfile]
      * @param {boolean | null} [ssrfDenyPrivateExplicit]
      */
-    constructor(respectRobotsTxt, softHttpErrors, stayOnDomain, allowSubdomains, includePaths, excludePaths, customHeaders, requestTimeout, maxRedirects, retryCount, retryCodes, cookiesEnabled, removeTags, content, downloadAssets, assetTypes, browser, userAgents, captureScreenshot, followDocumentUrls, downloadDocuments, documentMimeTypes, saveBrowserProfile, ssrf, maxDepth, maxPages, maxLinksPerPage, maxConcurrent, userAgent, rateLimitMs, auth, maxBodySize, mapLimit, mapSearch, maxAssetSize, proxy, documentUrlDepth, documentMaxSize, documentOutputDir, documentContentEncoding, warcOutput, browserProfile, ssrfDenyPrivateExplicit) {
+    constructor(crawlStrategy, respectRobotsTxt, softHttpErrors, stayOnDomain, allowSubdomains, includePaths, excludePaths, customHeaders, requestTimeout, maxRedirects, retryCount, retryCodes, cookiesEnabled, removeTags, content, downloadAssets, assetTypes, browser, userAgents, captureScreenshot, followDocumentUrls, downloadDocuments, documentMimeTypes, saveBrowserProfile, ssrf, maxDepth, maxPages, maxLinksPerPage, maxConcurrent, contentFilter, bm25Query, bm25Threshold, userAgent, rateLimitMs, auth, maxBodySize, mapLimit, mapSearch, maxAssetSize, proxy, documentUrlDepth, documentMaxSize, documentOutputDir, documentContentEncoding, warcOutput, browserProfile, ssrfDenyPrivateExplicit) {
         var ptr0 = isLikeNone(includePaths) ? 0 : passArrayJsValueToWasm0(includePaths, wasm.__wbindgen_malloc);
         var len0 = WASM_VECTOR_LEN;
         var ptr1 = isLikeNone(excludePaths) ? 0 : passArrayJsValueToWasm0(excludePaths, wasm.__wbindgen_malloc);
@@ -2234,22 +2292,24 @@ export class WasmCrawlConfig {
             _assertClass(ssrf, WasmSsrfPolicy);
             ptr9 = ssrf.__destroy_into_raw();
         }
-        var ptr10 = isLikeNone(userAgent) ? 0 : passStringToWasm0(userAgent, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+        var ptr10 = isLikeNone(bm25Query) ? 0 : passStringToWasm0(bm25Query, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
         var len10 = WASM_VECTOR_LEN;
-        var ptr11 = isLikeNone(mapSearch) ? 0 : passStringToWasm0(mapSearch, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+        var ptr11 = isLikeNone(userAgent) ? 0 : passStringToWasm0(userAgent, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
         var len11 = WASM_VECTOR_LEN;
-        let ptr12 = 0;
+        var ptr12 = isLikeNone(mapSearch) ? 0 : passStringToWasm0(mapSearch, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+        var len12 = WASM_VECTOR_LEN;
+        let ptr13 = 0;
         if (!isLikeNone(proxy)) {
             _assertClass(proxy, WasmProxyConfig);
-            ptr12 = proxy.__destroy_into_raw();
+            ptr13 = proxy.__destroy_into_raw();
         }
-        var ptr13 = isLikeNone(documentOutputDir) ? 0 : passStringToWasm0(documentOutputDir, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
-        var len13 = WASM_VECTOR_LEN;
-        var ptr14 = isLikeNone(warcOutput) ? 0 : passStringToWasm0(warcOutput, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+        var ptr14 = isLikeNone(documentOutputDir) ? 0 : passStringToWasm0(documentOutputDir, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
         var len14 = WASM_VECTOR_LEN;
-        var ptr15 = isLikeNone(browserProfile) ? 0 : passStringToWasm0(browserProfile, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+        var ptr15 = isLikeNone(warcOutput) ? 0 : passStringToWasm0(warcOutput, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
         var len15 = WASM_VECTOR_LEN;
-        const ret = wasm.wasmcrawlconfig_new(isLikeNone(respectRobotsTxt) ? 0xFFFFFF : respectRobotsTxt ? 1 : 0, isLikeNone(softHttpErrors) ? 0xFFFFFF : softHttpErrors ? 1 : 0, isLikeNone(stayOnDomain) ? 0xFFFFFF : stayOnDomain ? 1 : 0, isLikeNone(allowSubdomains) ? 0xFFFFFF : allowSubdomains ? 1 : 0, ptr0, len0, ptr1, len1, isLikeNone(customHeaders) ? 0 : addToExternrefTable0(customHeaders), !isLikeNone(requestTimeout), isLikeNone(requestTimeout) ? BigInt(0) : requestTimeout, isLikeNone(maxRedirects) ? Number.MAX_SAFE_INTEGER : (maxRedirects) >>> 0, isLikeNone(retryCount) ? Number.MAX_SAFE_INTEGER : (retryCount) >>> 0, ptr2, len2, isLikeNone(cookiesEnabled) ? 0xFFFFFF : cookiesEnabled ? 1 : 0, ptr3, len3, ptr4, isLikeNone(downloadAssets) ? 0xFFFFFF : downloadAssets ? 1 : 0, ptr5, len5, ptr6, ptr7, len7, isLikeNone(captureScreenshot) ? 0xFFFFFF : captureScreenshot ? 1 : 0, isLikeNone(followDocumentUrls) ? 0xFFFFFF : followDocumentUrls ? 1 : 0, isLikeNone(downloadDocuments) ? 0xFFFFFF : downloadDocuments ? 1 : 0, ptr8, len8, isLikeNone(saveBrowserProfile) ? 0xFFFFFF : saveBrowserProfile ? 1 : 0, ptr9, isLikeNone(maxDepth) ? Number.MAX_SAFE_INTEGER : (maxDepth) >>> 0, isLikeNone(maxPages) ? Number.MAX_SAFE_INTEGER : (maxPages) >>> 0, isLikeNone(maxLinksPerPage) ? Number.MAX_SAFE_INTEGER : (maxLinksPerPage) >>> 0, isLikeNone(maxConcurrent) ? Number.MAX_SAFE_INTEGER : (maxConcurrent) >>> 0, ptr10, len10, !isLikeNone(rateLimitMs), isLikeNone(rateLimitMs) ? BigInt(0) : rateLimitMs, isLikeNone(auth) ? 0 : addToExternrefTable0(auth), isLikeNone(maxBodySize) ? Number.MAX_SAFE_INTEGER : (maxBodySize) >>> 0, isLikeNone(mapLimit) ? Number.MAX_SAFE_INTEGER : (mapLimit) >>> 0, ptr11, len11, isLikeNone(maxAssetSize) ? Number.MAX_SAFE_INTEGER : (maxAssetSize) >>> 0, ptr12, isLikeNone(documentUrlDepth) ? Number.MAX_SAFE_INTEGER : (documentUrlDepth) >>> 0, isLikeNone(documentMaxSize) ? Number.MAX_SAFE_INTEGER : (documentMaxSize) >>> 0, ptr13, len13, isLikeNone(documentContentEncoding) ? 1 : documentContentEncoding, ptr14, len14, ptr15, len15, isLikeNone(ssrfDenyPrivateExplicit) ? 0xFFFFFF : ssrfDenyPrivateExplicit ? 1 : 0);
+        var ptr16 = isLikeNone(browserProfile) ? 0 : passStringToWasm0(browserProfile, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+        var len16 = WASM_VECTOR_LEN;
+        const ret = wasm.wasmcrawlconfig_new(isLikeNone(crawlStrategy) ? 4 : crawlStrategy, isLikeNone(respectRobotsTxt) ? 0xFFFFFF : respectRobotsTxt ? 1 : 0, isLikeNone(softHttpErrors) ? 0xFFFFFF : softHttpErrors ? 1 : 0, isLikeNone(stayOnDomain) ? 0xFFFFFF : stayOnDomain ? 1 : 0, isLikeNone(allowSubdomains) ? 0xFFFFFF : allowSubdomains ? 1 : 0, ptr0, len0, ptr1, len1, isLikeNone(customHeaders) ? 0 : addToExternrefTable0(customHeaders), !isLikeNone(requestTimeout), isLikeNone(requestTimeout) ? BigInt(0) : requestTimeout, isLikeNone(maxRedirects) ? Number.MAX_SAFE_INTEGER : (maxRedirects) >>> 0, isLikeNone(retryCount) ? Number.MAX_SAFE_INTEGER : (retryCount) >>> 0, ptr2, len2, isLikeNone(cookiesEnabled) ? 0xFFFFFF : cookiesEnabled ? 1 : 0, ptr3, len3, ptr4, isLikeNone(downloadAssets) ? 0xFFFFFF : downloadAssets ? 1 : 0, ptr5, len5, ptr6, ptr7, len7, isLikeNone(captureScreenshot) ? 0xFFFFFF : captureScreenshot ? 1 : 0, isLikeNone(followDocumentUrls) ? 0xFFFFFF : followDocumentUrls ? 1 : 0, isLikeNone(downloadDocuments) ? 0xFFFFFF : downloadDocuments ? 1 : 0, ptr8, len8, isLikeNone(saveBrowserProfile) ? 0xFFFFFF : saveBrowserProfile ? 1 : 0, ptr9, isLikeNone(maxDepth) ? Number.MAX_SAFE_INTEGER : (maxDepth) >>> 0, isLikeNone(maxPages) ? Number.MAX_SAFE_INTEGER : (maxPages) >>> 0, isLikeNone(maxLinksPerPage) ? Number.MAX_SAFE_INTEGER : (maxLinksPerPage) >>> 0, isLikeNone(maxConcurrent) ? Number.MAX_SAFE_INTEGER : (maxConcurrent) >>> 0, isLikeNone(contentFilter) ? 1 : contentFilter, ptr10, len10, !isLikeNone(bm25Threshold), isLikeNone(bm25Threshold) ? 0 : bm25Threshold, ptr11, len11, !isLikeNone(rateLimitMs), isLikeNone(rateLimitMs) ? BigInt(0) : rateLimitMs, isLikeNone(auth) ? 0 : addToExternrefTable0(auth), isLikeNone(maxBodySize) ? Number.MAX_SAFE_INTEGER : (maxBodySize) >>> 0, isLikeNone(mapLimit) ? Number.MAX_SAFE_INTEGER : (mapLimit) >>> 0, ptr12, len12, isLikeNone(maxAssetSize) ? Number.MAX_SAFE_INTEGER : (maxAssetSize) >>> 0, ptr13, isLikeNone(documentUrlDepth) ? Number.MAX_SAFE_INTEGER : (documentUrlDepth) >>> 0, isLikeNone(documentMaxSize) ? Number.MAX_SAFE_INTEGER : (documentMaxSize) >>> 0, ptr14, len14, isLikeNone(documentContentEncoding) ? 1 : documentContentEncoding, ptr15, len15, ptr16, len16, isLikeNone(ssrfDenyPrivateExplicit) ? 0xFFFFFF : ssrfDenyPrivateExplicit ? 1 : 0);
         this.__wbg_ptr = ret;
         WasmCrawlConfigFinalization.register(this, this.__wbg_ptr, this);
         return this;
@@ -2334,6 +2394,20 @@ export class WasmCrawlConfig {
     set auth(value) {
         wasm.wasmcrawlconfig_set_auth(this.__wbg_ptr, isLikeNone(value) ? 0 : addToExternrefTable0(value));
     }
+    /**
+     * @param {string | null} [value]
+     */
+    set bm25Query(value) {
+        var ptr0 = isLikeNone(value) ? 0 : passStringToWasm0(value, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+        var len0 = WASM_VECTOR_LEN;
+        wasm.wasmcrawlconfig_set_bm25Query(this.__wbg_ptr, ptr0, len0);
+    }
+    /**
+     * @param {number | null} [value]
+     */
+    set bm25Threshold(value) {
+        wasm.wasmcrawlconfig_set_bm25Threshold(this.__wbg_ptr, !isLikeNone(value), isLikeNone(value) ? 0 : value);
+    }
     /**
      * @param {WasmBrowserConfig} value
      */
@@ -2364,12 +2438,24 @@ export class WasmCrawlConfig {
         var ptr0 = value.__destroy_into_raw();
         wasm.wasmcrawlconfig_set_content(this.__wbg_ptr, ptr0);
     }
+    /**
+     * @param {WasmContentFilterKind | null} [value]
+     */
+    set contentFilter(value) {
+        wasm.wasmcrawlconfig_set_contentFilter(this.__wbg_ptr, isLikeNone(value) ? 1 : value);
+    }
     /**
      * @param {boolean} value
      */
     set cookiesEnabled(value) {
         wasm.wasmcrawlconfig_set_cookiesEnabled(this.__wbg_ptr, value);
     }
+    /**
+     * @param {WasmCrawlStrategyKind} value
+     */
+    set crawlStrategy(value) {
+        wasm.wasmcrawlconfig_set_crawlStrategy(this.__wbg_ptr, value);
+    }
     /**
      * @param {any} value
      */
@@ -3360,6 +3446,21 @@ export class WasmCrawlResult {
 }
 if (Symbol.dispose) WasmCrawlResult.prototype[Symbol.dispose] = WasmCrawlResult.prototype.free;
 
+/**
+ * Traversal order for a crawl.
+ *
+ * Selects both the queue discipline and the selection strategy, because global order is a
+ * property of the frontier: the engine hands its bounded selection window to the strategy, so
+ * a strategy alone can only reorder URLs that have already been dequeued.
+ * @enum {0 | 1 | 2 | 3}
+ */
+export const WasmCrawlStrategyKind = Object.freeze({
+    Bfs: 0, "0": "Bfs",
+    Dfs: 1, "1": "Dfs",
+    BestFirst: 2, "2": "BestFirst",
+    Adaptive: 3, "3": "Adaptive",
+});
+
 /**
  * Opt-in encoding applied to a downloaded document's bytes for callers who need the
  * content available in a serializable field rather than reading it from disk.
@@ -7951,7 +8052,7 @@ export function __wbg_new_typed_cceaf62d8d95e9f2(arg0, arg1) {
             const a = state0.a;
             state0.a = 0;
             try {
-                return wasm_bindgen__convert__closures_____invoke__h06ea3d672ad56c9d(a, state0.b, arg0, arg1);
+                return wasm_bindgen__convert__closures_____invoke__h072ef20b96c52fc3(a, state0.b, arg0, arg1);
             } finally {
                 state0.a = a;
             }
@@ -7974,10 +8075,18 @@ export function __wbg_next_8f26b64fa5e9f64b(arg0) {
     const ret = arg0.next;
     return ret;
 }
+export function __wbg_now_e7c6795a7f81e10f(arg0) {
+    const ret = arg0.now();
+    return ret;
+}
 export function __wbg_parse_1cc93481b0865939() { return handleError(function (arg0, arg1) {
     const ret = JSON.parse(getStringFromWasm0(arg0, arg1));
     return ret;
 }, arguments); }
+export function __wbg_performance_3fcf6e32a7e1ed0a(arg0) {
+    const ret = arg0.performance;
+    return ret;
+}
 export function __wbg_prototypesetcall_de8e0d9553586985(arg0, arg1, arg2) {
     Uint8Array.prototype.set.call(getArrayU8FromWasm0(arg0, arg1), arg2);
 }
@@ -8231,13 +8340,13 @@ export function __wbg_wasmsitemapurl_unwrap(arg0) {
     return ret;
 }
 export function __wbindgen_cast_0000000000000001(arg0, arg1) {
-    // Cast intrinsic for `Closure(Closure { owned: true, function: Function { arguments: [Externref], shim_idx: 992, ret: Result(Unit), inner_ret: Some(Result(Unit)) }, mutable: true }) -> Externref`.
-    const ret = makeMutClosure(arg0, arg1, wasm_bindgen__convert__closures_____invoke__he29f1bb22c192c97);
+    // Cast intrinsic for `Closure(Closure { owned: true, function: Function { arguments: [Externref], shim_idx: 1069, ret: Result(Unit), inner_ret: Some(Result(Unit)) }, mutable: true }) -> Externref`.
+    const ret = makeMutClosure(arg0, arg1, wasm_bindgen__convert__closures_____invoke__h2deefa3c9140ef43);
     return ret;
 }
 export function __wbindgen_cast_0000000000000002(arg0, arg1) {
-    // Cast intrinsic for `Closure(Closure { owned: true, function: Function { arguments: [], shim_idx: 882, ret: Unit, inner_ret: Some(Unit) }, mutable: true }) -> Externref`.
-    const ret = makeMutClosure(arg0, arg1, wasm_bindgen__convert__closures_____invoke__hbe9f9227fe7d8a65);
+    // Cast intrinsic for `Closure(Closure { owned: true, function: Function { arguments: [], shim_idx: 954, ret: Unit, inner_ret: Some(Unit) }, mutable: true }) -> Externref`.
+    const ret = makeMutClosure(arg0, arg1, wasm_bindgen__convert__closures_____invoke__h99e891e0bdc2d97f);
     return ret;
 }
 export function __wbindgen_cast_0000000000000003(arg0) {
@@ -8269,19 +8378,19 @@ export function __wbindgen_init_externref_table() {
     table.set(offset + 2, true);
     table.set(offset + 3, false);
 }
-function wasm_bindgen__convert__closures_____invoke__hbe9f9227fe7d8a65(arg0, arg1) {
-    wasm.wasm_bindgen__convert__closures_____invoke__hbe9f9227fe7d8a65(arg0, arg1);
+function wasm_bindgen__convert__closures_____invoke__h99e891e0bdc2d97f(arg0, arg1) {
+    wasm.wasm_bindgen__convert__closures_____invoke__h99e891e0bdc2d97f(arg0, arg1);
 }
 
-function wasm_bindgen__convert__closures_____invoke__he29f1bb22c192c97(arg0, arg1, arg2) {
-    const ret = wasm.wasm_bindgen__convert__closures_____invoke__he29f1bb22c192c97(arg0, arg1, arg2);
+function wasm_bindgen__convert__closures_____invoke__h2deefa3c9140ef43(arg0, arg1, arg2) {
+    const ret = wasm.wasm_bindgen__convert__closures_____invoke__h2deefa3c9140ef43(arg0, arg1, arg2);
     if (ret[1]) {
         throw takeFromExternrefTable0(ret[0]);
     }
 }
 
-function wasm_bindgen__convert__closures_____invoke__h06ea3d672ad56c9d(arg0, arg1, arg2, arg3) {
-    wasm.wasm_bindgen__convert__closures_____invoke__h06ea3d672ad56c9d(arg0, arg1, arg2, arg3);
+function wasm_bindgen__convert__closures_____invoke__h072ef20b96c52fc3(arg0, arg1, arg2, arg3) {
+    wasm.wasm_bindgen__convert__closures_____invoke__h072ef20b96c52fc3(arg0, arg1, arg2, arg3);
 }
 
 

package/pkg/bundler/crawlberg_wasm_bg.wasm.d.ts

@@ -203,11 +203,15 @@ export const wasmcookieinfo_value: (a: number) => [number, number];
 export const wasmcrawlconfig_allowSubdomains: (a: number) => number;
 export const wasmcrawlconfig_assetTypes: (a: number) => [number, number];
 export const wasmcrawlconfig_auth: (a: number) => any;
+export const wasmcrawlconfig_bm25Query: (a: number) => [number, number];
+export const wasmcrawlconfig_bm25Threshold: (a: number) => [number, number];
 export const wasmcrawlconfig_browser: (a: number) => number;
 export const wasmcrawlconfig_browserProfile: (a: number) => [number, number];
 export const wasmcrawlconfig_captureScreenshot: (a: number) => number;
 export const wasmcrawlconfig_content: (a: number) => number;
+export const wasmcrawlconfig_contentFilter: (a: number) => [number, number];
 export const wasmcrawlconfig_cookiesEnabled: (a: number) => number;
+export const wasmcrawlconfig_crawlStrategy: (a: number) => [number, number];
 export const wasmcrawlconfig_customHeaders: (a: number) => any;
 export const wasmcrawlconfig_default: () => number;
 export const wasmcrawlconfig_documentContentEncoding: (a: number) => [number, number];
@@ -229,8 +233,9 @@ export const wasmcrawlconfig_maxDepth: (a: number) => number;
 export const wasmcrawlconfig_maxLinksPerPage: (a: number) => number;
 export const wasmcrawlconfig_maxPages: (a: number) => number;
 export const wasmcrawlconfig_maxRedirects: (a: number) => number;
-export const wasmcrawlconfig_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: bigint, l: number, m: number, n: number, o: number, p: number, q: number, r: number, s: number, t: number, u: number, v: number, w: number, x: number, y: number, z: number, a1: number, b1: number, c1: number, d1: number, e1: number, f1: number, g1: number, h1: number, i1: number, j1: number, k1: number, l1: number, m1: number, n1: bigint, o1: number, p1: number, q1: number, r1: number, s1: number, t1: number, u1: number, v1: number, w1: number, x1: number, y1: number, z1: number, a2: number, b2: number, c2: number, d2: number, e2: number) => number;
+export const wasmcrawlconfig_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number, j: number, k: number, l: bigint, m: number, n: number, o: number, p: number, q: number, r: number, s: number, t: number, u: number, v: number, w: number, x: number, y: number, z: number, a1: number, b1: number, c1: number, d1: number, e1: number, f1: number, g1: number, h1: number, i1: number, j1: number, k1: number, l1: number, m1: number, n1: number, o1: number, p1: number, q1: number, r1: number, s1: number, t1: bigint, u1: number, v1: number, w1: number, x1: number, y1: number, z1: number, a2: number, b2: number, c2: number, d2: number, e2: number, f2: number, g2: number, h2: number, i2: number, j2: number, k2: number) => number;
 export const wasmcrawlconfig_proxy: (a: number) => number;
+export const wasmcrawlconfig_rateLimitMs: (a: number) => [number, bigint];
 export const wasmcrawlconfig_removeTags: (a: number) => [number, number];
 export const wasmcrawlconfig_respectRobotsTxt: (a: number) => number;
 export const wasmcrawlconfig_retryCodes: (a: number) => [number, number];
@@ -239,11 +244,15 @@ export const wasmcrawlconfig_saveBrowserProfile: (a: number) => number;
 export const wasmcrawlconfig_set_allowSubdomains: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_assetTypes: (a: number, b: number, c: number) => void;
 export const wasmcrawlconfig_set_auth: (a: number, b: number) => void;
+export const wasmcrawlconfig_set_bm25Query: (a: number, b: number, c: number) => void;
+export const wasmcrawlconfig_set_bm25Threshold: (a: number, b: number, c: number) => void;
 export const wasmcrawlconfig_set_browser: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_browserProfile: (a: number, b: number, c: number) => void;
 export const wasmcrawlconfig_set_captureScreenshot: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_content: (a: number, b: number) => void;
+export const wasmcrawlconfig_set_contentFilter: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_cookiesEnabled: (a: number, b: number) => void;
+export const wasmcrawlconfig_set_crawlStrategy: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_customHeaders: (a: number, b: any) => void;
 export const wasmcrawlconfig_set_documentContentEncoding: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_documentMaxSize: (a: number, b: number) => void;
@@ -265,6 +274,7 @@ export const wasmcrawlconfig_set_maxLinksPerPage: (a: number, b: number) => void
 export const wasmcrawlconfig_set_maxPages: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_maxRedirects: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_proxy: (a: number, b: number) => void;
+export const wasmcrawlconfig_set_rateLimitMs: (a: number, b: number, c: bigint) => void;
 export const wasmcrawlconfig_set_removeTags: (a: number, b: number, c: number) => void;
 export const wasmcrawlconfig_set_respectRobotsTxt: (a: number, b: number) => void;
 export const wasmcrawlconfig_set_retryCodes: (a: number, b: number, c: number) => void;
@@ -381,14 +391,10 @@ export const wasmdownloadeddocument_size: (a: number) => number;
 export const wasmdownloadeddocument_truncated: (a: number) => number;
 export const wasmdownloadeddocument_url: (a: number) => [number, number];
 export const wasmextractionmeta_chunksProcessed: (a: number) => number;
-export const wasmextractionmeta_completionTokens: (a: number) => [number, bigint];
-export const wasmextractionmeta_cost: (a: number) => [number, number];
 export const wasmextractionmeta_default: () => number;
 export const wasmextractionmeta_model: (a: number) => [number, number];
 export const wasmextractionmeta_new: (a: number, b: number, c: number, d: number, e: bigint, f: number, g: bigint, h: number, i: number) => number;
 export const wasmextractionmeta_set_chunksProcessed: (a: number, b: number) => void;
-export const wasmextractionmeta_set_completionTokens: (a: number, b: number, c: bigint) => void;
-export const wasmextractionmeta_set_cost: (a: number, b: number, c: number) => void;
 export const wasmextractionmeta_set_model: (a: number, b: number, c: number) => void;
 export const wasmfaviconinfo_mimeType: (a: number) => [number, number];
 export const wasmfaviconinfo_rel: (a: number) => [number, number];
@@ -682,11 +688,13 @@ export const wasmssrfpolicy_set_allowlist: (a: number, b: any) => void;
 export const wasmssrfpolicy_set_denyPrivate: (a: number, b: number) => void;
 export const wasmssrfpolicy_set_maxRedirects: (a: number, b: number) => void;
 export const wasmhostmatcher_new: () => number;
-export const wasmcrawlconfig_set_rateLimitMs: (a: number, b: number, c: bigint) => void;
+export const wasmcrawlconfig_set_requestTimeout: (a: number, b: number, c: bigint) => void;
+export const wasmextractionmeta_set_completionTokens: (a: number, b: number, c: bigint) => void;
 export const wasmextractionmeta_set_promptTokens: (a: number, b: number, c: bigint) => void;
 export const wasmpageaction_set_milliseconds: (a: number, b: number, c: bigint) => void;
-export const wasmcrawlconfig_rateLimitMs: (a: number) => [number, bigint];
 export const wasmcrawlconfig_requestTimeout: (a: number) => [number, bigint];
+export const wasmextractionmeta_completionTokens: (a: number) => [number, bigint];
+export const wasmextractionmeta_cost: (a: number) => [number, number];
 export const wasmextractionmeta_promptTokens: (a: number) => [number, bigint];
 export const wasmpageaction_amount: (a: number) => [number, bigint];
 export const wasmpageaction_milliseconds: (a: number) => [number, bigint];
@@ -708,13 +716,13 @@ export const wasmimageinfo_width: (a: number) => number;
 export const wasmpagemetadata_wordCount: (a: number) => number;
 export const wasmlinkinfo_nofollow: (a: number) => number;
 export const wasmauthconfig_new: () => number;
-export const wasmcrawlconfig_set_requestTimeout: (a: number, b: number, c: bigint) => void;
+export const wasmextractionmeta_set_cost: (a: number, b: number, c: number) => void;
 export const wasmpageaction_set_amount: (a: number, b: number, c: bigint) => void;
 export const wasmscraperesult_set_crawlDelay: (a: number, b: number, c: bigint) => void;
 export const wasmpageaction_new: () => number;
-export const wasm_bindgen__convert__closures_____invoke__he29f1bb22c192c97: (a: number, b: number, c: any) => [number, number];
-export const wasm_bindgen__convert__closures_____invoke__h06ea3d672ad56c9d: (a: number, b: number, c: any, d: any) => void;
-export const wasm_bindgen__convert__closures_____invoke__hbe9f9227fe7d8a65: (a: number, b: number) => void;
+export const wasm_bindgen__convert__closures_____invoke__h2deefa3c9140ef43: (a: number, b: number, c: any) => [number, number];
+export const wasm_bindgen__convert__closures_____invoke__h072ef20b96c52fc3: (a: number, b: number, c: any, d: any) => void;
+export const wasm_bindgen__convert__closures_____invoke__h99e891e0bdc2d97f: (a: number, b: number) => void;
 export const __wbindgen_malloc: (a: number, b: number) => number;
 export const __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number;
 export const __wbindgen_exn_store: (a: number) => void;

package/pkg/bundler/package.json

@@ -2,7 +2,7 @@
   "name": "crawlberg-wasm",
   "type": "module",
   "description": "High-performance web crawling engine",
-  "version": "1.2.1",
+  "version": "1.3.2",
   "license": "MIT",
   "repository": {
     "type": "git",

package/pkg/deno/crawlberg_wasm.d.ts

@@ -267,6 +267,13 @@ export class WasmContentConfig {
     wrapWidth: number;
 }
 
+/**
+ * Content filter applied to each crawled page before it reaches the result.
+ */
+export enum WasmContentFilterKind {
+    Bm25 = 0,
+}
+
 /**
  * Information about an HTTP cookie received from a response.
  */
@@ -290,7 +297,7 @@ export class WasmCrawlConfig {
     free(): void;
     [Symbol.dispose](): void;
     static default(): WasmCrawlConfig;
-    constructor(respectRobotsTxt?: boolean | null, softHttpErrors?: boolean | null, stayOnDomain?: boolean | null, allowSubdomains?: boolean | null, includePaths?: string[] | null, excludePaths?: string[] | null, customHeaders?: any | null, requestTimeout?: bigint | null, maxRedirects?: number | null, retryCount?: number | null, retryCodes?: Uint16Array | null, cookiesEnabled?: boolean | null, removeTags?: string[] | null, content?: WasmContentConfig | null, downloadAssets?: boolean | null, assetTypes?: any[] | null, browser?: WasmBrowserConfig | null, userAgents?: string[] | null, captureScreenshot?: boolean | null, followDocumentUrls?: boolean | null, downloadDocuments?: boolean | null, documentMimeTypes?: string[] | null, saveBrowserProfile?: boolean | null, ssrf?: WasmSsrfPolicy | null, maxDepth?: number | null, maxPages?: number | null, maxLinksPerPage?: number | null, maxConcurrent?: number | null, userAgent?: string | null, rateLimitMs?: bigint | null, auth?: any | null, maxBodySize?: number | null, mapLimit?: number | null, mapSearch?: string | null, maxAssetSize?: number | null, proxy?: WasmProxyConfig | null, documentUrlDepth?: number | null, documentMaxSize?: number | null, documentOutputDir?: string | null, documentContentEncoding?: WasmDocumentContentEncoding | null, warcOutput?: string | null, browserProfile?: string | null, ssrfDenyPrivateExplicit?: boolean | null);
+    constructor(crawlStrategy?: WasmCrawlStrategyKind | null, respectRobotsTxt?: boolean | null, softHttpErrors?: boolean | null, stayOnDomain?: boolean | null, allowSubdomains?: boolean | null, includePaths?: string[] | null, excludePaths?: string[] | null, customHeaders?: any | null, requestTimeout?: bigint | null, maxRedirects?: number | null, retryCount?: number | null, retryCodes?: Uint16Array | null, cookiesEnabled?: boolean | null, removeTags?: string[] | null, content?: WasmContentConfig | null, downloadAssets?: boolean | null, assetTypes?: any[] | null, browser?: WasmBrowserConfig | null, userAgents?: string[] | null, captureScreenshot?: boolean | null, followDocumentUrls?: boolean | null, downloadDocuments?: boolean | null, documentMimeTypes?: string[] | null, saveBrowserProfile?: boolean | null, ssrf?: WasmSsrfPolicy | null, maxDepth?: number | null, maxPages?: number | null, maxLinksPerPage?: number | null, maxConcurrent?: number | null, contentFilter?: WasmContentFilterKind | null, bm25Query?: string | null, bm25Threshold?: number | null, userAgent?: string | null, rateLimitMs?: bigint | null, auth?: any | null, maxBodySize?: number | null, mapLimit?: number | null, mapSearch?: string | null, maxAssetSize?: number | null, proxy?: WasmProxyConfig | null, documentUrlDepth?: number | null, documentMaxSize?: number | null, documentOutputDir?: string | null, documentContentEncoding?: WasmDocumentContentEncoding | null, warcOutput?: string | null, browserProfile?: string | null, ssrfDenyPrivateExplicit?: boolean | null);
     /**
      * Validate the configuration, returning an error if any values are invalid.
      */
@@ -299,12 +306,20 @@ export class WasmCrawlConfig {
     assetTypes: string[];
     get auth(): any | undefined;
     set auth(value: any | null | undefined);
+    get bm25Query(): string | undefined;
+    set bm25Query(value: string | null | undefined);
+    get bm25Threshold(): number | undefined;
+    set bm25Threshold(value: number | null | undefined);
     browser: WasmBrowserConfig;
     get browserProfile(): string | undefined;
     set browserProfile(value: string | null | undefined);
     captureScreenshot: boolean;
     content: WasmContentConfig;
+    get contentFilter(): string | undefined;
+    set contentFilter(value: WasmContentFilterKind | null | undefined);
     cookiesEnabled: boolean;
+    get crawlStrategy(): string;
+    set crawlStrategy(value: WasmCrawlStrategyKind);
     customHeaders: any;
     get documentContentEncoding(): string | undefined;
     set documentContentEncoding(value: WasmDocumentContentEncoding | null | undefined);
@@ -438,6 +453,20 @@ export class WasmCrawlResult {
     wasSkipped: boolean;
 }
 
+/**
+ * Traversal order for a crawl.
+ *
+ * Selects both the queue discipline and the selection strategy, because global order is a
+ * property of the frontier: the engine hands its bounded selection window to the strategy, so
+ * a strategy alone can only reorder URLs that have already been dequeued.
+ */
+export enum WasmCrawlStrategyKind {
+    Bfs = 0,
+    Dfs = 1,
+    BestFirst = 2,
+    Adaptive = 3,
+}
+
 /**
  * Opt-in encoding applied to a downloaded document's bytes for callers who need the
  * content available in a serializable field rather than reading it from disk.