@xberg-io/crawlberg-wasm 1.1.4 → 1.3.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xberg-io/crawlberg-wasm",
-  "version": "1.1.4",
+  "version": "1.3.2",
   "private": false,
   "description": "High-performance web crawling engine",
   "license": "MIT",
@@ -13,15 +13,19 @@
     "access": "public"
   },
   "type": "module",
-  "files": [
-    "pkg",
-    "*.wasm",
-    "*.d.ts",
-    "README.md"
-  ],
+  "files": ["pkg", "*.wasm", "*.d.ts", "README.md"],
   "main": "pkg/nodejs/crawlberg_wasm.js",
   "module": "pkg/web/crawlberg_wasm.js",
   "types": "pkg/nodejs/crawlberg_wasm.d.ts",
+  "exports": {
+    ".": {
+      "types": "./pkg/nodejs/crawlberg_wasm.d.ts",
+      "browser": "./pkg/web/crawlberg_wasm.js",
+      "import": "./pkg/nodejs/crawlberg_wasm.js",
+      "require": "./pkg/nodejs/crawlberg_wasm.js",
+      "default": "./pkg/nodejs/crawlberg_wasm.js"
+    }
+  },
   "engines": {
     "node": ">= 22"
   },

package/pkg/bundler/crawlberg_wasm.d.ts

@@ -267,6 +267,13 @@ export class WasmContentConfig {
     wrapWidth: number;
 }
 
+/**
+ * Content filter applied to each crawled page before it reaches the result.
+ */
+export enum WasmContentFilterKind {
+    Bm25 = 0,
+}
+
 /**
  * Information about an HTTP cookie received from a response.
  */
@@ -290,7 +297,7 @@ export class WasmCrawlConfig {
     free(): void;
     [Symbol.dispose](): void;
     static default(): WasmCrawlConfig;
-    constructor(respectRobotsTxt?: boolean | null, softHttpErrors?: boolean | null, stayOnDomain?: boolean | null, allowSubdomains?: boolean | null, includePaths?: string[] | null, excludePaths?: string[] | null, customHeaders?: any | null, requestTimeout?: bigint | null, maxRedirects?: number | null, retryCount?: number | null, retryCodes?: Uint16Array | null, cookiesEnabled?: boolean | null, removeTags?: string[] | null, content?: WasmContentConfig | null, downloadAssets?: boolean | null, assetTypes?: any[] | null, browser?: WasmBrowserConfig | null, userAgents?: string[] | null, captureScreenshot?: boolean | null, followDocumentUrls?: boolean | null, downloadDocuments?: boolean | null, documentMimeTypes?: string[] | null, saveBrowserProfile?: boolean | null, ssrf?: WasmSsrfPolicy | null, maxDepth?: number | null, maxPages?: number | null, maxConcurrent?: number | null, userAgent?: string | null, rateLimitMs?: bigint | null, auth?: any | null, maxBodySize?: number | null, mapLimit?: number | null, mapSearch?: string | null, maxAssetSize?: number | null, proxy?: WasmProxyConfig | null, documentUrlDepth?: number | null, documentMaxSize?: number | null, warcOutput?: string | null, browserProfile?: string | null);
+    constructor(crawlStrategy?: WasmCrawlStrategyKind | null, respectRobotsTxt?: boolean | null, softHttpErrors?: boolean | null, stayOnDomain?: boolean | null, allowSubdomains?: boolean | null, includePaths?: string[] | null, excludePaths?: string[] | null, customHeaders?: any | null, requestTimeout?: bigint | null, maxRedirects?: number | null, retryCount?: number | null, retryCodes?: Uint16Array | null, cookiesEnabled?: boolean | null, removeTags?: string[] | null, content?: WasmContentConfig | null, downloadAssets?: boolean | null, assetTypes?: any[] | null, browser?: WasmBrowserConfig | null, userAgents?: string[] | null, captureScreenshot?: boolean | null, followDocumentUrls?: boolean | null, downloadDocuments?: boolean | null, documentMimeTypes?: string[] | null, saveBrowserProfile?: boolean | null, ssrf?: WasmSsrfPolicy | null, maxDepth?: number | null, maxPages?: number | null, maxLinksPerPage?: number | null, maxConcurrent?: number | null, contentFilter?: WasmContentFilterKind | null, bm25Query?: string | null, bm25Threshold?: number | null, userAgent?: string | null, rateLimitMs?: bigint | null, auth?: any | null, maxBodySize?: number | null, mapLimit?: number | null, mapSearch?: string | null, maxAssetSize?: number | null, proxy?: WasmProxyConfig | null, documentUrlDepth?: number | null, documentMaxSize?: number | null, documentOutputDir?: string | null, documentContentEncoding?: WasmDocumentContentEncoding | null, warcOutput?: string | null, browserProfile?: string | null, ssrfDenyPrivateExplicit?: boolean | null);
     /**
      * Validate the configuration, returning an error if any values are invalid.
      */
@@ -299,16 +306,28 @@ export class WasmCrawlConfig {
     assetTypes: string[];
     get auth(): any | undefined;
     set auth(value: any | null | undefined);
+    get bm25Query(): string | undefined;
+    set bm25Query(value: string | null | undefined);
+    get bm25Threshold(): number | undefined;
+    set bm25Threshold(value: number | null | undefined);
     browser: WasmBrowserConfig;
     get browserProfile(): string | undefined;
     set browserProfile(value: string | null | undefined);
     captureScreenshot: boolean;
     content: WasmContentConfig;
+    get contentFilter(): string | undefined;
+    set contentFilter(value: WasmContentFilterKind | null | undefined);
     cookiesEnabled: boolean;
+    get crawlStrategy(): string;
+    set crawlStrategy(value: WasmCrawlStrategyKind);
     customHeaders: any;
+    get documentContentEncoding(): string | undefined;
+    set documentContentEncoding(value: WasmDocumentContentEncoding | null | undefined);
     get documentMaxSize(): number | undefined;
     set documentMaxSize(value: number | null | undefined);
     documentMimeTypes: string[];
+    get documentOutputDir(): string | undefined;
+    set documentOutputDir(value: string | null | undefined);
     get documentUrlDepth(): number | undefined;
     set documentUrlDepth(value: number | null | undefined);
     downloadAssets: boolean;
@@ -328,6 +347,8 @@ export class WasmCrawlConfig {
     set maxConcurrent(value: number | null | undefined);
     get maxDepth(): number | undefined;
     set maxDepth(value: number | null | undefined);
+    get maxLinksPerPage(): number | undefined;
+    set maxLinksPerPage(value: number | null | undefined);
     get maxPages(): number | undefined;
     set maxPages(value: number | null | undefined);
     maxRedirects: number;
@@ -344,6 +365,8 @@ export class WasmCrawlConfig {
     saveBrowserProfile: boolean;
     softHttpErrors: boolean;
     ssrf: WasmSsrfPolicy;
+    get ssrfDenyPrivateExplicit(): boolean | undefined;
+    set ssrfDenyPrivateExplicit(value: boolean | null | undefined);
     stayOnDomain: boolean;
     get userAgent(): string | undefined;
     set userAgent(value: string | null | undefined);
@@ -410,6 +433,13 @@ export class WasmCrawlResult {
     constructor(pages?: WasmCrawlPageResult[] | null, finalUrl?: string | null, redirectCount?: number | null, wasSkipped?: boolean | null, cookies?: WasmCookieInfo[] | null, stayedOnDomain?: boolean | null, browserUsed?: boolean | null, error?: string | null);
     /**
      * Returns the count of unique normalized URLs encountered during crawling.
+     *
+     * Computed from `pages` (not the deprecated `normalized_urls` field) so it
+     * is correct across every binding that reconstructs `CrawlResult` from
+     * `pages` alone. In streaming mode `pages` is empty, so this returns 0 on
+     * the opaque-handle (C/Go/C#/Zig/Dart) path where it previously counted
+     * streamed pages — a known, accepted cost of making the other ten binding
+     * families correct.
      */
     uniqueNormalizedUrls(): number;
     browserUsed: boolean;
@@ -423,6 +453,32 @@ export class WasmCrawlResult {
     wasSkipped: boolean;
 }
 
+/**
+ * Traversal order for a crawl.
+ *
+ * Selects both the queue discipline and the selection strategy, because global order is a
+ * property of the frontier: the engine hands its bounded selection window to the strategy, so
+ * a strategy alone can only reorder URLs that have already been dequeued.
+ */
+export enum WasmCrawlStrategyKind {
+    Bfs = 0,
+    Dfs = 1,
+    BestFirst = 2,
+    Adaptive = 3,
+}
+
+/**
+ * Opt-in encoding applied to a downloaded document's bytes for callers who need the
+ * content available in a serializable field rather than reading it from disk.
+ *
+ * `None` (the `CrawlConfig.document_content_encoding` default) produces neither — unlike
+ * screenshots, base64-encoding a document by default would duplicate an already
+ * up-to-`document_max_size` buffer (50 MB default) in memory per document.
+ */
+export enum WasmDocumentContentEncoding {
+    Base64 = 0,
+}
+
 /**
  * A downloaded asset from a page.
  */
@@ -453,13 +509,18 @@ export class WasmDownloadedDocument {
     free(): void;
     [Symbol.dispose](): void;
     static default(): WasmDownloadedDocument;
-    constructor(url?: string | null, mimeType?: string | null, size?: number | null, contentHash?: string | null, headers?: any | null, filename?: string | null);
+    constructor(url?: string | null, mimeType?: string | null, size?: number | null, contentHash?: string | null, headers?: any | null, truncated?: boolean | null, filename?: string | null, contentPath?: string | null, contentBase64?: string | null);
+    get contentBase64(): string | undefined;
+    set contentBase64(value: string | null | undefined);
     contentHash: string;
+    get contentPath(): string | undefined;
+    set contentPath(value: string | null | undefined);
     get filename(): string | undefined;
     set filename(value: string | null | undefined);
     headers: any;
     mimeType: string;
     size: number;
+    truncated: boolean;
     url: string;
 }
 
@@ -534,6 +595,33 @@ export class WasmHeadingInfo {
     text: string;
 }
 
+/**
+ * Hostname/IP allowlist matcher for SSRF policy.
+ *
+ * Serializes as an internally-tagged object so each variant is distinguishable on the
+ * wire and round-trips losslessly:
+ *
+ * ```json
+ * {"type": "exact",  "value": "api.example.com"}
+ * {"type": "suffix", "value": ".example.com"}
+ * {"type": "cidr",   "value": "10.0.0.0/8"}
+ * ```
+ *
+ * A bare JSON string is still accepted on deserialization and resolves to `Exact`,
+ * preserving configs written against the previous untagged representation.
+ *
+ * `Exact`: HostMatcher.Exact
+ */
+export class WasmHostMatcher {
+    free(): void;
+    [Symbol.dispose](): void;
+    static default(): WasmHostMatcher;
+    constructor();
+    type: string;
+    get value(): string | undefined;
+    set value(value: string | null | undefined);
+}
+
 /**
  * An hreflang alternate link entry.
  */
@@ -582,10 +670,12 @@ export class WasmInteractionResult {
     free(): void;
     [Symbol.dispose](): void;
     static default(): WasmInteractionResult;
-    constructor(actionResults?: WasmActionResult[] | null, finalHtml?: string | null, finalUrl?: string | null);
+    constructor(actionResults?: WasmActionResult[] | null, finalHtml?: string | null, finalUrl?: string | null, screenshotBase64?: string | null);
     actionResults: WasmActionResult[];
     finalHtml: string;
     finalUrl: string;
+    get screenshotBase64(): string | undefined;
+    set screenshotBase64(value: string | null | undefined);
 }
 
 /**
@@ -830,7 +920,7 @@ export class WasmScrapeResult {
     free(): void;
     [Symbol.dispose](): void;
     static default(): WasmScrapeResult;
-    constructor(statusCode?: number | null, finalUrl?: string | null, contentType?: string | null, html?: string | null, bodySize?: number | null, metadata?: WasmPageMetadata | null, links?: WasmLinkInfo[] | null, images?: WasmImageInfo[] | null, feeds?: WasmFeedInfo[] | null, jsonLd?: WasmJsonLdEntry[] | null, isAllowed?: boolean | null, noindexDetected?: boolean | null, nofollowDetected?: boolean | null, isPdf?: boolean | null, wasSkipped?: boolean | null, authHeaderSent?: boolean | null, assets?: WasmDownloadedAsset[] | null, jsRenderHint?: boolean | null, browserUsed?: boolean | null, crawlDelay?: bigint | null, xRobotsTag?: string | null, detectedCharset?: string | null, responseMeta?: WasmResponseMeta | null, markdown?: WasmMarkdownResult | null, extractedData?: any | null, extractionMeta?: WasmExtractionMeta | null, downloadedDocument?: WasmDownloadedDocument | null, browser?: WasmBrowserExtras | null);
+    constructor(statusCode?: number | null, finalUrl?: string | null, contentType?: string | null, html?: string | null, bodySize?: number | null, metadata?: WasmPageMetadata | null, links?: WasmLinkInfo[] | null, images?: WasmImageInfo[] | null, feeds?: WasmFeedInfo[] | null, jsonLd?: WasmJsonLdEntry[] | null, isAllowed?: boolean | null, noindexDetected?: boolean | null, nofollowDetected?: boolean | null, isPdf?: boolean | null, wasSkipped?: boolean | null, authHeaderSent?: boolean | null, assets?: WasmDownloadedAsset[] | null, jsRenderHint?: boolean | null, browserUsed?: boolean | null, crawlDelay?: bigint | null, xRobotsTag?: string | null, detectedCharset?: string | null, responseMeta?: WasmResponseMeta | null, markdown?: WasmMarkdownResult | null, extractedData?: any | null, extractionMeta?: WasmExtractionMeta | null, screenshotBase64?: string | null, downloadedDocument?: WasmDownloadedDocument | null, browser?: WasmBrowserExtras | null);
     assets: WasmDownloadedAsset[];
     authHeaderSent: boolean;
     bodySize: number;
@@ -864,6 +954,8 @@ export class WasmScrapeResult {
     noindexDetected: boolean;
     get responseMeta(): WasmResponseMeta | undefined;
     set responseMeta(value: WasmResponseMeta | null | undefined);
+    get screenshotBase64(): string | undefined;
+    set screenshotBase64(value: string | null | undefined);
     statusCode: number;
     wasSkipped: boolean;
     get xRobotsTag(): string | undefined;
@@ -913,9 +1005,18 @@ export class WasmSsrfPolicy {
      * - Outbound requests in a browser go through the fetch API, which enforces its own network policies.
      * - Rust-side SSRF checking is unenforceable and redundant in a wasm32 context.
      * - For testing and localhost access, the host's network sandbox is the enforcing boundary.
+     *
+     * **Node.js caveat:** `deny_private` (whatever its value) has no effect on hostname-based
+     * requests under `wasm32`. There is no DNS resolution on this target, so `validate_url`
+     * only ever checks a literal IP host; a domain name falls straight through to `Ok(())`. In a
+     * browser this is covered by same-origin/CORS. Node's `fetch` enforces no CORS, so a Node
+     * service embedding this wasm module can be driven to internal hosts by domain name even
+     * though `deny_private = true`. Do not rely on this policy to stop that in Node — enforce
+     * egress restrictions (network policy, firewall, proxy allowlist) outside the process.
      */
     static fromEnv(): WasmSsrfPolicy;
-    constructor(denyPrivate?: boolean | null, maxRedirects?: number | null);
+    constructor(denyPrivate?: boolean | null, allowlist?: any | null, maxRedirects?: number | null);
+    allowlist: any;
     denyPrivate: boolean;
     maxRedirects: number;
 }

package/pkg/bundler/crawlberg_wasm.js

@@ -5,5 +5,5 @@ import { __wbg_set_wasm } from "./crawlberg_wasm_bg.js";
 __wbg_set_wasm(wasm);
 wasm.__wbindgen_start();
 export {
-    WasmActionResult, WasmArticleMetadata, WasmAssetCategory, WasmAuthConfig, WasmBatchCrawlResult, WasmBatchCrawlResults, WasmBatchScrapeResult, WasmBatchScrapeResults, WasmBrowserBackend, WasmBrowserConfig, WasmBrowserExtras, WasmBrowserMode, WasmBrowserWait, WasmCitationReference, WasmCitationResult, WasmContentConfig, WasmCookieInfo, WasmCrawlConfig, WasmCrawlEngineHandle, WasmCrawlPageResult, WasmCrawlResult, WasmDownloadedAsset, WasmDownloadedDocument, WasmExtractionMeta, WasmFaviconInfo, WasmFeedInfo, WasmFeedType, WasmHeadingInfo, WasmHreflangEntry, WasmImageInfo, WasmImageSource, WasmInteractionResult, WasmJsonLdEntry, WasmLinkInfo, WasmLinkType, WasmMapResult, WasmMarkdownResult, WasmPageAction, WasmPageMetadata, WasmProxyConfig, WasmResponseMeta, WasmScrapeResult, WasmScrollDirection, WasmSitemapUrl, WasmSsrfPolicy, batchCrawl, batchScrape, crawl, createEngine, generateCitations, interact, mapUrls, scrape
+    WasmActionResult, WasmArticleMetadata, WasmAssetCategory, WasmAuthConfig, WasmBatchCrawlResult, WasmBatchCrawlResults, WasmBatchScrapeResult, WasmBatchScrapeResults, WasmBrowserBackend, WasmBrowserConfig, WasmBrowserExtras, WasmBrowserMode, WasmBrowserWait, WasmCitationReference, WasmCitationResult, WasmContentConfig, WasmContentFilterKind, WasmCookieInfo, WasmCrawlConfig, WasmCrawlEngineHandle, WasmCrawlPageResult, WasmCrawlResult, WasmCrawlStrategyKind, WasmDocumentContentEncoding, WasmDownloadedAsset, WasmDownloadedDocument, WasmExtractionMeta, WasmFaviconInfo, WasmFeedInfo, WasmFeedType, WasmHeadingInfo, WasmHostMatcher, WasmHreflangEntry, WasmImageInfo, WasmImageSource, WasmInteractionResult, WasmJsonLdEntry, WasmLinkInfo, WasmLinkType, WasmMapResult, WasmMarkdownResult, WasmPageAction, WasmPageMetadata, WasmProxyConfig, WasmResponseMeta, WasmScrapeResult, WasmScrollDirection, WasmSitemapUrl, WasmSsrfPolicy, batchCrawl, batchScrape, crawl, createEngine, generateCitations, interact, mapUrls, scrape
 } from "./crawlberg_wasm_bg.js";