@xberg-io/crawlberg-wasm 1.10.2 → 1.11.0

package/README.md

@@ -1,5 +1,5 @@
 <!-- This file is auto-generated by alef — DO NOT EDIT. -->
-<!-- alef:hash:3e2f4a488c2c856faef7619c7da3856ab9b8cdad4534264dea7b27a2ebf8bf35 -->
+<!-- alef:hash:317aa594b0d20c83a18e8cdb5342b90b90f56e94e51ba85fc80ebb44ab9c1331 -->
 <!-- To regenerate: alef readme -->
 <!-- To verify freshness: alef verify -->
 
@@ -101,14 +101,14 @@ pnpm add @xberg-io/crawlberg-wasm
 
 ## Agent plugin
 
-The `crawlberg` plugin is available via the `xberg-io/plugins` marketplace.
+The `crawlberg` plugin is available via the `xberg-io/crawlberg` marketplace.
 
 ```text
-/plugin marketplace add xberg-io/plugins
-/plugin install crawlberg@xberg
+/plugin marketplace add xberg-io/crawlberg
+/plugin install crawlberg@crawlberg
 ```
 
-Works with Claude Code, Codex, Cursor, Gemini CLI, Factory Droid, GitHub Copilot CLI, and opencode. See [the marketplace README](https://github.com/xberg-io/plugins) for harness-specific install instructions.
+See [the plugin guide](https://github.com/xberg-io/crawlberg/blob/main/plugin/README.md) for all supported runtimes, CLI prerequisites and standalone public skills.
 
 ## Quick Start
 

package/package.json

@@ -1,6 +1,16 @@
 {
+  "author": {
+    "name": "Na'aman Hirschfeld",
+    "email": "naaman@xberg.io"
+  },
+  "contributors": [
+    {
+      "name": "Tobias Perlstein",
+      "email": "tobias@xberg.io"
+    }
+  ],
   "name": "@xberg-io/crawlberg-wasm",
-  "version": "1.10.2",
+  "version": "1.11.0",
   "private": false,
   "description": "High-performance web crawling engine",
   "license": "MIT",
@@ -13,7 +23,12 @@
     "access": "public"
   },
   "type": "module",
-  "files": ["pkg", "*.wasm", "*.d.ts", "README.md"],
+  "files": [
+    "pkg",
+    "*.wasm",
+    "*.d.ts",
+    "README.md"
+  ],
   "main": "pkg/nodejs/crawlberg_wasm.js",
   "module": "pkg/web/crawlberg_wasm.js",
   "types": "pkg/nodejs/crawlberg_wasm.d.ts",
@@ -29,6 +44,10 @@
   "engines": {
     "node": ">= 22"
   },
+  "devDependencies": {
+    "@vitest/coverage-v8": "^5.0.4",
+    "vitest": "^5.0.4"
+  },
   "scripts": {
     "build": "wasm-pack build --target nodejs --out-dir pkg/nodejs",
     "build:ci": "wasm-pack build --release --target nodejs --out-dir pkg/nodejs",
@@ -36,14 +55,10 @@
     "build:wasm:bundler": "wasm-pack build --release --target bundler --out-dir pkg/bundler",
     "build:wasm:nodejs": "wasm-pack build --release --target nodejs --out-dir pkg/nodejs",
     "build:wasm:deno": "wasm-pack build --release --target deno --out-dir pkg/deno",
-    "build:all": "npm run build:wasm:web && npm run build:wasm:bundler && npm run build:wasm:nodejs && npm run build:wasm:deno && find pkg -name .gitignore -delete",
+    "build:all": "pnpm run build:wasm:web && pnpm run build:wasm:bundler && pnpm run build:wasm:nodejs && pnpm run build:wasm:deno && find pkg -name .gitignore -delete",
     "test": "vitest run",
     "test:watch": "vitest watch",
     "test:coverage": "vitest run --coverage",
     "clean": "rm -rf pkg dist"
-  },
-  "devDependencies": {
-    "vitest": "^5.0.0",
-    "@vitest/coverage-v8": "^5.0.0"
   }
-}
+}

package/pkg/bundler/README.md

@@ -1,5 +1,5 @@
 <!-- This file is auto-generated by alef — DO NOT EDIT. -->
-<!-- alef:hash:3e2f4a488c2c856faef7619c7da3856ab9b8cdad4534264dea7b27a2ebf8bf35 -->
+<!-- alef:hash:317aa594b0d20c83a18e8cdb5342b90b90f56e94e51ba85fc80ebb44ab9c1331 -->
 <!-- To regenerate: alef readme -->
 <!-- To verify freshness: alef verify -->
 
@@ -101,14 +101,14 @@ pnpm add @xberg-io/crawlberg-wasm
 
 ## Agent plugin
 
-The `crawlberg` plugin is available via the `xberg-io/plugins` marketplace.
+The `crawlberg` plugin is available via the `xberg-io/crawlberg` marketplace.
 
 ```text
-/plugin marketplace add xberg-io/plugins
-/plugin install crawlberg@xberg
+/plugin marketplace add xberg-io/crawlberg
+/plugin install crawlberg@crawlberg
 ```
 
-Works with Claude Code, Codex, Cursor, Gemini CLI, Factory Droid, GitHub Copilot CLI, and opencode. See [the marketplace README](https://github.com/xberg-io/plugins) for harness-specific install instructions.
+See [the plugin guide](https://github.com/xberg-io/crawlberg/blob/main/plugin/README.md) for all supported runtimes, CLI prerequisites and standalone public skills.
 
 ## Quick Start
 

package/pkg/bundler/crawlberg_wasm.d.ts

@@ -427,10 +427,15 @@ export class WasmContentConfig {
      */
     copyForTransfer(): WasmContentConfig;
     static default(): WasmContentConfig;
-    constructor(outputFormat?: string | null, preprocessingPreset?: string | null, removeNavigation?: boolean | null, removeForms?: boolean | null, stripTags?: string[] | null, preserveTags?: string[] | null, excludeSelectors?: string[] | null, skipImages?: boolean | null, wrap?: boolean | null, wrapWidth?: number | null, includeDocumentStructure?: boolean | null, extractMetadata?: boolean | null, maxDepth?: number | null);
+    constructor(outputFormat?: string | null, preprocessingPreset?: string | null, removeNavigation?: boolean | null, removeForms?: boolean | null, stripTags?: string[] | null, preserveTags?: string[] | null, excludeSelectors?: string[] | null, skipImages?: boolean | null, wrap?: boolean | null, wrapWidth?: number | null, includeDocumentStructure?: boolean | null, extractMetadata?: boolean | null, maxDepth?: number | null, stripNewlines?: boolean | null, bullets?: string | null, listIndentWidth?: number | null, strongEmSymbol?: string | null, compactTables?: boolean | null, keepInlineImagesIn?: string[] | null, keepHidden?: boolean | null);
+    bullets: string;
+    compactTables: boolean;
     excludeSelectors: string[];
     extractMetadata: boolean;
     includeDocumentStructure: boolean;
+    keepHidden: boolean;
+    keepInlineImagesIn: string[];
+    listIndentWidth: number;
     get maxDepth(): number | undefined;
     set maxDepth(value: number | null | undefined);
     outputFormat: string;
@@ -439,7 +444,9 @@ export class WasmContentConfig {
     removeForms: boolean;
     removeNavigation: boolean;
     skipImages: boolean;
+    stripNewlines: boolean;
     stripTags: string[];
+    strongEmSymbol: string;
     wrapWidth: number;
     wrap: boolean;
 }
@@ -452,7 +459,10 @@ export enum WasmContentFilterKind {
 }
 
 /**
- * Information about an HTTP cookie received from a response.
+ * Information about a cookie that a crawled page set.
+ *
+ * A crawl result gives `domain` and `path` as the browser stores the cookie, so a cookie has
+ * the same fields whether an HTTP fetch or the browser received it.
  */
 export class WasmCookieInfo {
     free(): void;
@@ -655,7 +665,7 @@ export class WasmCrawlPageResult {
      * wasm-bindgen cannot borrow class values nested in an array. To keep those handles,
      * copy each retained element with `copyForTransfer()` before passing the array.
      */
-    constructor(url?: string | null, normalizedUrl?: string | null, statusCode?: number | null, contentType?: string | null, html?: string | null, bodySize?: number | null, metadata?: WasmPageMetadata | null, links?: WasmLinkInfo[] | null, images?: WasmImageInfo[] | null, feeds?: WasmFeedInfo[] | null, jsonLd?: WasmJsonLdEntry[] | null, depth?: number | null, stayedOnDomain?: boolean | null, wasSkipped?: boolean | null, isPdf?: boolean | null, browserUsed?: boolean | null, finalUrl?: string | null, redirectCount?: number | null, noindexDetected?: boolean | null, nofollowDetected?: boolean | null, ssrfRefusedUrls?: string[] | null, detectedCharset?: string | null, markdown?: WasmMarkdownResult | null, extractedData?: any | null, extractionMeta?: WasmExtractionMeta | null, downloadedDocument?: WasmDownloadedDocument | null);
+    constructor(url?: string | null, normalizedUrl?: string | null, statusCode?: number | null, contentType?: string | null, html?: string | null, bodySize?: number | null, metadata?: WasmPageMetadata | null, links?: WasmLinkInfo[] | null, images?: WasmImageInfo[] | null, feeds?: WasmFeedInfo[] | null, jsonLd?: WasmJsonLdEntry[] | null, depth?: number | null, stayedOnDomain?: boolean | null, wasSkipped?: boolean | null, isPdf?: boolean | null, browserUsed?: boolean | null, finalUrl?: string | null, redirectCount?: number | null, noindexDetected?: boolean | null, nofollowDetected?: boolean | null, ssrfRefusedUrls?: string[] | null, detectedCharset?: string | null, markdown?: WasmMarkdownResult | null, extractedData?: any | null, extractionMeta?: WasmExtractionMeta | null, downloadedDocument?: WasmDownloadedDocument | null, originalUrl?: string | null);
     bodySize: number;
     browserUsed: boolean;
     contentType: string;
@@ -729,6 +739,7 @@ export class WasmCrawlPageResult {
     nofollowDetected: boolean;
     noindexDetected: boolean;
     normalizedUrl: string;
+    originalUrl: string;
     redirectCount: number;
     ssrfRefusedUrls: string[];
     statusCode: number;
@@ -1126,10 +1137,11 @@ export class WasmLinkInfo {
      */
     copyForTransfer(): WasmLinkInfo;
     static default(): WasmLinkInfo;
-    constructor(url?: string | null, text?: string | null, linkType?: WasmLinkType | null, nofollow?: boolean | null, rel?: string | null);
+    constructor(url?: string | null, text?: string | null, linkType?: WasmLinkType | null, nofollow?: boolean | null, rel?: string | null, originalUrl?: string | null);
     get linkType(): string;
     set linkType(value: WasmLinkType);
     nofollow: boolean;
+    originalUrl: string;
     get rel(): string | undefined;
     set rel(value: string | null | undefined);
     text: string;

package/pkg/bundler/crawlberg_wasm_bg.js

@@ -1790,6 +1790,28 @@ export class WasmContentConfig {
         const ptr = this.__destroy_into_raw();
         wasm.__wbg_wasmcontentconfig_free(ptr, 0);
     }
+    /**
+     * @returns {string}
+     */
+    get bullets() {
+        let deferred1_0;
+        let deferred1_1;
+        try {
+            const ret = wasm.wasmcontentconfig_bullets(this.__wbg_ptr);
+            deferred1_0 = ret[0];
+            deferred1_1 = ret[1];
+            return getStringFromWasm0(ret[0], ret[1]);
+        } finally {
+            wasm.__wbindgen_free(deferred1_0, deferred1_1, 1);
+        }
+    }
+    /**
+     * @returns {boolean}
+     */
+    get compactTables() {
+        const ret = wasm.wasmcontentconfig_compactTables(this.__wbg_ptr);
+        return ret !== 0;
+    }
     /**
      * Return an independent handle for an ownership-taking API.
      *
@@ -1831,6 +1853,29 @@ export class WasmContentConfig {
         const ret = wasm.wasmcontentconfig_includeDocumentStructure(this.__wbg_ptr);
         return ret !== 0;
     }
+    /**
+     * @returns {boolean}
+     */
+    get keepHidden() {
+        const ret = wasm.wasmcontentconfig_keepHidden(this.__wbg_ptr);
+        return ret !== 0;
+    }
+    /**
+     * @returns {string[]}
+     */
+    get keepInlineImagesIn() {
+        const ret = wasm.wasmcontentconfig_keepInlineImagesIn(this.__wbg_ptr);
+        var v1 = getArrayJsValueFromWasm0(ret[0], ret[1]);
+        wasm.__wbindgen_free(ret[0], ret[1] * 4, 4);
+        return v1;
+    }
+    /**
+     * @returns {number}
+     */
+    get listIndentWidth() {
+        const ret = wasm.wasmcontentconfig_listIndentWidth(this.__wbg_ptr);
+        return ret >>> 0;
+    }
     /**
      * @returns {number | undefined}
      */
@@ -1852,8 +1897,15 @@ export class WasmContentConfig {
      * @param {boolean | null} [includeDocumentStructure]
      * @param {boolean | null} [extractMetadata]
      * @param {number | null} [maxDepth]
-     */
-    constructor(outputFormat, preprocessingPreset, removeNavigation, removeForms, stripTags, preserveTags, excludeSelectors, skipImages, wrap, wrapWidth, includeDocumentStructure, extractMetadata, maxDepth) {
+     * @param {boolean | null} [stripNewlines]
+     * @param {string | null} [bullets]
+     * @param {number | null} [listIndentWidth]
+     * @param {string | null} [strongEmSymbol]
+     * @param {boolean | null} [compactTables]
+     * @param {string[] | null} [keepInlineImagesIn]
+     * @param {boolean | null} [keepHidden]
+     */
+    constructor(outputFormat, preprocessingPreset, removeNavigation, removeForms, stripTags, preserveTags, excludeSelectors, skipImages, wrap, wrapWidth, includeDocumentStructure, extractMetadata, maxDepth, stripNewlines, bullets, listIndentWidth, strongEmSymbol, compactTables, keepInlineImagesIn, keepHidden) {
         var ptr0 = isLikeNone(outputFormat) ? 0 : passStringToWasm0(outputFormat, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
         var len0 = WASM_VECTOR_LEN;
         var ptr1 = isLikeNone(preprocessingPreset) ? 0 : passStringToWasm0(preprocessingPreset, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
@@ -1864,7 +1916,13 @@ export class WasmContentConfig {
         var len3 = WASM_VECTOR_LEN;
         var ptr4 = isLikeNone(excludeSelectors) ? 0 : passArrayJsValueToWasm0(excludeSelectors, wasm.__wbindgen_malloc);
         var len4 = WASM_VECTOR_LEN;
-        const ret = wasm.wasmcontentconfig_new(ptr0, len0, ptr1, len1, isLikeNone(removeNavigation) ? 0xFFFFFF : removeNavigation ? 1 : 0, isLikeNone(removeForms) ? 0xFFFFFF : removeForms ? 1 : 0, ptr2, len2, ptr3, len3, ptr4, len4, isLikeNone(skipImages) ? 0xFFFFFF : skipImages ? 1 : 0, isLikeNone(wrap) ? 0xFFFFFF : wrap ? 1 : 0, isLikeNone(wrapWidth) ? Number.MAX_SAFE_INTEGER : (wrapWidth) >>> 0, isLikeNone(includeDocumentStructure) ? 0xFFFFFF : includeDocumentStructure ? 1 : 0, isLikeNone(extractMetadata) ? 0xFFFFFF : extractMetadata ? 1 : 0, isLikeNone(maxDepth) ? Number.MAX_SAFE_INTEGER : (maxDepth) >>> 0);
+        var ptr5 = isLikeNone(bullets) ? 0 : passStringToWasm0(bullets, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+        var len5 = WASM_VECTOR_LEN;
+        var ptr6 = isLikeNone(strongEmSymbol) ? 0 : passStringToWasm0(strongEmSymbol, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+        var len6 = WASM_VECTOR_LEN;
+        var ptr7 = isLikeNone(keepInlineImagesIn) ? 0 : passArrayJsValueToWasm0(keepInlineImagesIn, wasm.__wbindgen_malloc);
+        var len7 = WASM_VECTOR_LEN;
+        const ret = wasm.wasmcontentconfig_new(ptr0, len0, ptr1, len1, isLikeNone(removeNavigation) ? 0xFFFFFF : removeNavigation ? 1 : 0, isLikeNone(removeForms) ? 0xFFFFFF : removeForms ? 1 : 0, ptr2, len2, ptr3, len3, ptr4, len4, isLikeNone(skipImages) ? 0xFFFFFF : skipImages ? 1 : 0, isLikeNone(wrap) ? 0xFFFFFF : wrap ? 1 : 0, isLikeNone(wrapWidth) ? Number.MAX_SAFE_INTEGER : (wrapWidth) >>> 0, isLikeNone(includeDocumentStructure) ? 0xFFFFFF : includeDocumentStructure ? 1 : 0, isLikeNone(extractMetadata) ? 0xFFFFFF : extractMetadata ? 1 : 0, isLikeNone(maxDepth) ? Number.MAX_SAFE_INTEGER : (maxDepth) >>> 0, isLikeNone(stripNewlines) ? 0xFFFFFF : stripNewlines ? 1 : 0, ptr5, len5, isLikeNone(listIndentWidth) ? Number.MAX_SAFE_INTEGER : (listIndentWidth) >>> 0, ptr6, len6, isLikeNone(compactTables) ? 0xFFFFFF : compactTables ? 1 : 0, ptr7, len7, isLikeNone(keepHidden) ? 0xFFFFFF : keepHidden ? 1 : 0);
         this.__wbg_ptr = ret;
         WasmContentConfigFinalization.register(this, this.__wbg_ptr, this);
         return this;
@@ -1922,6 +1980,20 @@ export class WasmContentConfig {
         const ret = wasm.wasmcontentconfig_removeNavigation(this.__wbg_ptr);
         return ret !== 0;
     }
+    /**
+     * @param {string} value
+     */
+    set bullets(value) {
+        const ptr0 = passStringToWasm0(value, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+        const len0 = WASM_VECTOR_LEN;
+        wasm.wasmcontentconfig_set_bullets(this.__wbg_ptr, ptr0, len0);
+    }
+    /**
+     * @param {boolean} value
+     */
+    set compactTables(value) {
+        wasm.wasmcontentconfig_set_compactTables(this.__wbg_ptr, value);
+    }
     /**
      * @param {string[]} value
      */
@@ -1942,6 +2014,26 @@ export class WasmContentConfig {
     set includeDocumentStructure(value) {
         wasm.wasmcontentconfig_set_includeDocumentStructure(this.__wbg_ptr, value);
     }
+    /**
+     * @param {boolean} value
+     */
+    set keepHidden(value) {
+        wasm.wasmcontentconfig_set_keepHidden(this.__wbg_ptr, value);
+    }
+    /**
+     * @param {string[]} value
+     */
+    set keepInlineImagesIn(value) {
+        const ptr0 = passArrayJsValueToWasm0(value, wasm.__wbindgen_malloc);
+        const len0 = WASM_VECTOR_LEN;
+        wasm.wasmcontentconfig_set_keepInlineImagesIn(this.__wbg_ptr, ptr0, len0);
+    }
+    /**
+     * @param {number} value
+     */
+    set listIndentWidth(value) {
+        wasm.wasmcontentconfig_set_listIndentWidth(this.__wbg_ptr, value);
+    }
     /**
      * @param {number | null} [value]
      */
@@ -1990,6 +2082,12 @@ export class WasmContentConfig {
     set skipImages(value) {
         wasm.wasmcontentconfig_set_skipImages(this.__wbg_ptr, value);
     }
+    /**
+     * @param {boolean} value
+     */
+    set stripNewlines(value) {
+        wasm.wasmcontentconfig_set_stripNewlines(this.__wbg_ptr, value);
+    }
     /**
      * @param {string[]} value
      */
@@ -1998,6 +2096,14 @@ export class WasmContentConfig {
         const len0 = WASM_VECTOR_LEN;
         wasm.wasmcontentconfig_set_stripTags(this.__wbg_ptr, ptr0, len0);
     }
+    /**
+     * @param {string} value
+     */
+    set strongEmSymbol(value) {
+        const ptr0 = passStringToWasm0(value, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+        const len0 = WASM_VECTOR_LEN;
+        wasm.wasmcontentconfig_set_strongEmSymbol(this.__wbg_ptr, ptr0, len0);
+    }
     /**
      * @param {number} value
      */
@@ -2017,6 +2123,13 @@ export class WasmContentConfig {
         const ret = wasm.wasmcontentconfig_skipImages(this.__wbg_ptr);
         return ret !== 0;
     }
+    /**
+     * @returns {boolean}
+     */
+    get stripNewlines() {
+        const ret = wasm.wasmcontentconfig_stripNewlines(this.__wbg_ptr);
+        return ret !== 0;
+    }
     /**
      * @returns {string[]}
      */
@@ -2026,6 +2139,21 @@ export class WasmContentConfig {
         wasm.__wbindgen_free(ret[0], ret[1] * 4, 4);
         return v1;
     }
+    /**
+     * @returns {string}
+     */
+    get strongEmSymbol() {
+        let deferred1_0;
+        let deferred1_1;
+        try {
+            const ret = wasm.wasmcontentconfig_strongEmSymbol(this.__wbg_ptr);
+            deferred1_0 = ret[0];
+            deferred1_1 = ret[1];
+            return getStringFromWasm0(ret[0], ret[1]);
+        } finally {
+            wasm.__wbindgen_free(deferred1_0, deferred1_1, 1);
+        }
+    }
     /**
      * @returns {number}
      */
@@ -2052,7 +2180,10 @@ export const WasmContentFilterKind = Object.freeze({
 });
 
 /**
- * Information about an HTTP cookie received from a response.
+ * Information about a cookie that a crawled page set.
+ *
+ * A crawl result gives `domain` and `path` as the browser stores the cookie, so a cookie has
+ * the same fields whether an HTTP fetch or the browser received it.
  */
 export class WasmCookieInfo {
     static __wrap(ptr) {
@@ -3516,8 +3647,9 @@ export class WasmCrawlPageResult {
      * @param {any | null} [extractedData]
      * @param {WasmExtractionMeta | null} [extractionMeta]
      * @param {WasmDownloadedDocument | null} [downloadedDocument]
+     * @param {string | null} [originalUrl]
      */
-    constructor(url, normalizedUrl, statusCode, contentType, html, bodySize, metadata, links, images, feeds, jsonLd, depth, stayedOnDomain, wasSkipped, isPdf, browserUsed, finalUrl, redirectCount, noindexDetected, nofollowDetected, ssrfRefusedUrls, detectedCharset, markdown, extractedData, extractionMeta, downloadedDocument) {
+    constructor(url, normalizedUrl, statusCode, contentType, html, bodySize, metadata, links, images, feeds, jsonLd, depth, stayedOnDomain, wasSkipped, isPdf, browserUsed, finalUrl, redirectCount, noindexDetected, nofollowDetected, ssrfRefusedUrls, detectedCharset, markdown, extractedData, extractionMeta, downloadedDocument, originalUrl) {
         var ptr0 = isLikeNone(url) ? 0 : passStringToWasm0(url, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
         var len0 = WASM_VECTOR_LEN;
         var ptr1 = isLikeNone(normalizedUrl) ? 0 : passStringToWasm0(normalizedUrl, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
@@ -3560,7 +3692,9 @@ export class WasmCrawlPageResult {
             _assertClass(downloadedDocument, WasmDownloadedDocument);
             ptr14 = downloadedDocument.__destroy_into_raw();
         }
-        const ret = wasm.wasmcrawlpageresult_new(ptr0, len0, ptr1, len1, isLikeNone(statusCode) ? 0xFFFFFF : statusCode, ptr2, len2, ptr3, len3, isLikeNone(bodySize) ? Number.MAX_SAFE_INTEGER : (bodySize) >>> 0, ptr4, ptr5, len5, ptr6, len6, ptr7, len7, ptr8, len8, isLikeNone(depth) ? Number.MAX_SAFE_INTEGER : (depth) >>> 0, isLikeNone(stayedOnDomain) ? 0xFFFFFF : stayedOnDomain ? 1 : 0, isLikeNone(wasSkipped) ? 0xFFFFFF : wasSkipped ? 1 : 0, isLikeNone(isPdf) ? 0xFFFFFF : isPdf ? 1 : 0, isLikeNone(browserUsed) ? 0xFFFFFF : browserUsed ? 1 : 0, ptr9, len9, isLikeNone(redirectCount) ? Number.MAX_SAFE_INTEGER : (redirectCount) >>> 0, isLikeNone(noindexDetected) ? 0xFFFFFF : noindexDetected ? 1 : 0, isLikeNone(nofollowDetected) ? 0xFFFFFF : nofollowDetected ? 1 : 0, ptr10, len10, ptr11, len11, ptr12, isLikeNone(extractedData) ? 0 : addToExternrefTable0(extractedData), ptr13, ptr14);
+        var ptr15 = isLikeNone(originalUrl) ? 0 : passStringToWasm0(originalUrl, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+        var len15 = WASM_VECTOR_LEN;
+        const ret = wasm.wasmcrawlpageresult_new(ptr0, len0, ptr1, len1, isLikeNone(statusCode) ? 0xFFFFFF : statusCode, ptr2, len2, ptr3, len3, isLikeNone(bodySize) ? Number.MAX_SAFE_INTEGER : (bodySize) >>> 0, ptr4, ptr5, len5, ptr6, len6, ptr7, len7, ptr8, len8, isLikeNone(depth) ? Number.MAX_SAFE_INTEGER : (depth) >>> 0, isLikeNone(stayedOnDomain) ? 0xFFFFFF : stayedOnDomain ? 1 : 0, isLikeNone(wasSkipped) ? 0xFFFFFF : wasSkipped ? 1 : 0, isLikeNone(isPdf) ? 0xFFFFFF : isPdf ? 1 : 0, isLikeNone(browserUsed) ? 0xFFFFFF : browserUsed ? 1 : 0, ptr9, len9, isLikeNone(redirectCount) ? Number.MAX_SAFE_INTEGER : (redirectCount) >>> 0, isLikeNone(noindexDetected) ? 0xFFFFFF : noindexDetected ? 1 : 0, isLikeNone(nofollowDetected) ? 0xFFFFFF : nofollowDetected ? 1 : 0, ptr10, len10, ptr11, len11, ptr12, isLikeNone(extractedData) ? 0 : addToExternrefTable0(extractedData), ptr13, ptr14, ptr15, len15);
         this.__wbg_ptr = ret;
         WasmCrawlPageResultFinalization.register(this, this.__wbg_ptr, this);
         return this;
@@ -3594,6 +3728,21 @@ export class WasmCrawlPageResult {
             wasm.__wbindgen_free(deferred1_0, deferred1_1, 1);
         }
     }
+    /**
+     * @returns {string}
+     */
+    get originalUrl() {
+        let deferred1_0;
+        let deferred1_1;
+        try {
+            const ret = wasm.wasmcrawlpageresult_originalUrl(this.__wbg_ptr);
+            deferred1_0 = ret[0];
+            deferred1_1 = ret[1];
+            return getStringFromWasm0(ret[0], ret[1]);
+        } finally {
+            wasm.__wbindgen_free(deferred1_0, deferred1_1, 1);
+        }
+    }
     /**
      * @returns {number}
      */
@@ -3755,6 +3904,14 @@ export class WasmCrawlPageResult {
         const len0 = WASM_VECTOR_LEN;
         wasm.wasmcrawlpageresult_set_normalizedUrl(this.__wbg_ptr, ptr0, len0);
     }
+    /**
+     * @param {string} value
+     */
+    set originalUrl(value) {
+        const ptr0 = passStringToWasm0(value, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+        const len0 = WASM_VECTOR_LEN;
+        wasm.wasmcrawlpageresult_set_originalUrl(this.__wbg_ptr, ptr0, len0);
+    }
     /**
      * @param {number} value
      */
@@ -5774,15 +5931,18 @@ export class WasmLinkInfo {
      * @param {WasmLinkType | null} [linkType]
      * @param {boolean | null} [nofollow]
      * @param {string | null} [rel]
+     * @param {string | null} [originalUrl]
      */
-    constructor(url, text, linkType, nofollow, rel) {
+    constructor(url, text, linkType, nofollow, rel, originalUrl) {
         var ptr0 = isLikeNone(url) ? 0 : passStringToWasm0(url, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
         var len0 = WASM_VECTOR_LEN;
         var ptr1 = isLikeNone(text) ? 0 : passStringToWasm0(text, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
         var len1 = WASM_VECTOR_LEN;
         var ptr2 = isLikeNone(rel) ? 0 : passStringToWasm0(rel, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
         var len2 = WASM_VECTOR_LEN;
-        const ret = wasm.wasmlinkinfo_new(ptr0, len0, ptr1, len1, isLikeNone(linkType) ? 4 : linkType, isLikeNone(nofollow) ? 0xFFFFFF : nofollow ? 1 : 0, ptr2, len2);
+        var ptr3 = isLikeNone(originalUrl) ? 0 : passStringToWasm0(originalUrl, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+        var len3 = WASM_VECTOR_LEN;
+        const ret = wasm.wasmlinkinfo_new(ptr0, len0, ptr1, len1, isLikeNone(linkType) ? 4 : linkType, isLikeNone(nofollow) ? 0xFFFFFF : nofollow ? 1 : 0, ptr2, len2, ptr3, len3);
         this.__wbg_ptr = ret;
         WasmLinkInfoFinalization.register(this, this.__wbg_ptr, this);
         return this;
@@ -5794,6 +5954,21 @@ export class WasmLinkInfo {
         const ret = wasm.wasmlinkinfo_nofollow(this.__wbg_ptr);
         return ret !== 0;
     }
+    /**
+     * @returns {string}
+     */
+    get originalUrl() {
+        let deferred1_0;
+        let deferred1_1;
+        try {
+            const ret = wasm.wasmlinkinfo_originalUrl(this.__wbg_ptr);
+            deferred1_0 = ret[0];
+            deferred1_1 = ret[1];
+            return getStringFromWasm0(ret[0], ret[1]);
+        } finally {
+            wasm.__wbindgen_free(deferred1_0, deferred1_1, 1);
+        }
+    }
     /**
      * @returns {string | undefined}
      */
@@ -5818,6 +5993,14 @@ export class WasmLinkInfo {
     set nofollow(value) {
         wasm.wasmlinkinfo_set_nofollow(this.__wbg_ptr, value);
     }
+    /**
+     * @param {string} value
+     */
+    set originalUrl(value) {
+        const ptr0 = passStringToWasm0(value, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+        const len0 = WASM_VECTOR_LEN;
+        wasm.wasmlinkinfo_set_originalUrl(this.__wbg_ptr, ptr0, len0);
+    }
     /**
      * @param {string | null} [value]
      */
@@ -9065,7 +9248,7 @@ export function __wbg_new_typed_b01cb72a8af741a3(arg0, arg1) {
             const a = state0.a;
             state0.a = 0;
             try {
-                return wasm_bindgen__convert__closures_____invoke__h8ad11faaa2905452(a, state0.b, arg0, arg1);
+                return wasm_bindgen_ceaac9754e66b2fe___convert__closures_____invoke___js_sys_6b40a5061b778857___Function_fn_wasm_bindgen_ceaac9754e66b2fe___JsValue_____wasm_bindgen_ceaac9754e66b2fe___sys__Undefined___js_sys_6b40a5061b778857___Function_fn_wasm_bindgen_ceaac9754e66b2fe___JsValue_____wasm_bindgen_ceaac9754e66b2fe___sys__Undefined_______true_(a, state0.b, arg0, arg1);
             } finally {
                 state0.a = a;
             }
@@ -9359,13 +9542,13 @@ export function __wbg_wasmsitemapurl_unwrap(arg0) {
     return ret;
 }
 export function __wbindgen_generic_0000000000000001(arg0, arg1) {
-    // Cast intrinsic for `Closure(Closure { owned: true, function: Function { arguments: [Externref], shim_idx: 1331, ret: Result(Unit), inner_ret: Some(Result(Unit)) }, mutable: true }) -> Externref`.
-    const ret = makeMutClosure(arg0, arg1, wasm_bindgen__convert__closures_____invoke__h2eedacf566f3748c);
+    // Cast intrinsic for `Closure(Closure { owned: true, function: Function { arguments: [Externref], shim_idx: 1467, ret: Result(Unit), inner_ret: Some(Result(Unit)) }, mutable: true }) -> Externref`.
+    const ret = makeMutClosure(arg0, arg1, wasm_bindgen_ceaac9754e66b2fe___convert__closures_____invoke___wasm_bindgen_ceaac9754e66b2fe___JsValue__core_608f92abc48d28da___result__Result_____wasm_bindgen_ceaac9754e66b2fe___JsError___true_);
     return ret;
 }
 export function __wbindgen_generic_0000000000000002(arg0, arg1) {
-    // Cast intrinsic for `Closure(Closure { owned: true, function: Function { arguments: [], shim_idx: 1222, ret: Unit, inner_ret: Some(Unit) }, mutable: true }) -> Externref`.
-    const ret = makeMutClosure(arg0, arg1, wasm_bindgen__convert__closures_____invoke__h3a7ccac3e94bbb5c);
+    // Cast intrinsic for `Closure(Closure { owned: true, function: Function { arguments: [], shim_idx: 1309, ret: Unit, inner_ret: Some(Unit) }, mutable: true }) -> Externref`.
+    const ret = makeMutClosure(arg0, arg1, wasm_bindgen_ceaac9754e66b2fe___convert__closures_____invoke_______true_);
     return ret;
 }
 export function __wbindgen_generic_0000000000000003(arg0) {
@@ -9397,19 +9580,19 @@ export function __wbindgen_init_externref_table() {
     table.set(offset + 2, true);
     table.set(offset + 3, false);
 }
-function wasm_bindgen__convert__closures_____invoke__h3a7ccac3e94bbb5c(arg0, arg1) {
-    wasm.wasm_bindgen__convert__closures_____invoke__h3a7ccac3e94bbb5c(arg0, arg1);
+function wasm_bindgen_ceaac9754e66b2fe___convert__closures_____invoke_______true_(arg0, arg1) {
+    wasm.wasm_bindgen_ceaac9754e66b2fe___convert__closures_____invoke_______true_(arg0, arg1);
 }
 
-function wasm_bindgen__convert__closures_____invoke__h2eedacf566f3748c(arg0, arg1, arg2) {
-    const ret = wasm.wasm_bindgen__convert__closures_____invoke__h2eedacf566f3748c(arg0, arg1, arg2);
+function wasm_bindgen_ceaac9754e66b2fe___convert__closures_____invoke___wasm_bindgen_ceaac9754e66b2fe___JsValue__core_608f92abc48d28da___result__Result_____wasm_bindgen_ceaac9754e66b2fe___JsError___true_(arg0, arg1, arg2) {
+    const ret = wasm.wasm_bindgen_ceaac9754e66b2fe___convert__closures_____invoke___wasm_bindgen_ceaac9754e66b2fe___JsValue__core_608f92abc48d28da___result__Result_____wasm_bindgen_ceaac9754e66b2fe___JsError___true_(arg0, arg1, arg2);
     if (ret[1]) {
         throw takeFromExternrefTable0(ret[0]);
     }
 }
 
-function wasm_bindgen__convert__closures_____invoke__h8ad11faaa2905452(arg0, arg1, arg2, arg3) {
-    wasm.wasm_bindgen__convert__closures_____invoke__h8ad11faaa2905452(arg0, arg1, arg2, arg3);
+function wasm_bindgen_ceaac9754e66b2fe___convert__closures_____invoke___js_sys_6b40a5061b778857___Function_fn_wasm_bindgen_ceaac9754e66b2fe___JsValue_____wasm_bindgen_ceaac9754e66b2fe___sys__Undefined___js_sys_6b40a5061b778857___Function_fn_wasm_bindgen_ceaac9754e66b2fe___JsValue_____wasm_bindgen_ceaac9754e66b2fe___sys__Undefined_______true_(arg0, arg1, arg2, arg3) {
+    wasm.wasm_bindgen_ceaac9754e66b2fe___convert__closures_____invoke___js_sys_6b40a5061b778857___Function_fn_wasm_bindgen_ceaac9754e66b2fe___JsValue_____wasm_bindgen_ceaac9754e66b2fe___sys__Undefined___js_sys_6b40a5061b778857___Function_fn_wasm_bindgen_ceaac9754e66b2fe___JsValue_____wasm_bindgen_ceaac9754e66b2fe___sys__Undefined_______true_(arg0, arg1, arg2, arg3);
 }