@xberg-io/crawlberg-wasm 1.7.2 → 1.9.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xberg-io/crawlberg-wasm",
-  "version": "1.7.2",
+  "version": "1.9.0",
   "private": false,
   "description": "High-performance web crawling engine",
   "license": "MIT",

package/pkg/bundler/crawlberg_wasm.d.ts

@@ -153,11 +153,14 @@ export class WasmBrowserConfig {
     free(): void;
     [Symbol.dispose](): void;
     static default(): WasmBrowserConfig;
-    constructor(mode?: WasmBrowserMode | null, backend?: WasmBrowserBackend | null, timeout?: bigint | null, wait?: WasmBrowserWait | null, blockUrlPatterns?: string[] | null, captureNetworkEvents?: boolean | null, sessionAffinity?: boolean | null, endpoint?: string | null, waitSelector?: string | null, extraWait?: bigint | null, proxy?: WasmProxyConfig | null, evalScript?: string | null, robotsUserAgent?: string | null);
+    constructor(mode?: WasmBrowserMode | null, backend?: WasmBrowserBackend | null, timeout?: bigint | null, overallTimeout?: bigint | null, shutdownTimeout?: bigint | null, wait?: WasmBrowserWait | null, blockUrlPatterns?: string[] | null, captureNetworkEvents?: boolean | null, sessionAffinity?: boolean | null, chromeArgs?: string[] | null, endpoint?: string | null, waitSelector?: string | null, extraWait?: bigint | null, proxy?: WasmProxyConfig | null, evalScript?: string | null, robotsUserAgent?: string | null, chromePath?: string | null);
     get backend(): string;
     set backend(value: WasmBrowserBackend);
     blockUrlPatterns: string[];
     captureNetworkEvents: boolean;
+    chromeArgs: string[];
+    get chromePath(): string | undefined;
+    set chromePath(value: string | null | undefined);
     get endpoint(): string | undefined;
     set endpoint(value: string | null | undefined);
     get evalScript(): string | undefined;
@@ -166,11 +169,15 @@ export class WasmBrowserConfig {
     set extraWait(value: bigint | null | undefined);
     get mode(): string;
     set mode(value: WasmBrowserMode);
+    get overallTimeout(): bigint | undefined;
+    set overallTimeout(value: bigint | null | undefined);
     get proxy(): WasmProxyConfig | undefined;
     set proxy(value: WasmProxyConfig | null | undefined);
     get robotsUserAgent(): string | undefined;
     set robotsUserAgent(value: string | null | undefined);
     sessionAffinity: boolean;
+    get shutdownTimeout(): bigint | undefined;
+    set shutdownTimeout(value: bigint | null | undefined);
     get timeout(): bigint | undefined;
     set timeout(value: bigint | null | undefined);
     get waitSelector(): string | undefined;
@@ -251,8 +258,9 @@ export class WasmContentConfig {
     free(): void;
     [Symbol.dispose](): void;
     static default(): WasmContentConfig;
-    constructor(outputFormat?: string | null, preprocessingPreset?: string | null, removeNavigation?: boolean | null, removeForms?: boolean | null, stripTags?: string[] | null, preserveTags?: string[] | null, excludeSelectors?: string[] | null, skipImages?: boolean | null, wrap?: boolean | null, wrapWidth?: number | null, includeDocumentStructure?: boolean | null, maxDepth?: number | null);
+    constructor(outputFormat?: string | null, preprocessingPreset?: string | null, removeNavigation?: boolean | null, removeForms?: boolean | null, stripTags?: string[] | null, preserveTags?: string[] | null, excludeSelectors?: string[] | null, skipImages?: boolean | null, wrap?: boolean | null, wrapWidth?: number | null, includeDocumentStructure?: boolean | null, extractMetadata?: boolean | null, maxDepth?: number | null);
     excludeSelectors: string[];
+    extractMetadata: boolean;
     includeDocumentStructure: boolean;
     get maxDepth(): number | undefined;
     set maxDepth(value: number | null | undefined);
@@ -297,7 +305,7 @@ export class WasmCrawlConfig {
     free(): void;
     [Symbol.dispose](): void;
     static default(): WasmCrawlConfig;
-    constructor(crawlStrategy?: WasmCrawlStrategyKind | null, respectRobotsTxt?: boolean | null, softHttpErrors?: boolean | null, stayOnDomain?: boolean | null, allowSubdomains?: boolean | null, includePaths?: string[] | null, excludePaths?: string[] | null, customHeaders?: any | null, requestTimeout?: bigint | null, maxRedirects?: number | null, retryCount?: number | null, retryCodes?: Uint16Array | null, cookiesEnabled?: boolean | null, removeTags?: string[] | null, content?: WasmContentConfig | null, downloadAssets?: boolean | null, assetTypes?: WasmAssetCategory[] | null, browser?: WasmBrowserConfig | null, userAgents?: string[] | null, captureScreenshot?: boolean | null, followDocumentUrls?: boolean | null, downloadDocuments?: boolean | null, documentMimeTypes?: string[] | null, saveBrowserProfile?: boolean | null, ssrf?: WasmSsrfPolicy | null, maxDepth?: number | null, maxPages?: number | null, maxLinksPerPage?: number | null, maxConcurrent?: number | null, contentFilter?: WasmContentFilterKind | null, bm25Query?: string | null, bm25Threshold?: number | null, userAgent?: string | null, rateLimitMs?: bigint | null, auth?: any | null, maxBodySize?: number | null, mapLimit?: number | null, mapSearch?: string | null, maxAssetSize?: number | null, proxy?: WasmProxyConfig | null, documentUrlDepth?: number | null, documentMaxSize?: number | null, documentOutputDir?: string | null, documentContentEncoding?: WasmDocumentContentEncoding | null, warcOutput?: string | null, browserProfile?: string | null, ssrfDenyPrivateExplicit?: boolean | null);
+    constructor(crawlStrategy?: WasmCrawlStrategyKind | null, respectRobotsTxt?: boolean | null, softHttpErrors?: boolean | null, stayOnDomain?: boolean | null, allowSubdomains?: boolean | null, includePaths?: string[] | null, excludePaths?: string[] | null, pathPatternsMatchQuery?: boolean | null, pathPatternsMatchUrl?: boolean | null, dedupIncludeQuery?: boolean | null, stripTrackingParams?: boolean | null, trackingParams?: string[] | null, customHeaders?: any | null, requestTimeout?: bigint | null, maxRedirects?: number | null, retryCount?: number | null, retryCodes?: Uint16Array | null, retryInitialDelayMs?: bigint | null, retryMaxDelayMs?: bigint | null, rateLimitJitterRatio?: number | null, cookiesEnabled?: boolean | null, removeTags?: string[] | null, content?: WasmContentConfig | null, downloadAssets?: boolean | null, assetTypes?: WasmAssetCategory[] | null, browser?: WasmBrowserConfig | null, userAgents?: string[] | null, captureScreenshot?: boolean | null, followDocumentUrls?: boolean | null, downloadDocuments?: boolean | null, documentMimeTypes?: string[] | null, saveBrowserProfile?: boolean | null, ssrf?: WasmSsrfPolicy | null, maxDepth?: number | null, maxPages?: number | null, maxLinksPerPage?: number | null, maxConcurrent?: number | null, contentFilter?: WasmContentFilterKind | null, bm25Query?: string | null, bm25Threshold?: number | null, userAgent?: string | null, rateLimitMs?: bigint | null, auth?: any | null, maxBodySize?: number | null, mapLimit?: number | null, mapSearch?: string | null, maxAssetSize?: number | null, proxy?: WasmProxyConfig | null, documentUrlDepth?: number | null, documentMaxSize?: number | null, documentOutputDir?: string | null, documentContentEncoding?: WasmDocumentContentEncoding | null, warcOutput?: string | null, browserProfile?: string | null, ssrfDenyPrivateExplicit?: boolean | null);
     /**
      * Validate the configuration, returning an error if any values are invalid.
      *
@@ -324,6 +332,7 @@ export class WasmCrawlConfig {
     get crawlStrategy(): string;
     set crawlStrategy(value: WasmCrawlStrategyKind);
     customHeaders: any;
+    dedupIncludeQuery: boolean;
     get documentContentEncoding(): string | undefined;
     set documentContentEncoding(value: WasmDocumentContentEncoding | null | undefined);
     get documentMaxSize(): number | undefined;
@@ -355,8 +364,11 @@ export class WasmCrawlConfig {
     get maxPages(): number | undefined;
     set maxPages(value: number | null | undefined);
     maxRedirects: number;
+    pathPatternsMatchQuery: boolean;
+    pathPatternsMatchUrl: boolean;
     get proxy(): WasmProxyConfig | undefined;
     set proxy(value: WasmProxyConfig | null | undefined);
+    rateLimitJitterRatio: number;
     get rateLimitMs(): bigint | undefined;
     set rateLimitMs(value: bigint | null | undefined);
     removeTags: string[];
@@ -365,12 +377,16 @@ export class WasmCrawlConfig {
     respectRobotsTxt: boolean;
     retryCodes: Uint16Array;
     retryCount: number;
+    retryInitialDelayMs: bigint;
+    retryMaxDelayMs: bigint;
     saveBrowserProfile: boolean;
     softHttpErrors: boolean;
     get ssrfDenyPrivateExplicit(): boolean | undefined;
     set ssrfDenyPrivateExplicit(value: boolean | null | undefined);
     ssrf: WasmSsrfPolicy;
     stayOnDomain: boolean;
+    stripTrackingParams: boolean;
+    trackingParams: string[];
     get userAgent(): string | undefined;
     set userAgent(value: string | null | undefined);
     userAgents: string[];
@@ -397,7 +413,7 @@ export class WasmCrawlPageResult {
     free(): void;
     [Symbol.dispose](): void;
     static default(): WasmCrawlPageResult;
-    constructor(url?: string | null, normalizedUrl?: string | null, statusCode?: number | null, contentType?: string | null, html?: string | null, bodySize?: number | null, metadata?: WasmPageMetadata | null, links?: WasmLinkInfo[] | null, images?: WasmImageInfo[] | null, feeds?: WasmFeedInfo[] | null, jsonLd?: WasmJsonLdEntry[] | null, depth?: number | null, stayedOnDomain?: boolean | null, wasSkipped?: boolean | null, isPdf?: boolean | null, browserUsed?: boolean | null, detectedCharset?: string | null, markdown?: WasmMarkdownResult | null, extractedData?: any | null, extractionMeta?: WasmExtractionMeta | null, downloadedDocument?: WasmDownloadedDocument | null);
+    constructor(url?: string | null, normalizedUrl?: string | null, statusCode?: number | null, contentType?: string | null, html?: string | null, bodySize?: number | null, metadata?: WasmPageMetadata | null, links?: WasmLinkInfo[] | null, images?: WasmImageInfo[] | null, feeds?: WasmFeedInfo[] | null, jsonLd?: WasmJsonLdEntry[] | null, depth?: number | null, stayedOnDomain?: boolean | null, wasSkipped?: boolean | null, isPdf?: boolean | null, browserUsed?: boolean | null, finalUrl?: string | null, redirectCount?: number | null, noindexDetected?: boolean | null, nofollowDetected?: boolean | null, ssrfRefusedUrls?: string[] | null, detectedCharset?: string | null, markdown?: WasmMarkdownResult | null, extractedData?: any | null, extractionMeta?: WasmExtractionMeta | null, downloadedDocument?: WasmDownloadedDocument | null);
     bodySize: number;
     browserUsed: boolean;
     contentType: string;
@@ -411,6 +427,7 @@ export class WasmCrawlPageResult {
     get extractionMeta(): WasmExtractionMeta | undefined;
     set extractionMeta(value: WasmExtractionMeta | null | undefined);
     feeds: WasmFeedInfo[];
+    finalUrl: string;
     html: string;
     images: WasmImageInfo[];
     isPdf: boolean;
@@ -419,7 +436,11 @@ export class WasmCrawlPageResult {
     get markdown(): WasmMarkdownResult | undefined;
     set markdown(value: WasmMarkdownResult | null | undefined);
     metadata: WasmPageMetadata;
+    nofollowDetected: boolean;
+    noindexDetected: boolean;
     normalizedUrl: string;
+    redirectCount: number;
+    ssrfRefusedUrls: string[];
     statusCode: number;
     stayedOnDomain: boolean;
     url: string;
@@ -673,12 +694,13 @@ export class WasmInteractionResult {
     free(): void;
     [Symbol.dispose](): void;
     static default(): WasmInteractionResult;
-    constructor(actionResults?: WasmActionResult[] | null, finalHtml?: string | null, finalUrl?: string | null, screenshotBase64?: string | null);
+    constructor(actionResults?: WasmActionResult[] | null, finalHtml?: string | null, finalUrl?: string | null, ssrfRefusedUrls?: string[] | null, screenshotBase64?: string | null);
     actionResults: WasmActionResult[];
     finalHtml: string;
     finalUrl: string;
     get screenshotBase64(): string | undefined;
     set screenshotBase64(value: string | null | undefined);
+    ssrfRefusedUrls: string[];
 }
 
 /**
@@ -923,7 +945,7 @@ export class WasmScrapeResult {
     free(): void;
     [Symbol.dispose](): void;
     static default(): WasmScrapeResult;
-    constructor(statusCode?: number | null, finalUrl?: string | null, contentType?: string | null, html?: string | null, bodySize?: number | null, metadata?: WasmPageMetadata | null, links?: WasmLinkInfo[] | null, images?: WasmImageInfo[] | null, feeds?: WasmFeedInfo[] | null, jsonLd?: WasmJsonLdEntry[] | null, isAllowed?: boolean | null, noindexDetected?: boolean | null, nofollowDetected?: boolean | null, isPdf?: boolean | null, wasSkipped?: boolean | null, authHeaderSent?: boolean | null, assets?: WasmDownloadedAsset[] | null, jsRenderHint?: boolean | null, browserUsed?: boolean | null, crawlDelay?: bigint | null, xRobotsTag?: string | null, detectedCharset?: string | null, responseMeta?: WasmResponseMeta | null, markdown?: WasmMarkdownResult | null, extractedData?: any | null, extractionMeta?: WasmExtractionMeta | null, screenshotBase64?: string | null, downloadedDocument?: WasmDownloadedDocument | null, browser?: WasmBrowserExtras | null);
+    constructor(statusCode?: number | null, finalUrl?: string | null, contentType?: string | null, html?: string | null, bodySize?: number | null, metadata?: WasmPageMetadata | null, links?: WasmLinkInfo[] | null, images?: WasmImageInfo[] | null, feeds?: WasmFeedInfo[] | null, jsonLd?: WasmJsonLdEntry[] | null, isAllowed?: boolean | null, noindexDetected?: boolean | null, nofollowDetected?: boolean | null, isPdf?: boolean | null, wasSkipped?: boolean | null, authHeaderSent?: boolean | null, assets?: WasmDownloadedAsset[] | null, jsRenderHint?: boolean | null, browserUsed?: boolean | null, ssrfRefusedUrls?: string[] | null, crawlDelay?: bigint | null, xRobotsTag?: string | null, detectedCharset?: string | null, responseMeta?: WasmResponseMeta | null, markdown?: WasmMarkdownResult | null, extractedData?: any | null, extractionMeta?: WasmExtractionMeta | null, screenshotBase64?: string | null, downloadedDocument?: WasmDownloadedDocument | null, browser?: WasmBrowserExtras | null);
     assets: WasmDownloadedAsset[];
     authHeaderSent: boolean;
     bodySize: number;
@@ -959,6 +981,7 @@ export class WasmScrapeResult {
     set responseMeta(value: WasmResponseMeta | null | undefined);
     get screenshotBase64(): string | undefined;
     set screenshotBase64(value: string | null | undefined);
+    ssrfRefusedUrls: string[];
     statusCode: number;
     wasSkipped: boolean;
     get xRobotsTag(): string | undefined;