@kreuzberg/kreuzcrawl 0.1.1 → 0.2.0

package/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "kreuzcrawl-node"
-version = "0.1.1"
+version = "0.2.0"
 edition = "2024"
 license = "Elastic-2.0"
 
@@ -16,3 +16,7 @@ serde_json = "1"
 
 [build-dependencies]
 napi-build = "2"
+
+[lints.clippy]
+duplicated_attributes = "allow"
+unwrap_used = "allow"

package/index.d.ts

@@ -12,14 +12,6 @@ export declare function crawl(engine: JsCrawlEngineHandle, url: string): Promise
 
 export declare function createEngine(config?: JsCrawlConfig | undefined | null): JsCrawlEngineHandle
 
-export interface JsActionResult {
-  actionIndex?: number
-  actionType?: string
-  success?: boolean
-  data?: string
-  error?: string
-}
-
 export interface JsArticleMetadata {
   publishedTime?: string
   modifiedTime?: string
@@ -83,16 +75,6 @@ export declare const enum JsBrowserWait {
   Fixed = 'fixed'
 }
 
-export interface JsCachedPage {
-  url?: string
-  statusCode?: number
-  contentType?: string
-  body?: string
-  etag?: string
-  lastModified?: string
-  cachedAt?: number
-}
-
 export interface JsCitationReference {
   index?: number
   url?: string
@@ -104,6 +86,21 @@ export interface JsCitationResult {
   references?: Array<JsCitationReference>
 }
 
+export interface JsContentConfig {
+  outputFormat?: string
+  preprocessingPreset?: string
+  removeNavigation?: boolean
+  removeForms?: boolean
+  stripTags?: Array<string>
+  preserveTags?: Array<string>
+  excludeSelectors?: Array<string>
+  skipImages?: boolean
+  maxDepth?: number
+  wrap?: boolean
+  wrapWidth?: number
+  includeDocumentStructure?: boolean
+}
+
 export interface JsCookieInfo {
   name?: string
   value?: string
@@ -123,14 +120,15 @@ export interface JsCrawlConfig {
   excludePaths?: Array<string>
   customHeaders?: Record<string, string>
   requestTimeout?: number
+  rateLimitMs?: number
   maxRedirects?: number
   retryCount?: number
   retryCodes?: Array<number>
   cookiesEnabled?: boolean
   auth?: JsAuthConfig
   maxBodySize?: number
-  mainContentOnly?: boolean
   removeTags?: Array<string>
+  content?: JsContentConfig
   mapLimit?: number
   mapSearch?: string
   downloadAssets?: boolean
@@ -148,12 +146,6 @@ export interface JsCrawlConfig {
   saveBrowserProfile?: boolean
 }
 
-export declare const enum JsCrawlEvent {
-  Page = 'Page',
-  Error = 'Error',
-  Complete = 'Complete'
-}
-
 export interface JsCrawlPageResult {
   url?: string
   normalizedUrl?: string
@@ -258,13 +250,6 @@ export declare const enum JsImageSource {
   TwitterImage = 'twitter_image'
 }
 
-export interface JsInteractionResult {
-  actionResults?: Array<JsActionResult>
-  finalHtml?: string
-  finalUrl?: string
-  screenshot?: Array<number>
-}
-
 export interface JsJsonLdEntry {
   schemaType?: string
   name?: string
@@ -379,7 +364,6 @@ export interface JsScrapeResult {
   isPdf?: boolean
   wasSkipped?: boolean
   detectedCharset?: string
-  mainContentOnly?: boolean
   authHeaderSent?: boolean
   responseMeta?: JsResponseMeta
   assets?: Array<JsDownloadedAsset>

package/index.js

@@ -77,8 +77,8 @@ function requireNative() {
       try {
         const binding = require('@kreuzberg/kreuzcrawl-android-arm64')
         const bindingPackageVersion = require('@kreuzberg/kreuzcrawl-android-arm64/package.json').version
-        if (bindingPackageVersion !== '0.1.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -93,8 +93,8 @@ function requireNative() {
       try {
         const binding = require('@kreuzberg/kreuzcrawl-android-arm-eabi')
         const bindingPackageVersion = require('@kreuzberg/kreuzcrawl-android-arm-eabi/package.json').version
-        if (bindingPackageVersion !== '0.1.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -114,8 +114,8 @@ function requireNative() {
       try {
         const binding = require('@kreuzberg/kreuzcrawl-win32-x64-gnu')
         const bindingPackageVersion = require('@kreuzberg/kreuzcrawl-win32-x64-gnu/package.json').version
-        if (bindingPackageVersion !== '0.1.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -130,8 +130,8 @@ function requireNative() {
       try {
         const binding = require('@kreuzberg/kreuzcrawl-win32-x64-msvc')
         const bindingPackageVersion = require('@kreuzberg/kreuzcrawl-win32-x64-msvc/package.json').version
-        if (bindingPackageVersion !== '0.1.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -147,8 +147,8 @@ function requireNative() {
       try {
         const binding = require('@kreuzberg/kreuzcrawl-win32-ia32-msvc')
         const bindingPackageVersion = require('@kreuzberg/kreuzcrawl-win32-ia32-msvc/package.json').version
-        if (bindingPackageVersion !== '0.1.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -163,8 +163,8 @@ function requireNative() {
       try {
         const binding = require('@kreuzberg/kreuzcrawl-win32-arm64-msvc')
         const bindingPackageVersion = require('@kreuzberg/kreuzcrawl-win32-arm64-msvc/package.json').version
-        if (bindingPackageVersion !== '0.1.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -182,8 +182,8 @@ function requireNative() {
     try {
       const binding = require('@kreuzberg/kreuzcrawl-darwin-universal')
       const bindingPackageVersion = require('@kreuzberg/kreuzcrawl-darwin-universal/package.json').version
-      if (bindingPackageVersion !== '0.1.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-        throw new Error(`Native binding package version mismatch, expected 0.1.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+      if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+        throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
       }
       return binding
     } catch (e) {
@@ -198,8 +198,8 @@ function requireNative() {
       try {
         const binding = require('@kreuzberg/kreuzcrawl-darwin-x64')
         const bindingPackageVersion = require('@kreuzberg/kreuzcrawl-darwin-x64/package.json').version
-        if (bindingPackageVersion !== '0.1.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -214,8 +214,8 @@ function requireNative() {
       try {
         const binding = require('@kreuzberg/kreuzcrawl-darwin-arm64')
         const bindingPackageVersion = require('@kreuzberg/kreuzcrawl-darwin-arm64/package.json').version
-        if (bindingPackageVersion !== '0.1.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -234,8 +234,8 @@ function requireNative() {
       try {
         const binding = require('@kreuzberg/kreuzcrawl-freebsd-x64')
         const bindingPackageVersion = require('@kreuzberg/kreuzcrawl-freebsd-x64/package.json').version
-        if (bindingPackageVersion !== '0.1.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -250,8 +250,8 @@ function requireNative() {
       try {
         const binding = require('@kreuzberg/kreuzcrawl-freebsd-arm64')
         const bindingPackageVersion = require('@kreuzberg/kreuzcrawl-freebsd-arm64/package.json').version
-        if (bindingPackageVersion !== '0.1.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -271,8 +271,8 @@ function requireNative() {
         try {
           const binding = require('@kreuzberg/kreuzcrawl-linux-x64-musl')
           const bindingPackageVersion = require('@kreuzberg/kreuzcrawl-linux-x64-musl/package.json').version
-          if (bindingPackageVersion !== '0.1.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 0.1.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -287,8 +287,8 @@ function requireNative() {
         try {
           const binding = require('@kreuzberg/kreuzcrawl-linux-x64-gnu')
           const bindingPackageVersion = require('@kreuzberg/kreuzcrawl-linux-x64-gnu/package.json').version
-          if (bindingPackageVersion !== '0.1.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 0.1.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -305,8 +305,8 @@ function requireNative() {
         try {
           const binding = require('@kreuzberg/kreuzcrawl-linux-arm64-musl')
           const bindingPackageVersion = require('@kreuzberg/kreuzcrawl-linux-arm64-musl/package.json').version
-          if (bindingPackageVersion !== '0.1.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 0.1.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -321,8 +321,8 @@ function requireNative() {
         try {
           const binding = require('@kreuzberg/kreuzcrawl-linux-arm64-gnu')
           const bindingPackageVersion = require('@kreuzberg/kreuzcrawl-linux-arm64-gnu/package.json').version
-          if (bindingPackageVersion !== '0.1.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 0.1.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -339,8 +339,8 @@ function requireNative() {
         try {
           const binding = require('@kreuzberg/kreuzcrawl-linux-arm-musleabihf')
           const bindingPackageVersion = require('@kreuzberg/kreuzcrawl-linux-arm-musleabihf/package.json').version
-          if (bindingPackageVersion !== '0.1.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 0.1.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -355,8 +355,8 @@ function requireNative() {
         try {
           const binding = require('@kreuzberg/kreuzcrawl-linux-arm-gnueabihf')
           const bindingPackageVersion = require('@kreuzberg/kreuzcrawl-linux-arm-gnueabihf/package.json').version
-          if (bindingPackageVersion !== '0.1.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 0.1.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -373,8 +373,8 @@ function requireNative() {
         try {
           const binding = require('@kreuzberg/kreuzcrawl-linux-loong64-musl')
           const bindingPackageVersion = require('@kreuzberg/kreuzcrawl-linux-loong64-musl/package.json').version
-          if (bindingPackageVersion !== '0.1.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 0.1.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -389,8 +389,8 @@ function requireNative() {
         try {
           const binding = require('@kreuzberg/kreuzcrawl-linux-loong64-gnu')
           const bindingPackageVersion = require('@kreuzberg/kreuzcrawl-linux-loong64-gnu/package.json').version
-          if (bindingPackageVersion !== '0.1.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 0.1.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -407,8 +407,8 @@ function requireNative() {
         try {
           const binding = require('@kreuzberg/kreuzcrawl-linux-riscv64-musl')
           const bindingPackageVersion = require('@kreuzberg/kreuzcrawl-linux-riscv64-musl/package.json').version
-          if (bindingPackageVersion !== '0.1.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 0.1.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -423,8 +423,8 @@ function requireNative() {
         try {
           const binding = require('@kreuzberg/kreuzcrawl-linux-riscv64-gnu')
           const bindingPackageVersion = require('@kreuzberg/kreuzcrawl-linux-riscv64-gnu/package.json').version
-          if (bindingPackageVersion !== '0.1.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 0.1.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -440,8 +440,8 @@ function requireNative() {
       try {
         const binding = require('@kreuzberg/kreuzcrawl-linux-ppc64-gnu')
         const bindingPackageVersion = require('@kreuzberg/kreuzcrawl-linux-ppc64-gnu/package.json').version
-        if (bindingPackageVersion !== '0.1.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -456,8 +456,8 @@ function requireNative() {
       try {
         const binding = require('@kreuzberg/kreuzcrawl-linux-s390x-gnu')
         const bindingPackageVersion = require('@kreuzberg/kreuzcrawl-linux-s390x-gnu/package.json').version
-        if (bindingPackageVersion !== '0.1.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -476,8 +476,8 @@ function requireNative() {
       try {
         const binding = require('@kreuzberg/kreuzcrawl-openharmony-arm64')
         const bindingPackageVersion = require('@kreuzberg/kreuzcrawl-openharmony-arm64/package.json').version
-        if (bindingPackageVersion !== '0.1.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -492,8 +492,8 @@ function requireNative() {
       try {
         const binding = require('@kreuzberg/kreuzcrawl-openharmony-x64')
         const bindingPackageVersion = require('@kreuzberg/kreuzcrawl-openharmony-x64/package.json').version
-        if (bindingPackageVersion !== '0.1.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -508,8 +508,8 @@ function requireNative() {
       try {
         const binding = require('@kreuzberg/kreuzcrawl-openharmony-arm')
         const bindingPackageVersion = require('@kreuzberg/kreuzcrawl-openharmony-arm/package.json').version
-        if (bindingPackageVersion !== '0.1.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -584,7 +584,6 @@ module.exports.createEngine = nativeBinding.createEngine
 module.exports.JsAssetCategory = nativeBinding.JsAssetCategory
 module.exports.JsBrowserMode = nativeBinding.JsBrowserMode
 module.exports.JsBrowserWait = nativeBinding.JsBrowserWait
-module.exports.JsCrawlEvent = nativeBinding.JsCrawlEvent
 module.exports.JsFeedType = nativeBinding.JsFeedType
 module.exports.JsImageSource = nativeBinding.JsImageSource
 module.exports.JsLinkType = nativeBinding.JsLinkType

package/npm/darwin-arm64/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kreuzberg/kreuzcrawl-darwin-arm64",
-  "version": "0.1.1",
+  "version": "0.2.0",
   "cpu": [
     "arm64"
   ],

package/npm/linux-arm64-gnu/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kreuzberg/kreuzcrawl-linux-arm64-gnu",
-  "version": "0.1.1",
+  "version": "0.2.0",
   "cpu": [
     "arm64"
   ],

package/npm/linux-x64-gnu/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kreuzberg/kreuzcrawl-linux-x64-gnu",
-  "version": "0.1.1",
+  "version": "0.2.0",
   "cpu": [
     "x64"
   ],

package/npm/win32-x64-msvc/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kreuzberg/kreuzcrawl-win32-x64-msvc",
-  "version": "0.1.1",
+  "version": "0.2.0",
   "cpu": [
     "x64"
   ],

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kreuzberg/kreuzcrawl",
-  "version": "0.1.1",
+  "version": "0.2.0",
   "main": "index.js",
   "types": "index.d.ts",
   "repository": {
@@ -35,9 +35,9 @@
     "@napi-rs/cli": "^3.6.2"
   },
   "optionalDependencies": {
-    "@kreuzberg/kreuzcrawl-darwin-arm64": "0.1.1",
-    "@kreuzberg/kreuzcrawl-linux-arm64-gnu": "0.1.1",
-    "@kreuzberg/kreuzcrawl-linux-x64-gnu": "0.1.1",
-    "@kreuzberg/kreuzcrawl-win32-x64-msvc": "0.1.1"
+    "@kreuzberg/kreuzcrawl-darwin-arm64": "0.2.0",
+    "@kreuzberg/kreuzcrawl-linux-arm64-gnu": "0.2.0",
+    "@kreuzberg/kreuzcrawl-linux-x64-gnu": "0.2.0",
+    "@kreuzberg/kreuzcrawl-win32-x64-msvc": "0.2.0"
   }
 }

package/src/lib.rs

@@ -6,7 +6,13 @@
     clippy::let_unit_value,
     clippy::needless_borrow,
     clippy::map_identity,
-    clippy::just_underscores_and_digits
+    clippy::just_underscores_and_digits,
+    clippy::unnecessary_cast,
+    clippy::unused_unit,
+    clippy::unwrap_or_default,
+    clippy::derivable_impls,
+    clippy::needless_borrows_for_generic_args,
+    clippy::unnecessary_fallible_conversions
 )]
 
 use napi::*;
@@ -42,6 +48,34 @@ pub struct JsProxyConfig {
     pub password: Option<String>,
 }
 
+#[derive(Clone, Default, serde::Serialize, serde::Deserialize)]
+#[napi(object)]
+pub struct JsContentConfig {
+    #[napi(js_name = "outputFormat")]
+    pub output_format: Option<String>,
+    #[napi(js_name = "preprocessingPreset")]
+    pub preprocessing_preset: Option<String>,
+    #[napi(js_name = "removeNavigation")]
+    pub remove_navigation: Option<bool>,
+    #[napi(js_name = "removeForms")]
+    pub remove_forms: Option<bool>,
+    #[napi(js_name = "stripTags")]
+    pub strip_tags: Option<Vec<String>>,
+    #[napi(js_name = "preserveTags")]
+    pub preserve_tags: Option<Vec<String>>,
+    #[napi(js_name = "excludeSelectors")]
+    pub exclude_selectors: Option<Vec<String>>,
+    #[napi(js_name = "skipImages")]
+    pub skip_images: Option<bool>,
+    #[napi(js_name = "maxDepth")]
+    pub max_depth: Option<i64>,
+    pub wrap: Option<bool>,
+    #[napi(js_name = "wrapWidth")]
+    pub wrap_width: Option<i64>,
+    #[napi(js_name = "includeDocumentStructure")]
+    pub include_document_structure: Option<bool>,
+}
+
 #[derive(Clone, Default, serde::Serialize, serde::Deserialize)]
 #[napi(object)]
 pub struct JsBrowserConfig {
@@ -80,6 +114,8 @@ pub struct JsCrawlConfig {
     pub custom_headers: Option<HashMap<String, String>>,
     #[napi(js_name = "requestTimeout")]
     pub request_timeout: Option<i64>,
+    #[napi(js_name = "rateLimitMs")]
+    pub rate_limit_ms: Option<i64>,
     #[napi(js_name = "maxRedirects")]
     pub max_redirects: Option<i64>,
     #[napi(js_name = "retryCount")]
@@ -91,10 +127,9 @@ pub struct JsCrawlConfig {
     pub auth: Option<JsAuthConfig>,
     #[napi(js_name = "maxBodySize")]
     pub max_body_size: Option<i64>,
-    #[napi(js_name = "mainContentOnly")]
-    pub main_content_only: Option<bool>,
     #[napi(js_name = "removeTags")]
     pub remove_tags: Option<Vec<String>>,
+    pub content: Option<JsContentConfig>,
     #[napi(js_name = "mapLimit")]
     pub map_limit: Option<i64>,
     #[napi(js_name = "mapSearch")]
@@ -139,30 +174,6 @@ pub struct JsDownloadedDocument {
     pub headers: Option<HashMap<String, String>>,
 }
 
-#[derive(Clone, Default, serde::Serialize, serde::Deserialize)]
-#[napi(object)]
-pub struct JsInteractionResult {
-    #[napi(js_name = "actionResults")]
-    pub action_results: Option<Vec<JsActionResult>>,
-    #[napi(js_name = "finalHtml")]
-    pub final_html: Option<String>,
-    #[napi(js_name = "finalUrl")]
-    pub final_url: Option<String>,
-    pub screenshot: Option<Vec<u8>>,
-}
-
-#[derive(Clone, Default, serde::Serialize, serde::Deserialize)]
-#[napi(object)]
-pub struct JsActionResult {
-    #[napi(js_name = "actionIndex")]
-    pub action_index: Option<i64>,
-    #[napi(js_name = "actionType")]
-    pub action_type: Option<String>,
-    pub success: Option<bool>,
-    pub data: Option<String>,
-    pub error: Option<String>,
-}
-
 #[derive(Clone, Default, serde::Serialize, serde::Deserialize)]
 #[napi(object)]
 pub struct JsScrapeResult {
@@ -195,8 +206,6 @@ pub struct JsScrapeResult {
     pub was_skipped: Option<bool>,
     #[napi(js_name = "detectedCharset")]
     pub detected_charset: Option<String>,
-    #[napi(js_name = "mainContentOnly")]
-    pub main_content_only: Option<bool>,
     #[napi(js_name = "authHeaderSent")]
     pub auth_header_sent: Option<bool>,
     #[napi(js_name = "responseMeta")]
@@ -297,22 +306,6 @@ pub struct JsMarkdownResult {
     pub fit_content: Option<String>,
 }
 
-#[derive(Clone, Default, serde::Serialize, serde::Deserialize)]
-#[napi(object)]
-pub struct JsCachedPage {
-    pub url: Option<String>,
-    #[napi(js_name = "statusCode")]
-    pub status_code: Option<u16>,
-    #[napi(js_name = "contentType")]
-    pub content_type: Option<String>,
-    pub body: Option<String>,
-    pub etag: Option<String>,
-    #[napi(js_name = "lastModified")]
-    pub last_modified: Option<String>,
-    #[napi(js_name = "cachedAt")]
-    pub cached_at: Option<i64>,
-}
-
 #[derive(Clone, Default, serde::Serialize, serde::Deserialize)]
 #[napi(object)]
 pub struct JsLinkInfo {
@@ -674,21 +667,6 @@ impl Default for JsAssetCategory {
     }
 }
 
-#[napi(string_enum)]
-#[derive(Clone, serde::Serialize, serde::Deserialize)]
-pub enum JsCrawlEvent {
-    Page,
-    Error,
-    Complete,
-}
-
-#[allow(clippy::derivable_impls)]
-impl Default for JsCrawlEvent {
-    fn default() -> Self {
-        Self::Page
-    }
-}
-
 #[allow(clippy::missing_errors_doc)]
 #[napi(js_name = "createEngine")]
 pub fn create_engine(config: Option<JsCrawlConfig>) -> Result<JsCrawlEngineHandle> {
@@ -781,6 +759,44 @@ impl From<kreuzcrawl::ProxyConfig> for JsProxyConfig {
     }
 }
 
+impl From<JsContentConfig> for kreuzcrawl::ContentConfig {
+    fn from(val: JsContentConfig) -> Self {
+        Self {
+            output_format: val.output_format.unwrap_or_default(),
+            preprocessing_preset: val.preprocessing_preset.unwrap_or_default(),
+            remove_navigation: val.remove_navigation.unwrap_or_default(),
+            remove_forms: val.remove_forms.unwrap_or_default(),
+            strip_tags: val.strip_tags.unwrap_or_default(),
+            preserve_tags: val.preserve_tags.unwrap_or_default(),
+            exclude_selectors: val.exclude_selectors.unwrap_or_default(),
+            skip_images: val.skip_images.unwrap_or_default(),
+            max_depth: val.max_depth.map(|v| v as usize),
+            wrap: val.wrap.unwrap_or_default(),
+            wrap_width: val.wrap_width.map(|v| v as usize).unwrap_or_default(),
+            include_document_structure: val.include_document_structure.unwrap_or_default(),
+        }
+    }
+}
+
+impl From<kreuzcrawl::ContentConfig> for JsContentConfig {
+    fn from(val: kreuzcrawl::ContentConfig) -> Self {
+        Self {
+            output_format: Some(val.output_format),
+            preprocessing_preset: Some(val.preprocessing_preset),
+            remove_navigation: Some(val.remove_navigation),
+            remove_forms: Some(val.remove_forms),
+            strip_tags: Some(val.strip_tags),
+            preserve_tags: Some(val.preserve_tags),
+            exclude_selectors: Some(val.exclude_selectors),
+            skip_images: Some(val.skip_images),
+            max_depth: val.max_depth.map(|v| v as i64),
+            wrap: Some(val.wrap),
+            wrap_width: Some(val.wrap_width as i64),
+            include_document_structure: Some(val.include_document_structure),
+        }
+    }
+}
+
 #[allow(clippy::field_reassign_with_default)]
 impl From<JsBrowserConfig> for kreuzcrawl::BrowserConfig {
     fn from(val: JsBrowserConfig) -> Self {
@@ -828,14 +844,15 @@ impl From<JsCrawlConfig> for kreuzcrawl::CrawlConfig {
         if let Some(__v) = val.request_timeout {
             __result.request_timeout = std::time::Duration::from_millis(__v as u64);
         }
+        __result.rate_limit_ms = val.rate_limit_ms.map(|v| v as u64);
         __result.max_redirects = val.max_redirects.map(|v| v as usize).unwrap_or_default();
         __result.retry_count = val.retry_count.map(|v| v as usize).unwrap_or_default();
         __result.retry_codes = val.retry_codes.unwrap_or_default();
         __result.cookies_enabled = val.cookies_enabled.unwrap_or_default();
         __result.auth = val.auth.map(Into::into);
         __result.max_body_size = val.max_body_size.map(|v| v as usize);
-        __result.main_content_only = val.main_content_only.unwrap_or_default();
         __result.remove_tags = val.remove_tags.unwrap_or_default();
+        __result.content = val.content.map(Into::into).unwrap_or_default();
         __result.map_limit = val.map_limit.map(|v| v as usize);
         __result.map_search = val.map_search;
         __result.download_assets = val.download_assets.unwrap_or_default();
@@ -872,14 +889,15 @@ impl From<kreuzcrawl::CrawlConfig> for JsCrawlConfig {
             exclude_paths: Some(val.exclude_paths),
             custom_headers: Some(val.custom_headers.into_iter().collect()),
             request_timeout: Some(val.request_timeout.as_millis() as u64 as i64),
+            rate_limit_ms: val.rate_limit_ms.map(|v| v as i64),
             max_redirects: Some(val.max_redirects as i64),
             retry_count: Some(val.retry_count as i64),
             retry_codes: Some(val.retry_codes),
             cookies_enabled: Some(val.cookies_enabled),
             auth: val.auth.map(Into::into),
             max_body_size: val.max_body_size.map(|v| v as i64),
-            main_content_only: Some(val.main_content_only),
             remove_tags: Some(val.remove_tags),
+            content: Some(val.content.into()),
             map_limit: val.map_limit.map(|v| v as i64),
             map_search: val.map_search,
             download_assets: Some(val.download_assets),
@@ -920,41 +938,18 @@ impl From<kreuzcrawl::DownloadedDocument> for JsDownloadedDocument {
             mime_type: Some(format!("{:?}", val.mime_type)),
             content: Some(val.content.to_vec()),
             size: Some(val.size as i64),
-            filename: val.filename.as_ref().map(|v| format!("{:?}", v)),
+            filename: val.filename.as_ref().map(|v| format!("{v:?}")),
             content_hash: Some(format!("{:?}", val.content_hash)),
             headers: Some(
                 val.headers
                     .into_iter()
-                    .map(|(k, v)| (format!("{:?}", k), format!("{:?}", v)))
+                    .map(|(k, v)| (k.to_string(), v.to_string()))
                     .collect(),
             ),
         }
     }
 }
 
-impl From<kreuzcrawl::InteractionResult> for JsInteractionResult {
-    fn from(val: kreuzcrawl::InteractionResult) -> Self {
-        Self {
-            action_results: Some(val.action_results.into_iter().map(Into::into).collect()),
-            final_html: Some(val.final_html),
-            final_url: Some(val.final_url),
-            screenshot: val.screenshot.map(|v| v.to_vec()),
-        }
-    }
-}
-
-impl From<kreuzcrawl::ActionResult> for JsActionResult {
-    fn from(val: kreuzcrawl::ActionResult) -> Self {
-        Self {
-            action_index: Some(val.action_index as i64),
-            action_type: Some(format!("{:?}", val.action_type)),
-            success: Some(val.success),
-            data: val.data.as_ref().map(ToString::to_string),
-            error: val.error,
-        }
-    }
-}
-
 impl From<JsScrapeResult> for kreuzcrawl::ScrapeResult {
     fn from(val: JsScrapeResult) -> Self {
         Self {
@@ -987,7 +982,6 @@ impl From<JsScrapeResult> for kreuzcrawl::ScrapeResult {
             is_pdf: val.is_pdf.unwrap_or_default(),
             was_skipped: val.was_skipped.unwrap_or_default(),
             detected_charset: val.detected_charset,
-            main_content_only: val.main_content_only.unwrap_or_default(),
             auth_header_sent: val.auth_header_sent.unwrap_or_default(),
             response_meta: val.response_meta.map(Into::into),
             assets: val
@@ -1025,7 +1019,6 @@ impl From<kreuzcrawl::ScrapeResult> for JsScrapeResult {
             is_pdf: Some(val.is_pdf),
             was_skipped: Some(val.was_skipped),
             detected_charset: val.detected_charset,
-            main_content_only: Some(val.main_content_only),
             auth_header_sent: Some(val.auth_header_sent),
             response_meta: val.response_meta.map(Into::into),
             assets: Some(val.assets.into_iter().map(Into::into).collect()),
@@ -1213,20 +1206,6 @@ impl From<kreuzcrawl::MarkdownResult> for JsMarkdownResult {
     }
 }
 
-impl From<kreuzcrawl::CachedPage> for JsCachedPage {
-    fn from(val: kreuzcrawl::CachedPage) -> Self {
-        Self {
-            url: Some(val.url),
-            status_code: Some(val.status_code),
-            content_type: Some(val.content_type),
-            body: Some(val.body),
-            etag: val.etag,
-            last_modified: val.last_modified,
-            cached_at: Some(val.cached_at as i64),
-        }
-    }
-}
-
 impl From<JsLinkInfo> for kreuzcrawl::LinkInfo {
     fn from(val: JsLinkInfo) -> Self {
         Self {
@@ -1845,16 +1824,6 @@ impl From<kreuzcrawl::AssetCategory> for JsAssetCategory {
     }
 }
 
-impl From<kreuzcrawl::CrawlEvent> for JsCrawlEvent {
-    fn from(val: kreuzcrawl::CrawlEvent) -> Self {
-        match val {
-            kreuzcrawl::CrawlEvent::Page(..) => Self::Page,
-            kreuzcrawl::CrawlEvent::Error { .. } => Self::Error,
-            kreuzcrawl::CrawlEvent::Complete { .. } => Self::Complete,
-        }
-    }
-}
-
 // Error variant name constants
 pub const CRAWL_ERROR_ERROR_NOT_FOUND: &str = "NotFound";
 pub const CRAWL_ERROR_ERROR_UNAUTHORIZED: &str = "Unauthorized";