@kreuzberg/kreuzcrawl 0.1.2 → 0.2.0

package/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "kreuzcrawl-node"
-version = "0.1.2"
+version = "0.2.0"
 edition = "2024"
 license = "Elastic-2.0"
 

package/index.d.ts

@@ -86,6 +86,21 @@ export interface JsCitationResult {
   references?: Array<JsCitationReference>
 }
 
+export interface JsContentConfig {
+  outputFormat?: string
+  preprocessingPreset?: string
+  removeNavigation?: boolean
+  removeForms?: boolean
+  stripTags?: Array<string>
+  preserveTags?: Array<string>
+  excludeSelectors?: Array<string>
+  skipImages?: boolean
+  maxDepth?: number
+  wrap?: boolean
+  wrapWidth?: number
+  includeDocumentStructure?: boolean
+}
+
 export interface JsCookieInfo {
   name?: string
   value?: string
@@ -112,8 +127,8 @@ export interface JsCrawlConfig {
   cookiesEnabled?: boolean
   auth?: JsAuthConfig
   maxBodySize?: number
-  mainContentOnly?: boolean
   removeTags?: Array<string>
+  content?: JsContentConfig
   mapLimit?: number
   mapSearch?: string
   downloadAssets?: boolean
@@ -349,7 +364,6 @@ export interface JsScrapeResult {
   isPdf?: boolean
   wasSkipped?: boolean
   detectedCharset?: string
-  mainContentOnly?: boolean
   authHeaderSent?: boolean
   responseMeta?: JsResponseMeta
   assets?: Array<JsDownloadedAsset>

package/index.js

@@ -77,8 +77,8 @@ function requireNative() {
       try {
         const binding = require('@kreuzberg/kreuzcrawl-android-arm64')
         const bindingPackageVersion = require('@kreuzberg/kreuzcrawl-android-arm64/package.json').version
-        if (bindingPackageVersion !== '0.1.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -93,8 +93,8 @@ function requireNative() {
       try {
         const binding = require('@kreuzberg/kreuzcrawl-android-arm-eabi')
         const bindingPackageVersion = require('@kreuzberg/kreuzcrawl-android-arm-eabi/package.json').version
-        if (bindingPackageVersion !== '0.1.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -114,8 +114,8 @@ function requireNative() {
       try {
         const binding = require('@kreuzberg/kreuzcrawl-win32-x64-gnu')
         const bindingPackageVersion = require('@kreuzberg/kreuzcrawl-win32-x64-gnu/package.json').version
-        if (bindingPackageVersion !== '0.1.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -130,8 +130,8 @@ function requireNative() {
       try {
         const binding = require('@kreuzberg/kreuzcrawl-win32-x64-msvc')
         const bindingPackageVersion = require('@kreuzberg/kreuzcrawl-win32-x64-msvc/package.json').version
-        if (bindingPackageVersion !== '0.1.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -147,8 +147,8 @@ function requireNative() {
       try {
         const binding = require('@kreuzberg/kreuzcrawl-win32-ia32-msvc')
         const bindingPackageVersion = require('@kreuzberg/kreuzcrawl-win32-ia32-msvc/package.json').version
-        if (bindingPackageVersion !== '0.1.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -163,8 +163,8 @@ function requireNative() {
       try {
         const binding = require('@kreuzberg/kreuzcrawl-win32-arm64-msvc')
         const bindingPackageVersion = require('@kreuzberg/kreuzcrawl-win32-arm64-msvc/package.json').version
-        if (bindingPackageVersion !== '0.1.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -182,8 +182,8 @@ function requireNative() {
     try {
       const binding = require('@kreuzberg/kreuzcrawl-darwin-universal')
       const bindingPackageVersion = require('@kreuzberg/kreuzcrawl-darwin-universal/package.json').version
-      if (bindingPackageVersion !== '0.1.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-        throw new Error(`Native binding package version mismatch, expected 0.1.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+      if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+        throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
       }
       return binding
     } catch (e) {
@@ -198,8 +198,8 @@ function requireNative() {
       try {
         const binding = require('@kreuzberg/kreuzcrawl-darwin-x64')
         const bindingPackageVersion = require('@kreuzberg/kreuzcrawl-darwin-x64/package.json').version
-        if (bindingPackageVersion !== '0.1.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -214,8 +214,8 @@ function requireNative() {
       try {
         const binding = require('@kreuzberg/kreuzcrawl-darwin-arm64')
         const bindingPackageVersion = require('@kreuzberg/kreuzcrawl-darwin-arm64/package.json').version
-        if (bindingPackageVersion !== '0.1.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -234,8 +234,8 @@ function requireNative() {
       try {
         const binding = require('@kreuzberg/kreuzcrawl-freebsd-x64')
         const bindingPackageVersion = require('@kreuzberg/kreuzcrawl-freebsd-x64/package.json').version
-        if (bindingPackageVersion !== '0.1.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -250,8 +250,8 @@ function requireNative() {
       try {
         const binding = require('@kreuzberg/kreuzcrawl-freebsd-arm64')
         const bindingPackageVersion = require('@kreuzberg/kreuzcrawl-freebsd-arm64/package.json').version
-        if (bindingPackageVersion !== '0.1.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -271,8 +271,8 @@ function requireNative() {
         try {
           const binding = require('@kreuzberg/kreuzcrawl-linux-x64-musl')
           const bindingPackageVersion = require('@kreuzberg/kreuzcrawl-linux-x64-musl/package.json').version
-          if (bindingPackageVersion !== '0.1.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 0.1.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -287,8 +287,8 @@ function requireNative() {
         try {
           const binding = require('@kreuzberg/kreuzcrawl-linux-x64-gnu')
           const bindingPackageVersion = require('@kreuzberg/kreuzcrawl-linux-x64-gnu/package.json').version
-          if (bindingPackageVersion !== '0.1.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 0.1.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -305,8 +305,8 @@ function requireNative() {
         try {
           const binding = require('@kreuzberg/kreuzcrawl-linux-arm64-musl')
           const bindingPackageVersion = require('@kreuzberg/kreuzcrawl-linux-arm64-musl/package.json').version
-          if (bindingPackageVersion !== '0.1.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 0.1.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -321,8 +321,8 @@ function requireNative() {
         try {
           const binding = require('@kreuzberg/kreuzcrawl-linux-arm64-gnu')
           const bindingPackageVersion = require('@kreuzberg/kreuzcrawl-linux-arm64-gnu/package.json').version
-          if (bindingPackageVersion !== '0.1.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 0.1.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -339,8 +339,8 @@ function requireNative() {
         try {
           const binding = require('@kreuzberg/kreuzcrawl-linux-arm-musleabihf')
           const bindingPackageVersion = require('@kreuzberg/kreuzcrawl-linux-arm-musleabihf/package.json').version
-          if (bindingPackageVersion !== '0.1.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 0.1.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -355,8 +355,8 @@ function requireNative() {
         try {
           const binding = require('@kreuzberg/kreuzcrawl-linux-arm-gnueabihf')
           const bindingPackageVersion = require('@kreuzberg/kreuzcrawl-linux-arm-gnueabihf/package.json').version
-          if (bindingPackageVersion !== '0.1.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 0.1.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -373,8 +373,8 @@ function requireNative() {
         try {
           const binding = require('@kreuzberg/kreuzcrawl-linux-loong64-musl')
           const bindingPackageVersion = require('@kreuzberg/kreuzcrawl-linux-loong64-musl/package.json').version
-          if (bindingPackageVersion !== '0.1.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 0.1.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -389,8 +389,8 @@ function requireNative() {
         try {
           const binding = require('@kreuzberg/kreuzcrawl-linux-loong64-gnu')
           const bindingPackageVersion = require('@kreuzberg/kreuzcrawl-linux-loong64-gnu/package.json').version
-          if (bindingPackageVersion !== '0.1.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 0.1.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -407,8 +407,8 @@ function requireNative() {
         try {
           const binding = require('@kreuzberg/kreuzcrawl-linux-riscv64-musl')
           const bindingPackageVersion = require('@kreuzberg/kreuzcrawl-linux-riscv64-musl/package.json').version
-          if (bindingPackageVersion !== '0.1.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 0.1.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -423,8 +423,8 @@ function requireNative() {
         try {
           const binding = require('@kreuzberg/kreuzcrawl-linux-riscv64-gnu')
           const bindingPackageVersion = require('@kreuzberg/kreuzcrawl-linux-riscv64-gnu/package.json').version
-          if (bindingPackageVersion !== '0.1.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 0.1.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -440,8 +440,8 @@ function requireNative() {
       try {
         const binding = require('@kreuzberg/kreuzcrawl-linux-ppc64-gnu')
         const bindingPackageVersion = require('@kreuzberg/kreuzcrawl-linux-ppc64-gnu/package.json').version
-        if (bindingPackageVersion !== '0.1.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -456,8 +456,8 @@ function requireNative() {
       try {
         const binding = require('@kreuzberg/kreuzcrawl-linux-s390x-gnu')
         const bindingPackageVersion = require('@kreuzberg/kreuzcrawl-linux-s390x-gnu/package.json').version
-        if (bindingPackageVersion !== '0.1.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -476,8 +476,8 @@ function requireNative() {
       try {
         const binding = require('@kreuzberg/kreuzcrawl-openharmony-arm64')
         const bindingPackageVersion = require('@kreuzberg/kreuzcrawl-openharmony-arm64/package.json').version
-        if (bindingPackageVersion !== '0.1.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -492,8 +492,8 @@ function requireNative() {
       try {
         const binding = require('@kreuzberg/kreuzcrawl-openharmony-x64')
         const bindingPackageVersion = require('@kreuzberg/kreuzcrawl-openharmony-x64/package.json').version
-        if (bindingPackageVersion !== '0.1.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -508,8 +508,8 @@ function requireNative() {
       try {
         const binding = require('@kreuzberg/kreuzcrawl-openharmony-arm')
         const bindingPackageVersion = require('@kreuzberg/kreuzcrawl-openharmony-arm/package.json').version
-        if (bindingPackageVersion !== '0.1.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 0.1.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '0.2.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 0.2.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {

package/npm/darwin-arm64/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kreuzberg/kreuzcrawl-darwin-arm64",
-  "version": "0.1.2",
+  "version": "0.2.0",
   "cpu": [
     "arm64"
   ],

package/npm/linux-arm64-gnu/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kreuzberg/kreuzcrawl-linux-arm64-gnu",
-  "version": "0.1.2",
+  "version": "0.2.0",
   "cpu": [
     "arm64"
   ],

package/npm/linux-x64-gnu/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kreuzberg/kreuzcrawl-linux-x64-gnu",
-  "version": "0.1.2",
+  "version": "0.2.0",
   "cpu": [
     "x64"
   ],

package/npm/win32-x64-msvc/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kreuzberg/kreuzcrawl-win32-x64-msvc",
-  "version": "0.1.2",
+  "version": "0.2.0",
   "cpu": [
     "x64"
   ],

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kreuzberg/kreuzcrawl",
-  "version": "0.1.2",
+  "version": "0.2.0",
   "main": "index.js",
   "types": "index.d.ts",
   "repository": {
@@ -35,9 +35,9 @@
     "@napi-rs/cli": "^3.6.2"
   },
   "optionalDependencies": {
-    "@kreuzberg/kreuzcrawl-darwin-arm64": "0.1.2",
-    "@kreuzberg/kreuzcrawl-linux-arm64-gnu": "0.1.2",
-    "@kreuzberg/kreuzcrawl-linux-x64-gnu": "0.1.2",
-    "@kreuzberg/kreuzcrawl-win32-x64-msvc": "0.1.2"
+    "@kreuzberg/kreuzcrawl-darwin-arm64": "0.2.0",
+    "@kreuzberg/kreuzcrawl-linux-arm64-gnu": "0.2.0",
+    "@kreuzberg/kreuzcrawl-linux-x64-gnu": "0.2.0",
+    "@kreuzberg/kreuzcrawl-win32-x64-msvc": "0.2.0"
   }
 }

package/src/lib.rs

@@ -48,6 +48,34 @@ pub struct JsProxyConfig {
     pub password: Option<String>,
 }
 
+#[derive(Clone, Default, serde::Serialize, serde::Deserialize)]
+#[napi(object)]
+pub struct JsContentConfig {
+    #[napi(js_name = "outputFormat")]
+    pub output_format: Option<String>,
+    #[napi(js_name = "preprocessingPreset")]
+    pub preprocessing_preset: Option<String>,
+    #[napi(js_name = "removeNavigation")]
+    pub remove_navigation: Option<bool>,
+    #[napi(js_name = "removeForms")]
+    pub remove_forms: Option<bool>,
+    #[napi(js_name = "stripTags")]
+    pub strip_tags: Option<Vec<String>>,
+    #[napi(js_name = "preserveTags")]
+    pub preserve_tags: Option<Vec<String>>,
+    #[napi(js_name = "excludeSelectors")]
+    pub exclude_selectors: Option<Vec<String>>,
+    #[napi(js_name = "skipImages")]
+    pub skip_images: Option<bool>,
+    #[napi(js_name = "maxDepth")]
+    pub max_depth: Option<i64>,
+    pub wrap: Option<bool>,
+    #[napi(js_name = "wrapWidth")]
+    pub wrap_width: Option<i64>,
+    #[napi(js_name = "includeDocumentStructure")]
+    pub include_document_structure: Option<bool>,
+}
+
 #[derive(Clone, Default, serde::Serialize, serde::Deserialize)]
 #[napi(object)]
 pub struct JsBrowserConfig {
@@ -99,10 +127,9 @@ pub struct JsCrawlConfig {
     pub auth: Option<JsAuthConfig>,
     #[napi(js_name = "maxBodySize")]
     pub max_body_size: Option<i64>,
-    #[napi(js_name = "mainContentOnly")]
-    pub main_content_only: Option<bool>,
     #[napi(js_name = "removeTags")]
     pub remove_tags: Option<Vec<String>>,
+    pub content: Option<JsContentConfig>,
     #[napi(js_name = "mapLimit")]
     pub map_limit: Option<i64>,
     #[napi(js_name = "mapSearch")]
@@ -179,8 +206,6 @@ pub struct JsScrapeResult {
     pub was_skipped: Option<bool>,
     #[napi(js_name = "detectedCharset")]
     pub detected_charset: Option<String>,
-    #[napi(js_name = "mainContentOnly")]
-    pub main_content_only: Option<bool>,
     #[napi(js_name = "authHeaderSent")]
     pub auth_header_sent: Option<bool>,
     #[napi(js_name = "responseMeta")]
@@ -734,6 +759,44 @@ impl From<kreuzcrawl::ProxyConfig> for JsProxyConfig {
     }
 }
 
+impl From<JsContentConfig> for kreuzcrawl::ContentConfig {
+    fn from(val: JsContentConfig) -> Self {
+        Self {
+            output_format: val.output_format.unwrap_or_default(),
+            preprocessing_preset: val.preprocessing_preset.unwrap_or_default(),
+            remove_navigation: val.remove_navigation.unwrap_or_default(),
+            remove_forms: val.remove_forms.unwrap_or_default(),
+            strip_tags: val.strip_tags.unwrap_or_default(),
+            preserve_tags: val.preserve_tags.unwrap_or_default(),
+            exclude_selectors: val.exclude_selectors.unwrap_or_default(),
+            skip_images: val.skip_images.unwrap_or_default(),
+            max_depth: val.max_depth.map(|v| v as usize),
+            wrap: val.wrap.unwrap_or_default(),
+            wrap_width: val.wrap_width.map(|v| v as usize).unwrap_or_default(),
+            include_document_structure: val.include_document_structure.unwrap_or_default(),
+        }
+    }
+}
+
+impl From<kreuzcrawl::ContentConfig> for JsContentConfig {
+    fn from(val: kreuzcrawl::ContentConfig) -> Self {
+        Self {
+            output_format: Some(val.output_format),
+            preprocessing_preset: Some(val.preprocessing_preset),
+            remove_navigation: Some(val.remove_navigation),
+            remove_forms: Some(val.remove_forms),
+            strip_tags: Some(val.strip_tags),
+            preserve_tags: Some(val.preserve_tags),
+            exclude_selectors: Some(val.exclude_selectors),
+            skip_images: Some(val.skip_images),
+            max_depth: val.max_depth.map(|v| v as i64),
+            wrap: Some(val.wrap),
+            wrap_width: Some(val.wrap_width as i64),
+            include_document_structure: Some(val.include_document_structure),
+        }
+    }
+}
+
 #[allow(clippy::field_reassign_with_default)]
 impl From<JsBrowserConfig> for kreuzcrawl::BrowserConfig {
     fn from(val: JsBrowserConfig) -> Self {
@@ -788,8 +851,8 @@ impl From<JsCrawlConfig> for kreuzcrawl::CrawlConfig {
         __result.cookies_enabled = val.cookies_enabled.unwrap_or_default();
         __result.auth = val.auth.map(Into::into);
         __result.max_body_size = val.max_body_size.map(|v| v as usize);
-        __result.main_content_only = val.main_content_only.unwrap_or_default();
         __result.remove_tags = val.remove_tags.unwrap_or_default();
+        __result.content = val.content.map(Into::into).unwrap_or_default();
         __result.map_limit = val.map_limit.map(|v| v as usize);
         __result.map_search = val.map_search;
         __result.download_assets = val.download_assets.unwrap_or_default();
@@ -833,8 +896,8 @@ impl From<kreuzcrawl::CrawlConfig> for JsCrawlConfig {
             cookies_enabled: Some(val.cookies_enabled),
             auth: val.auth.map(Into::into),
             max_body_size: val.max_body_size.map(|v| v as i64),
-            main_content_only: Some(val.main_content_only),
             remove_tags: Some(val.remove_tags),
+            content: Some(val.content.into()),
             map_limit: val.map_limit.map(|v| v as i64),
             map_search: val.map_search,
             download_assets: Some(val.download_assets),
@@ -919,7 +982,6 @@ impl From<JsScrapeResult> for kreuzcrawl::ScrapeResult {
             is_pdf: val.is_pdf.unwrap_or_default(),
             was_skipped: val.was_skipped.unwrap_or_default(),
             detected_charset: val.detected_charset,
-            main_content_only: val.main_content_only.unwrap_or_default(),
             auth_header_sent: val.auth_header_sent.unwrap_or_default(),
             response_meta: val.response_meta.map(Into::into),
             assets: val
@@ -957,7 +1019,6 @@ impl From<kreuzcrawl::ScrapeResult> for JsScrapeResult {
             is_pdf: Some(val.is_pdf),
             was_skipped: Some(val.was_skipped),
             detected_charset: val.detected_charset,
-            main_content_only: Some(val.main_content_only),
             auth_header_sent: Some(val.auth_header_sent),
             response_meta: val.response_meta.map(Into::into),
             assets: Some(val.assets.into_iter().map(Into::into).collect()),