kreuzberg 4.1.2 → 4.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6550daabf58e5e396576e5a83c6a53f226e677f9c129920c9990bba309fbd7ba
-  data.tar.gz: 9595aa468666391d08a0962db589bbbc50d5bd1c8532e101efa234f6c523d7c5
+  metadata.gz: '010412940492f83bc170a4a4efc644ac7e3818502734035523796287837a3893'
+  data.tar.gz: af24cba007cc58283d678d4b15251ebae3a5740f60ade9a54cc19973a2022a82
 SHA512:
-  metadata.gz: 0dea911deebe061515dd4cbff2b76b3a7947c68f196fcc576001d42d80386f6c53f8ed63e0e4acb8e719ad6f95c21e689df7aef5f6cbbbc0d1c92ef96ddb673c
-  data.tar.gz: 0df091f80f7c73dda0c17d89d4aa0571cd01f0f2b697b187fd9bae28f8dbcf96cd2e3a269f9831a442b8cf46ce40608586d3d6a242d84bb394fe6056cba3b492
+  metadata.gz: ad67348bec54a01ca3592ed72e9b2b8bc9e711a37e11b40ada31466c67f834132fc0de278c53a1c014fa6751da7abebae934cff2a9cc1835f7e056c895a273cb
+  data.tar.gz: ca2cdb076a5d1af67f0e807978a966d1a391cc286bcdf5499544e3403196140918a54674beab77ea09fc0e8bc7ab66f357da5d984326a511b1d21643a3d6cf41

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    kreuzberg (4.1.2)
+    kreuzberg (4.2.1)
 
 GEM
   remote: https://rubygems.org/
@@ -207,7 +207,7 @@ CHECKSUMS
   i18n (1.14.8) sha256=285778639134865c5e0f6269e0b818256017e8cde89993fdfcbfb64d088824a5
   io-console (0.8.2) sha256=d6e3ae7a7cc7574f4b8893b4fca2162e57a825b223a177b7afa236c5ef9814cc
   json (2.18.0) sha256=b10506aee4183f5cf49e0efc48073d7b75843ce3782c68dbeb763351c08fd505
-  kreuzberg (4.1.2)
+  kreuzberg (4.2.1)
   language_server-protocol (3.17.0.5) sha256=fd1e39a51a28bf3eec959379985a72e296e9f9acfce46f6a79d31ca8760803cc
   lint_roller (1.1.0) sha256=2c0c845b632a7d172cb849cc90c1bce937a28c5c8ccccb50dfd46a485003cc87
   listen (3.10.0) sha256=c6e182db62143aeccc2e1960033bebe7445309c7272061979bb098d03760c9d2

data/README.md

@@ -22,7 +22,7 @@
     <img src="https://img.shields.io/maven-central/v/dev.kreuzberg/kreuzberg?label=Java&color=007ec6" alt="Java">
   </a>
   <a href="https://github.com/kreuzberg-dev/kreuzberg/releases">
-    <img src="https://img.shields.io/github/v/tag/kreuzberg-dev/kreuzberg?label=Go&color=007ec6&filter=v4.1.2" alt="Go">
+    <img src="https://img.shields.io/github/v/tag/kreuzberg-dev/kreuzberg?label=Go&color=007ec6&filter=v4.2.1" alt="Go">
   </a>
   <a href="https://www.nuget.org/packages/Kreuzberg/">
     <img src="https://img.shields.io/nuget/v/Kreuzberg?label=C%23&color=007ec6" alt="C#">

data/ext/kreuzberg_rb/native/Cargo.lock

@@ -716,9 +716,9 @@ dependencies = [
 
 [[package]]
 name = "cc"
-version = "1.2.53"
+version = "1.2.54"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "755d2fce177175ffca841e9a06afdb2c4ab0f593d53b4dee48147dfaade85932"
+checksum = "6354c81bbfd62d9cfa9cb3c773c2b7b2a3a482d569de977fd0e961f6e7c00583"
 dependencies = [
  "find-msvc-tools",
  "jobserver",
@@ -2027,9 +2027,9 @@ dependencies = [
 
 [[package]]
 name = "html-to-markdown-rs"
-version = "2.23.1"
+version = "2.23.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e1f4d6781ac8dd203853803d27054ca4153c7fd0f3956cb7fc95dc06f42a1c46"
+checksum = "ffbf49450676163bdf69fac2acf72674fcf2d2aaf690aa247368c567cc9afb2a"
 dependencies = [
  "astral-tl",
  "base64 0.22.1",
@@ -2627,7 +2627,7 @@ dependencies = [
 
 [[package]]
 name = "kreuzberg"
-version = "4.1.0"
+version = "4.2.0"
 dependencies = [
  "ahash",
  "async-trait",
@@ -2706,17 +2706,18 @@ dependencies = [
  "uuid",
  "whatlang",
  "yake-rust",
- "zip 7.1.0",
+ "zip 7.2.0",
 ]
 
 [[package]]
 name = "kreuzberg-ffi"
-version = "4.1.0"
+version = "4.2.0"
 dependencies = [
  "async-trait",
  "cbindgen",
  "html-to-markdown-rs",
  "kreuzberg",
+ "log",
  "serde",
  "serde_json",
  "tokio",
@@ -2767,14 +2768,14 @@ dependencies = [
 
 [[package]]
 name = "kreuzberg-tesseract"
-version = "4.1.0"
+version = "4.2.0"
 dependencies = [
  "cc",
  "cmake",
  "libc",
  "reqwest 0.13.1",
  "thiserror 2.0.18",
- "zip 7.1.0",
+ "zip 7.2.0",
 ]
 
 [[package]]
@@ -5022,9 +5023,9 @@ dependencies = [
 
 [[package]]
 name = "rmcp"
-version = "0.13.0"
+version = "0.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d1815dbc06c414d720f8bc1951eccd66bc99efc6376331f1e7093a119b3eb508"
+checksum = "0a621b37a548ff6ab6292d57841eb25785a7f146d89391a19c9f199414bd13da"
 dependencies = [
  "async-trait",
  "axum",
@@ -5054,9 +5055,9 @@ dependencies = [
 
 [[package]]
 name = "rmcp-macros"
-version = "0.13.0"
+version = "0.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "11f0bc7008fa102e771a76c6d2c9b253be3f2baa5964e060464d038ae1cbc573"
+checksum = "6b79ed92303f9262db79575aa8c3652581668e9d136be6fd0b9ededa78954c95"
 dependencies = [
  "darling 0.23.0",
  "proc-macro2",
@@ -6013,6 +6014,7 @@ checksum = "91e7d9e3bb61134e77bde20dd4825b97c010155709965fedf0f49bb138e52a9d"
 dependencies = [
  "deranged",
  "itoa",
+ "js-sys",
  "num-conv",
  "powerfmt",
  "serde",
@@ -6405,6 +6407,12 @@ version = "0.25.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d2df906b07856748fa3f6e0ad0cbaa047052d4a7dd609e231c4f72cee8c36f31"
 
+[[package]]
+name = "typed-path"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e43ffa54726cdc9ea78392023ffe9fe9cf9ac779e1c6fcb0d23f9862e3879d20"
+
 [[package]]
 name = "typeid"
 version = "1.0.3"
@@ -6698,9 +6706,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
 
 [[package]]
 name = "uuid"
-version = "1.19.0"
+version = "1.20.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e2e054861b4bd027cd373e18e8d8d8e6548085000e41290d95ce0c373a654b4a"
+checksum = "ee48d38b119b0cd71fe4141b30f5ba9c7c5d9f4e7a3a8b4a674e4b6ef789976f"
 dependencies = [
  "getrandom 0.3.4",
  "js-sys",
@@ -7498,9 +7506,9 @@ dependencies = [
 
 [[package]]
 name = "zip"
-version = "7.1.0"
+version = "7.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9013f1222db8a6d680f13a7ccdc60a781199cd09c2fa4eff58e728bb181757fc"
+checksum = "c42e33efc22a0650c311c2ef19115ce232583abbe80850bc8b66509ebef02de0"
 dependencies = [
  "aes",
  "bzip2",
@@ -7518,6 +7526,7 @@ dependencies = [
  "ppmd-rust",
  "sha1",
  "time",
+ "typed-path",
  "zeroize",
  "zopfli",
  "zstd",

data/kreuzberg.gemspec

@@ -130,10 +130,22 @@ vendor_files = Dir.chdir(__dir__) do
   kreuzberg_files + kreuzberg_ffi_files + kreuzberg_tesseract_files + rb_sys_files + workspace_toml
 end
 
+# When vendor files exist, get ext/ files from filesystem (to include modified Cargo.toml
+# with vendor paths) instead of from git (which has original 5-level crate paths)
+ext_files_from_fs = Dir.chdir(__dir__) do
+  Dir.glob('ext/**/*', File::FNM_DOTMATCH)
+     .reject { |f| File.directory?(f) }
+     .reject { |f| f.include?('/target/') }
+     .grep_v(/\.(swp|bak|tmp)$/)
+     .grep_v(/~$/)
+end
+
 files = if (ruby_files + core_files + ffi_files).empty?
           fallback_files
         elsif vendor_files.any?
-          ruby_files + vendor_files
+          # Use ext/ files from filesystem (modified by vendor script) + non-ext ruby files from git
+          non_ext_ruby_files = ruby_files.reject { |f| f.start_with?('ext/') }
+          non_ext_ruby_files + ext_files_from_fs + vendor_files
         else
           ruby_files + core_files + ffi_files
         end

data/lib/kreuzberg/cli.rb

@@ -8,24 +8,34 @@ module Kreuzberg
 
     # Extract content from a file using the CLI
     #
-    # @param path [String] Path to the file
+    # @param path_or_nil [String, nil] Path to the file (positional, for backward compatibility)
+    # @param path [String] Path to the file (keyword argument)
     # @param output [String] Output format ("text", "json", "markdown")
     # @param ocr [Boolean] Enable OCR
     # @return [String] Extracted content
     #
-    def extract(path, output: 'text', ocr: false)
-      args = ['extract', path, '--format', output]
+    def extract(path_or_nil = nil, path: nil, output: 'text', ocr: false)
+      # Support both positional and keyword argument for path (backward compatibility)
+      actual_path = path_or_nil || path
+      raise ArgumentError, 'path is required' if actual_path.nil?
+
+      args = ['extract', actual_path, '--format', output]
       args.push('--ocr', ocr ? 'true' : 'false')
       CLIProxy.call(args)
     end
 
     # Detect MIME type of a file using the CLI
     #
-    # @param path [String] Path to the file
+    # @param path_or_nil [String, nil] Path to the file (positional, for backward compatibility)
+    # @param path [String] Path to the file (keyword argument)
     # @return [String] MIME type
     #
-    def detect(path)
-      CLIProxy.call(['detect', path]).strip
+    def detect(path_or_nil = nil, path: nil)
+      # Support both positional and keyword argument for path (backward compatibility)
+      actual_path = path_or_nil || path
+      raise ArgumentError, 'path is required' if actual_path.nil?
+
+      CLIProxy.call(['detect', actual_path]).strip
     end
 
     # Get CLI version

data/lib/kreuzberg/cli_proxy.rb

@@ -91,11 +91,13 @@ module Kreuzberg
         lib_path.join('bin', binary_name),
         lib_path.join(binary_name),
         root_path.join('../../crates/kreuzberg-cli/target/release', binary_name),
-        root_path.join('../../target/release', binary_name)
+        root_path.join('../../target/release', binary_name),
+        root_path.join('../../target/debug', binary_name)
       ]
 
       workspace_root = root_path.parent&.parent
       paths << workspace_root.join('target', 'release', binary_name) if workspace_root
+      paths << workspace_root.join('target', 'debug', binary_name) if workspace_root
 
       paths
     end

data/lib/kreuzberg/config.rb

@@ -715,9 +715,12 @@ module Kreuzberg
     class Extraction
       attr_reader :use_cache, :enable_quality_processing, :force_ocr,
                   :ocr, :chunking, :language_detection, :pdf_options,
-                  :image_extraction, :image_preprocessing, :postprocessor,
+                  :images, :image_preprocessing, :postprocessor,
                   :token_reduction, :keywords, :html_options, :pages,
-                  :max_concurrent_extractions
+                  :max_concurrent_extractions, :output_format, :result_format
+
+      # Alias for backward compatibility - image_extraction is the canonical name
+      alias image_extraction images
 
       # Load configuration from a file.
       #
@@ -738,7 +741,7 @@ module Kreuzberg
         use_cache enable_quality_processing force_ocr ocr chunking
         language_detection pdf_options image_extraction image_preprocessing
         postprocessor token_reduction keywords html_options pages
-        max_concurrent_extractions
+        max_concurrent_extractions output_format result_format
       ].freeze
 
       # Aliases for backward compatibility
@@ -746,6 +749,12 @@ module Kreuzberg
         images: :image_extraction
       }.freeze
 
+      # Valid output format values (case-insensitive, normalized internally)
+      VALID_OUTPUT_FORMATS = %w[plain markdown html djot].freeze
+
+      # Valid result format values (case-insensitive, normalized internally)
+      VALID_RESULT_FORMATS = %w[unified elements element_based].freeze
+
       def self.from_file(path)
         hash = Kreuzberg._config_from_file_native(path)
         new(**normalize_hash_keys(hash))
@@ -789,41 +798,87 @@ module Kreuzberg
         new(**normalize_hash_keys(hash))
       end
 
-      def initialize(
-        use_cache: true,
-        enable_quality_processing: false,
-        force_ocr: false,
-        ocr: nil,
-        chunking: nil,
-        language_detection: nil,
-        pdf_options: nil,
-        image_extraction: nil,
-        image_preprocessing: nil,
-        postprocessor: nil,
-        token_reduction: nil,
-        keywords: nil,
-        html_options: nil,
-        pages: nil,
-        max_concurrent_extractions: nil
-      )
-        @use_cache = use_cache ? true : false
-        @enable_quality_processing = enable_quality_processing ? true : false
-        @force_ocr = force_ocr ? true : false
-        @ocr = normalize_config(ocr, OCR)
-        @chunking = normalize_config(chunking, Chunking)
-        @language_detection = normalize_config(language_detection, LanguageDetection)
-        @pdf_options = normalize_config(pdf_options, PDF)
-        @image_extraction = normalize_config(image_extraction, ImageExtraction)
-        @image_preprocessing = normalize_config(image_preprocessing, ImagePreprocessing)
-        @postprocessor = normalize_config(postprocessor, PostProcessor)
-        @token_reduction = normalize_config(token_reduction, TokenReduction)
-        @keywords = normalize_config(keywords, Keywords)
-        @html_options = normalize_config(html_options, HtmlOptions)
-        @pages = normalize_config(pages, PageConfig)
-        @max_concurrent_extractions = max_concurrent_extractions&.to_i
+      def initialize(hash = nil,
+                     use_cache: true,
+                     enable_quality_processing: false,
+                     force_ocr: false,
+                     ocr: nil,
+                     chunking: nil,
+                     language_detection: nil,
+                     pdf_options: nil,
+                     image_extraction: nil,
+                     image_preprocessing: nil,
+                     postprocessor: nil,
+                     token_reduction: nil,
+                     keywords: nil,
+                     html_options: nil,
+                     pages: nil,
+                     max_concurrent_extractions: nil,
+                     output_format: nil,
+                     result_format: nil)
+        kwargs = {
+          use_cache: use_cache, enable_quality_processing: enable_quality_processing,
+          force_ocr: force_ocr, ocr: ocr, chunking: chunking, language_detection: language_detection,
+          pdf_options: pdf_options, image_extraction: image_extraction,
+          image_preprocessing: image_preprocessing, postprocessor: postprocessor,
+          token_reduction: token_reduction, keywords: keywords, html_options: html_options,
+          pages: pages, max_concurrent_extractions: max_concurrent_extractions,
+          output_format: output_format, result_format: result_format
+        }
+        extracted = extract_from_hash(hash, kwargs)
+
+        assign_attributes(extracted)
+      end
+
+      def extract_from_hash(hash, defaults)
+        return defaults unless hash.is_a?(Hash)
+
+        hash = hash.transform_keys(&:to_sym)
+        defaults.merge(hash.slice(*defaults.keys))
+      end
+
+      def assign_attributes(params)
+        @use_cache = params[:use_cache] ? true : false
+        @enable_quality_processing = params[:enable_quality_processing] ? true : false
+        @force_ocr = params[:force_ocr] ? true : false
+        @ocr = normalize_config(params[:ocr], OCR)
+        @chunking = normalize_config(params[:chunking], Chunking)
+        @language_detection = normalize_config(params[:language_detection], LanguageDetection)
+        @pdf_options = normalize_config(params[:pdf_options], PDF)
+        @images = normalize_config(params[:image_extraction], ImageExtraction)
+        @image_preprocessing = normalize_config(params[:image_preprocessing], ImagePreprocessing)
+        @postprocessor = normalize_config(params[:postprocessor], PostProcessor)
+        @token_reduction = normalize_config(params[:token_reduction], TokenReduction)
+        @keywords = normalize_config(params[:keywords], Keywords)
+        @html_options = normalize_config(params[:html_options], HtmlOptions)
+        @pages = normalize_config(params[:pages], PageConfig)
+        @max_concurrent_extractions = params[:max_concurrent_extractions]&.to_i
+        @output_format = validate_output_format(params[:output_format])
+        @result_format = validate_result_format(params[:result_format])
+      end
+
+      def validate_output_format(value)
+        return nil if value.nil?
+
+        str_value = value.to_s.downcase
+        return str_value if VALID_OUTPUT_FORMATS.include?(str_value)
+
+        raise ArgumentError,
+              "Invalid output_format: #{value}. Valid values: #{VALID_OUTPUT_FORMATS.join(', ')}"
+      end
+
+      def validate_result_format(value)
+        return nil if value.nil?
+
+        str_value = value.to_s.downcase
+        return str_value if VALID_RESULT_FORMATS.include?(str_value)
+
+        raise ArgumentError,
+              "Invalid result_format: #{value}. Valid values: #{VALID_RESULT_FORMATS.join(', ')}"
       end
 
       # rubocop:disable Metrics/CyclomaticComplexity
+      # rubocop:disable Metrics/MethodLength
       def to_h
         {
           use_cache: @use_cache,
@@ -833,16 +888,19 @@ module Kreuzberg
           chunking: @chunking&.to_h,
           language_detection: @language_detection&.to_h,
           pdf_options: @pdf_options&.to_h,
-          image_extraction: @image_extraction&.to_h,
+          images: @images&.to_h,
           image_preprocessing: @image_preprocessing&.to_h,
           postprocessor: @postprocessor&.to_h,
           token_reduction: @token_reduction&.to_h,
           keywords: @keywords&.to_h,
           html_options: @html_options&.to_h,
           pages: @pages&.to_h,
-          max_concurrent_extractions: @max_concurrent_extractions
+          max_concurrent_extractions: @max_concurrent_extractions,
+          output_format: @output_format,
+          result_format: @result_format
         }.compact
       end
+      # rubocop:enable Metrics/MethodLength
       # rubocop:enable Metrics/CyclomaticComplexity
 
       # Serialize configuration to JSON string
@@ -966,7 +1024,7 @@ module Kreuzberg
         when :pdf_options
           @pdf_options = normalize_config(value, PDF)
         when :image_extraction
-          @image_extraction = normalize_config(value, ImageExtraction)
+          @images = normalize_config(value, ImageExtraction)
         when :image_preprocessing
           @image_preprocessing = normalize_config(value, ImagePreprocessing)
         when :postprocessor
@@ -981,6 +1039,10 @@ module Kreuzberg
           @pages = normalize_config(value, PageConfig)
         when :max_concurrent_extractions
           @max_concurrent_extractions = value&.to_i
+        when :output_format
+          @output_format = validate_output_format(value)
+        when :result_format
+          @result_format = validate_result_format(value)
         else
           raise ArgumentError, "Unknown configuration key: #{key}"
         end
@@ -1002,6 +1064,24 @@ module Kreuzberg
         nil
       end
 
+      # Set output_format attribute
+      #
+      # @param value [String, nil] Output format value
+      # @return [String, nil] The value that was set
+      #
+      def output_format=(value)
+        @output_format = validate_output_format(value)
+      end
+
+      # Set result_format attribute
+      #
+      # @param value [String, nil] Result format value
+      # @return [String, nil] The value that was set
+      #
+      def result_format=(value)
+        @result_format = validate_result_format(value)
+      end
+
       private
 
       def normalize_config(value, klass)
@@ -1020,7 +1100,7 @@ module Kreuzberg
         @chunking = merged.chunking
         @language_detection = merged.language_detection
         @pdf_options = merged.pdf_options
-        @image_extraction = merged.image_extraction
+        @images = merged.image_extraction
         @image_preprocessing = merged.image_preprocessing
         @postprocessor = merged.postprocessor
         @token_reduction = merged.token_reduction
@@ -1028,6 +1108,8 @@ module Kreuzberg
         @html_options = merged.html_options
         @pages = merged.pages
         @max_concurrent_extractions = merged.max_concurrent_extractions
+        @output_format = merged.output_format
+        @result_format = merged.result_format
       end
     end
   end