kreuzberg 4.0.0.pre.rc.29 → 4.0.0.rc1

data/lib/kreuzberg/result.rb

@@ -7,58 +7,66 @@ rescue LoadError
 end
 
 module Kreuzberg
+  # Extraction result wrapper
+  #
+  # Provides structured access to extraction results from the native extension.
+  #
   # @example
+  #   result = Kreuzberg.extract_file_sync("document.pdf")
+  #   puts result.content
+  #   puts "MIME type: #{result.mime_type}"
+  #   puts "Metadata: #{result.metadata.inspect}"
+  #   result.tables.each { |table| puts table.inspect }
+  #
   # rubocop:disable Metrics/ClassLength
   class Result
     attr_reader :content, :mime_type, :metadata, :metadata_json, :tables,
-                :detected_languages, :chunks, :images, :pages
+                :detected_languages, :chunks, :images
 
+    # Table structure
+    #
     # @!attribute [r] cells
     #   @return [Array<Array<String>>] Table cells (2D array)
     # @!attribute [r] markdown
     #   @return [String] Markdown representation
     # @!attribute [r] page_number
     #   @return [Integer] Page number where table was found
+    #
     Table = Struct.new(:cells, :markdown, :page_number, keyword_init: true) do
       def to_h
         { cells: cells, markdown: markdown, page_number: page_number }
       end
     end
 
+    # Text chunk
+    #
     # @!attribute [r] content
     #   @return [String] Chunk content
-    # @!attribute [r] byte_start
-    #   @return [Integer] Starting byte offset (UTF-8)
-    # @!attribute [r] byte_end
-    #   @return [Integer] Ending byte offset (UTF-8)
+    # @!attribute [r] char_start
+    #   @return [Integer] Starting character index
+    # @!attribute [r] char_end
+    #   @return [Integer] Ending character index
     # @!attribute [r] token_count
     #   @return [Integer, nil] Approximate token count (may be nil)
-    # @!attribute [r] first_page
-    #   @return [Integer, nil] First page number (1-indexed)
-    # @!attribute [r] last_page
-    #   @return [Integer, nil] Last page number (1-indexed)
+    #
     Chunk = Struct.new(
       :content,
-      :byte_start,
-      :byte_end,
+      :char_start,
+      :char_end,
       :token_count,
       :chunk_index,
       :total_chunks,
-      :first_page,
-      :last_page,
       :embedding,
       keyword_init: true
     ) do
       def to_h
         {
           content: content,
-          byte_start: byte_start,
-          byte_end: byte_end,
+          char_start: char_start,
+          char_end: char_end,
           token_count: token_count,
           chunk_index: chunk_index,
           total_chunks: total_chunks,
-          first_page: first_page,
-          last_page: last_page,
           embedding: embedding
         }
       end
@@ -95,30 +103,12 @@ module Kreuzberg
       end
     end
 
-    # @!attribute [r] page_number
-    #   @return [Integer] Page number (1-indexed)
-    # @!attribute [r] content
-    #   @return [String] Text content for this page
-    # @!attribute [r] tables
-    #   @return [Array<Table>] Tables on this page
-    # @!attribute [r] images
-    #   @return [Array<Image>] Images on this page
-    PageContent = Struct.new(:page_number, :content, :tables, :images, keyword_init: true) do
-      def to_h
-        {
-          page_number: page_number,
-          content: content,
-          tables: tables.map(&:to_h),
-          images: images.map(&:to_h)
-        }
-      end
-    end
-
     # Initialize from native hash result
     #
     # @param hash [Hash] Hash returned from native extension
     #
     def initialize(hash)
+      # Handle both string and symbol keys for flexibility
       @content = get_value(hash, 'content', '')
       @mime_type = get_value(hash, 'mime_type', '')
       @metadata_json = get_value(hash, 'metadata_json', '{}')
@@ -127,7 +117,6 @@ module Kreuzberg
       @detected_languages = parse_detected_languages(get_value(hash, 'detected_languages'))
       @chunks = parse_chunks(get_value(hash, 'chunks'))
       @images = parse_images(get_value(hash, 'images'))
-      @pages = parse_pages(get_value(hash, 'pages'))
     end
 
     # Convert to hash
@@ -139,11 +128,10 @@ module Kreuzberg
         content: @content,
         mime_type: @mime_type,
         metadata: @metadata,
-        tables: serialize_tables,
+        tables: @tables.map(&:to_h),
         detected_languages: @detected_languages,
-        chunks: serialize_chunks,
-        images: serialize_images,
-        pages: serialize_pages
+        chunks: @chunks&.map(&:to_h),
+        images: @images&.map(&:to_h)
       }
     end
 
@@ -155,100 +143,8 @@ module Kreuzberg
       to_h.to_json(*)
     end
 
-    # Get the total number of pages in the document
-    #
-    # @return [Integer] Total page count (>= 0), or -1 on error
-    #
-    # @example
-    #   result = Kreuzberg.extract_file_sync("document.pdf")
-    #   puts "Document has #{result.page_count} pages"
-    #
-    def page_count
-      if @metadata.is_a?(Hash) && @metadata['pages'].is_a?(Hash)
-        @metadata['pages']['total_count'] || 0
-      else
-        0
-      end
-    end
-
-    # Get the total number of text chunks
-    #
-    # Returns 0 if chunking was not performed.
-    #
-    # @return [Integer] Total chunk count (>= 0), or -1 on error
-    #
-    # @example
-    #   result = Kreuzberg.extract_file_sync("document.pdf")
-    #   puts "Document has #{result.chunk_count} chunks"
-    #
-    def chunk_count
-      @chunks&.length || 0
-    end
-
-    # Get the primary detected language
-    #
-    # @return [String, nil] ISO 639 language code (e.g., "en", "de"), or nil if not detected
-    #
-    # @example
-    #   result = Kreuzberg.extract_file_sync("document.pdf")
-    #   lang = result.detected_language
-    #   puts "Language: #{lang}" if lang
-    #
-    def detected_language
-      return @metadata['language'] if @metadata.is_a?(Hash) && @metadata['language']
-      return @detected_languages&.first if @detected_languages&.any?
-
-      nil
-    end
-
-    # Get a metadata field by name
-    #
-    # Supports dot notation for nested fields (e.g., "format.pages").
-    #
-    # @param name [String, Symbol] Field name
-    # @return [Object, nil] Field value, or nil if field doesn't exist
-    #
-    # @example Get a top-level field
-    #   result = Kreuzberg.extract_file_sync("document.pdf")
-    #   title = result.metadata_field("title")
-    #   puts "Title: #{title}" if title
-    #
-    # @example Get a nested field
-    #   format_info = result.metadata_field("format.pages")
-    #
-    def metadata_field(name)
-      return nil unless @metadata.is_a?(Hash)
-
-      parts = name.to_s.split('.')
-      value = @metadata
-
-      parts.each do |part|
-        return nil unless value.is_a?(Hash)
-
-        value = value[part]
-      end
-
-      value
-    end
-
     private
 
-    def serialize_tables
-      @tables.map(&:to_h)
-    end
-
-    def serialize_chunks
-      @chunks&.map(&:to_h)
-    end
-
-    def serialize_images
-      @images&.map(&:to_h)
-    end
-
-    def serialize_pages
-      @pages&.map(&:to_h)
-    end
-
     def get_value(hash, key, default = nil)
       hash[key] || hash[key.to_sym] || default
     end
@@ -274,22 +170,21 @@ module Kreuzberg
     def parse_detected_languages(langs_data)
       return nil if langs_data.nil?
 
+      # Detected languages is now just an array of strings
       langs_data.is_a?(Array) ? langs_data : []
     end
 
     def parse_chunks(chunks_data)
-      return [] if chunks_data.nil? || chunks_data.empty?
+      return nil if chunks_data.nil?
 
       chunks_data.map do |chunk_hash|
         Chunk.new(
           content: chunk_hash['content'],
-          byte_start: chunk_hash['byte_start'],
-          byte_end: chunk_hash['byte_end'],
+          char_start: chunk_hash['char_start'],
+          char_end: chunk_hash['char_end'],
           token_count: chunk_hash['token_count'],
           chunk_index: chunk_hash['chunk_index'],
           total_chunks: chunk_hash['total_chunks'],
-          first_page: chunk_hash['first_page'],
-          last_page: chunk_hash['last_page'],
           embedding: chunk_hash['embedding']
         )
       end
@@ -316,19 +211,6 @@ module Kreuzberg
         )
       end
     end
-
-    def parse_pages(pages_data)
-      return nil if pages_data.nil?
-
-      pages_data.map do |page_hash|
-        PageContent.new(
-          page_number: page_hash['page_number'],
-          content: page_hash['content'],
-          tables: parse_tables(page_hash['tables']),
-          images: parse_images(page_hash['images'])
-        )
-      end
-    end
   end
   # rubocop:enable Metrics/ClassLength
 end

data/lib/kreuzberg/setup_lib_path.rb

@@ -19,31 +19,10 @@ module Kreuzberg
       when /linux/
         prepend_env('LD_LIBRARY_PATH', lib_dir)
       when /mswin|mingw|cygwin/
-        # Windows uses PATH to locate DLLs
         prepend_env('PATH', lib_dir, separator: ';')
-        # Also check common locations for PDFium on Windows
-        setup_windows_library_paths(lib_dir)
       end
     end
 
-    def setup_windows_library_paths(lib_dir)
-      # Add target/release to PATH for DLL lookup during development
-      target_release = File.expand_path('../../target/release', lib_dir)
-      prepend_env('PATH', target_release, separator: ';') if Dir.exist?(target_release)
-
-      # Check for short path CARGO_TARGET_DIR (CI uses C:\t)
-      cargo_target_dir = ENV.fetch('CARGO_TARGET_DIR', nil)
-      return unless cargo_target_dir
-
-      target_release_alt = File.join(cargo_target_dir, 'release')
-      prepend_env('PATH', target_release_alt, separator: ';') if Dir.exist?(target_release_alt)
-
-      # Also check for target-specific subdirectory (Windows GNU builds)
-      gnu_release = File.join(cargo_target_dir, 'x86_64-pc-windows-gnu', 'release')
-      prepend_env('PATH', gnu_release, separator: ';') if Dir.exist?(gnu_release)
-    end
-    private_class_method :setup_windows_library_paths
-
     def prepend_env(key, value, separator: ':')
       current = ENV.fetch(key, nil)
       return if current&.split(separator)&.include?(value)
@@ -58,7 +37,8 @@ module Kreuzberg
 
       ensure_install_name(bundle)
       ensure_loader_rpath(bundle)
-    rescue Errno::ENOENT, IOError # rubocop:disable Lint/SuppressedException
+    rescue Errno::ENOENT, IOError
+      # Tool not available (e.g., on CI). The dynamic loader can still use the updated env vars.
     end
     private_class_method :fix_macos_install_name
 

data/lib/kreuzberg/validator_protocol.rb

@@ -1,14 +1,87 @@
 # frozen_string_literal: true
 
 module Kreuzberg
+  # Validator protocol interface.
+  #
+  # This module defines the protocol that all Ruby validators must implement
+  # to be registered with the Rust core via the FFI bridge.
+  #
+  # Validators are called during extraction to validate results. If validation fails,
+  # the validator should raise a Kreuzberg::Errors::ValidationError, which will
+  # cause the extraction to fail.
+  #
   # @example Implementing a minimum length validator
+  #   class MinimumLengthValidator
+  #     include Kreuzberg::ValidatorProtocol
+  #
+  #     def initialize(min_length = 10)
+  #       @min_length = min_length
+  #     end
+  #
+  #     def call(result)
+  #       if result["content"].length < @min_length
+  #         raise Kreuzberg::Errors::ValidationError.new(
+  #           "Content too short: #{result["content"].length} < #{@min_length}"
+  #         )
+  #       end
+  #     end
+  #   end
+  #
+  #   Kreuzberg.register_validator("min_length", MinimumLengthValidator.new(100))
+  #
   # @example Implementing a content quality validator
+  #   class QualityValidator
+  #     include Kreuzberg::ValidatorProtocol
+  #
+  #     def call(result)
+  #       # Check if content has sufficient quality
+  #       if result["content"].strip.empty?
+  #         raise Kreuzberg::Errors::ValidationError.new("Empty content extracted")
+  #       end
+  #
+  #       # Check if metadata is present
+  #       if result["metadata"].empty?
+  #         raise Kreuzberg::Errors::ValidationError.new("No metadata extracted")
+  #       end
+  #     end
+  #   end
+  #
+  #   Kreuzberg.register_validator("quality", QualityValidator.new)
+  #
   # @example Using a Proc as a validator
+  #   Kreuzberg.register_validator("not_empty", ->(result) {
+  #     if result["content"].strip.empty?
+  #       raise Kreuzberg::Errors::ValidationError.new("Content cannot be empty")
+  #     end
+  #   })
+  #
   module ValidatorProtocol
+    # Validate an extraction result.
+    #
+    # This method is called during extraction to validate results. If validation fails,
+    # raise a Kreuzberg::Errors::ValidationError with a descriptive message explaining
+    # why validation failed. If validation passes, return without raising.
+    #
+    # The validator receives the extraction result as a hash with the same structure
+    # as post-processors (see PostProcessorProtocol for details).
+    #
     # @param result [Hash] Extraction result to validate with the following structure:
+    #   - "content" [String] - Extracted text content
+    #   - "mime_type" [String] - MIME type of the source document
+    #   - "metadata" [Hash] - Document metadata (title, author, etc.)
+    #   - "tables" [Array<Hash>] - Extracted tables
+    #   - "detected_languages" [Array<String>, nil] - Detected language codes
+    #   - "chunks" [Array<String>, nil] - Content chunks (if chunking enabled)
+    #
     # @return [void]
     # @raise [Kreuzberg::Errors::ValidationError] if validation fails
+    #
     # @example
+    #   def call(result)
+    #     if result["content"].length < 10
+    #       raise Kreuzberg::Errors::ValidationError.new("Content too short")
+    #     end
+    #   end
     def call(result)
       raise NotImplementedError, "#{self.class} must implement #call(result)"
     end

data/lib/kreuzberg/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Kreuzberg
-  VERSION = '4.0.0-rc.29'
+  VERSION = '4.0.0.rc1'
 end

data/lib/kreuzberg.rb

@@ -6,8 +6,6 @@ Kreuzberg::SetupLibPath.configure
 require_relative 'kreuzberg/version'
 require 'kreuzberg_rb'
 
-# Kreuzberg is a Ruby binding for the Rust core library providing document extraction,
-# text extraction, and OCR capabilities.
 module Kreuzberg
   autoload :Config, 'kreuzberg/config'
   autoload :Result, 'kreuzberg/result'
@@ -16,28 +14,17 @@ module Kreuzberg
   autoload :APIProxy, 'kreuzberg/api_proxy'
   autoload :MCPProxy, 'kreuzberg/mcp_proxy'
   autoload :Errors, 'kreuzberg/errors'
-  autoload :ErrorContext, 'kreuzberg/error_context'
   autoload :PostProcessorProtocol, 'kreuzberg/post_processor_protocol'
   autoload :ValidatorProtocol, 'kreuzberg/validator_protocol'
   autoload :OcrBackendProtocol, 'kreuzberg/ocr_backend_protocol'
 
-  autoload :HtmlMetadata, 'kreuzberg/types'
-  autoload :HeaderMetadata, 'kreuzberg/types'
-  autoload :LinkMetadata, 'kreuzberg/types'
-  autoload :ImageMetadata, 'kreuzberg/types'
-  autoload :StructuredData, 'kreuzberg/types'
-
+  # Alias for API consistency with other language bindings
   ExtractionConfig = Config::Extraction
-  PageConfig = Config::PageConfig
-
-  module KeywordAlgorithm
-    YAKE = :yake
-    RAKE = :rake
-  end
 
   @__cache_tracker = { entries: 0, bytes: 0 }
 
   class << self
+    # Store native methods as private methods
     alias native_extract_file_sync extract_file_sync
     alias native_extract_bytes_sync extract_bytes_sync
     alias native_batch_extract_files_sync batch_extract_files_sync
@@ -54,39 +41,38 @@ module Kreuzberg
     private :native_batch_extract_bytes_sync, :native_batch_extract_bytes
   end
 
+  # Register a Ruby post-processor that conforms to PostProcessorProtocol.
   module_function :register_post_processor
 
+  # Remove a post-processor by name.
   module_function :unregister_post_processor
 
+  # Purge all registered post-processors.
   module_function :clear_post_processors
 
+  # Register a validator that follows ValidatorProtocol.
   module_function :register_validator
 
+  # Remove a validator by name.
   module_function :unregister_validator
 
+  # Purge all validators.
   module_function :clear_validators
 
+  # List all registered validators.
   module_function :list_validators
 
+  # List all registered post-processors.
   module_function :list_post_processors
 
+  # Register an OCR backend instance implementing OcrBackendProtocol.
   module_function :register_ocr_backend
 
+  # Unregister an OCR backend by name.
   module_function :unregister_ocr_backend
 
+  # List all registered OCR backends.
   module_function :list_ocr_backends
-
-  module_function :detect_mime_type
-
-  module_function :detect_mime_type_from_path
-
-  module_function :validate_mime_type
-
-  module_function :get_extensions_for_mime
-
-  module_function :list_embedding_presets
-
-  module_function :get_embedding_preset
 end
 
 require_relative 'kreuzberg/cache_api'