kreuzberg 4.0.0.rc2 → 4.0.0

data/lib/kreuzberg/setup_lib_path.rb

@@ -1,80 +1,99 @@
-# frozen_string_literal: true
-
-require 'rbconfig'
-require 'open3'
-
-module Kreuzberg
-  # Configures library paths for dynamic linking on different platforms.
-  module SetupLibPath
-    module_function
-
-    def configure
-      lib_dir = File.expand_path('..', __dir__ || '.')
-      host_os = RbConfig::CONFIG['host_os']
-
-      case host_os
-      when /darwin/
-        prepend_env('DYLD_LIBRARY_PATH', lib_dir)
-        prepend_env('DYLD_FALLBACK_LIBRARY_PATH', "#{lib_dir}:/usr/local/lib:/usr/lib")
-        fix_macos_install_name(lib_dir)
-      when /linux/
-        prepend_env('LD_LIBRARY_PATH', lib_dir)
-      when /mswin|mingw|cygwin/
-        prepend_env('PATH', lib_dir, separator: ';')
-      end
-    end
-
-    def prepend_env(key, value, separator: ':')
-      current = ENV.fetch(key, nil)
-      return if current&.split(separator)&.include?(value)
-
-      ENV[key] = current.nil? || current.empty? ? value : "#{value}#{separator}#{current}"
-    end
-    private_class_method :prepend_env
-
-    def fix_macos_install_name(lib_dir)
-      bundle = macos_bundle(lib_dir)
-      return unless bundle
-
-      ensure_install_name(bundle)
-      ensure_loader_rpath(bundle)
-    rescue Errno::ENOENT, IOError
-      # Tool not available (e.g., on CI). The dynamic loader can still use the updated env vars.
-    end
-    private_class_method :fix_macos_install_name
-
-    def macos_bundle(lib_dir)
-      bundle = File.join(lib_dir, 'kreuzberg_rb.bundle')
-      pdfium = File.join(lib_dir, 'libpdfium.dylib')
-      return unless File.exist?(bundle) && File.exist?(pdfium)
-
-      bundle
-    end
-    private_class_method :macos_bundle
-
-    def ensure_install_name(bundle)
-      output, status = Open3.capture2('otool', '-L', bundle)
-      return unless status.success?
-
-      replacements = {
-        './libpdfium.dylib' => '@loader_path/libpdfium.dylib',
-        '@rpath/libpdfium.dylib' => '@loader_path/libpdfium.dylib'
-      }
-
-      replacements.each do |current, desired|
-        next unless output.include?(current)
-
-        Open3.capture2('install_name_tool', '-change', current, desired, bundle)
-      end
-    end
-    private_class_method :ensure_install_name
-
-    def ensure_loader_rpath(bundle)
-      rpath_output, rpath_status = Open3.capture2('otool', '-l', bundle)
-      return unless rpath_status.success? && !rpath_output.include?('@loader_path')
-
-      Open3.capture2('install_name_tool', '-add_rpath', '@loader_path', bundle)
-    end
-    private_class_method :ensure_loader_rpath
-  end
-end
+# frozen_string_literal: true
+
+require 'rbconfig'
+require 'open3'
+
+module Kreuzberg
+  module SetupLibPath
+    module_function
+
+    def configure
+      lib_dir = File.expand_path('..', __dir__ || '.')
+      host_os = RbConfig::CONFIG['host_os']
+
+      case host_os
+      when /darwin/
+        prepend_env('DYLD_LIBRARY_PATH', lib_dir)
+        prepend_env('DYLD_FALLBACK_LIBRARY_PATH', "#{lib_dir}:/usr/local/lib:/usr/lib")
+        fix_macos_install_name(lib_dir)
+      when /linux/
+        prepend_env('LD_LIBRARY_PATH', lib_dir)
+      when /mswin|mingw|cygwin/
+        # Windows uses PATH to locate DLLs
+        prepend_env('PATH', lib_dir, separator: ';')
+        # Also check common locations for PDFium on Windows
+        setup_windows_library_paths(lib_dir)
+      end
+    end
+
+    def setup_windows_library_paths(lib_dir)
+      # Add target/release to PATH for DLL lookup during development
+      target_release = File.expand_path('../../target/release', lib_dir)
+      prepend_env('PATH', target_release, separator: ';') if Dir.exist?(target_release)
+
+      # Check for short path CARGO_TARGET_DIR (CI uses C:\t)
+      cargo_target_dir = ENV.fetch('CARGO_TARGET_DIR', nil)
+      return unless cargo_target_dir
+
+      target_release_alt = File.join(cargo_target_dir, 'release')
+      prepend_env('PATH', target_release_alt, separator: ';') if Dir.exist?(target_release_alt)
+
+      # Also check for target-specific subdirectory (Windows GNU builds)
+      gnu_release = File.join(cargo_target_dir, 'x86_64-pc-windows-gnu', 'release')
+      prepend_env('PATH', gnu_release, separator: ';') if Dir.exist?(gnu_release)
+    end
+    private_class_method :setup_windows_library_paths
+
+    def prepend_env(key, value, separator: ':')
+      current = ENV.fetch(key, nil)
+      return if current&.split(separator)&.include?(value)
+
+      ENV[key] = current.nil? || current.empty? ? value : "#{value}#{separator}#{current}"
+    end
+    private_class_method :prepend_env
+
+    def fix_macos_install_name(lib_dir)
+      bundle = macos_bundle(lib_dir)
+      return unless bundle
+
+      ensure_install_name(bundle)
+      ensure_loader_rpath(bundle)
+    rescue Errno::ENOENT, IOError # rubocop:disable Lint/SuppressedException
+    end
+    private_class_method :fix_macos_install_name
+
+    def macos_bundle(lib_dir)
+      bundle = File.join(lib_dir, 'kreuzberg_rb.bundle')
+      pdfium = File.join(lib_dir, 'libpdfium.dylib')
+      return unless File.exist?(bundle) && File.exist?(pdfium)
+
+      bundle
+    end
+    private_class_method :macos_bundle
+
+    def ensure_install_name(bundle)
+      output, status = Open3.capture2('otool', '-L', bundle)
+      return unless status.success?
+
+      replacements = {
+        './libpdfium.dylib' => '@loader_path/libpdfium.dylib',
+        '@rpath/libpdfium.dylib' => '@loader_path/libpdfium.dylib'
+      }
+
+      replacements.each do |current, desired|
+        next unless output.include?(current)
+
+        Open3.capture2('install_name_tool', '-change', current, desired, bundle)
+      end
+    end
+    private_class_method :ensure_install_name
+
+    def ensure_loader_rpath(bundle)
+      rpath_output, rpath_status = Open3.capture2('otool', '-l', bundle)
+      return unless rpath_status.success? && !rpath_output.include?('@loader_path')
+
+      Open3.capture2('install_name_tool', '-add_rpath', '@loader_path', bundle)
+    end
+    private_class_method :ensure_loader_rpath
+  end
+end

data/lib/kreuzberg/types.rb

@@ -0,0 +1,170 @@
+# frozen_string_literal: true
+
+require 'sorbet-runtime'
+
+module Kreuzberg
+  # @example
+  class HtmlMetadata < T::Struct
+    extend T::Sig
+
+    const :title, T.nilable(String)
+
+    const :description, T.nilable(String)
+
+    const :author, T.nilable(String)
+
+    const :copyright, T.nilable(String)
+
+    const :keywords, T::Array[String]
+
+    const :canonical_url, T.nilable(String)
+
+    const :language, T.nilable(String)
+
+    const :text_direction, T.nilable(String)
+
+    const :mime_type, T.nilable(String)
+
+    const :charset, T.nilable(String)
+
+    const :generator, T.nilable(String)
+
+    const :viewport, T.nilable(String)
+
+    const :theme_color, T.nilable(String)
+
+    const :application_name, T.nilable(String)
+
+    const :robots, T.nilable(String)
+
+    const :open_graph, T::Hash[String, String]
+
+    const :twitter_card, T::Hash[String, String]
+
+    const :meta_tags, T::Hash[String, String]
+
+    const :headers, T::Array[HeaderMetadata]
+
+    const :links, T::Array[LinkMetadata]
+
+    const :images, T::Array[ImageMetadata]
+
+    const :structured_data, T::Array[StructuredData]
+  end
+
+  # Header/Heading metadata
+  #
+  # Represents a heading element found in the HTML document
+  #
+  # @example
+  #   header = Kreuzberg::HeaderMetadata.new(
+  #     level: 1,
+  #     text: "Main Title",
+  #     id: "main-title",
+  #     depth: 0,
+  #     html_offset: 245
+  #   )
+  #   puts "#{header.text} (H#{header.level})"
+  #
+  class HeaderMetadata < T::Struct
+    extend T::Sig
+
+    const :level, Integer
+
+    const :text, String
+
+    const :id, T.nilable(String)
+
+    const :depth, Integer
+
+    const :html_offset, Integer
+  end
+
+  # Link metadata
+  #
+  # Represents a link element found in the HTML document
+  #
+  # @example
+  #   link = Kreuzberg::LinkMetadata.new(
+  #     href: "https://example.com",
+  #     text: "Example",
+  #     title: "Example Site",
+  #     link_type: "external",
+  #     rel: ["noopener", "noreferrer"],
+  #     attributes: { "data-id" => "123" }
+  #   )
+  #   puts "#{link.text} -> #{link.href}"
+  #
+  class LinkMetadata < T::Struct
+    extend T::Sig
+
+    const :href, String
+
+    const :text, String
+
+    const :title, T.nilable(String)
+
+    const :link_type, String
+
+    const :rel, T::Array[String]
+
+    const :attributes, T::Hash[String, String]
+  end
+
+  # Image metadata
+  #
+  # Represents an image element found in the HTML document
+  #
+  # @example
+  #   image = Kreuzberg::ImageMetadata.new(
+  #     src: "images/logo.png",
+  #     alt: "Company Logo",
+  #     title: nil,
+  #     dimensions: [200, 100],
+  #     image_type: "png",
+  #     attributes: { "loading" => "lazy" }
+  #   )
+  #   if image.dimensions
+  #     width, height = image.dimensions
+  #     puts "#{width}x#{height}"
+  #   end
+  #
+  class ImageMetadata < T::Struct
+    extend T::Sig
+
+    const :src, String
+
+    const :alt, T.nilable(String)
+
+    const :title, T.nilable(String)
+
+    const :dimensions, T.nilable(T::Array[Integer])
+
+    const :image_type, String
+
+    const :attributes, T::Hash[String, String]
+  end
+
+  # Structured data metadata
+  #
+  # Represents structured data (JSON-LD, microdata, etc.) found in the HTML document
+  #
+  # @example
+  #   structured = Kreuzberg::StructuredData.new(
+  #     data_type: "json-ld",
+  #     raw_json: '{"@context":"https://schema.org","@type":"Article",...}',
+  #     schema_type: "Article"
+  #   )
+  #   data = JSON.parse(structured.raw_json)
+  #   puts data['@type']
+  #
+  class StructuredData < T::Struct
+    extend T::Sig
+
+    const :data_type, String
+
+    const :raw_json, String
+
+    const :schema_type, T.nilable(String)
+  end
+end

data/lib/kreuzberg/validator_protocol.rb

@@ -1,89 +1,16 @@
-# frozen_string_literal: true
-
-module Kreuzberg
-  # Validator protocol interface.
-  #
-  # This module defines the protocol that all Ruby validators must implement
-  # to be registered with the Rust core via the FFI bridge.
-  #
-  # Validators are called during extraction to validate results. If validation fails,
-  # the validator should raise a Kreuzberg::Errors::ValidationError, which will
-  # cause the extraction to fail.
-  #
-  # @example Implementing a minimum length validator
-  #   class MinimumLengthValidator
-  #     include Kreuzberg::ValidatorProtocol
-  #
-  #     def initialize(min_length = 10)
-  #       @min_length = min_length
-  #     end
-  #
-  #     def call(result)
-  #       if result["content"].length < @min_length
-  #         raise Kreuzberg::Errors::ValidationError.new(
-  #           "Content too short: #{result["content"].length} < #{@min_length}"
-  #         )
-  #       end
-  #     end
-  #   end
-  #
-  #   Kreuzberg.register_validator("min_length", MinimumLengthValidator.new(100))
-  #
-  # @example Implementing a content quality validator
-  #   class QualityValidator
-  #     include Kreuzberg::ValidatorProtocol
-  #
-  #     def call(result)
-  #       # Check if content has sufficient quality
-  #       if result["content"].strip.empty?
-  #         raise Kreuzberg::Errors::ValidationError.new("Empty content extracted")
-  #       end
-  #
-  #       # Check if metadata is present
-  #       if result["metadata"].empty?
-  #         raise Kreuzberg::Errors::ValidationError.new("No metadata extracted")
-  #       end
-  #     end
-  #   end
-  #
-  #   Kreuzberg.register_validator("quality", QualityValidator.new)
-  #
-  # @example Using a Proc as a validator
-  #   Kreuzberg.register_validator("not_empty", ->(result) {
-  #     if result["content"].strip.empty?
-  #       raise Kreuzberg::Errors::ValidationError.new("Content cannot be empty")
-  #     end
-  #   })
-  #
-  module ValidatorProtocol
-    # Validate an extraction result.
-    #
-    # This method is called during extraction to validate results. If validation fails,
-    # raise a Kreuzberg::Errors::ValidationError with a descriptive message explaining
-    # why validation failed. If validation passes, return without raising.
-    #
-    # The validator receives the extraction result as a hash with the same structure
-    # as post-processors (see PostProcessorProtocol for details).
-    #
-    # @param result [Hash] Extraction result to validate with the following structure:
-    #   - "content" [String] - Extracted text content
-    #   - "mime_type" [String] - MIME type of the source document
-    #   - "metadata" [Hash] - Document metadata (title, author, etc.)
-    #   - "tables" [Array<Hash>] - Extracted tables
-    #   - "detected_languages" [Array<String>, nil] - Detected language codes
-    #   - "chunks" [Array<String>, nil] - Content chunks (if chunking enabled)
-    #
-    # @return [void]
-    # @raise [Kreuzberg::Errors::ValidationError] if validation fails
-    #
-    # @example
-    #   def call(result)
-    #     if result["content"].length < 10
-    #       raise Kreuzberg::Errors::ValidationError.new("Content too short")
-    #     end
-    #   end
-    def call(result)
-      raise NotImplementedError, "#{self.class} must implement #call(result)"
-    end
-  end
-end
+# frozen_string_literal: true
+
+module Kreuzberg
+  # @example Implementing a minimum length validator
+  # @example Implementing a content quality validator
+  # @example Using a Proc as a validator
+  module ValidatorProtocol
+    # @param result [Hash] Extraction result to validate with the following structure:
+    # @return [void]
+    # @raise [Kreuzberg::Errors::ValidationError] if validation fails
+    # @example
+    def call(result)
+      raise NotImplementedError, "#{self.class} must implement #call(result)"
+    end
+  end
+end

data/lib/kreuzberg/version.rb

@@ -1,5 +1,5 @@
-# frozen_string_literal: true
-
-module Kreuzberg
-  VERSION = '4.0.0.rc2'
-end
+# frozen_string_literal: true
+
+module Kreuzberg
+  VERSION = '4.0.0'
+end

data/lib/kreuzberg.rb

@@ -1,103 +1,96 @@
-# frozen_string_literal: true
-
-require_relative 'kreuzberg/setup_lib_path'
-Kreuzberg::SetupLibPath.configure
-
-require_relative 'kreuzberg/version'
-require 'kreuzberg_rb'
-
-# Kreuzberg is a Ruby binding for the Rust core library providing document extraction,
-# text extraction, and OCR capabilities.
-module Kreuzberg
-  autoload :Config, 'kreuzberg/config'
-  autoload :Result, 'kreuzberg/result'
-  autoload :CLI, 'kreuzberg/cli'
-  autoload :CLIProxy, 'kreuzberg/cli_proxy'
-  autoload :APIProxy, 'kreuzberg/api_proxy'
-  autoload :MCPProxy, 'kreuzberg/mcp_proxy'
-  autoload :Errors, 'kreuzberg/errors'
-  autoload :ErrorContext, 'kreuzberg/error_context'
-  autoload :PostProcessorProtocol, 'kreuzberg/post_processor_protocol'
-  autoload :ValidatorProtocol, 'kreuzberg/validator_protocol'
-  autoload :OcrBackendProtocol, 'kreuzberg/ocr_backend_protocol'
-
-  # Alias for API consistency with other language bindings
-  ExtractionConfig = Config::Extraction
-
-  @__cache_tracker = { entries: 0, bytes: 0 }
-
-  class << self
-    # Store native methods as private methods
-    alias native_extract_file_sync extract_file_sync
-    alias native_extract_bytes_sync extract_bytes_sync
-    alias native_batch_extract_files_sync batch_extract_files_sync
-    alias native_extract_file extract_file
-    alias native_extract_bytes extract_bytes
-    alias native_batch_extract_files batch_extract_files
-    alias native_batch_extract_bytes_sync batch_extract_bytes_sync
-    alias native_batch_extract_bytes batch_extract_bytes
-    alias native_clear_cache clear_cache
-    alias native_cache_stats cache_stats
-
-    private :native_extract_file_sync, :native_extract_bytes_sync, :native_batch_extract_files_sync
-    private :native_extract_file, :native_extract_bytes, :native_batch_extract_files
-    private :native_batch_extract_bytes_sync, :native_batch_extract_bytes
-  end
-
-  # Register a Ruby post-processor that conforms to PostProcessorProtocol.
-  module_function :register_post_processor
-
-  # Remove a post-processor by name.
-  module_function :unregister_post_processor
-
-  # Purge all registered post-processors.
-  module_function :clear_post_processors
-
-  # Register a validator that follows ValidatorProtocol.
-  module_function :register_validator
-
-  # Remove a validator by name.
-  module_function :unregister_validator
-
-  # Purge all validators.
-  module_function :clear_validators
-
-  # List all registered validators.
-  module_function :list_validators
-
-  # List all registered post-processors.
-  module_function :list_post_processors
-
-  # Register an OCR backend instance implementing OcrBackendProtocol.
-  module_function :register_ocr_backend
-
-  # Unregister an OCR backend by name.
-  module_function :unregister_ocr_backend
-
-  # List all registered OCR backends.
-  module_function :list_ocr_backends
-
-  # Detect MIME type from file bytes.
-  module_function :detect_mime_type
-
-  # Detect MIME type from a file path.
-  module_function :detect_mime_type_from_path
-
-  # Validate a MIME type string.
-  module_function :validate_mime_type
-
-  # Get file extensions for a given MIME type.
-  module_function :get_extensions_for_mime
-
-  # List all available embedding presets.
-  module_function :list_embedding_presets
-
-  # Get a specific embedding preset by name.
-  module_function :get_embedding_preset
-end
-
-require_relative 'kreuzberg/cache_api'
-require_relative 'kreuzberg/extraction_api'
-
-Kreuzberg.singleton_class.prepend(Kreuzberg::CacheAPI)
-Kreuzberg.singleton_class.prepend(Kreuzberg::ExtractionAPI)
+# frozen_string_literal: true
+
+require_relative 'kreuzberg/setup_lib_path'
+Kreuzberg::SetupLibPath.configure
+
+require_relative 'kreuzberg/version'
+require 'kreuzberg_rb'
+
+# Kreuzberg is a Ruby binding for the Rust core library providing document extraction,
+# text extraction, and OCR capabilities.
+module Kreuzberg
+  autoload :Config, 'kreuzberg/config'
+  autoload :Result, 'kreuzberg/result'
+  autoload :CLI, 'kreuzberg/cli'
+  autoload :CLIProxy, 'kreuzberg/cli_proxy'
+  autoload :APIProxy, 'kreuzberg/api_proxy'
+  autoload :MCPProxy, 'kreuzberg/mcp_proxy'
+  autoload :Errors, 'kreuzberg/errors'
+  autoload :ErrorContext, 'kreuzberg/error_context'
+  autoload :PostProcessorProtocol, 'kreuzberg/post_processor_protocol'
+  autoload :ValidatorProtocol, 'kreuzberg/validator_protocol'
+  autoload :OcrBackendProtocol, 'kreuzberg/ocr_backend_protocol'
+
+  autoload :HtmlMetadata, 'kreuzberg/types'
+  autoload :HeaderMetadata, 'kreuzberg/types'
+  autoload :LinkMetadata, 'kreuzberg/types'
+  autoload :ImageMetadata, 'kreuzberg/types'
+  autoload :StructuredData, 'kreuzberg/types'
+
+  ExtractionConfig = Config::Extraction
+  PageConfig = Config::PageConfig
+
+  module KeywordAlgorithm
+    YAKE = :yake
+    RAKE = :rake
+  end
+
+  @__cache_tracker = { entries: 0, bytes: 0 }
+
+  class << self
+    alias native_extract_file_sync extract_file_sync
+    alias native_extract_bytes_sync extract_bytes_sync
+    alias native_batch_extract_files_sync batch_extract_files_sync
+    alias native_extract_file extract_file
+    alias native_extract_bytes extract_bytes
+    alias native_batch_extract_files batch_extract_files
+    alias native_batch_extract_bytes_sync batch_extract_bytes_sync
+    alias native_batch_extract_bytes batch_extract_bytes
+    alias native_clear_cache clear_cache
+    alias native_cache_stats cache_stats
+
+    private :native_extract_file_sync, :native_extract_bytes_sync, :native_batch_extract_files_sync
+    private :native_extract_file, :native_extract_bytes, :native_batch_extract_files
+    private :native_batch_extract_bytes_sync, :native_batch_extract_bytes
+  end
+
+  module_function :register_post_processor
+
+  module_function :unregister_post_processor
+
+  module_function :clear_post_processors
+
+  module_function :register_validator
+
+  module_function :unregister_validator
+
+  module_function :clear_validators
+
+  module_function :list_validators
+
+  module_function :list_post_processors
+
+  module_function :register_ocr_backend
+
+  module_function :unregister_ocr_backend
+
+  module_function :list_ocr_backends
+
+  module_function :detect_mime_type
+
+  module_function :detect_mime_type_from_path
+
+  module_function :validate_mime_type
+
+  module_function :get_extensions_for_mime
+
+  module_function :list_embedding_presets
+
+  module_function :get_embedding_preset
+end
+
+require_relative 'kreuzberg/cache_api'
+require_relative 'kreuzberg/extraction_api'
+
+Kreuzberg.singleton_class.prepend(Kreuzberg::CacheAPI)
+Kreuzberg.singleton_class.prepend(Kreuzberg::ExtractionAPI)