kreuzberg 4.0.0.rc1

data/lib/kreuzberg/result.rb

@@ -0,0 +1,216 @@
+# frozen_string_literal: true
+
+begin
+  require 'json'
+rescue LoadError
+  require 'json/pure'
+end
+
+module Kreuzberg
+  # Extraction result wrapper
+  #
+  # Provides structured access to extraction results from the native extension.
+  #
+  # @example
+  #   result = Kreuzberg.extract_file_sync("document.pdf")
+  #   puts result.content
+  #   puts "MIME type: #{result.mime_type}"
+  #   puts "Metadata: #{result.metadata.inspect}"
+  #   result.tables.each { |table| puts table.inspect }
+  #
+  # rubocop:disable Metrics/ClassLength
+  class Result
+    attr_reader :content, :mime_type, :metadata, :metadata_json, :tables,
+                :detected_languages, :chunks, :images
+
+    # Table structure
+    #
+    # @!attribute [r] cells
+    #   @return [Array<Array<String>>] Table cells (2D array)
+    # @!attribute [r] markdown
+    #   @return [String] Markdown representation
+    # @!attribute [r] page_number
+    #   @return [Integer] Page number where table was found
+    #
+    Table = Struct.new(:cells, :markdown, :page_number, keyword_init: true) do
+      def to_h
+        { cells: cells, markdown: markdown, page_number: page_number }
+      end
+    end
+
+    # Text chunk
+    #
+    # @!attribute [r] content
+    #   @return [String] Chunk content
+    # @!attribute [r] char_start
+    #   @return [Integer] Starting character index
+    # @!attribute [r] char_end
+    #   @return [Integer] Ending character index
+    # @!attribute [r] token_count
+    #   @return [Integer, nil] Approximate token count (may be nil)
+    #
+    Chunk = Struct.new(
+      :content,
+      :char_start,
+      :char_end,
+      :token_count,
+      :chunk_index,
+      :total_chunks,
+      :embedding,
+      keyword_init: true
+    ) do
+      def to_h
+        {
+          content: content,
+          char_start: char_start,
+          char_end: char_end,
+          token_count: token_count,
+          chunk_index: chunk_index,
+          total_chunks: total_chunks,
+          embedding: embedding
+        }
+      end
+    end
+
+    Image = Struct.new(
+      :data,
+      :format,
+      :image_index,
+      :page_number,
+      :width,
+      :height,
+      :colorspace,
+      :bits_per_component,
+      :is_mask,
+      :description,
+      :ocr_result,
+      keyword_init: true
+    ) do
+      def to_h
+        {
+          data: data,
+          format: format,
+          image_index: image_index,
+          page_number: page_number,
+          width: width,
+          height: height,
+          colorspace: colorspace,
+          bits_per_component: bits_per_component,
+          is_mask: is_mask,
+          description: description,
+          ocr_result: ocr_result&.to_h
+        }
+      end
+    end
+
+    # Initialize from native hash result
+    #
+    # @param hash [Hash] Hash returned from native extension
+    #
+    def initialize(hash)
+      # Handle both string and symbol keys for flexibility
+      @content = get_value(hash, 'content', '')
+      @mime_type = get_value(hash, 'mime_type', '')
+      @metadata_json = get_value(hash, 'metadata_json', '{}')
+      @metadata = parse_metadata(@metadata_json)
+      @tables = parse_tables(get_value(hash, 'tables'))
+      @detected_languages = parse_detected_languages(get_value(hash, 'detected_languages'))
+      @chunks = parse_chunks(get_value(hash, 'chunks'))
+      @images = parse_images(get_value(hash, 'images'))
+    end
+
+    # Convert to hash
+    #
+    # @return [Hash] Hash representation
+    #
+    def to_h
+      {
+        content: @content,
+        mime_type: @mime_type,
+        metadata: @metadata,
+        tables: @tables.map(&:to_h),
+        detected_languages: @detected_languages,
+        chunks: @chunks&.map(&:to_h),
+        images: @images&.map(&:to_h)
+      }
+    end
+
+    # Convert to JSON
+    #
+    # @return [String] JSON representation
+    #
+    def to_json(*)
+      to_h.to_json(*)
+    end
+
+    private
+
+    def get_value(hash, key, default = nil)
+      hash[key] || hash[key.to_sym] || default
+    end
+
+    def parse_metadata(metadata_json)
+      JSON.parse(metadata_json)
+    rescue JSON::ParserError
+      {}
+    end
+
+    def parse_tables(tables_data)
+      return [] if tables_data.nil? || tables_data.empty?
+
+      tables_data.map do |table_hash|
+        Table.new(
+          cells: table_hash['cells'] || [],
+          markdown: table_hash['markdown'] || '',
+          page_number: table_hash['page_number'] || 0
+        )
+      end
+    end
+
+    def parse_detected_languages(langs_data)
+      return nil if langs_data.nil?
+
+      # Detected languages is now just an array of strings
+      langs_data.is_a?(Array) ? langs_data : []
+    end
+
+    def parse_chunks(chunks_data)
+      return nil if chunks_data.nil?
+
+      chunks_data.map do |chunk_hash|
+        Chunk.new(
+          content: chunk_hash['content'],
+          char_start: chunk_hash['char_start'],
+          char_end: chunk_hash['char_end'],
+          token_count: chunk_hash['token_count'],
+          chunk_index: chunk_hash['chunk_index'],
+          total_chunks: chunk_hash['total_chunks'],
+          embedding: chunk_hash['embedding']
+        )
+      end
+    end
+
+    def parse_images(images_data)
+      return nil if images_data.nil?
+
+      images_data.map do |image_hash|
+        data = image_hash['data']
+        data = data.dup.force_encoding(Encoding::BINARY) if data.respond_to?(:force_encoding)
+        Image.new(
+          data: data,
+          format: image_hash['format'],
+          image_index: image_hash['image_index'],
+          page_number: image_hash['page_number'],
+          width: image_hash['width'],
+          height: image_hash['height'],
+          colorspace: image_hash['colorspace'],
+          bits_per_component: image_hash['bits_per_component'],
+          is_mask: image_hash['is_mask'],
+          description: image_hash['description'],
+          ocr_result: image_hash['ocr_result'] ? Result.new(image_hash['ocr_result']) : nil
+        )
+      end
+    end
+  end
+  # rubocop:enable Metrics/ClassLength
+end

data/lib/kreuzberg/setup_lib_path.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require 'rbconfig'
+require 'open3'
+
+module Kreuzberg
+  module SetupLibPath
+    module_function
+
+    def configure
+      lib_dir = File.expand_path('..', __dir__ || '.')
+      host_os = RbConfig::CONFIG['host_os']
+
+      case host_os
+      when /darwin/
+        prepend_env('DYLD_LIBRARY_PATH', lib_dir)
+        prepend_env('DYLD_FALLBACK_LIBRARY_PATH', "#{lib_dir}:/usr/local/lib:/usr/lib")
+        fix_macos_install_name(lib_dir)
+      when /linux/
+        prepend_env('LD_LIBRARY_PATH', lib_dir)
+      when /mswin|mingw|cygwin/
+        prepend_env('PATH', lib_dir, separator: ';')
+      end
+    end
+
+    def prepend_env(key, value, separator: ':')
+      current = ENV.fetch(key, nil)
+      return if current&.split(separator)&.include?(value)
+
+      ENV[key] = current.nil? || current.empty? ? value : "#{value}#{separator}#{current}"
+    end
+    private_class_method :prepend_env
+
+    def fix_macos_install_name(lib_dir)
+      bundle = macos_bundle(lib_dir)
+      return unless bundle
+
+      ensure_install_name(bundle)
+      ensure_loader_rpath(bundle)
+    rescue Errno::ENOENT, IOError
+      # Tool not available (e.g., on CI). The dynamic loader can still use the updated env vars.
+    end
+    private_class_method :fix_macos_install_name
+
+    def macos_bundle(lib_dir)
+      bundle = File.join(lib_dir, 'kreuzberg_rb.bundle')
+      pdfium = File.join(lib_dir, 'libpdfium.dylib')
+      return unless File.exist?(bundle) && File.exist?(pdfium)
+
+      bundle
+    end
+    private_class_method :macos_bundle
+
+    def ensure_install_name(bundle)
+      output, status = Open3.capture2('otool', '-L', bundle)
+      return unless status.success?
+
+      replacements = {
+        './libpdfium.dylib' => '@loader_path/libpdfium.dylib',
+        '@rpath/libpdfium.dylib' => '@loader_path/libpdfium.dylib'
+      }
+
+      replacements.each do |current, desired|
+        next unless output.include?(current)
+
+        Open3.capture2('install_name_tool', '-change', current, desired, bundle)
+      end
+    end
+    private_class_method :ensure_install_name
+
+    def ensure_loader_rpath(bundle)
+      rpath_output, rpath_status = Open3.capture2('otool', '-l', bundle)
+      return unless rpath_status.success? && !rpath_output.include?('@loader_path')
+
+      Open3.capture2('install_name_tool', '-add_rpath', '@loader_path', bundle)
+    end
+    private_class_method :ensure_loader_rpath
+  end
+end

data/lib/kreuzberg/validator_protocol.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+module Kreuzberg
+  # Validator protocol interface.
+  #
+  # This module defines the protocol that all Ruby validators must implement
+  # to be registered with the Rust core via the FFI bridge.
+  #
+  # Validators are called during extraction to validate results. If validation fails,
+  # the validator should raise a Kreuzberg::Errors::ValidationError, which will
+  # cause the extraction to fail.
+  #
+  # @example Implementing a minimum length validator
+  #   class MinimumLengthValidator
+  #     include Kreuzberg::ValidatorProtocol
+  #
+  #     def initialize(min_length = 10)
+  #       @min_length = min_length
+  #     end
+  #
+  #     def call(result)
+  #       if result["content"].length < @min_length
+  #         raise Kreuzberg::Errors::ValidationError.new(
+  #           "Content too short: #{result["content"].length} < #{@min_length}"
+  #         )
+  #       end
+  #     end
+  #   end
+  #
+  #   Kreuzberg.register_validator("min_length", MinimumLengthValidator.new(100))
+  #
+  # @example Implementing a content quality validator
+  #   class QualityValidator
+  #     include Kreuzberg::ValidatorProtocol
+  #
+  #     def call(result)
+  #       # Check if content has sufficient quality
+  #       if result["content"].strip.empty?
+  #         raise Kreuzberg::Errors::ValidationError.new("Empty content extracted")
+  #       end
+  #
+  #       # Check if metadata is present
+  #       if result["metadata"].empty?
+  #         raise Kreuzberg::Errors::ValidationError.new("No metadata extracted")
+  #       end
+  #     end
+  #   end
+  #
+  #   Kreuzberg.register_validator("quality", QualityValidator.new)
+  #
+  # @example Using a Proc as a validator
+  #   Kreuzberg.register_validator("not_empty", ->(result) {
+  #     if result["content"].strip.empty?
+  #       raise Kreuzberg::Errors::ValidationError.new("Content cannot be empty")
+  #     end
+  #   })
+  #
+  module ValidatorProtocol
+    # Validate an extraction result.
+    #
+    # This method is called during extraction to validate results. If validation fails,
+    # raise a Kreuzberg::Errors::ValidationError with a descriptive message explaining
+    # why validation failed. If validation passes, return without raising.
+    #
+    # The validator receives the extraction result as a hash with the same structure
+    # as post-processors (see PostProcessorProtocol for details).
+    #
+    # @param result [Hash] Extraction result to validate with the following structure:
+    #   - "content" [String] - Extracted text content
+    #   - "mime_type" [String] - MIME type of the source document
+    #   - "metadata" [Hash] - Document metadata (title, author, etc.)
+    #   - "tables" [Array<Hash>] - Extracted tables
+    #   - "detected_languages" [Array<String>, nil] - Detected language codes
+    #   - "chunks" [Array<String>, nil] - Content chunks (if chunking enabled)
+    #
+    # @return [void]
+    # @raise [Kreuzberg::Errors::ValidationError] if validation fails
+    #
+    # @example
+    #   def call(result)
+    #     if result["content"].length < 10
+    #       raise Kreuzberg::Errors::ValidationError.new("Content too short")
+    #     end
+    #   end
+    def call(result)
+      raise NotImplementedError, "#{self.class} must implement #call(result)"
+    end
+  end
+end

data/lib/kreuzberg/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Kreuzberg
+  VERSION = '4.0.0.rc1'
+end

data/lib/kreuzberg.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+require_relative 'kreuzberg/setup_lib_path'
+Kreuzberg::SetupLibPath.configure
+
+require_relative 'kreuzberg/version'
+require 'kreuzberg_rb'
+
+module Kreuzberg
+  autoload :Config, 'kreuzberg/config'
+  autoload :Result, 'kreuzberg/result'
+  autoload :CLI, 'kreuzberg/cli'
+  autoload :CLIProxy, 'kreuzberg/cli_proxy'
+  autoload :APIProxy, 'kreuzberg/api_proxy'
+  autoload :MCPProxy, 'kreuzberg/mcp_proxy'
+  autoload :Errors, 'kreuzberg/errors'
+  autoload :PostProcessorProtocol, 'kreuzberg/post_processor_protocol'
+  autoload :ValidatorProtocol, 'kreuzberg/validator_protocol'
+  autoload :OcrBackendProtocol, 'kreuzberg/ocr_backend_protocol'
+
+  # Alias for API consistency with other language bindings
+  ExtractionConfig = Config::Extraction
+
+  @__cache_tracker = { entries: 0, bytes: 0 }
+
+  class << self
+    # Store native methods as private methods
+    alias native_extract_file_sync extract_file_sync
+    alias native_extract_bytes_sync extract_bytes_sync
+    alias native_batch_extract_files_sync batch_extract_files_sync
+    alias native_extract_file extract_file
+    alias native_extract_bytes extract_bytes
+    alias native_batch_extract_files batch_extract_files
+    alias native_batch_extract_bytes_sync batch_extract_bytes_sync
+    alias native_batch_extract_bytes batch_extract_bytes
+    alias native_clear_cache clear_cache
+    alias native_cache_stats cache_stats
+
+    private :native_extract_file_sync, :native_extract_bytes_sync, :native_batch_extract_files_sync
+    private :native_extract_file, :native_extract_bytes, :native_batch_extract_files
+    private :native_batch_extract_bytes_sync, :native_batch_extract_bytes
+  end
+
+  # Register a Ruby post-processor that conforms to PostProcessorProtocol.
+  module_function :register_post_processor
+
+  # Remove a post-processor by name.
+  module_function :unregister_post_processor
+
+  # Purge all registered post-processors.
+  module_function :clear_post_processors
+
+  # Register a validator that follows ValidatorProtocol.
+  module_function :register_validator
+
+  # Remove a validator by name.
+  module_function :unregister_validator
+
+  # Purge all validators.
+  module_function :clear_validators
+
+  # List all registered validators.
+  module_function :list_validators
+
+  # List all registered post-processors.
+  module_function :list_post_processors
+
+  # Register an OCR backend instance implementing OcrBackendProtocol.
+  module_function :register_ocr_backend
+
+  # Unregister an OCR backend by name.
+  module_function :unregister_ocr_backend
+
+  # List all registered OCR backends.
+  module_function :list_ocr_backends
+end
+
+require_relative 'kreuzberg/cache_api'
+require_relative 'kreuzberg/extraction_api'
+
+Kreuzberg.singleton_class.prepend(Kreuzberg::CacheAPI)
+Kreuzberg.singleton_class.prepend(Kreuzberg::ExtractionAPI)

data/sig/kreuzberg/internal.rbs

@@ -0,0 +1,184 @@
+# Internal modules - not part of public API but fully typed
+
+module Kreuzberg
+  module SetupLibPath
+    # Public method
+    def configure: () -> void
+    def self.configure: () -> void
+
+    # Private methods (module_function makes them both instance and class methods)
+    def prepend_env: (String key, String value, ?separator: String) -> void
+    def self.prepend_env: (String key, String value, ?separator: String) -> void
+    def fix_macos_install_name: (String lib_dir) -> void
+    def self.fix_macos_install_name: (String lib_dir) -> void
+    def macos_bundle: (String lib_dir) -> String?
+    def self.macos_bundle: (String lib_dir) -> String?
+    def ensure_install_name: (String bundle) -> void
+    def self.ensure_install_name: (String bundle) -> void
+    def ensure_loader_rpath: (String bundle) -> void
+    def self.ensure_loader_rpath: (String bundle) -> void
+  end
+
+  module CLI
+    # All methods are both instance and class methods due to module_function
+    def extract: (String path, ?output: String, ?ocr: bool) -> String
+    def self.extract: (String path, ?output: String, ?ocr: bool) -> String
+    def detect: (String path) -> String
+    def self.detect: (String path) -> String
+    def version: () -> String
+    def self.version: () -> String
+    def help: () -> String
+    def self.help: () -> String
+  end
+
+  module CLIProxy
+    class Error < Kreuzberg::Errors::Error
+    end
+
+    class MissingBinaryError < Error
+    end
+
+    class CLIExecutionError < Error
+      attr_reader stderr: String
+      attr_reader status: Integer
+
+      def initialize: (String message, stderr: String, status: Integer) -> void
+    end
+
+    # All methods are both instance and class methods due to module_function
+    def call: (Array[String] argv) -> String
+    def self.call: (Array[String] argv) -> String
+    def find_cli_binary: () -> Pathname
+    def self.find_cli_binary: () -> Pathname
+    def root_path: () -> Pathname
+    def self.root_path: () -> Pathname
+    def lib_path: () -> Pathname
+    def self.lib_path: () -> Pathname
+    def search_paths: (String binary_name) -> Array[Pathname]
+    def self.search_paths: (String binary_name) -> Array[Pathname]
+    def missing_binary_message: () -> String
+    def self.missing_binary_message: () -> String
+  end
+
+  module APIProxy
+    class Error < Kreuzberg::Errors::Error
+    end
+
+    class MissingBinaryError < Error
+    end
+
+    class ServerError < Error
+    end
+
+    class Server
+      attr_reader port: Integer
+      attr_reader host: String
+      attr_reader process: Process::Status?
+
+      def initialize: (?port: Integer, ?host: String) -> void
+      def start: () -> void
+      def stop: () -> void
+      def running?: () -> bool
+      def health_check: () -> bool
+    end
+
+    # Module function methods are both instance and class methods
+    def run: (?port: Integer, ?host: String) { (Server) -> untyped } -> untyped
+    def self.run: (?port: Integer, ?host: String) { (Server) -> untyped } -> untyped
+    def start_server: (?port: Integer?) -> Server
+    def self.start_server: (?port: Integer?) -> Server
+    def extract_file: (String path, ?mime_type: String?, ?config: Hash[Symbol, untyped]?, ?server: Server?) -> Kreuzberg::Result
+    def self.extract_file: (String path, ?mime_type: String?, ?config: Hash[Symbol, untyped]?, ?server: Server?) -> Kreuzberg::Result
+    def batch_extract: (Array[String] paths, ?config: Hash[Symbol, untyped]?, ?server: Server?) -> Array[Kreuzberg::Result]
+    def self.batch_extract: (Array[String] paths, ?config: Hash[Symbol, untyped]?, ?server: Server?) -> Array[Kreuzberg::Result]
+    def find_api_binary: () -> Pathname
+    def self.find_api_binary: () -> Pathname
+    def missing_binary_message: () -> String
+    def self.missing_binary_message: () -> String
+  end
+
+  module MCPProxy
+    class Error < Kreuzberg::Errors::Error
+    end
+
+    class MissingBinaryError < Error
+    end
+
+    class ServerError < Error
+    end
+
+    class Server
+      attr_reader pid: Integer?
+      attr_reader transport: String
+
+      def initialize: (?transport: String) -> void
+      def start: () -> (Integer | nil)
+      def stop: () -> void
+      def running?: () -> bool
+      def send_message: (Hash[untyped, untyped] message) -> void
+      def read_message: () -> (Hash[untyped, untyped] | nil)
+
+      private
+
+      def start_stdio: (Pathname binary) -> nil
+      def start_sse: (Pathname binary) -> (Integer | nil)
+      def close_pipes: () -> void
+    end
+
+    # Module function methods are both instance and class methods
+    def run: (?transport: String) { (Server) -> untyped } -> untyped
+    def self.run: (?transport: String) { (Server) -> untyped } -> untyped
+    def extract_file: (String path, ?mime_type: String?, ?config: Hash[Symbol, untyped]?) -> Kreuzberg::Result
+    def self.extract_file: (String path, ?mime_type: String?, ?config: Hash[Symbol, untyped]?) -> Kreuzberg::Result
+    def batch_extract: (Array[String] paths, ?config: Hash[Symbol, untyped]?) -> Array[Kreuzberg::Result]
+    def self.batch_extract: (Array[String] paths, ?config: Hash[Symbol, untyped]?) -> Array[Kreuzberg::Result]
+    def find_mcp_binary: () -> Pathname
+    def self.find_mcp_binary: () -> Pathname
+    def missing_binary_message: () -> String
+    def self.missing_binary_message: () -> String
+  end
+
+  # Cache API module (prepended to Kreuzberg singleton class)
+  module CacheAPI
+    @__cache_tracker: Hash[Symbol, Integer]
+
+    def clear_cache: () -> void
+    def cache_stats: () -> Hash[Symbol | String, Integer]
+
+    private
+
+    def record_cache_entry!: (Kreuzberg::Result | Array[Kreuzberg::Result] results, Hash[Symbol, untyped] opts) -> void
+    def reset_cache_tracker!: () -> nil
+
+    # Native methods (defined in Rust)
+    def native_clear_cache: () -> void
+    def native_cache_stats: () -> Hash[Symbol | String, Integer]
+  end
+
+  # Extraction API module (prepended to Kreuzberg singleton class)
+  module ExtractionAPI
+    def extract_file_sync: (String | Pathname path, ?mime_type: String?, ?config: config_input?) -> Result
+    def extract_bytes_sync: (String data, String mime_type, ?config: config_input?) -> Result
+    def batch_extract_files_sync: (Array[String | Pathname] paths, ?config: config_input?) -> Array[Result]
+    def extract_file: (String | Pathname path, ?mime_type: String?, ?config: config_input?) -> Result
+    def extract_bytes: (String data, String mime_type, ?config: config_input?) -> Result
+    def batch_extract_files: (Array[String | Pathname] paths, ?config: config_input?) -> Array[Result]
+    def batch_extract_bytes_sync: (Array[String] data_array, Array[String] mime_types, ?config: config_input?) -> Array[Result]
+    def batch_extract_bytes: (Array[String] data_array, Array[String] mime_types, ?config: config_input?) -> Array[Result]
+
+    def normalize_config: (config_input? config) -> Hash[Symbol, untyped]
+
+    # Native methods (defined in Rust)
+    def native_extract_file_sync: (String path, ?String? mime_type, **untyped opts) -> extraction_result_hash
+    def native_extract_bytes_sync: (String data, String mime_type, **untyped opts) -> extraction_result_hash
+    def native_batch_extract_files_sync: (Array[String] paths, **untyped opts) -> Array[extraction_result_hash]
+    def native_extract_file: (String path, ?String? mime_type, **untyped opts) -> extraction_result_hash
+    def native_extract_bytes: (String data, String mime_type, **untyped opts) -> extraction_result_hash
+    def native_batch_extract_files: (Array[String] paths, **untyped opts) -> Array[extraction_result_hash]
+    def native_batch_extract_bytes_sync: (Array[String] data_array, Array[String] mime_types, **untyped opts) -> Array[extraction_result_hash]
+    def native_batch_extract_bytes: (Array[String] data_array, Array[String] mime_types, **untyped opts) -> Array[extraction_result_hash]
+
+    # Cache API methods (from prepended CacheAPI module)
+    def record_cache_entry!: (Kreuzberg::Result | Array[Kreuzberg::Result] results, Hash[Symbol, untyped] opts) -> void
+  end
+end