kreuzberg 4.0.0.pre.rc.6

data/lib/kreuzberg/errors.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Kreuzberg
+  # Error code constants matching kreuzberg-ffi error codes
+  ERROR_CODE_SUCCESS = 0
+  ERROR_CODE_GENERIC = 1
+  ERROR_CODE_PANIC = 2
+  ERROR_CODE_INVALID_ARGUMENT = 3
+  ERROR_CODE_IO = 4
+  ERROR_CODE_PARSING = 5
+  ERROR_CODE_OCR = 6
+  ERROR_CODE_MISSING_DEPENDENCY = 7
+
+  module Errors
+    # Panic context information from FFI error introspection
+    class PanicContext
+      attr_reader :file, :line, :function, :message, :timestamp_secs
+
+      def initialize(file:, line:, function:, message:, timestamp_secs:)
+        @file = file
+        @line = line
+        @function = function
+        @message = message
+        @timestamp_secs = timestamp_secs
+      end
+
+      def to_s
+        "#{file}:#{line}:#{function}: #{message}"
+      end
+
+      def to_h
+        {
+          file:,
+          line:,
+          function:,
+          message:,
+          timestamp_secs:
+        }
+      end
+
+      def self.from_json(json_string)
+        return nil if json_string.nil? || json_string.empty?
+
+        data = JSON.parse(json_string, symbolize_names: true)
+        sliced = data.slice(:file, :line, :function, :message, :timestamp_secs)
+        new(**with_defaults(sliced))
+      rescue JSON::ParserError
+        nil
+      end
+
+      def self.with_defaults(sliced)
+        {
+          file: sliced[:file] || '',
+          line: sliced[:line] || 0,
+          function: sliced[:function] || '',
+          message: sliced[:message] || '',
+          timestamp_secs: sliced[:timestamp_secs] || 0
+        }
+      end
+      private_class_method :with_defaults
+    end
+
+    # Base error class for all Kreuzberg errors
+    class Error < StandardError
+      attr_reader :panic_context, :error_code
+
+      def initialize(message, panic_context: nil, error_code: nil)
+        super(message)
+        @panic_context = panic_context
+        @error_code = error_code
+      end
+    end
+
+    # Raised when validation fails
+    class ValidationError < Error; end
+
+    # Raised when document parsing fails
+    class ParsingError < Error
+      attr_reader :context
+
+      def initialize(message, context: nil, panic_context: nil, error_code: nil)
+        super(message, panic_context:, error_code:)
+        @context = context
+      end
+    end
+
+    # Raised when OCR processing fails
+    class OCRError < Error
+      attr_reader :context
+
+      def initialize(message, context: nil, panic_context: nil, error_code: nil)
+        super(message, panic_context:, error_code:)
+        @context = context
+      end
+    end
+
+    # Raised when a required dependency is missing
+    class MissingDependencyError < Error
+      attr_reader :dependency
+
+      def initialize(message, dependency: nil, panic_context: nil, error_code: nil)
+        super(message, panic_context:, error_code:)
+        @dependency = dependency
+      end
+    end
+
+    # Raised when an I/O operation fails
+    class IOError < Error; end
+
+    # Raised when plugin operations fail
+    class PluginError < Error; end
+
+    # Raised when an unsupported file format or MIME type is encountered
+    class UnsupportedFormatError < Error; end
+  end
+end

data/lib/kreuzberg/extraction_api.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+module Kreuzberg
+  # Provides extraction methods for documents and text.
+  module ExtractionAPI
+    def extract_file_sync(path, mime_type: nil, config: nil)
+      opts = normalize_config(config)
+      hash = if mime_type
+               native_extract_file_sync(path.to_s, mime_type.to_s, **opts)
+             else
+               native_extract_file_sync(path.to_s, **opts)
+             end
+      result = Result.new(hash)
+      record_cache_entry!(result, opts)
+      result
+    end
+
+    def extract_bytes_sync(data, mime_type, config: nil)
+      opts = normalize_config(config)
+      hash = native_extract_bytes_sync(data.to_s, mime_type.to_s, **opts)
+      result = Result.new(hash)
+      record_cache_entry!(result, opts)
+      result
+    end
+
+    def batch_extract_files_sync(paths, config: nil)
+      opts = normalize_config(config)
+      hashes = native_batch_extract_files_sync(paths.map(&:to_s), **opts)
+      results = hashes.map { |hash| Result.new(hash) }
+      record_cache_entry!(results, opts)
+      results
+    end
+
+    def extract_file(path, mime_type: nil, config: nil)
+      opts = normalize_config(config)
+      hash = if mime_type
+               native_extract_file(path.to_s, mime_type.to_s, **opts)
+             else
+               native_extract_file(path.to_s, **opts)
+             end
+      result = Result.new(hash)
+      record_cache_entry!(result, opts)
+      result
+    end
+
+    def extract_bytes(data, mime_type, config: nil)
+      opts = normalize_config(config)
+      hash = native_extract_bytes(data.to_s, mime_type.to_s, **opts)
+      result = Result.new(hash)
+      record_cache_entry!(result, opts)
+      result
+    end
+
+    def batch_extract_files(paths, config: nil)
+      opts = normalize_config(config)
+      hashes = native_batch_extract_files(paths.map(&:to_s), **opts)
+      results = hashes.map { |hash| Result.new(hash) }
+      record_cache_entry!(results, opts)
+      results
+    end
+
+    def batch_extract_bytes_sync(data_array, mime_types, config: nil)
+      opts = normalize_config(config)
+      hashes = native_batch_extract_bytes_sync(data_array.map(&:to_s), mime_types.map(&:to_s), **opts)
+      results = hashes.map { |hash| Result.new(hash) }
+      record_cache_entry!(results, opts)
+      results
+    end
+
+    def batch_extract_bytes(data_array, mime_types, config: nil)
+      opts = normalize_config(config)
+      hashes = native_batch_extract_bytes(data_array.map(&:to_s), mime_types.map(&:to_s), **opts)
+      results = hashes.map { |hash| Result.new(hash) }
+      record_cache_entry!(results, opts)
+      results
+    end
+
+    def normalize_config(config)
+      return {} if config.nil?
+      return config if config.is_a?(Hash)
+
+      config.to_h
+    end
+  end
+end

data/lib/kreuzberg/mcp_proxy.rb

@@ -0,0 +1,186 @@
+# frozen_string_literal: true
+
+require 'open3'
+require 'pathname'
+require 'json'
+
+module Kreuzberg
+  # MCP (Model Context Protocol) server proxy
+  #
+  # Starts and manages the Kreuzberg MCP server for Claude Desktop integration.
+  #
+  # @example Start MCP server
+  #   server = Kreuzberg::MCPProxy.new
+  #   server.start
+  #
+  module MCPProxy
+    Error = Class.new(Kreuzberg::Errors::Error)
+    MissingBinaryError = Class.new(Error)
+    ServerError = Class.new(Error)
+
+    # MCP server instance
+    class Server
+      attr_reader :pid, :transport
+
+      # Initialize MCP server
+      #
+      # @param transport [String] Transport method ("stdio" or "sse")
+      #
+      def initialize(transport: 'stdio')
+        @transport = transport
+        @pid = nil
+        @stdin = nil
+        @stdout = nil
+        @stderr = nil
+      end
+
+      # Start the MCP server
+      #
+      # @return [Integer, nil] Process ID (for SSE) or nil (for stdio)
+      #
+      def start
+        binary = MCPProxy.find_mcp_binary
+
+        case @transport
+        when 'stdio'
+          start_stdio(binary)
+        when 'sse'
+          start_sse(binary)
+        else
+          raise ServerError, "Unknown transport: #{@transport}"
+        end
+      end
+
+      # Stop the server
+      #
+      # @return [void]
+      #
+      def stop
+        return unless @pid
+
+        Process.kill('TERM', @pid)
+        Process.wait(@pid)
+      rescue Errno::ESRCH, Errno::ECHILD
+        # Process already dead
+      ensure
+        @pid = nil
+        close_pipes
+      end
+
+      # Send a message to the server (stdio only)
+      #
+      # @param message [Hash] JSON-RPC message
+      # @return [void]
+      #
+      def send_message(message)
+        raise ServerError, 'Can only send messages in stdio mode' unless @transport == 'stdio'
+        raise ServerError, 'Server not started' unless @stdin
+
+        @stdin.puts(JSON.generate(message))
+        @stdin.flush
+      end
+
+      # Read a message from the server (stdio only)
+      #
+      # @return [Hash] JSON-RPC message
+      #
+      def read_message
+        raise ServerError, 'Can only read messages in stdio mode' unless @transport == 'stdio'
+        raise ServerError, 'Server not started' unless @stdout
+
+        line = @stdout.gets
+        JSON.parse(line) if line
+      end
+
+      # Check if server is running
+      #
+      # @return [Boolean]
+      #
+      def running?
+        return false unless @pid
+
+        Process.kill(0, @pid)
+        true
+      rescue Errno::ESRCH, Errno::EPERM
+        false
+      end
+
+      private
+
+      def start_stdio(binary)
+        @stdin, @stdout, @stderr, wait_thr = Open3.popen3(binary.to_s, 'mcp', '--transport', 'stdio')
+        @pid = wait_thr.pid
+        nil
+      end
+
+      def start_sse(binary)
+        @pid = spawn(
+          binary.to_s,
+          'mcp',
+          '--transport', 'sse',
+          out: $stdout,
+          err: $stderr
+        )
+        Process.detach(@pid)
+        sleep 1 # Give server time to start
+        @pid
+      end
+
+      def close_pipes
+        @stdin&.close
+        @stdout&.close
+        @stderr&.close
+        @stdin = @stdout = @stderr = nil
+      end
+    end
+
+    module_function
+
+    # Run MCP server with a block
+    #
+    # @param transport [String] Transport method
+    # @yield [Server] Yields server instance
+    # @return [Object] Block result
+    #
+    # @example
+    #   Kreuzberg::MCPProxy.run(transport: 'stdio') do |server|
+    #     server.send_message({ method: 'tools/list' })
+    #     response = server.read_message
+    #   end
+    #
+    def run(transport: 'stdio')
+      server = Server.new(transport: transport)
+      server.start
+      yield server
+    ensure
+      server&.stop
+    end
+
+    # Find the MCP binary
+    #
+    # @return [Pathname] Path to binary
+    # @raise [MissingBinaryError] If not found
+    #
+    def find_mcp_binary
+      # MCP is served by kreuzberg CLI
+      binary_name = Gem.win_platform? ? 'kreuzberg.exe' : 'kreuzberg'
+      found = CLIProxy.search_paths(binary_name).find(&:file?)
+      return found if found
+
+      raise MissingBinaryError, missing_binary_message
+    end
+
+    # Error message for missing binary
+    #
+    # @return [String]
+    #
+    def missing_binary_message
+      <<~MSG.strip
+        kreuzberg binary not found for MCP server. Build it with:
+        `cargo build --release --package kreuzberg-cli`
+
+        Or ensure kreuzberg is installed with MCP support.
+      MSG
+    end
+  end
+end

data/lib/kreuzberg/ocr_backend_protocol.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+module Kreuzberg
+  # OCR backend protocol interface.
+  #
+  # This module defines the protocol that all Ruby OCR backends must implement
+  # to be registered with the Rust core via the FFI bridge.
+  #
+  # OCR backends implement optical character recognition for images and scanned documents.
+  # They are called when OCR is enabled in the extraction configuration.
+  #
+  # @example Implementing a custom OCR backend
+  #   class CustomOcrBackend
+  #     include Kreuzberg::OcrBackendProtocol
+  #
+  #     def name
+  #       "custom-ocr"
+  #     end
+  #
+  #     def process_image(image_bytes, config)
+  #       # Perform OCR on image_bytes
+  #       # This is a placeholder - integrate with a real OCR engine
+  #       text = my_ocr_engine.recognize(image_bytes, language: config["language"])
+  #       text
+  #     end
+  #   end
+  #
+  #   backend = CustomOcrBackend.new
+  #   Kreuzberg.register_ocr_backend(backend.name, backend)
+  #
+  #   # Use in extraction
+  #   result = Kreuzberg.extract_file_sync(
+  #     "scanned.pdf",
+  #     config: { ocr: { backend: "custom-ocr", language: "eng" } }
+  #   )
+  #
+  # @example Implementing an OCR backend with initialization
+  #   class ModelBasedOcr
+  #     include Kreuzberg::OcrBackendProtocol
+  #
+  #     def initialize
+  #       @model = nil
+  #     end
+  #
+  #     def name
+  #       "model-ocr"
+  #     end
+  #
+  #     def process_image(image_bytes, config)
+  #       # Load model on first use (lazy initialization)
+  #       @model ||= load_model
+  #
+  #       # Run OCR
+  #       @model.recognize(image_bytes, config)
+  #     end
+  #
+  #     private
+  #
+  #     def load_model
+  #       # Load ML model for OCR
+  #       MyOcrModel.load("path/to/model")
+  #     end
+  #   end
+  #
+  #   Kreuzberg.register_ocr_backend("model-ocr", ModelBasedOcr.new)
+  #
+  module OcrBackendProtocol
+    # Return the unique name of this OCR backend.
+    #
+    # This name is used in ExtractionConfig to select the backend:
+    #
+    #   config = { ocr: { backend: "custom-ocr", language: "eng" } }
+    #
+    # The name should be a lowercase string with hyphens (e.g., "custom-ocr", "tesseract").
+    #
+    # @return [String] Unique backend identifier
+    #
+    # @example
+    #   def name
+    #     "custom-ocr"
+    #   end
+    def name
+      raise NotImplementedError, "#{self.class} must implement #name"
+    end
+
+    # Process image bytes and extract text via OCR.
+    #
+    # This method receives raw image data (PNG, JPEG, TIFF, etc.) and an OCR configuration
+    # hash. It must return the extracted text as a string.
+    #
+    # The config hash contains OCR settings such as:
+    # - "language" [String] - Language code (e.g., "eng", "deu", "fra")
+    # - "backend" [String] - Backend name (same as #name)
+    # - Additional backend-specific settings
+    #
+    # @param image_bytes [String] Binary image data (PNG, JPEG, TIFF, etc.)
+    # @param config [Hash] OCR configuration with the following keys:
+    #   - "language" [String] - Language code for OCR (e.g., "eng", "deu")
+    #   - "backend" [String] - Backend name
+    #
+    # @return [String] Extracted text content
+    #
+    # @example
+    #   def process_image(image_bytes, config)
+    #     language = config["language"] || "eng"
+    #     text = my_ocr_engine.recognize(image_bytes, language: language)
+    #     text
+    #   end
+    def process_image(image_bytes, config)
+      raise NotImplementedError, "#{self.class} must implement #process_image(image_bytes, config)"
+    end
+  end
+end

data/lib/kreuzberg/post_processor_protocol.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+module Kreuzberg
+  # PostProcessor protocol interface.
+  #
+  # This module defines the protocol that all Ruby post-processors must implement
+  # to be registered with the Rust core via the FFI bridge.
+  #
+  # Post-processors enrich extraction results by adding metadata, transforming content,
+  # or performing additional analysis. They are called after extraction completes.
+  #
+  # @example Implementing a simple post-processor
+  #   class UpcaseProcessor
+  #     include Kreuzberg::PostProcessorProtocol
+  #
+  #     def call(result)
+  #       result["content"] = result["content"].upcase
+  #       result
+  #     end
+  #   end
+  #
+  #   Kreuzberg.register_post_processor("upcase", UpcaseProcessor.new)
+  #
+  # @example Implementing a post-processor that adds metadata
+  #   class EntityExtractor
+  #     include Kreuzberg::PostProcessorProtocol
+  #
+  #     def call(result)
+  #       entities = extract_entities(result["content"])
+  #       result["metadata"]["entities"] = entities
+  #       result
+  #     end
+  #
+  #     private
+  #
+  #     def extract_entities(text)
+  #       # Extract named entities from text
+  #       # This is a placeholder - use a real NER library in production
+  #       text.scan(/[A-Z][a-z]+(?:\s[A-Z][a-z]+)*/)
+  #     end
+  #   end
+  #
+  #   Kreuzberg.register_post_processor("entities", EntityExtractor.new)
+  #
+  # @example Using a Proc as a post-processor
+  #   Kreuzberg.register_post_processor("word_count", ->(result) {
+  #     word_count = result["content"].split.length
+  #     result["metadata"]["word_count"] = word_count
+  #     result
+  #   })
+  #
+  module PostProcessorProtocol
+    # Process and enrich an extraction result.
+    #
+    # This method is called after extraction completes. It receives the extraction result
+    # as a hash and must return the modified hash. The processor can:
+    # - Add new keys to result["metadata"]
+    # - Transform result["content"]
+    # - Add entries to result["tables"]
+    # - Modify any other result fields
+    #
+    # Existing metadata keys will not be overwritten by the FFI bridge, so it's safe
+    # to add new keys without worrying about conflicts.
+    #
+    # @param result [Hash] Extraction result with the following structure:
+    #   - "content" [String] - Extracted text content
+    #   - "mime_type" [String] - MIME type of the source document
+    #   - "metadata" [Hash] - Document metadata (title, author, etc.)
+    #   - "tables" [Array<Hash>] - Extracted tables
+    #   - "detected_languages" [Array<String>, nil] - Detected language codes
+    #   - "chunks" [Array<String>, nil] - Content chunks (if chunking enabled)
+    #
+    # @return [Hash] Modified extraction result with enriched metadata
+    #
+    # @example
+    #   def call(result)
+    #     text = result["content"]
+    #     entities = extract_entities(text)
+    #     result["metadata"]["entities"] = entities
+    #     result
+    #   end
+    def call(result)
+      raise NotImplementedError, "#{self.class} must implement #call(result)"
+    end
+  end
+end