kreuzberg 4.0.0.pre.rc.29 → 4.0.0.rc1

data/lib/kreuzberg/extraction_api.rb

@@ -2,19 +2,7 @@
 
 module Kreuzberg
   module ExtractionAPI
-    # @param path [String, Pathname] Path to the document file to extract
-    # @param mime_type [String, nil] Optional MIME type for the file (e.g., 'application/pdf').
-    # @param config [Config::Extraction, Hash, nil] Extraction configuration controlling
-    # @return [Result] Extraction result containing content, metadata, tables, and images
-    # @raise [Errors::IOError] If the file cannot be read or access is denied
-    # @raise [Errors::ParsingError] If document parsing fails
-    # @raise [Errors::UnsupportedFormatError] If the file format is not supported
-    # @raise [Errors::OCRError] If OCR is enabled and fails
-    # @raise [Errors::MissingDependencyError] If a required dependency is missing
-    # @example Extract a PDF file
-    # @example Extract with explicit MIME type
-    # @example Extract with OCR enabled
-    def extract_file_sync(path:, mime_type: nil, config: nil)
+    def extract_file_sync(path, mime_type: nil, config: nil)
       opts = normalize_config(config)
       hash = if mime_type
                native_extract_file_sync(path.to_s, mime_type.to_s, **opts)
@@ -26,33 +14,7 @@ module Kreuzberg
       result
     end
 
-    # Synchronously extract content from byte data.
-    #
-    # Performs document extraction directly from binary data in memory. Useful for
-    # extracting content from files already loaded into memory or from network streams.
-    #
-    # @param data [String] Binary document data (can contain any byte values)
-    # @param mime_type [String] MIME type of the data (required, e.g., 'application/pdf').
-    #   This parameter is mandatory to guide the extraction engine.
-    # @param config [Config::Extraction, Hash, nil] Extraction configuration. Accepts
-    #   either a {Config::Extraction} object or a configuration hash.
-    #
-    # @return [Result] Extraction result containing content, metadata, tables, and images
-    #
-    # @raise [Errors::ParsingError] If document parsing fails
-    # @raise [Errors::UnsupportedFormatError] If the MIME type is not supported
-    # @raise [Errors::OCRError] If OCR is enabled and fails
-    # @raise [Errors::MissingDependencyError] If a required dependency is missing
-    #
-    # @example Extract PDF from memory
-    #   pdf_data = File.read("document.pdf", binmode: true)
-    #   result = Kreuzberg.extract_bytes_sync(pdf_data, "application/pdf")
-    #   puts result.content
-    #
-    # @example Extract from a network stream
-    #   response = HTTParty.get("https://example.com/document.docx")
-    #   result = Kreuzberg.extract_bytes_sync(response.body, "application/vnd.openxmlformats-officedocument.wordprocessingml.document")
-    def extract_bytes_sync(data:, mime_type:, config: nil)
+    def extract_bytes_sync(data, mime_type, config: nil)
       opts = normalize_config(config)
       hash = native_extract_bytes_sync(data.to_s, mime_type.to_s, **opts)
       result = Result.new(hash)
@@ -60,38 +22,7 @@ module Kreuzberg
       result
     end
 
-    # Synchronously extract content from multiple files.
-    #
-    # Processes multiple files in a single batch operation. Files are extracted sequentially,
-    # and results maintain the same order as the input paths. This is useful for bulk
-    # processing multiple documents with consistent configuration.
-    #
-    # @param paths [Array<String, Pathname>] Array of file paths to extract. Each path
-    #   is converted to a string and MIME type is auto-detected from extension.
-    # @param config [Config::Extraction, Hash, nil] Extraction configuration applied to all files.
-    #   Accepts either a {Config::Extraction} object or a configuration hash.
-    #
-    # @return [Array<Result>] Array of extraction results in the same order as input paths.
-    #   Array length matches the input paths length.
-    #
-    # @raise [Errors::IOError] If any file cannot be read
-    # @raise [Errors::ParsingError] If any document parsing fails
-    # @raise [Errors::UnsupportedFormatError] If any file format is not supported
-    # @raise [Errors::OCRError] If OCR is enabled and fails on any document
-    # @raise [Errors::MissingDependencyError] If a required dependency is missing
-    #
-    # @example Batch extract multiple PDFs
-    #   paths = ["doc1.pdf", "doc2.pdf", "doc3.pdf"]
-    #   results = Kreuzberg.batch_extract_files_sync(paths)
-    #   results.each_with_index do |result, idx|
-    #     puts "File #{idx}: #{result.content.length} characters"
-    #   end
-    #
-    # @example Batch extract with consistent configuration
-    #   paths = Dir.glob("documents/*.pdf")
-    #   config = Kreuzberg::Config::Extraction.new(force_ocr: true)
-    #   results = Kreuzberg.batch_extract_files_sync(paths, config: config)
-    def batch_extract_files_sync(paths:, config: nil)
+    def batch_extract_files_sync(paths, config: nil)
       opts = normalize_config(config)
       hashes = native_batch_extract_files_sync(paths.map(&:to_s), **opts)
       results = hashes.map { |hash| Result.new(hash) }
@@ -99,37 +30,7 @@ module Kreuzberg
       results
     end
 
-    # Asynchronously extract content from a file.
-    #
-    # Non-blocking extraction that returns a {Result} promise. Extraction is performed
-    # in the background using native threads or the Tokio runtime. This method is
-    # preferred for I/O-bound operations and integrating with async workflows.
-    #
-    # @param path [String, Pathname] Path to the document file to extract
-    # @param mime_type [String, nil] Optional MIME type for the file (e.g., 'application/pdf').
-    #   If omitted, type is detected from file extension.
-    # @param config [Config::Extraction, Hash, nil] Extraction configuration. Accepts
-    #   either a {Config::Extraction} object or a configuration hash.
-    #
-    # @return [Result] Extraction result containing content, metadata, tables, and images.
-    #   In async contexts, this result is available upon method return.
-    #
-    # @raise [Errors::IOError] If the file cannot be read or access is denied
-    # @raise [Errors::ParsingError] If document parsing fails
-    # @raise [Errors::UnsupportedFormatError] If the file format is not supported
-    # @raise [Errors::OCRError] If OCR is enabled and fails
-    # @raise [Errors::MissingDependencyError] If a required dependency is missing
-    #
-    # @example Extract a PDF file asynchronously
-    #   result = Kreuzberg.extract_file("large_document.pdf")
-    #   puts result.content
-    #
-    # @example Extract with custom OCR configuration
-    #   config = Kreuzberg::Config::Extraction.new(
-    #     ocr: Kreuzberg::Config::OCR.new(language: "deu")
-    #   )
-    #   result = Kreuzberg.extract_file("document.pdf", config: config)
-    def extract_file(path:, mime_type: nil, config: nil)
+    def extract_file(path, mime_type: nil, config: nil)
       opts = normalize_config(config)
       hash = if mime_type
                native_extract_file(path.to_s, mime_type.to_s, **opts)
@@ -141,37 +42,7 @@ module Kreuzberg
       result
     end
 
-    # Asynchronously extract content from byte data.
-    #
-    # Non-blocking extraction from in-memory binary data. Like {#extract_file},
-    # this performs extraction in the background, making it suitable for handling
-    # high-volume extraction workloads without blocking the main thread.
-    #
-    # @param data [String] Binary document data (can contain any byte values)
-    # @param mime_type [String] MIME type of the data (required, e.g., 'application/pdf').
-    #   This parameter is mandatory to guide the extraction engine.
-    # @param config [Config::Extraction, Hash, nil] Extraction configuration. Accepts
-    #   either a {Config::Extraction} object or a configuration hash.
-    #
-    # @return [Result] Extraction result containing content, metadata, tables, and images
-    #
-    # @raise [Errors::ParsingError] If document parsing fails
-    # @raise [Errors::UnsupportedFormatError] If the MIME type is not supported
-    # @raise [Errors::OCRError] If OCR is enabled and fails
-    # @raise [Errors::MissingDependencyError] If a required dependency is missing
-    #
-    # @example Extract PDF from memory asynchronously
-    #   pdf_data = File.read("document.pdf", binmode: true)
-    #   result = Kreuzberg.extract_bytes(pdf_data, "application/pdf")
-    #   puts result.content
-    #
-    # @example Extract with image extraction
-    #   data = File.read("file.docx", binmode: true)
-    #   config = Kreuzberg::Config::Extraction.new(
-    #     image_extraction: Kreuzberg::Config::ImageExtraction.new(extract_images: true)
-    #   )
-    #   result = Kreuzberg.extract_bytes(data, "application/vnd.openxmlformats-officedocument.wordprocessingml.document", config: config)
-    def extract_bytes(data:, mime_type:, config: nil)
+    def extract_bytes(data, mime_type, config: nil)
       opts = normalize_config(config)
       hash = native_extract_bytes(data.to_s, mime_type.to_s, **opts)
       result = Result.new(hash)
@@ -179,40 +50,7 @@ module Kreuzberg
       result
     end
 
-    # Asynchronously extract content from multiple files.
-    #
-    # Non-blocking batch extraction from multiple files. Results maintain the same order
-    # as input paths. This is the preferred method for bulk processing when non-blocking
-    # I/O is required (e.g., in web servers or async applications).
-    #
-    # @param paths [Array<String, Pathname>] Array of file paths to extract. Each path
-    #   is converted to a string and MIME type is auto-detected from extension.
-    # @param config [Config::Extraction, Hash, nil] Extraction configuration applied to all files.
-    #   Accepts either a {Config::Extraction} object or a configuration hash.
-    #
-    # @return [Array<Result>] Array of extraction results in the same order as input paths.
-    #   Array length matches the input paths length.
-    #
-    # @raise [Errors::IOError] If any file cannot be read
-    # @raise [Errors::ParsingError] If any document parsing fails
-    # @raise [Errors::UnsupportedFormatError] If any file format is not supported
-    # @raise [Errors::OCRError] If OCR is enabled and fails on any document
-    # @raise [Errors::MissingDependencyError] If a required dependency is missing
-    #
-    # @example Batch extract multiple files asynchronously
-    #   paths = ["invoice_1.pdf", "invoice_2.pdf", "invoice_3.pdf"]
-    #   results = Kreuzberg.batch_extract_files(paths)
-    #   results.each_with_index do |result, idx|
-    #     puts "Invoice #{idx}: #{result.detected_languages}"
-    #   end
-    #
-    # @example Batch extract with chunking
-    #   paths = Dir.glob("reports/*.docx")
-    #   config = Kreuzberg::Config::Extraction.new(
-    #     chunking: Kreuzberg::Config::Chunking.new(max_chars: 1000, max_overlap: 200)
-    #   )
-    #   results = Kreuzberg.batch_extract_files(paths, config: config)
-    def batch_extract_files(paths:, config: nil)
+    def batch_extract_files(paths, config: nil)
       opts = normalize_config(config)
       hashes = native_batch_extract_files(paths.map(&:to_s), **opts)
       results = hashes.map { |hash| Result.new(hash) }
@@ -220,38 +58,7 @@ module Kreuzberg
       results
     end
 
-    # Synchronously extract content from multiple byte data sources.
-    #
-    # Processes multiple in-memory binary documents in a single batch operation. Results
-    # maintain the same order as the input data array. The mime_types array must have
-    # the same length as the data_array.
-    #
-    # @param data_array [Array<String>] Array of binary document data. Each element can
-    #   contain any byte values (e.g., PDF binary data).
-    # @param mime_types [Array<String>] Array of MIME types corresponding to each data item.
-    #   Must be the same length as data_array (e.g., ["application/pdf", "application/msword"]).
-    # @param config [Config::Extraction, Hash, nil] Extraction configuration applied to all items.
-    #   Accepts either a {Config::Extraction} object or a configuration hash.
-    #
-    # @return [Array<Result>] Array of extraction results in the same order as input data.
-    #   Array length matches the data_array length.
-    #
-    # @raise [ArgumentError] If data_array and mime_types have different lengths
-    # @raise [Errors::ParsingError] If any document parsing fails
-    # @raise [Errors::UnsupportedFormatError] If any MIME type is not supported
-    # @raise [Errors::OCRError] If OCR is enabled and fails on any document
-    # @raise [Errors::MissingDependencyError] If a required dependency is missing
-    #
-    # @example Batch extract binary documents
-    #   pdf_data_1 = File.read("doc1.pdf", binmode: true)
-    #   pdf_data_2 = File.read("doc2.pdf", binmode: true)
-    #   docx_data = File.read("report.docx", binmode: true)
-    #
-    #   data = [pdf_data_1, pdf_data_2, docx_data]
-    #   types = ["application/pdf", "application/pdf", "application/vnd.openxmlformats-officedocument.wordprocessingml.document"]
-    #   results = Kreuzberg.batch_extract_bytes_sync(data, types)
-    #   results.each { |r| puts r.content }
-    def batch_extract_bytes_sync(data_array:, mime_types:, config: nil)
+    def batch_extract_bytes_sync(data_array, mime_types, config: nil)
       opts = normalize_config(config)
       hashes = native_batch_extract_bytes_sync(data_array.map(&:to_s), mime_types.map(&:to_s), **opts)
       results = hashes.map { |hash| Result.new(hash) }
@@ -259,43 +66,7 @@ module Kreuzberg
       results
     end
 
-    # Asynchronously extract content from multiple byte data sources.
-    #
-    # Non-blocking batch extraction from multiple in-memory binary documents. Results
-    # maintain the same order as the input data array. This method is preferred when
-    # processing multiple documents without blocking (e.g., handling multiple uploads).
-    #
-    # @param data_array [Array<String>] Array of binary document data. Each element can
-    #   contain any byte values (e.g., PDF binary data).
-    # @param mime_types [Array<String>] Array of MIME types corresponding to each data item.
-    #   Must be the same length as data_array (e.g., ["application/pdf", "application/msword"]).
-    # @param config [Config::Extraction, Hash, nil] Extraction configuration applied to all items.
-    #   Accepts either a {Config::Extraction} object or a configuration hash.
-    #
-    # @return [Array<Result>] Array of extraction results in the same order as input data.
-    #   Array length matches the data_array length.
-    #
-    # @raise [ArgumentError] If data_array and mime_types have different lengths
-    # @raise [Errors::ParsingError] If any document parsing fails
-    # @raise [Errors::UnsupportedFormatError] If any MIME type is not supported
-    # @raise [Errors::OCRError] If OCR is enabled and fails on any document
-    # @raise [Errors::MissingDependencyError] If a required dependency is missing
-    #
-    # @example Batch extract uploaded documents asynchronously
-    #   # From a web request with multiple file uploads
-    #   uploaded_files = params[:files]  # Array of uploaded file objects
-    #   data = uploaded_files.map(&:read)
-    #   types = uploaded_files.map(&:content_type)
-    #
-    #   results = Kreuzberg.batch_extract_bytes(data, types)
-    #   results.each { |r| puts r.content }
-    #
-    # @example Batch extract with OCR
-    #   data = [scan_1_bytes, scan_2_bytes, scan_3_bytes]
-    #   types = ["image/png", "image/png", "image/png"]
-    #   config = Kreuzberg::Config::Extraction.new(force_ocr: true)
-    #   results = Kreuzberg.batch_extract_bytes(data, types, config: config)
-    def batch_extract_bytes(data_array:, mime_types:, config: nil)
+    def batch_extract_bytes(data_array, mime_types, config: nil)
       opts = normalize_config(config)
       hashes = native_batch_extract_bytes(data_array.map(&:to_s), mime_types.map(&:to_s), **opts)
       results = hashes.map { |hash| Result.new(hash) }

data/lib/kreuzberg/mcp_proxy.rb

@@ -5,7 +5,14 @@ require 'pathname'
 require 'json'
 
 module Kreuzberg
+  # MCP (Model Context Protocol) server proxy
+  #
+  # Starts and manages the Kreuzberg MCP server for Claude Desktop integration.
+  #
   # @example Start MCP server
+  #   server = Kreuzberg::MCPProxy.new
+  #   server.start
+  #
   module MCPProxy
     Error = Class.new(Kreuzberg::Errors::Error)
     MissingBinaryError = Class.new(Error)
@@ -53,7 +60,8 @@ module Kreuzberg
 
         Process.kill('TERM', @pid)
         Process.wait(@pid)
-      rescue Errno::ESRCH, Errno::ECHILD # rubocop:disable Lint/SuppressedException
+      rescue Errno::ESRCH, Errno::ECHILD
+        # Process already dead
       ensure
         @pid = nil
         close_pipes
@@ -114,7 +122,7 @@ module Kreuzberg
           err: $stderr
         )
         Process.detach(@pid)
-        sleep 1
+        sleep 1 # Give server time to start
         @pid
       end
 
@@ -154,6 +162,7 @@ module Kreuzberg
     # @raise [MissingBinaryError] If not found
     #
     def find_mcp_binary
+      # MCP is served by kreuzberg CLI
       binary_name = Gem.win_platform? ? 'kreuzberg.exe' : 'kreuzberg'
       found = CLIProxy.search_paths(binary_name).find(&:file?)
       return found if found

data/lib/kreuzberg/ocr_backend_protocol.rb

@@ -1,11 +1,84 @@
 # frozen_string_literal: true
 
 module Kreuzberg
+  # OCR backend protocol interface.
+  #
+  # This module defines the protocol that all Ruby OCR backends must implement
+  # to be registered with the Rust core via the FFI bridge.
+  #
+  # OCR backends implement optical character recognition for images and scanned documents.
+  # They are called when OCR is enabled in the extraction configuration.
+  #
   # @example Implementing a custom OCR backend
+  #   class CustomOcrBackend
+  #     include Kreuzberg::OcrBackendProtocol
+  #
+  #     def name
+  #       "custom-ocr"
+  #     end
+  #
+  #     def process_image(image_bytes, config)
+  #       # Perform OCR on image_bytes
+  #       # This is a placeholder - integrate with a real OCR engine
+  #       text = my_ocr_engine.recognize(image_bytes, language: config["language"])
+  #       text
+  #     end
+  #   end
+  #
+  #   backend = CustomOcrBackend.new
+  #   Kreuzberg.register_ocr_backend(backend.name, backend)
+  #
+  #   # Use in extraction
+  #   result = Kreuzberg.extract_file_sync(
+  #     "scanned.pdf",
+  #     config: { ocr: { backend: "custom-ocr", language: "eng" } }
+  #   )
+  #
   # @example Implementing an OCR backend with initialization
+  #   class ModelBasedOcr
+  #     include Kreuzberg::OcrBackendProtocol
+  #
+  #     def initialize
+  #       @model = nil
+  #     end
+  #
+  #     def name
+  #       "model-ocr"
+  #     end
+  #
+  #     def process_image(image_bytes, config)
+  #       # Load model on first use (lazy initialization)
+  #       @model ||= load_model
+  #
+  #       # Run OCR
+  #       @model.recognize(image_bytes, config)
+  #     end
+  #
+  #     private
+  #
+  #     def load_model
+  #       # Load ML model for OCR
+  #       MyOcrModel.load("path/to/model")
+  #     end
+  #   end
+  #
+  #   Kreuzberg.register_ocr_backend("model-ocr", ModelBasedOcr.new)
+  #
   module OcrBackendProtocol
+    # Return the unique name of this OCR backend.
+    #
+    # This name is used in ExtractionConfig to select the backend:
+    #
+    #   config = { ocr: { backend: "custom-ocr", language: "eng" } }
+    #
+    # The name should be a lowercase string with hyphens (e.g., "custom-ocr", "tesseract").
+    #
     # @return [String] Unique backend identifier
+    #
     # @example
+    #   def name
+    #     "custom-ocr"
+    #   end
     def name
       raise NotImplementedError, "#{self.class} must implement #name"
     end

data/lib/kreuzberg/post_processor_protocol.rb

@@ -1,13 +1,84 @@
 # frozen_string_literal: true
 
 module Kreuzberg
+  # PostProcessor protocol interface.
+  #
+  # This module defines the protocol that all Ruby post-processors must implement
+  # to be registered with the Rust core via the FFI bridge.
+  #
+  # Post-processors enrich extraction results by adding metadata, transforming content,
+  # or performing additional analysis. They are called after extraction completes.
+  #
   # @example Implementing a simple post-processor
+  #   class UpcaseProcessor
+  #     include Kreuzberg::PostProcessorProtocol
+  #
+  #     def call(result)
+  #       result["content"] = result["content"].upcase
+  #       result
+  #     end
+  #   end
+  #
+  #   Kreuzberg.register_post_processor("upcase", UpcaseProcessor.new)
+  #
   # @example Implementing a post-processor that adds metadata
+  #   class EntityExtractor
+  #     include Kreuzberg::PostProcessorProtocol
+  #
+  #     def call(result)
+  #       entities = extract_entities(result["content"])
+  #       result["metadata"]["entities"] = entities
+  #       result
+  #     end
+  #
+  #     private
+  #
+  #     def extract_entities(text)
+  #       # Extract named entities from text
+  #       # This is a placeholder - use a real NER library in production
+  #       text.scan(/[A-Z][a-z]+(?:\s[A-Z][a-z]+)*/)
+  #     end
+  #   end
+  #
+  #   Kreuzberg.register_post_processor("entities", EntityExtractor.new)
+  #
   # @example Using a Proc as a post-processor
+  #   Kreuzberg.register_post_processor("word_count", ->(result) {
+  #     word_count = result["content"].split.length
+  #     result["metadata"]["word_count"] = word_count
+  #     result
+  #   })
+  #
   module PostProcessorProtocol
+    # Process and enrich an extraction result.
+    #
+    # This method is called after extraction completes. It receives the extraction result
+    # as a hash and must return the modified hash. The processor can:
+    # - Add new keys to result["metadata"]
+    # - Transform result["content"]
+    # - Add entries to result["tables"]
+    # - Modify any other result fields
+    #
+    # Existing metadata keys will not be overwritten by the FFI bridge, so it's safe
+    # to add new keys without worrying about conflicts.
+    #
     # @param result [Hash] Extraction result with the following structure:
+    #   - "content" [String] - Extracted text content
+    #   - "mime_type" [String] - MIME type of the source document
+    #   - "metadata" [Hash] - Document metadata (title, author, etc.)
+    #   - "tables" [Array<Hash>] - Extracted tables
+    #   - "detected_languages" [Array<String>, nil] - Detected language codes
+    #   - "chunks" [Array<String>, nil] - Content chunks (if chunking enabled)
+    #
     # @return [Hash] Modified extraction result with enriched metadata
+    #
     # @example
+    #   def call(result)
+    #     text = result["content"]
+    #     entities = extract_entities(text)
+    #     result["metadata"]["entities"] = entities
+    #     result
+    #   end
     def call(result)
       raise NotImplementedError, "#{self.class} must implement #call(result)"
     end