kreuzberg 4.0.8 → 4.1.0

data/ext/kreuzberg_rb/native/src/result.rs

@@ -0,0 +1,326 @@
+//! ExtractionResult conversion to Ruby values
+//!
+//! Handles conversion of Kreuzberg ExtractionResult to Ruby Hash,
+//! including complex nested structures like chunks, images, tables, and elements.
+
+use crate::error_handling::runtime_error;
+use crate::helpers::{json_value_to_ruby, set_hash_entry};
+
+use kreuzberg::ExtractionResult as RustExtractionResult;
+use magnus::{Error, RHash, Ruby, IntoValue};
+use magnus::value::ReprValue;
+
+/// Convert Kreuzberg ExtractionResult to Ruby Hash
+///
+/// Converts the Rust extraction result into a Ruby hash with all fields including:
+/// - content, mime_type, metadata
+/// - tables (with cells and markdown)
+/// - detected_languages
+/// - chunks (with embeddings)
+/// - images (including OCR results)
+/// - pages (with per-page content)
+/// - elements (for element-based format)
+pub fn extraction_result_to_ruby(ruby: &Ruby, result: RustExtractionResult) -> Result<RHash, Error> {
+    let hash = ruby.hash_new();
+
+    // Set content and MIME type
+    let content_value = ruby.str_new(result.content.as_str()).into_value_with(ruby);
+    set_hash_entry(ruby, &hash, "content", content_value)?;
+
+    let mime_value = ruby.str_new(result.mime_type.as_str()).into_value_with(ruby);
+    set_hash_entry(ruby, &hash, "mime_type", mime_value)?;
+
+    // Set metadata both as JSON string and parsed hash
+    let metadata_json = serde_json::to_string(&result.metadata)
+        .map_err(|e| runtime_error(format!("Failed to serialize metadata: {}", e)))?;
+    let metadata_json_value = ruby.str_new(&metadata_json).into_value_with(ruby);
+    set_hash_entry(ruby, &hash, "metadata_json", metadata_json_value)?;
+    let metadata_value = serde_json::to_value(&result.metadata)
+        .map_err(|e| runtime_error(format!("Failed to serialize metadata: {}", e)))?;
+    let metadata_hash = json_value_to_ruby(ruby, &metadata_value)?;
+    set_hash_entry(ruby, &hash, "metadata", metadata_hash)?;
+
+    // Convert tables
+    let tables_array = ruby.ary_new();
+    for table in result.tables {
+        let table_hash = ruby.hash_new();
+
+        let cells_array = ruby.ary_new();
+        for row in table.cells {
+            let row_array = ruby.ary_from_vec(row);
+            cells_array.push(row_array)?;
+        }
+        table_hash.aset("cells", cells_array)?;
+        table_hash.aset("markdown", table.markdown)?;
+        table_hash.aset("page_number", table.page_number)?;
+
+        tables_array.push(table_hash)?;
+    }
+    let tables_value = tables_array.into_value_with(ruby);
+    set_hash_entry(ruby, &hash, "tables", tables_value)?;
+
+    // Convert detected languages
+    if let Some(langs) = result.detected_languages {
+        let langs_array = ruby.ary_from_vec(langs);
+        let langs_value = langs_array.into_value_with(ruby);
+        set_hash_entry(ruby, &hash, "detected_languages", langs_value)?;
+    } else {
+        set_hash_entry(ruby, &hash, "detected_languages", ruby.qnil().as_value())?;
+    }
+
+    // Convert chunks
+    if let Some(chunks) = result.chunks {
+        let chunks_array = ruby.ary_new();
+        for chunk in chunks {
+            let chunk_hash = ruby.hash_new();
+            chunk_hash.aset("content", chunk.content)?;
+            chunk_hash.aset("byte_start", chunk.metadata.byte_start)?;
+            chunk_hash.aset("byte_end", chunk.metadata.byte_end)?;
+            if let Some(token_count) = chunk.metadata.token_count {
+                chunk_hash.aset("token_count", token_count)?;
+            } else {
+                chunk_hash.aset("token_count", ruby.qnil().as_value())?;
+            }
+            chunk_hash.aset("chunk_index", chunk.metadata.chunk_index)?;
+            chunk_hash.aset("total_chunks", chunk.metadata.total_chunks)?;
+            if let Some(first_page) = chunk.metadata.first_page {
+                chunk_hash.aset("first_page", first_page as i64)?;
+            } else {
+                chunk_hash.aset("first_page", ruby.qnil().as_value())?;
+            }
+            if let Some(last_page) = chunk.metadata.last_page {
+                chunk_hash.aset("last_page", last_page as i64)?;
+            } else {
+                chunk_hash.aset("last_page", ruby.qnil().as_value())?;
+            }
+            if let Some(embedding) = chunk.embedding {
+                let embedding_array = ruby.ary_new();
+                for value in embedding {
+                    embedding_array.push(ruby.float_from_f64(value as f64).into_value_with(ruby))?;
+                }
+                chunk_hash.aset("embedding", embedding_array)?;
+            } else {
+                chunk_hash.aset("embedding", ruby.qnil().as_value())?;
+            }
+            chunks_array.push(chunk_hash)?;
+        }
+        let chunks_value = chunks_array.into_value_with(ruby);
+        set_hash_entry(ruby, &hash, "chunks", chunks_value)?;
+    } else {
+        set_hash_entry(ruby, &hash, "chunks", ruby.qnil().as_value())?;
+    }
+
+    // Convert images
+    if let Some(images) = result.images {
+        let images_array = ruby.ary_new();
+        for image in images {
+            let image_hash = ruby.hash_new();
+            let data_value = ruby.str_from_slice(&image.data).into_value_with(ruby);
+            image_hash.aset("data", data_value)?;
+            image_hash.aset("format", image.format)?;
+            image_hash.aset("image_index", image.image_index as i64)?;
+            if let Some(page) = image.page_number {
+                image_hash.aset("page_number", page as i64)?;
+            } else {
+                image_hash.aset("page_number", ruby.qnil().as_value())?;
+            }
+            if let Some(width) = image.width {
+                image_hash.aset("width", width as i64)?;
+            } else {
+                image_hash.aset("width", ruby.qnil().as_value())?;
+            }
+            if let Some(height) = image.height {
+                image_hash.aset("height", height as i64)?;
+            } else {
+                image_hash.aset("height", ruby.qnil().as_value())?;
+            }
+            if let Some(colorspace) = image.colorspace {
+                image_hash.aset("colorspace", colorspace)?;
+            } else {
+                image_hash.aset("colorspace", ruby.qnil().as_value())?;
+            }
+            if let Some(bits) = image.bits_per_component {
+                image_hash.aset("bits_per_component", bits as i64)?;
+            } else {
+                image_hash.aset("bits_per_component", ruby.qnil().as_value())?;
+            }
+            image_hash.aset(
+                "is_mask",
+                if image.is_mask {
+                    ruby.qtrue().as_value()
+                } else {
+                    ruby.qfalse().as_value()
+                },
+            )?;
+            if let Some(description) = image.description {
+                image_hash.aset("description", description)?;
+            } else {
+                image_hash.aset("description", ruby.qnil().as_value())?;
+            }
+            if let Some(ocr_result) = image.ocr_result {
+                let nested = extraction_result_to_ruby(ruby, *ocr_result)?;
+                image_hash.aset("ocr_result", nested.into_value_with(ruby))?;
+            } else {
+                image_hash.aset("ocr_result", ruby.qnil().as_value())?;
+            }
+            images_array.push(image_hash)?;
+        }
+        set_hash_entry(ruby, &hash, "images", images_array.into_value_with(ruby))?;
+    } else {
+        set_hash_entry(ruby, &hash, "images", ruby.qnil().as_value())?;
+    }
+
+    // Convert pages
+    if let Some(page_content_list) = result.pages {
+        let pages_array = ruby.ary_new();
+        for page_content in page_content_list {
+            let page_hash = ruby.hash_new();
+            page_hash.aset("page_number", page_content.page_number as i64)?;
+            page_hash.aset("content", page_content.content)?;
+
+            let tables_array = ruby.ary_new();
+            for table in page_content.tables {
+                let table_hash = ruby.hash_new();
+
+                let cells_array = ruby.ary_new();
+                for row in table.cells.clone() {
+                    let row_array = ruby.ary_from_vec(row);
+                    cells_array.push(row_array)?;
+                }
+                table_hash.aset("cells", cells_array)?;
+                table_hash.aset("markdown", table.markdown.clone())?;
+                table_hash.aset("page_number", table.page_number as i64)?;
+
+                tables_array.push(table_hash)?;
+            }
+            page_hash.aset("tables", tables_array)?;
+
+            let images_array = ruby.ary_new();
+            for image in page_content.images {
+                let image_hash = ruby.hash_new();
+                let data_value = ruby.str_from_slice(&image.data).into_value_with(ruby);
+                image_hash.aset("data", data_value)?;
+                image_hash.aset("format", image.format.clone())?;
+                image_hash.aset("image_index", image.image_index as i64)?;
+                if let Some(page) = image.page_number {
+                    image_hash.aset("page_number", page as i64)?;
+                } else {
+                    image_hash.aset("page_number", ruby.qnil().as_value())?;
+                }
+                if let Some(width) = image.width {
+                    image_hash.aset("width", width as i64)?;
+                } else {
+                    image_hash.aset("width", ruby.qnil().as_value())?;
+                }
+                if let Some(height) = image.height {
+                    image_hash.aset("height", height as i64)?;
+                } else {
+                    image_hash.aset("height", ruby.qnil().as_value())?;
+                }
+                if let Some(colorspace) = &image.colorspace {
+                    image_hash.aset("colorspace", colorspace.clone())?;
+                } else {
+                    image_hash.aset("colorspace", ruby.qnil().as_value())?;
+                }
+                if let Some(bits) = image.bits_per_component {
+                    image_hash.aset("bits_per_component", bits as i64)?;
+                } else {
+                    image_hash.aset("bits_per_component", ruby.qnil().as_value())?;
+                }
+                image_hash.aset(
+                    "is_mask",
+                    if image.is_mask {
+                        ruby.qtrue().as_value()
+                    } else {
+                        ruby.qfalse().as_value()
+                    },
+                )?;
+                if let Some(description) = &image.description {
+                    image_hash.aset("description", description.clone())?;
+                } else {
+                    image_hash.aset("description", ruby.qnil().as_value())?;
+                }
+                if let Some(ocr_result) = &image.ocr_result {
+                    let nested = extraction_result_to_ruby(ruby, (**ocr_result).clone())?;
+                    image_hash.aset("ocr_result", nested.into_value_with(ruby))?;
+                } else {
+                    image_hash.aset("ocr_result", ruby.qnil().as_value())?;
+                }
+                images_array.push(image_hash)?;
+            }
+            page_hash.aset("images", images_array)?;
+
+            pages_array.push(page_hash)?;
+        }
+        set_hash_entry(ruby, &hash, "pages", pages_array.into_value_with(ruby))?;
+    } else {
+        set_hash_entry(ruby, &hash, "pages", ruby.qnil().as_value())?;
+    }
+
+    // Convert elements (element-based format)
+    if let Some(elements_list) = result.elements {
+        let elements_array = ruby.ary_new();
+        for element in elements_list {
+            let element_hash = ruby.hash_new();
+            element_hash.aset("element_id", element.element_id.as_ref())?;
+
+            // Convert ElementType to snake_case string
+            use kreuzberg::types::ElementType as ET;
+            let element_type_str = match element.element_type {
+                ET::Title => "title",
+                ET::NarrativeText => "narrative_text",
+                ET::Heading => "heading",
+                ET::ListItem => "list_item",
+                ET::Table => "table",
+                ET::Image => "image",
+                ET::PageBreak => "page_break",
+                ET::CodeBlock => "code_block",
+                ET::BlockQuote => "block_quote",
+                ET::Footer => "footer",
+                ET::Header => "header",
+            };
+            element_hash.aset("element_type", element_type_str)?;
+            element_hash.aset("text", element.text)?;
+
+            let metadata_hash = ruby.hash_new();
+            if let Some(page_num) = element.metadata.page_number {
+                metadata_hash.aset("page_number", page_num as i64)?;
+            } else {
+                metadata_hash.aset("page_number", ruby.qnil().as_value())?;
+            }
+            if let Some(filename) = &element.metadata.filename {
+                metadata_hash.aset("filename", filename.as_str())?;
+            } else {
+                metadata_hash.aset("filename", ruby.qnil().as_value())?;
+            }
+            if let Some(coords) = element.metadata.coordinates {
+                let coords_hash = ruby.hash_new();
+                coords_hash.aset("x0", coords.x0)?;
+                coords_hash.aset("y0", coords.y0)?;
+                coords_hash.aset("x1", coords.x1)?;
+                coords_hash.aset("y1", coords.y1)?;
+                metadata_hash.aset("coordinates", coords_hash)?;
+            } else {
+                metadata_hash.aset("coordinates", ruby.qnil().as_value())?;
+            }
+            if let Some(elem_idx) = element.metadata.element_index {
+                metadata_hash.aset("element_index", elem_idx as i64)?;
+            } else {
+                metadata_hash.aset("element_index", ruby.qnil().as_value())?;
+            }
+            let additional_hash = ruby.hash_new();
+            for (key, value) in &element.metadata.additional {
+                additional_hash.aset(key.as_str(), value.as_str())?;
+            }
+            metadata_hash.aset("additional", additional_hash)?;
+
+            element_hash.aset("metadata", metadata_hash)?;
+            elements_array.push(element_hash)?;
+        }
+        set_hash_entry(ruby, &hash, "elements", elements_array.into_value_with(ruby))?;
+    } else {
+        set_hash_entry(ruby, &hash, "elements", ruby.qnil().as_value())?;
+    }
+
+    Ok(hash)
+}

data/ext/kreuzberg_rb/native/src/validation.rs

@@ -0,0 +1,4 @@
+//! Validation functions for configuration and formats
+//!
+//! Provides validation for MIME types, formats, and other configuration parameters.
+//! These validation functions are re-exported directly from lib.rs via kreuzberg_ffi.

data/lib/kreuzberg/config.rb

@@ -905,6 +905,72 @@ module Kreuzberg
         self
       end
 
+      # Set a configuration field using hash-like syntax
+      #
+      # @param key [Symbol, String] Field name to set
+      # @param value [Object] Value to set
+      # @return [Object] The value that was set
+      #
+      # @example
+      #   config = Extraction.new(use_cache: true)
+      #   config[:use_cache] = false
+      #   config[:force_ocr] = true
+      #
+      # rubocop:disable Metrics/CyclomaticComplexity, Metrics/MethodLength
+      def []=(key, value)
+        key_sym = key.to_sym
+        case key_sym
+        when :use_cache
+          @use_cache = value ? true : false
+        when :enable_quality_processing
+          @enable_quality_processing = value ? true : false
+        when :force_ocr
+          @force_ocr = value ? true : false
+        when :ocr
+          @ocr = normalize_config(value, OCR)
+        when :chunking
+          @chunking = normalize_config(value, Chunking)
+        when :language_detection
+          @language_detection = normalize_config(value, LanguageDetection)
+        when :pdf_options
+          @pdf_options = normalize_config(value, PDF)
+        when :image_extraction
+          @image_extraction = normalize_config(value, ImageExtraction)
+        when :image_preprocessing
+          @image_preprocessing = normalize_config(value, ImagePreprocessing)
+        when :postprocessor
+          @postprocessor = normalize_config(value, PostProcessor)
+        when :token_reduction
+          @token_reduction = normalize_config(value, TokenReduction)
+        when :keywords
+          @keywords = normalize_config(value, Keywords)
+        when :html_options
+          @html_options = normalize_config(value, HtmlOptions)
+        when :pages
+          @pages = normalize_config(value, PageConfig)
+        when :max_concurrent_extractions
+          @max_concurrent_extractions = value&.to_i
+        else
+          raise ArgumentError, "Unknown configuration key: #{key}"
+        end
+      end
+      # rubocop:enable Metrics/CyclomaticComplexity, Metrics/MethodLength
+
+      # Get a configuration field using hash-like syntax
+      #
+      # @param key [Symbol, String] Field name to get
+      # @return [Object, nil] The field value
+      #
+      # @example
+      #   config = Extraction.new(use_cache: true)
+      #   config[:use_cache]  # => true
+      #
+      def [](key)
+        send(key.to_sym)
+      rescue NoMethodError
+        nil
+      end
+
       private
 
       def normalize_config(value, klass)

data/lib/kreuzberg/result.rb

@@ -11,7 +11,7 @@ module Kreuzberg
   # rubocop:disable Metrics/ClassLength
   class Result
     attr_reader :content, :mime_type, :metadata, :metadata_json, :tables,
-                :detected_languages, :chunks, :images, :pages
+                :detected_languages, :chunks, :images, :pages, :elements
 
     # @!attribute [r] cells
     #   @return [Array<Array<String>>] Table cells (2D array)
@@ -114,6 +114,68 @@ module Kreuzberg
       end
     end
 
+    # @!attribute [r] x0
+    #   @return [Float] Left x-coordinate
+    # @!attribute [r] y0
+    #   @return [Float] Bottom y-coordinate
+    # @!attribute [r] x1
+    #   @return [Float] Right x-coordinate
+    # @!attribute [r] y1
+    #   @return [Float] Top y-coordinate
+    ElementBoundingBox = Struct.new(:x0, :y0, :x1, :y1, keyword_init: true) do
+      def to_h
+        { x0: x0, y0: y0, x1: x1, y1: y1 }
+      end
+    end
+
+    # @!attribute [r] page_number
+    #   @return [Integer, nil] Page number (1-indexed)
+    # @!attribute [r] filename
+    #   @return [String, nil] Source filename or document name
+    # @!attribute [r] coordinates
+    #   @return [ElementBoundingBox, nil] Bounding box coordinates if available
+    # @!attribute [r] element_index
+    #   @return [Integer, nil] Position index in the element sequence
+    # @!attribute [r] additional
+    #   @return [Hash<String, String>] Additional custom metadata
+    ElementMetadataStruct = Struct.new(
+      :page_number,
+      :filename,
+      :coordinates,
+      :element_index,
+      :additional,
+      keyword_init: true
+    ) do
+      def to_h
+        {
+          page_number: page_number,
+          filename: filename,
+          coordinates: coordinates&.to_h,
+          element_index: element_index,
+          additional: additional
+        }
+      end
+    end
+
+    # @!attribute [r] element_id
+    #   @return [String] Unique element identifier
+    # @!attribute [r] element_type
+    #   @return [String] Semantic type of the element
+    # @!attribute [r] text
+    #   @return [String] Text content of the element
+    # @!attribute [r] metadata
+    #   @return [ElementMetadataStruct] Metadata about the element
+    ElementStruct = Struct.new(:element_id, :element_type, :text, :metadata, keyword_init: true) do
+      def to_h
+        {
+          element_id: element_id,
+          element_type: element_type,
+          text: text,
+          metadata: metadata&.to_h
+        }
+      end
+    end
+
     # Initialize from native hash result
     #
     # @param hash [Hash] Hash returned from native extension
@@ -128,6 +190,7 @@ module Kreuzberg
       @chunks = parse_chunks(get_value(hash, 'chunks'))
       @images = parse_images(get_value(hash, 'images'))
       @pages = parse_pages(get_value(hash, 'pages'))
+      @elements = parse_elements(get_value(hash, 'elements'))
     end
 
     # Convert to hash
@@ -143,7 +206,8 @@ module Kreuzberg
         detected_languages: @detected_languages,
         chunks: serialize_chunks,
         images: serialize_images,
-        pages: serialize_pages
+        pages: serialize_pages,
+        elements: serialize_elements
       }
     end
 
@@ -249,6 +313,10 @@ module Kreuzberg
       @pages&.map(&:to_h)
     end
 
+    def serialize_elements
+      @elements&.map(&:to_h)
+    end
+
     def get_value(hash, key, default = nil)
       hash[key] || hash[key.to_sym] || default
     end
@@ -329,6 +397,43 @@ module Kreuzberg
         )
       end
     end
+
+    def parse_elements(elements_data)
+      return nil if elements_data.nil?
+
+      elements_data.map { |element_hash| parse_element(element_hash) }
+    end
+
+    def parse_element(element_hash)
+      metadata_hash = element_hash['metadata'] || {}
+      coordinates = parse_element_coordinates(metadata_hash['coordinates'])
+
+      metadata = ElementMetadataStruct.new(
+        page_number: metadata_hash['page_number'],
+        filename: metadata_hash['filename'],
+        coordinates: coordinates,
+        element_index: metadata_hash['element_index'],
+        additional: metadata_hash['additional'] || {}
+      )
+
+      ElementStruct.new(
+        element_id: element_hash['element_id'],
+        element_type: element_hash['element_type'],
+        text: element_hash['text'],
+        metadata: metadata
+      )
+    end
+
+    def parse_element_coordinates(coordinates_data)
+      return nil if coordinates_data.nil?
+
+      ElementBoundingBox.new(
+        x0: coordinates_data['x0'].to_f,
+        y0: coordinates_data['y0'].to_f,
+        x1: coordinates_data['x1'].to_f,
+        y1: coordinates_data['y1'].to_f
+      )
+    end
   end
   # rubocop:enable Metrics/ClassLength
 end

data/lib/kreuzberg/types.rb

@@ -3,6 +3,110 @@
 require 'sorbet-runtime'
 
 module Kreuzberg
+  # Semantic element type classification.
+  #
+  # Categorizes text content into semantic units for downstream processing.
+  # Supports the element types commonly found in Unstructured documents.
+  #
+  # @example
+  #   type = Kreuzberg::ElementType::TITLE
+  #
+  ElementType = T.type_alias do
+    T.any(
+      'title',
+      'narrative_text',
+      'heading',
+      'list_item',
+      'table',
+      'image',
+      'page_break',
+      'code_block',
+      'block_quote',
+      'footer',
+      'header'
+    )
+  end
+
+  # Bounding box coordinates for element positioning.
+  #
+  # Represents rectangular coordinates for an element within a page.
+  #
+  # @example
+  #   bbox = Kreuzberg::BoundingBox.new(
+  #     x0: 10.0,
+  #     y0: 20.0,
+  #     x1: 100.0,
+  #     y1: 50.0
+  #   )
+  #   puts "Width: #{bbox.x1 - bbox.x0}"
+  #
+  class BoundingBox < T::Struct
+    extend T::Sig
+
+    const :x0, Float
+
+    const :y0, Float
+
+    const :x1, Float
+
+    const :y1, Float
+  end
+
+  # Metadata for a semantic element.
+  #
+  # Provides contextual information about an extracted element including
+  # its position within the document and custom metadata fields.
+  #
+  # @example
+  #   metadata = Kreuzberg::ElementMetadata.new(
+  #     page_number: 1,
+  #     filename: "document.pdf",
+  #     coordinates: bbox,
+  #     element_index: 5,
+  #     additional: { "style" => "bold" }
+  #   )
+  #
+  class ElementMetadata < T::Struct
+    extend T::Sig
+
+    const :page_number, T.nilable(Integer)
+
+    const :filename, T.nilable(String)
+
+    const :coordinates, T.nilable(BoundingBox)
+
+    const :element_index, T.nilable(Integer)
+
+    const :additional, T::Hash[String, String]
+  end
+
+  # Semantic element extracted from document.
+  #
+  # Represents a logical unit of content with semantic classification,
+  # unique identifier, and metadata for tracking origin and position.
+  # Compatible with Unstructured.io element format when output_format='element_based'.
+  #
+  # @example
+  #   element = Kreuzberg::Element.new(
+  #     element_id: "elem-abc123",
+  #     element_type: "narrative_text",
+  #     text: "This is the main content.",
+  #     metadata: metadata
+  #   )
+  #   puts "#{element.element_type}: #{element.text}"
+  #
+  class Element < T::Struct
+    extend T::Sig
+
+    const :element_id, String
+
+    const :element_type, String
+
+    const :text, String
+
+    const :metadata, ElementMetadata
+  end
+
   # Header/Heading metadata
   #
   # Represents a heading element found in the HTML document

data/lib/kreuzberg/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Kreuzberg
-  VERSION = '4.0.8'
+  VERSION = '4.1.0'
 end

data/lib/kreuzberg.rb

@@ -83,10 +83,6 @@ module Kreuzberg
   module_function :validate_mime_type
 
   module_function :get_extensions_for_mime
-
-  module_function :list_embedding_presets
-
-  module_function :get_embedding_preset
 end
 
 require_relative 'kreuzberg/cache_api'