kreuzberg 4.0.0.pre.rc.6 → 4.0.0.pre.rc.8

data/ext/kreuzberg_rb/native/Cargo.toml

@@ -7,7 +7,7 @@ rb-sys = { path = "../../../vendor/rb-sys" }
 
 [package]
 name = "kreuzberg-rb"
-version = "4.0.0-rc.6"
+version = "4.0.0-rc.8"
 edition = "2024"
 rust-version = "1.91"
 authors = ["Na'aman Hirschfeld <nhirschfeld@gmail.com>"]
@@ -30,6 +30,7 @@ default = []
 [dependencies]
 async-trait = "0.1.89"
 kreuzberg = { path = "../../../vendor/kreuzberg", features = ["full", "embeddings"] }
+kreuzberg-ffi = { path = "../../../vendor/kreuzberg-ffi", features = ["embeddings"] }
 magnus = { git = "https://github.com/matsadler/magnus", rev = "f6db11769efb517427bf7f121f9c32e18b059b38", features = [
     "rb-sys",
 ] }
@@ -37,8 +38,17 @@ rb-sys = { version = "0.9.119", default-features = false, features = [
     "stable-api-compiled-fallback",
 ] }
 serde_json = "1.0.145"
-tokio = { version = "1.48.0", features = ["rt", "rt-multi-thread", "macros", "sync", "process", "fs", "time", "io-util"] }
-html-to-markdown-rs = { version = "2.11.1", default-features = false }
+tokio = { version = "1.48.0", features = [
+    "rt",
+    "rt-multi-thread",
+    "macros",
+    "sync",
+    "process",
+    "fs",
+    "time",
+    "io-util",
+] }
+html-to-markdown-rs = { version = "2.14.1", default-features = false }
 
 [dev-dependencies]
 pretty_assertions = "1.4"

data/ext/kreuzberg_rb/native/src/lib.rs

@@ -1,3 +1,5 @@
+#![allow(unpredictable_function_pointer_comparisons)]
+
 //! Kreuzberg Ruby Bindings (Magnus)
 //!
 //! High-performance document intelligence framework bindings for Ruby.
@@ -7,6 +9,7 @@ use html_to_markdown_rs::options::{
     CodeBlockStyle, ConversionOptions, HeadingStyle, HighlightStyle, ListIndentType, NewlineStyle, PreprocessingPreset,
     WhitespaceMode,
 };
+use kreuzberg::core::config::PageConfig;
 use kreuzberg::keywords::{
     KeywordAlgorithm as RustKeywordAlgorithm, KeywordConfig as RustKeywordConfig, RakeParams as RustRakeParams,
     YakeParams as RustYakeParams,
@@ -1050,6 +1053,36 @@ fn html_options_to_ruby_hash(ruby: &Ruby, options: &ConversionOptions) -> Result
 
     Ok(hash)
 }
+
+/// Parse PageConfig from Ruby Hash
+fn parse_page_config(ruby: &Ruby, hash: RHash) -> Result<PageConfig, Error> {
+    let extract_pages = if let Some(val) = get_kw(ruby, hash, "extract_pages") {
+        bool::try_convert(val)?
+    } else {
+        false
+    };
+
+    let insert_page_markers = if let Some(val) = get_kw(ruby, hash, "insert_page_markers") {
+        bool::try_convert(val)?
+    } else {
+        false
+    };
+
+    let marker_format = if let Some(val) = get_kw(ruby, hash, "marker_format") {
+        String::try_convert(val)?
+    } else {
+        "\n\n<!-- PAGE {page_num} -->\n\n".to_string()
+    };
+
+    let config = PageConfig {
+        extract_pages,
+        insert_page_markers,
+        marker_format,
+    };
+
+    Ok(config)
+}
+
 /// Parse ExtractionConfig from Ruby Hash
 fn parse_extraction_config(ruby: &Ruby, opts: Option<RHash>) -> Result<ExtractionConfig, Error> {
     let mut config = ExtractionConfig::default();
@@ -1130,6 +1163,13 @@ fn parse_extraction_config(ruby: &Ruby, opts: Option<RHash>) -> Result<Extractio
             config.html_options = Some(parse_html_options(ruby, html_hash)?);
         }
 
+        if let Some(val) = get_kw(ruby, hash, "pages")
+            && !val.is_nil()
+        {
+            let pages_hash = RHash::try_convert(val)?;
+            config.pages = Some(parse_page_config(ruby, pages_hash)?);
+        }
+
         if let Some(val) = get_kw(ruby, hash, "max_concurrent_extractions") {
             let value = usize::try_convert(val)?;
             config.max_concurrent_extractions = Some(value);
@@ -1532,8 +1572,8 @@ fn extraction_result_to_ruby(ruby: &Ruby, result: RustExtractionResult) -> Resul
         for chunk in chunks {
             let chunk_hash = ruby.hash_new();
             chunk_hash.aset("content", chunk.content)?;
-            chunk_hash.aset("char_start", chunk.metadata.char_start)?;
-            chunk_hash.aset("char_end", chunk.metadata.char_end)?;
+            chunk_hash.aset("byte_start", chunk.metadata.byte_start)?;
+            chunk_hash.aset("byte_end", chunk.metadata.byte_end)?;
             if let Some(token_count) = chunk.metadata.token_count {
                 chunk_hash.aset("token_count", token_count)?;
             } else {
@@ -1541,6 +1581,16 @@ fn extraction_result_to_ruby(ruby: &Ruby, result: RustExtractionResult) -> Resul
             }
             chunk_hash.aset("chunk_index", chunk.metadata.chunk_index)?;
             chunk_hash.aset("total_chunks", chunk.metadata.total_chunks)?;
+            if let Some(first_page) = chunk.metadata.first_page {
+                chunk_hash.aset("first_page", first_page as i64)?;
+            } else {
+                chunk_hash.aset("first_page", ruby.qnil().as_value())?;
+            }
+            if let Some(last_page) = chunk.metadata.last_page {
+                chunk_hash.aset("last_page", last_page as i64)?;
+            } else {
+                chunk_hash.aset("last_page", ruby.qnil().as_value())?;
+            }
             if let Some(embedding) = chunk.embedding {
                 let embedding_array = ruby.ary_new();
                 for value in embedding {
@@ -1617,6 +1667,92 @@ fn extraction_result_to_ruby(ruby: &Ruby, result: RustExtractionResult) -> Resul
         set_hash_entry(ruby, &hash, "images", ruby.qnil().as_value())?;
     }
 
+    if let Some(page_content_list) = result.pages {
+        let pages_array = ruby.ary_new();
+        for page_content in page_content_list {
+            let page_hash = ruby.hash_new();
+            page_hash.aset("page_number", page_content.page_number as i64)?;
+            page_hash.aset("content", page_content.content)?;
+
+            let tables_array = ruby.ary_new();
+            for table in page_content.tables {
+                let table_hash = ruby.hash_new();
+
+                let cells_array = ruby.ary_new();
+                for row in table.cells {
+                    let row_array = ruby.ary_from_vec(row);
+                    cells_array.push(row_array)?;
+                }
+                table_hash.aset("cells", cells_array)?;
+                table_hash.aset("markdown", table.markdown)?;
+                table_hash.aset("page_number", table.page_number as i64)?;
+
+                tables_array.push(table_hash)?;
+            }
+            page_hash.aset("tables", tables_array)?;
+
+            let images_array = ruby.ary_new();
+            for image in page_content.images {
+                let image_hash = ruby.hash_new();
+                let data_value = ruby.str_from_slice(&image.data).into_value_with(ruby);
+                image_hash.aset("data", data_value)?;
+                image_hash.aset("format", image.format)?;
+                image_hash.aset("image_index", image.image_index as i64)?;
+                if let Some(page) = image.page_number {
+                    image_hash.aset("page_number", page as i64)?;
+                } else {
+                    image_hash.aset("page_number", ruby.qnil().as_value())?;
+                }
+                if let Some(width) = image.width {
+                    image_hash.aset("width", width as i64)?;
+                } else {
+                    image_hash.aset("width", ruby.qnil().as_value())?;
+                }
+                if let Some(height) = image.height {
+                    image_hash.aset("height", height as i64)?;
+                } else {
+                    image_hash.aset("height", ruby.qnil().as_value())?;
+                }
+                if let Some(colorspace) = image.colorspace {
+                    image_hash.aset("colorspace", colorspace)?;
+                } else {
+                    image_hash.aset("colorspace", ruby.qnil().as_value())?;
+                }
+                if let Some(bits) = image.bits_per_component {
+                    image_hash.aset("bits_per_component", bits as i64)?;
+                } else {
+                    image_hash.aset("bits_per_component", ruby.qnil().as_value())?;
+                }
+                image_hash.aset(
+                    "is_mask",
+                    if image.is_mask {
+                        ruby.qtrue().as_value()
+                    } else {
+                        ruby.qfalse().as_value()
+                    },
+                )?;
+                if let Some(description) = image.description {
+                    image_hash.aset("description", description)?;
+                } else {
+                    image_hash.aset("description", ruby.qnil().as_value())?;
+                }
+                if let Some(ocr_result) = image.ocr_result {
+                    let nested = extraction_result_to_ruby(ruby, *ocr_result)?;
+                    image_hash.aset("ocr_result", nested.into_value_with(ruby))?;
+                } else {
+                    image_hash.aset("ocr_result", ruby.qnil().as_value())?;
+                }
+                images_array.push(image_hash)?;
+            }
+            page_hash.aset("images", images_array)?;
+
+            pages_array.push(page_hash)?;
+        }
+        set_hash_entry(ruby, &hash, "pages", pages_array.into_value_with(ruby))?;
+    } else {
+        set_hash_entry(ruby, &hash, "pages", ruby.qnil().as_value())?;
+    }
+
     Ok(hash)
 }
 
@@ -2366,6 +2502,7 @@ fn register_ocr_backend(name: String, backend: Value) -> Result<(), Error> {
                 detected_languages: None,
                 chunks: None,
                 images: None,
+                pages: None,
             })
         }
 

data/kreuzberg.gemspec

@@ -21,6 +21,15 @@ core_files =
                .map { |path| path.delete_prefix('crates/') }
                .map { |path| "vendor/#{path}" }
 
+# Include the kreuzberg-ffi crate
+ffi_prefix = 'crates/kreuzberg-ffi/'
+ffi_cmd = %(git -C "#{repo_root}" ls-files -z #{ffi_prefix})
+ffi_files =
+  `#{ffi_cmd}`.split("\x0")
+              .select { |path| path.start_with?(ffi_prefix) }
+              .map { |path| path.delete_prefix('crates/') }
+              .map { |path| "vendor/#{path}" }
+
 fallback_files = Dir.chdir(__dir__) do
   ruby_fallback = Dir.glob(
     %w[
@@ -45,10 +54,24 @@ fallback_files = Dir.chdir(__dir__) do
   core_fallback = Dir.chdir(repo_root) do
     Dir.glob('crates/kreuzberg/**/*', File::FNM_DOTMATCH)
        .reject { |f| File.directory?(f) }
+       .reject { |f| f.include?('/.fastembed_cache/') }
+       .reject { |f| f.include?('/target/') }
+       .grep_v(/\.(swp|bak|tmp)$/)
+       .grep_v(/~$/)
+       .map { |path| "vendor/#{path.delete_prefix('crates/')}" }
+  end
+
+  # Fallback for FFI crate - copy from repo root
+  ffi_fallback = Dir.chdir(repo_root) do
+    Dir.glob('crates/kreuzberg-ffi/**/*', File::FNM_DOTMATCH)
+       .reject { |f| File.directory?(f) }
+       .reject { |f| f.include?('/target/') }
+       .grep_v(/\.(swp|bak|tmp)$/)
+       .grep_v(/~$/)
        .map { |path| "vendor/#{path.delete_prefix('crates/')}" }
   end
 
-  ruby_fallback + core_fallback
+  ruby_fallback + core_fallback + ffi_fallback
 end
 
 # Check for vendored crates (copied during CI/packaging)
@@ -57,6 +80,7 @@ vendor_files = Dir.chdir(__dir__) do
                       Dir.glob('vendor/kreuzberg/**/*', File::FNM_DOTMATCH)
                          .reject { |f| File.directory?(f) }
                          .reject { |f| f.include?('/.fastembed_cache/') }
+                         .reject { |f| f.include?('/.kreuzberg/') }
                          .reject { |f| f.include?('/target/') }
                          .grep_v(/\.(swp|bak|tmp)$/)
                          .grep_v(/~$/)
@@ -64,6 +88,16 @@ vendor_files = Dir.chdir(__dir__) do
                       []
                     end
 
+  kreuzberg_ffi_files = if Dir.exist?('vendor/kreuzberg-ffi')
+                          Dir.glob('vendor/kreuzberg-ffi/**/*', File::FNM_DOTMATCH)
+                             .reject { |f| File.directory?(f) }
+                             .reject { |f| f.include?('/target/') }
+                             .grep_v(/\.(swp|bak|tmp)$/)
+                             .grep_v(/~$/)
+                        else
+                          []
+                        end
+
   rb_sys_files = if Dir.exist?('vendor/rb-sys')
                    Dir.glob('vendor/rb-sys/**/*', File::FNM_DOTMATCH)
                       .reject { |f| File.directory?(f) }
@@ -80,17 +114,17 @@ vendor_files = Dir.chdir(__dir__) do
                      []
                    end
 
-  kreuzberg_files + rb_sys_files + workspace_toml
+  kreuzberg_files + kreuzberg_ffi_files + rb_sys_files + workspace_toml
 end
 
 # Use git-tracked files if available, otherwise fallback to glob
 # Always include vendored files if they exist on disk (for CI packaging)
-files = if (ruby_files + core_files).empty?
+files = if (ruby_files + core_files + ffi_files).empty?
           fallback_files
         elsif vendor_files.any?
           ruby_files + vendor_files
         else
-          ruby_files + core_files
+          ruby_files + core_files + ffi_files
         end
 
 # Filter to only include files that actually exist

data/lib/kreuzberg/config.rb

@@ -492,6 +492,36 @@ module Kreuzberg
       end
     end
 
+    # Page tracking configuration for multi-page documents
+    #
+    # @example Enable page extraction
+    #   pages = PageConfig.new(extract_pages: true)
+    #
+    # @example Enable page markers in content
+    #   pages = PageConfig.new(insert_page_markers: true, marker_format: "--- PAGE {page_num} ---")
+    #
+    class PageConfig
+      attr_reader :extract_pages, :insert_page_markers, :marker_format
+
+      def initialize(
+        extract_pages: false,
+        insert_page_markers: false,
+        marker_format: "\n\n<!-- PAGE {page_num} -->\n\n"
+      )
+        @extract_pages = extract_pages ? true : false
+        @insert_page_markers = insert_page_markers ? true : false
+        @marker_format = marker_format.to_s
+      end
+
+      def to_h
+        {
+          extract_pages: @extract_pages,
+          insert_page_markers: @insert_page_markers,
+          marker_format: @marker_format
+        }
+      end
+    end
+
     # Post-processor configuration
     #
     # @example Enable all post-processors
@@ -576,7 +606,7 @@ module Kreuzberg
       attr_reader :use_cache, :enable_quality_processing, :force_ocr,
                   :ocr, :chunking, :language_detection, :pdf_options,
                   :image_extraction, :image_preprocessing, :postprocessor,
-                  :token_reduction, :keywords, :html_options,
+                  :token_reduction, :keywords, :html_options, :pages,
                   :max_concurrent_extractions
 
       # Load configuration from a file.
@@ -634,6 +664,7 @@ module Kreuzberg
         token_reduction: nil,
         keywords: nil,
         html_options: nil,
+        pages: nil,
         max_concurrent_extractions: nil
       )
         @use_cache = use_cache ? true : false
@@ -649,6 +680,7 @@ module Kreuzberg
         @token_reduction = normalize_config(token_reduction, TokenReduction)
         @keywords = normalize_config(keywords, Keywords)
         @html_options = normalize_config(html_options, HtmlOptions)
+        @pages = normalize_config(pages, PageConfig)
         @max_concurrent_extractions = max_concurrent_extractions&.to_i
       end
 
@@ -668,6 +700,7 @@ module Kreuzberg
           token_reduction: @token_reduction&.to_h,
           keywords: @keywords&.to_h,
           html_options: @html_options&.to_h,
+          pages: @pages&.to_h,
           max_concurrent_extractions: @max_concurrent_extractions
         }.compact
       end

data/lib/kreuzberg/result.rb

@@ -21,7 +21,7 @@ module Kreuzberg
   # rubocop:disable Metrics/ClassLength
   class Result
     attr_reader :content, :mime_type, :metadata, :metadata_json, :tables,
-                :detected_languages, :chunks, :images
+                :detected_languages, :chunks, :images, :pages
 
     # Table structure
     #
@@ -42,31 +42,39 @@ module Kreuzberg
     #
     # @!attribute [r] content
     #   @return [String] Chunk content
-    # @!attribute [r] char_start
-    #   @return [Integer] Starting character index
-    # @!attribute [r] char_end
-    #   @return [Integer] Ending character index
+    # @!attribute [r] byte_start
+    #   @return [Integer] Starting byte offset (UTF-8)
+    # @!attribute [r] byte_end
+    #   @return [Integer] Ending byte offset (UTF-8)
     # @!attribute [r] token_count
     #   @return [Integer, nil] Approximate token count (may be nil)
+    # @!attribute [r] first_page
+    #   @return [Integer, nil] First page number (1-indexed)
+    # @!attribute [r] last_page
+    #   @return [Integer, nil] Last page number (1-indexed)
     #
     Chunk = Struct.new(
       :content,
-      :char_start,
-      :char_end,
+      :byte_start,
+      :byte_end,
       :token_count,
       :chunk_index,
       :total_chunks,
+      :first_page,
+      :last_page,
       :embedding,
       keyword_init: true
     ) do
       def to_h
         {
           content: content,
-          char_start: char_start,
-          char_end: char_end,
+          byte_start: byte_start,
+          byte_end: byte_end,
           token_count: token_count,
           chunk_index: chunk_index,
           total_chunks: total_chunks,
+          first_page: first_page,
+          last_page: last_page,
           embedding: embedding
         }
       end
@@ -103,6 +111,28 @@ module Kreuzberg
       end
     end
 
+    # Per-page content
+    #
+    # @!attribute [r] page_number
+    #   @return [Integer] Page number (1-indexed)
+    # @!attribute [r] content
+    #   @return [String] Text content for this page
+    # @!attribute [r] tables
+    #   @return [Array<Table>] Tables on this page
+    # @!attribute [r] images
+    #   @return [Array<Image>] Images on this page
+    #
+    PageContent = Struct.new(:page_number, :content, :tables, :images, keyword_init: true) do
+      def to_h
+        {
+          page_number: page_number,
+          content: content,
+          tables: tables.map(&:to_h),
+          images: images.map(&:to_h)
+        }
+      end
+    end
+
     # Initialize from native hash result
     #
     # @param hash [Hash] Hash returned from native extension
@@ -117,6 +147,7 @@ module Kreuzberg
       @detected_languages = parse_detected_languages(get_value(hash, 'detected_languages'))
       @chunks = parse_chunks(get_value(hash, 'chunks'))
       @images = parse_images(get_value(hash, 'images'))
+      @pages = parse_pages(get_value(hash, 'pages'))
     end
 
     # Convert to hash
@@ -128,10 +159,11 @@ module Kreuzberg
         content: @content,
         mime_type: @mime_type,
         metadata: @metadata,
-        tables: @tables.map(&:to_h),
+        tables: serialize_tables,
         detected_languages: @detected_languages,
-        chunks: @chunks&.map(&:to_h),
-        images: @images&.map(&:to_h)
+        chunks: serialize_chunks,
+        images: serialize_images,
+        pages: serialize_pages
       }
     end
 
@@ -145,6 +177,22 @@ module Kreuzberg
 
     private
 
+    def serialize_tables
+      @tables.map(&:to_h)
+    end
+
+    def serialize_chunks
+      @chunks&.map(&:to_h)
+    end
+
+    def serialize_images
+      @images&.map(&:to_h)
+    end
+
+    def serialize_pages
+      @pages&.map(&:to_h)
+    end
+
     def get_value(hash, key, default = nil)
       hash[key] || hash[key.to_sym] || default
     end
@@ -180,11 +228,13 @@ module Kreuzberg
       chunks_data.map do |chunk_hash|
         Chunk.new(
           content: chunk_hash['content'],
-          char_start: chunk_hash['char_start'],
-          char_end: chunk_hash['char_end'],
+          byte_start: chunk_hash['byte_start'],
+          byte_end: chunk_hash['byte_end'],
           token_count: chunk_hash['token_count'],
           chunk_index: chunk_hash['chunk_index'],
           total_chunks: chunk_hash['total_chunks'],
+          first_page: chunk_hash['first_page'],
+          last_page: chunk_hash['last_page'],
           embedding: chunk_hash['embedding']
         )
       end
@@ -211,6 +261,19 @@ module Kreuzberg
         )
       end
     end
+
+    def parse_pages(pages_data)
+      return nil if pages_data.nil?
+
+      pages_data.map do |page_hash|
+        PageContent.new(
+          page_number: page_hash['page_number'],
+          content: page_hash['content'],
+          tables: parse_tables(page_hash['tables']),
+          images: parse_images(page_hash['images'])
+        )
+      end
+    end
   end
   # rubocop:enable Metrics/ClassLength
 end

data/lib/kreuzberg/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Kreuzberg
-  VERSION = '4.0.0-rc.6'
+  VERSION = '4.0.0-rc.8'
 end

data/sig/kreuzberg.rbs

@@ -168,6 +168,15 @@ module Kreuzberg
       def to_h: () -> Hash[Symbol, untyped]
     end
 
+    class PageConfig
+      attr_reader extract_pages: bool
+      attr_reader insert_page_markers: bool
+      attr_reader marker_format: String
+
+      def initialize: (?extract_pages: bool, ?insert_page_markers: bool, ?marker_format: String) -> void
+      def to_h: () -> Hash[Symbol, untyped]
+    end
+
     class Extraction
       attr_reader use_cache: bool
       attr_reader enable_quality_processing: bool
@@ -182,6 +191,7 @@ module Kreuzberg
       attr_reader token_reduction: TokenReduction?
       attr_reader keywords: Keywords?
       attr_reader html_options: HtmlOptions?
+      attr_reader pages: PageConfig?
       attr_reader max_concurrent_extractions: Integer?
 
       def self.from_file: (String path) -> Extraction
@@ -199,6 +209,7 @@ module Kreuzberg
         ?token_reduction: (TokenReduction | Hash[Symbol, untyped])?,
         ?keywords: (Keywords | Hash[Symbol, untyped])?,
         ?html_options: (HtmlOptions | Hash[Symbol, untyped])?,
+        ?pages: (PageConfig | Hash[Symbol, untyped])?,
         ?max_concurrent_extractions: Integer?
       ) -> void
       def to_h: () -> Hash[Symbol, untyped]
@@ -234,11 +245,13 @@ module Kreuzberg
 
   type chunk_hash = {
     content: String,
-    char_start: Integer,
-    char_end: Integer,
+    byte_start: Integer,
+    byte_end: Integer,
     token_count: Integer?,
     chunk_index: Integer?,
     total_chunks: Integer?,
+    first_page: Integer?,
+    last_page: Integer?,
     embedding: Array[Float]?
   }
 
@@ -278,20 +291,24 @@ module Kreuzberg
     # Text chunk
     class Chunk
       attr_reader content: String
-      attr_reader char_start: Integer
-      attr_reader char_end: Integer
+      attr_reader byte_start: Integer
+      attr_reader byte_end: Integer
       attr_reader token_count: Integer?
       attr_reader chunk_index: Integer?
       attr_reader total_chunks: Integer?
+      attr_reader first_page: Integer?
+      attr_reader last_page: Integer?
       attr_reader embedding: Array[Float]?
 
       def initialize: (
         content: String,
-        char_start: Integer,
-        char_end: Integer,
+        byte_start: Integer,
+        byte_end: Integer,
         token_count: Integer?,
         chunk_index: Integer?,
         total_chunks: Integer?,
+        first_page: Integer?,
+        last_page: Integer?,
         embedding: Array[Float]?
       ) -> void
       def to_h: () -> chunk_hash