kreuzberg 4.0.0.pre.rc.29 → 4.0.0.rc1

data/ext/kreuzberg_rb/native/src/lib.rs

@@ -1,5 +1,3 @@
-#![allow(unpredictable_function_pointer_comparisons)]
-
 //! Kreuzberg Ruby Bindings (Magnus)
 //!
 //! High-performance document intelligence framework bindings for Ruby.
@@ -9,59 +7,23 @@ use html_to_markdown_rs::options::{
     CodeBlockStyle, ConversionOptions, HeadingStyle, HighlightStyle, ListIndentType, NewlineStyle, PreprocessingPreset,
     WhitespaceMode,
 };
-use kreuzberg::core::config::PageConfig;
 use kreuzberg::keywords::{
     KeywordAlgorithm as RustKeywordAlgorithm, KeywordConfig as RustKeywordConfig, RakeParams as RustRakeParams,
     YakeParams as RustYakeParams,
 };
 use kreuzberg::types::TesseractConfig as RustTesseractConfig;
-use kreuzberg::pdf::HierarchyConfig;
 use kreuzberg::{
-    ChunkingConfig, EmbeddingConfig, ExtractionConfig, ExtractionResult as RustExtractionResult,
-    ImageExtractionConfig, ImagePreprocessingConfig, KreuzbergError, LanguageDetectionConfig, OcrConfig, PdfConfig,
-    PostProcessorConfig, TokenReductionConfig,
+    ChunkingConfig, EmbeddingConfig, ExtractionConfig, ExtractionResult as RustExtractionResult, ImageExtractionConfig,
+    ImagePreprocessingConfig, KreuzbergError, LanguageDetectionConfig, OcrConfig, PdfConfig, PostProcessorConfig,
+    TokenReductionConfig,
 };
 use magnus::exception::ExceptionClass;
 use magnus::r_hash::ForEach;
 use magnus::value::ReprValue;
-use magnus::{
-    Error, IntoValue, RArray, RHash, RString, Ruby, Symbol, TryConvert, Value, function, scan_args::scan_args,
-};
+use magnus::{Error, IntoValue, RArray, RHash, Ruby, Symbol, TryConvert, Value, function, scan_args::scan_args};
 use std::fs;
 use std::path::{Path, PathBuf};
 
-// Re-export FFI types and functions from kreuzberg_ffi crate.
-// This ensures proper linking by importing Rust symbols directly
-// instead of declaring them as external C symbols.
-pub use kreuzberg_ffi::{
-    // Types
-    CErrorDetails, CMetadataField,
-    // Panic/error handling (from panic_shield module)
-    get_last_error_code, get_last_error_message, get_last_panic_context,
-    // Error functions (from error module)
-    kreuzberg_get_error_details, kreuzberg_classify_error,
-    kreuzberg_error_code_name, kreuzberg_error_code_description,
-    // Result functions (from result module)
-    kreuzberg_result_get_page_count, kreuzberg_result_get_chunk_count,
-    kreuzberg_result_get_detected_language, kreuzberg_result_get_metadata_field,
-    // Memory and util functions (from lib.rs)
-    kreuzberg_free_string, kreuzberg_last_error, kreuzberg_last_error_code,
-    kreuzberg_last_panic_context,
-    // Validation functions (from lib.rs)
-    kreuzberg_validate_binarization_method, kreuzberg_validate_ocr_backend,
-    kreuzberg_validate_language_code, kreuzberg_validate_token_reduction_level,
-    kreuzberg_validate_tesseract_psm, kreuzberg_validate_tesseract_oem,
-    kreuzberg_validate_output_format, kreuzberg_validate_confidence,
-    kreuzberg_validate_dpi, kreuzberg_validate_chunking_params,
-    kreuzberg_get_valid_binarization_methods, kreuzberg_get_valid_language_codes,
-    kreuzberg_get_valid_ocr_backends, kreuzberg_get_valid_token_reduction_levels,
-    // Config functions (from config module, now re-exported through lib.rs)
-    kreuzberg_config_from_json, kreuzberg_config_free, kreuzberg_config_is_valid,
-    kreuzberg_config_to_json, kreuzberg_config_get_field, kreuzberg_config_merge,
-};
-
-use std::ffi::c_char;
-
 /// Keeps Ruby values alive across plugin registrations by informing the GC.
 struct GcGuardedValue {
     value: Value,
@@ -87,27 +49,6 @@ impl Drop for GcGuardedValue {
     }
 }
 
-/// Retrieve panic context from FFI if available
-fn get_panic_context() -> Option<String> {
-    unsafe {
-        let ctx_ptr = kreuzberg_last_panic_context();
-        if ctx_ptr.is_null() {
-            return None;
-        }
-
-        let c_str = std::ffi::CStr::from_ptr(ctx_ptr);
-        let context = c_str.to_string_lossy().to_string();
-        kreuzberg_free_string(ctx_ptr as *mut std::ffi::c_char);
-
-        if context.is_empty() { None } else { Some(context) }
-    }
-}
-
-/// Retrieve error code from FFI
-fn get_error_code() -> i32 {
-    unsafe { kreuzberg_last_error_code() }
-}
-
 /// Convert Kreuzberg errors to Ruby exceptions
 fn kreuzberg_error(err: KreuzbergError) -> Error {
     let ruby = Ruby::get().expect("Ruby not initialized");
@@ -314,10 +255,10 @@ fn ruby_value_to_json(value: Value) -> Result<serde_json::Value, Error> {
         return Ok(serde_json::Value::Number(serde_json::Number::from(unsigned)));
     }
 
-    if let Ok(float) = f64::try_convert(value)
-        && let Some(num) = serde_json::Number::from_f64(float)
-    {
-        return Ok(serde_json::Value::Number(num));
+    if let Ok(float) = f64::try_convert(value) {
+        if let Some(num) = serde_json::Number::from_f64(float) {
+            return Ok(serde_json::Value::Number(num));
+        }
     }
 
     if let Ok(sym) = Symbol::try_convert(value) {
@@ -455,46 +396,6 @@ fn parse_language_detection_config(ruby: &Ruby, hash: RHash) -> Result<LanguageD
     Ok(config)
 }
 
-/// Parse HierarchyConfig from Ruby Hash
-fn parse_hierarchy_config(ruby: &Ruby, hash: RHash) -> Result<HierarchyConfig, Error> {
-    let enabled = if let Some(val) = get_kw(ruby, hash, "enabled") {
-        bool::try_convert(val)?
-    } else {
-        true
-    };
-
-    let k_clusters = if let Some(val) = get_kw(ruby, hash, "k_clusters") {
-        usize::try_convert(val)?
-    } else {
-        6
-    };
-
-    let include_bbox = if let Some(val) = get_kw(ruby, hash, "include_bbox") {
-        bool::try_convert(val)?
-    } else {
-        true
-    };
-
-    let ocr_coverage_threshold = if let Some(val) = get_kw(ruby, hash, "ocr_coverage_threshold") {
-        if !val.is_nil() {
-            Some(f64::try_convert(val)? as f32)
-        } else {
-            None
-        }
-    } else {
-        None
-    };
-
-    let config = HierarchyConfig {
-        enabled,
-        k_clusters,
-        include_bbox,
-        ocr_coverage_threshold,
-    };
-
-    Ok(config)
-}
-
 /// Parse PdfConfig from Ruby Hash
 fn parse_pdf_config(ruby: &Ruby, hash: RHash) -> Result<PdfConfig, Error> {
     let extract_images = if let Some(val) = get_kw(ruby, hash, "extract_images") {
@@ -520,22 +421,10 @@ fn parse_pdf_config(ruby: &Ruby, hash: RHash) -> Result<PdfConfig, Error> {
         true
     };
 
-    let hierarchy = if let Some(val) = get_kw(ruby, hash, "hierarchy") {
-        if !val.is_nil() {
-            let h_hash = RHash::try_convert(val)?;
-            Some(parse_hierarchy_config(ruby, h_hash)?)
-        } else {
-            None
-        }
-    } else {
-        None
-    };
-
     let config = PdfConfig {
         extract_images,
         passwords,
         extract_metadata,
-        hierarchy,
     };
 
     Ok(config)
@@ -682,8 +571,6 @@ fn parse_postprocessor_config(ruby: &Ruby, hash: RHash) -> Result<PostProcessorC
         enabled,
         enabled_processors,
         disabled_processors,
-        enabled_set: None,
-        disabled_set: None,
     };
 
     Ok(config)
@@ -746,10 +633,10 @@ fn parse_keyword_config(ruby: &Ruby, hash: RHash) -> Result<RustKeywordConfig, E
         }
     }
 
-    if let Some(val) = get_kw(ruby, hash, "language")
-        && !val.is_nil()
-    {
-        config.language = Some(symbol_to_string(val)?);
+    if let Some(val) = get_kw(ruby, hash, "language") {
+        if !val.is_nil() {
+            config.language = Some(symbol_to_string(val)?);
+        }
     }
 
     if let Some(val) = get_kw(ruby, hash, "yake_params")
@@ -1136,36 +1023,6 @@ fn html_options_to_ruby_hash(ruby: &Ruby, options: &ConversionOptions) -> Result
 
     Ok(hash)
 }
-
-/// Parse PageConfig from Ruby Hash
-fn parse_page_config(ruby: &Ruby, hash: RHash) -> Result<PageConfig, Error> {
-    let extract_pages = if let Some(val) = get_kw(ruby, hash, "extract_pages") {
-        bool::try_convert(val)?
-    } else {
-        false
-    };
-
-    let insert_page_markers = if let Some(val) = get_kw(ruby, hash, "insert_page_markers") {
-        bool::try_convert(val)?
-    } else {
-        false
-    };
-
-    let marker_format = if let Some(val) = get_kw(ruby, hash, "marker_format") {
-        String::try_convert(val)?
-    } else {
-        "\n\n<!-- PAGE {page_num} -->\n\n".to_string()
-    };
-
-    let config = PageConfig {
-        extract_pages,
-        insert_page_markers,
-        marker_format,
-    };
-
-    Ok(config)
-}
-
 /// Parse ExtractionConfig from Ruby Hash
 fn parse_extraction_config(ruby: &Ruby, opts: Option<RHash>) -> Result<ExtractionConfig, Error> {
     let mut config = ExtractionConfig::default();
@@ -1246,13 +1103,6 @@ fn parse_extraction_config(ruby: &Ruby, opts: Option<RHash>) -> Result<Extractio
             config.html_options = Some(parse_html_options(ruby, html_hash)?);
         }
 
-        if let Some(val) = get_kw(ruby, hash, "pages")
-            && !val.is_nil()
-        {
-            let pages_hash = RHash::try_convert(val)?;
-            config.pages = Some(parse_page_config(ruby, pages_hash)?);
-        }
-
         if let Some(val) = get_kw(ruby, hash, "max_concurrent_extractions") {
             let value = usize::try_convert(val)?;
             config.max_concurrent_extractions = Some(value);
@@ -1655,8 +1505,8 @@ fn extraction_result_to_ruby(ruby: &Ruby, result: RustExtractionResult) -> Resul
         for chunk in chunks {
             let chunk_hash = ruby.hash_new();
             chunk_hash.aset("content", chunk.content)?;
-            chunk_hash.aset("byte_start", chunk.metadata.byte_start)?;
-            chunk_hash.aset("byte_end", chunk.metadata.byte_end)?;
+            chunk_hash.aset("char_start", chunk.metadata.char_start)?;
+            chunk_hash.aset("char_end", chunk.metadata.char_end)?;
             if let Some(token_count) = chunk.metadata.token_count {
                 chunk_hash.aset("token_count", token_count)?;
             } else {
@@ -1664,16 +1514,6 @@ fn extraction_result_to_ruby(ruby: &Ruby, result: RustExtractionResult) -> Resul
             }
             chunk_hash.aset("chunk_index", chunk.metadata.chunk_index)?;
             chunk_hash.aset("total_chunks", chunk.metadata.total_chunks)?;
-            if let Some(first_page) = chunk.metadata.first_page {
-                chunk_hash.aset("first_page", first_page as i64)?;
-            } else {
-                chunk_hash.aset("first_page", ruby.qnil().as_value())?;
-            }
-            if let Some(last_page) = chunk.metadata.last_page {
-                chunk_hash.aset("last_page", last_page as i64)?;
-            } else {
-                chunk_hash.aset("last_page", ruby.qnil().as_value())?;
-            }
             if let Some(embedding) = chunk.embedding {
                 let embedding_array = ruby.ary_new();
                 for value in embedding {
@@ -1750,92 +1590,6 @@ fn extraction_result_to_ruby(ruby: &Ruby, result: RustExtractionResult) -> Resul
         set_hash_entry(ruby, &hash, "images", ruby.qnil().as_value())?;
     }
 
-    if let Some(page_content_list) = result.pages {
-        let pages_array = ruby.ary_new();
-        for page_content in page_content_list {
-            let page_hash = ruby.hash_new();
-            page_hash.aset("page_number", page_content.page_number as i64)?;
-            page_hash.aset("content", page_content.content)?;
-
-            let tables_array = ruby.ary_new();
-            for table in page_content.tables {
-                let table_hash = ruby.hash_new();
-
-                let cells_array = ruby.ary_new();
-                for row in table.cells.clone() {
-                    let row_array = ruby.ary_from_vec(row);
-                    cells_array.push(row_array)?;
-                }
-                table_hash.aset("cells", cells_array)?;
-                table_hash.aset("markdown", table.markdown.clone())?;
-                table_hash.aset("page_number", table.page_number as i64)?;
-
-                tables_array.push(table_hash)?;
-            }
-            page_hash.aset("tables", tables_array)?;
-
-            let images_array = ruby.ary_new();
-            for image in page_content.images {
-                let image_hash = ruby.hash_new();
-                let data_value = ruby.str_from_slice(&image.data).into_value_with(ruby);
-                image_hash.aset("data", data_value)?;
-                image_hash.aset("format", image.format.clone())?;
-                image_hash.aset("image_index", image.image_index as i64)?;
-                if let Some(page) = image.page_number {
-                    image_hash.aset("page_number", page as i64)?;
-                } else {
-                    image_hash.aset("page_number", ruby.qnil().as_value())?;
-                }
-                if let Some(width) = image.width {
-                    image_hash.aset("width", width as i64)?;
-                } else {
-                    image_hash.aset("width", ruby.qnil().as_value())?;
-                }
-                if let Some(height) = image.height {
-                    image_hash.aset("height", height as i64)?;
-                } else {
-                    image_hash.aset("height", ruby.qnil().as_value())?;
-                }
-                if let Some(colorspace) = &image.colorspace {
-                    image_hash.aset("colorspace", colorspace.clone())?;
-                } else {
-                    image_hash.aset("colorspace", ruby.qnil().as_value())?;
-                }
-                if let Some(bits) = image.bits_per_component {
-                    image_hash.aset("bits_per_component", bits as i64)?;
-                } else {
-                    image_hash.aset("bits_per_component", ruby.qnil().as_value())?;
-                }
-                image_hash.aset(
-                    "is_mask",
-                    if image.is_mask {
-                        ruby.qtrue().as_value()
-                    } else {
-                        ruby.qfalse().as_value()
-                    },
-                )?;
-                if let Some(description) = &image.description {
-                    image_hash.aset("description", description.clone())?;
-                } else {
-                    image_hash.aset("description", ruby.qnil().as_value())?;
-                }
-                if let Some(ocr_result) = &image.ocr_result {
-                    let nested = extraction_result_to_ruby(ruby, (**ocr_result).clone())?;
-                    image_hash.aset("ocr_result", nested.into_value_with(ruby))?;
-                } else {
-                    image_hash.aset("ocr_result", ruby.qnil().as_value())?;
-                }
-                images_array.push(image_hash)?;
-            }
-            page_hash.aset("images", images_array)?;
-
-            pages_array.push(page_hash)?;
-        }
-        set_hash_entry(ruby, &hash, "pages", pages_array.into_value_with(ruby))?;
-    } else {
-        set_hash_entry(ruby, &hash, "pages", ruby.qnil().as_value())?;
-    }
-
     Ok(hash)
 }
 
@@ -1880,14 +1634,13 @@ fn extract_file_sync(args: &[Value]) -> Result<RHash, Error> {
 ///
 fn extract_bytes_sync(args: &[Value]) -> Result<RHash, Error> {
     let ruby = Ruby::get().expect("Ruby not initialized");
-    let args = scan_args::<(RString, String), (), (), (), RHash, ()>(args)?;
+    let args = scan_args::<(String, String), (), (), (), RHash, ()>(args)?;
     let (data, mime_type) = args.required;
     let opts = Some(args.keywords);
 
     let config = parse_extraction_config(&ruby, opts)?;
 
-    let bytes = unsafe { data.as_slice() };
-    let result = kreuzberg::extract_bytes_sync(bytes, &mime_type, &config).map_err(kreuzberg_error)?;
+    let result = kreuzberg::extract_bytes_sync(data.as_bytes(), &mime_type, &config).map_err(kreuzberg_error)?;
 
     extraction_result_to_ruby(&ruby, result)
 }
@@ -1961,7 +1714,7 @@ fn extract_file(args: &[Value]) -> Result<RHash, Error> {
 ///
 fn extract_bytes(args: &[Value]) -> Result<RHash, Error> {
     let ruby = Ruby::get().expect("Ruby not initialized");
-    let args = scan_args::<(RString, String), (), (), (), RHash, ()>(args)?;
+    let args = scan_args::<(String, String), (), (), (), RHash, ()>(args)?;
     let (data, mime_type) = args.required;
     let opts = Some(args.keywords);
 
@@ -1970,9 +1723,8 @@ fn extract_bytes(args: &[Value]) -> Result<RHash, Error> {
     let runtime =
         tokio::runtime::Runtime::new().map_err(|e| runtime_error(format!("Failed to create Tokio runtime: {}", e)))?;
 
-    let bytes = unsafe { data.as_slice() };
     let result = runtime
-        .block_on(async { kreuzberg::extract_bytes(bytes, &mime_type, &config).await })
+        .block_on(async { kreuzberg::extract_bytes(data.as_bytes(), &mime_type, &config).await })
         .map_err(kreuzberg_error)?;
 
     extraction_result_to_ruby(&ruby, result)
@@ -2029,10 +1781,7 @@ fn batch_extract_bytes_sync(args: &[Value]) -> Result<RArray, Error> {
 
     let config = parse_extraction_config(&ruby, opts)?;
 
-    let bytes_vec: Vec<RString> = bytes_array
-        .into_iter()
-        .map(RString::try_convert)
-        .collect::<Result<_, _>>()?;
+    let bytes_vec: Vec<String> = bytes_array.to_vec::<String>()?;
     let mime_types: Vec<String> = mime_types_array.to_vec::<String>()?;
 
     if bytes_vec.len() != mime_types.len() {
@@ -2043,10 +1792,10 @@ fn batch_extract_bytes_sync(args: &[Value]) -> Result<RArray, Error> {
         )));
     }
 
-    let contents: Vec<(Vec<u8>, String)> = bytes_vec
+    let contents: Vec<(&[u8], &str)> = bytes_vec
         .iter()
         .zip(mime_types.iter())
-        .map(|(bytes, mime)| (unsafe { bytes.as_slice() }.to_vec(), mime.clone()))
+        .map(|(bytes, mime)| (bytes.as_bytes(), mime.as_str()))
         .collect();
 
     let results = kreuzberg::batch_extract_bytes_sync(contents, &config).map_err(kreuzberg_error)?;
@@ -2074,10 +1823,7 @@ fn batch_extract_bytes(args: &[Value]) -> Result<RArray, Error> {
 
     let config = parse_extraction_config(&ruby, opts)?;
 
-    let bytes_vec: Vec<RString> = bytes_array
-        .into_iter()
-        .map(RString::try_convert)
-        .collect::<Result<_, _>>()?;
+    let bytes_vec: Vec<String> = bytes_array.to_vec::<String>()?;
     let mime_types: Vec<String> = mime_types_array.to_vec::<String>()?;
 
     if bytes_vec.len() != mime_types.len() {
@@ -2088,10 +1834,10 @@ fn batch_extract_bytes(args: &[Value]) -> Result<RArray, Error> {
         )));
     }
 
-    let contents: Vec<(Vec<u8>, String)> = bytes_vec
+    let contents: Vec<(&[u8], &str)> = bytes_vec
         .iter()
         .zip(mime_types.iter())
-        .map(|(bytes, mime)| (unsafe { bytes.as_slice() }.to_vec(), mime.clone()))
+        .map(|(bytes, mime)| (bytes.as_bytes(), mime.as_str()))
         .collect();
 
     let runtime =
@@ -2251,6 +1997,9 @@ fn register_post_processor(args: &[Value]) -> Result<(), Error> {
             let processor = self.processor.value();
             let result_clone = result.clone();
 
+            // Use block_in_place to avoid GVL deadlocks (same pattern as Python PostProcessor)
+            // See crates/kreuzberg-py/README.md:151-158 for explanation
+            // CRITICAL: spawn_blocking causes GVL deadlocks, must use block_in_place
             let updated_result = tokio::task::block_in_place(|| {
                 let ruby = Ruby::get().expect("Ruby not initialized");
                 let result_hash = extraction_result_to_ruby(&ruby, result_clone.clone()).map_err(|e| {
@@ -2457,6 +2206,9 @@ fn register_validator(args: &[Value]) -> Result<(), Error> {
             let validator = self.validator.value();
             let result_clone = result.clone();
 
+            // Use block_in_place to avoid GVL deadlocks (same pattern as Python Validator)
+            // See crates/kreuzberg-py/README.md:151-158 for explanation
+            // CRITICAL: spawn_blocking causes GVL deadlocks, must use block_in_place
             tokio::task::block_in_place(|| {
                 let ruby = Ruby::get().expect("Ruby not initialized");
                 let result_hash =
@@ -2593,7 +2345,6 @@ fn register_ocr_backend(name: String, backend: Value) -> Result<(), Error> {
                 detected_languages: None,
                 chunks: None,
                 images: None,
-                pages: None,
             })
         }
 
@@ -2864,7 +2615,6 @@ fn get_extensions_for_mime_native(mime_type: String) -> Result<Vec<String>, Erro
     kreuzberg::get_extensions_for_mime(&mime_type).map_err(kreuzberg_error)
 }
 
-#[cfg(feature = "embeddings")]
 /// List all available embedding preset names.
 ///
 /// Returns an array of preset names that can be used with get_embedding_preset.
@@ -2890,7 +2640,6 @@ fn list_embedding_presets(ruby: &Ruby) -> Result<RArray, Error> {
     Ok(array)
 }
 
-#[cfg(feature = "embeddings")]
 /// Get a specific embedding preset by name.
 ///
 /// Returns a preset configuration hash, or nil if the preset name is not found.
@@ -2931,6 +2680,8 @@ fn get_embedding_preset(ruby: &Ruby, name: String) -> Result<Value, Error> {
             set_hash_entry(ruby, &hash, "chunk_size", preset.chunk_size.into_value_with(ruby))?;
             set_hash_entry(ruby, &hash, "overlap", preset.overlap.into_value_with(ruby))?;
 
+            // Note: When embeddings feature is enabled in kreuzberg, the model field is EmbeddingModel
+            // Since Ruby bindings typically build with all features, we use the model field and format it.
             let model_name = format!("{:?}", preset.model);
 
             set_hash_entry(ruby, &hash, "model_name", ruby.str_new(&model_name).as_value())?;
@@ -2943,562 +2694,6 @@ fn get_embedding_preset(ruby: &Ruby, name: String) -> Result<Value, Error> {
     }
 }
 
-/// Get the last error code from FFI
-///
-/// Returns an i32 error code indicating the type of error that occurred:
-/// - 0: Success (no error)
-/// - 1: GenericError
-/// - 2: Panic
-/// - 3: InvalidArgument
-/// - 4: IoError
-/// - 5: ParsingError
-/// - 6: OcrError
-/// - 7: MissingDependency
-///
-/// @return [Integer] The error code
-fn last_error_code() -> i32 {
-    get_error_code()
-}
-
-/// Get the last panic context from FFI as a JSON string
-///
-/// Returns a JSON string containing panic context if the last error was a panic,
-/// or nil if no panic context is available.
-///
-/// The JSON structure contains:
-/// - file: Source file where panic occurred
-/// - line: Line number
-/// - function: Function name
-/// - message: Panic message
-/// - timestamp_secs: Unix timestamp
-///
-/// @return [String, nil] JSON string with panic context or nil
-fn last_panic_context_json(ruby: &Ruby) -> Value {
-    match get_panic_context() {
-        Some(json) => ruby.str_new(&json).as_value(),
-        None => ruby.qnil().as_value(),
-    }
-}
-
-/// Validates a binarization method string
-///
-/// @param method [String] The binarization method (e.g., "otsu", "adaptive", "sauvola")
-/// @return [Integer] 1 if valid, 0 if invalid (error message available via Kreuzberg::_last_error_code_native)
-fn validate_binarization_method(method: String) -> Result<i32, Error> {
-    let c_method = std::ffi::CString::new(method).map_err(|_| runtime_error("Invalid method string"))?;
-
-    Ok(unsafe { kreuzberg_validate_binarization_method(c_method.as_ptr()) })
-}
-
-/// Validates an OCR backend string
-///
-/// @param backend [String] The OCR backend (e.g., "tesseract", "easyocr", "paddleocr")
-/// @return [Integer] 1 if valid, 0 if invalid
-fn validate_ocr_backend(backend: String) -> Result<i32, Error> {
-    let c_backend = std::ffi::CString::new(backend).map_err(|_| runtime_error("Invalid backend string"))?;
-
-    Ok(unsafe { kreuzberg_validate_ocr_backend(c_backend.as_ptr()) })
-}
-
-/// Validates a language code (ISO 639-1 or 639-3)
-///
-/// @param code [String] The language code (e.g., "en", "eng", "de", "deu")
-/// @return [Integer] 1 if valid, 0 if invalid
-fn validate_language_code(code: String) -> Result<i32, Error> {
-    let c_code = std::ffi::CString::new(code).map_err(|_| runtime_error("Invalid language code string"))?;
-
-    Ok(unsafe { kreuzberg_validate_language_code(c_code.as_ptr()) })
-}
-
-/// Validates a token reduction level
-///
-/// @param level [String] The token reduction level (e.g., "off", "light", "moderate", "aggressive", "maximum")
-/// @return [Integer] 1 if valid, 0 if invalid
-fn validate_token_reduction_level(level: String) -> Result<i32, Error> {
-    let c_level = std::ffi::CString::new(level).map_err(|_| runtime_error("Invalid token reduction level string"))?;
-
-    Ok(unsafe { kreuzberg_validate_token_reduction_level(c_level.as_ptr()) })
-}
-
-/// Validates a tesseract PSM (Page Segmentation Mode) value
-///
-/// @param psm [Integer] The PSM value (0-13)
-/// @return [Integer] 1 if valid, 0 if invalid
-fn validate_tesseract_psm(psm: i32) -> Result<i32, Error> {
-    Ok(kreuzberg_validate_tesseract_psm(psm))
-}
-
-/// Validates a tesseract OEM (OCR Engine Mode) value
-///
-/// @param oem [Integer] The OEM value (0-3)
-/// @return [Integer] 1 if valid, 0 if invalid
-fn validate_tesseract_oem(oem: i32) -> Result<i32, Error> {
-    Ok(kreuzberg_validate_tesseract_oem(oem))
-}
-
-/// Validates an output format string
-///
-/// @param format [String] The output format (e.g., "text", "markdown")
-/// @return [Integer] 1 if valid, 0 if invalid
-fn validate_output_format(format: String) -> Result<i32, Error> {
-    let c_format = std::ffi::CString::new(format).map_err(|_| runtime_error("Invalid format string"))?;
-
-    Ok(unsafe { kreuzberg_validate_output_format(c_format.as_ptr()) })
-}
-
-/// Validates a confidence threshold value
-///
-/// @param confidence [Float] The confidence value (0.0-1.0)
-/// @return [Integer] 1 if valid, 0 if invalid
-fn validate_confidence(confidence: f64) -> Result<i32, Error> {
-    Ok(kreuzberg_validate_confidence(confidence))
-}
-
-/// Validates a DPI (dots per inch) value
-///
-/// @param dpi [Integer] The DPI value
-/// @return [Integer] 1 if valid, 0 if invalid
-fn validate_dpi(dpi: i32) -> Result<i32, Error> {
-    Ok(kreuzberg_validate_dpi(dpi))
-}
-
-/// Validates chunking parameters
-///
-/// @param max_chars [Integer] Maximum characters per chunk
-/// @param max_overlap [Integer] Maximum overlap between chunks
-/// @return [Integer] 1 if valid, 0 if invalid
-fn validate_chunking_params(max_chars: usize, max_overlap: usize) -> Result<i32, Error> {
-    Ok(kreuzberg_validate_chunking_params(max_chars, max_overlap))
-}
-
-/// Gets valid binarization methods as a JSON string
-///
-/// @return [String] JSON array of valid binarization methods
-fn get_valid_binarization_methods(_ruby: &Ruby) -> Result<String, Error> {
-    let ptr = kreuzberg_get_valid_binarization_methods();
-    if ptr.is_null() {
-        return Err(runtime_error("Failed to get valid binarization methods"));
-    }
-
-    let c_str = unsafe { std::ffi::CStr::from_ptr(ptr) };
-    let result = c_str
-        .to_str()
-        .map_err(|_| runtime_error("Invalid UTF-8 in binarization methods"))?
-        .to_string();
-
-    unsafe {
-        kreuzberg_free_string(ptr as *mut c_char);
-    }
-
-    Ok(result)
-}
-
-/// Gets valid language codes as a JSON string
-///
-/// @return [String] JSON array of valid language codes
-fn get_valid_language_codes(_ruby: &Ruby) -> Result<String, Error> {
-    let ptr = kreuzberg_get_valid_language_codes();
-    if ptr.is_null() {
-        return Err(runtime_error("Failed to get valid language codes"));
-    }
-
-    let c_str = unsafe { std::ffi::CStr::from_ptr(ptr) };
-    let result = c_str
-        .to_str()
-        .map_err(|_| runtime_error("Invalid UTF-8 in language codes"))?
-        .to_string();
-
-    unsafe {
-        kreuzberg_free_string(ptr as *mut c_char);
-    }
-
-    Ok(result)
-}
-
-/// Gets valid OCR backends as a JSON string
-///
-/// @return [String] JSON array of valid OCR backends
-fn get_valid_ocr_backends(_ruby: &Ruby) -> Result<String, Error> {
-    let ptr = kreuzberg_get_valid_ocr_backends();
-    if ptr.is_null() {
-        return Err(runtime_error("Failed to get valid OCR backends"));
-    }
-
-    let c_str = unsafe { std::ffi::CStr::from_ptr(ptr) };
-    let result = c_str
-        .to_str()
-        .map_err(|_| runtime_error("Invalid UTF-8 in OCR backends"))?
-        .to_string();
-
-    unsafe {
-        kreuzberg_free_string(ptr as *mut c_char);
-    }
-
-    Ok(result)
-}
-
-/// Gets valid token reduction levels as a JSON string
-///
-/// @return [String] JSON array of valid token reduction levels
-fn get_valid_token_reduction_levels(_ruby: &Ruby) -> Result<String, Error> {
-    let ptr = kreuzberg_get_valid_token_reduction_levels();
-    if ptr.is_null() {
-        return Err(runtime_error("Failed to get valid token reduction levels"));
-    }
-
-    let c_str = unsafe { std::ffi::CStr::from_ptr(ptr) };
-    let result = c_str
-        .to_str()
-        .map_err(|_| runtime_error("Invalid UTF-8 in token reduction levels"))?
-        .to_string();
-
-    unsafe {
-        kreuzberg_free_string(ptr as *mut c_char);
-    }
-
-    Ok(result)
-}
-
-/// Serialize a config to JSON string
-/// @param config_json [String] JSON string representing the config
-/// @return [String] Serialized JSON config
-fn config_to_json_wrapper(_ruby: &Ruby, config_json: String) -> Result<String, Error> {
-    let c_json =
-        std::ffi::CString::new(config_json).map_err(|e| runtime_error(format!("Invalid config JSON: {}", e)))?;
-
-    let config_ptr = unsafe { kreuzberg_config_from_json(c_json.as_ptr()) };
-    if config_ptr.is_null() {
-        return Err(runtime_error("Failed to parse config from JSON"));
-    }
-
-    let json_ptr = unsafe { kreuzberg_config_to_json(config_ptr) };
-    let result = if json_ptr.is_null() {
-        Err(runtime_error("Failed to serialize config to JSON"))
-    } else {
-        let c_str = unsafe { std::ffi::CStr::from_ptr(json_ptr) };
-        let json = c_str
-            .to_str()
-            .map_err(|_| runtime_error("Invalid UTF-8 in serialized config"))?
-            .to_string();
-        unsafe {
-            kreuzberg_free_string(json_ptr as *mut c_char);
-        }
-        Ok(json)
-    };
-
-    unsafe {
-        kreuzberg_config_free(config_ptr);
-    }
-    result
-}
-
-/// Get a field from config
-/// @param config_json [String] JSON string representing the config
-/// @param field_name [String] Field name (supports dot notation)
-/// @return [Object] Parsed JSON value, or nil if field doesn't exist
-fn config_get_field_wrapper(ruby: &Ruby, config_json: String, field_name: String) -> Result<Value, Error> {
-    let c_json =
-        std::ffi::CString::new(config_json).map_err(|e| runtime_error(format!("Invalid config JSON: {}", e)))?;
-    let c_field =
-        std::ffi::CString::new(field_name).map_err(|e| runtime_error(format!("Invalid field name: {}", e)))?;
-
-    let config_ptr = unsafe { kreuzberg_config_from_json(c_json.as_ptr()) };
-    if config_ptr.is_null() {
-        return Err(runtime_error("Failed to parse config from JSON"));
-    }
-
-    let field_ptr = unsafe { kreuzberg_config_get_field(config_ptr, c_field.as_ptr()) };
-    let result = if field_ptr.is_null() {
-        Ok(ruby.qnil().as_value())
-    } else {
-        let c_str = unsafe { std::ffi::CStr::from_ptr(field_ptr) };
-        let json_str = c_str
-            .to_str()
-            .map_err(|_| runtime_error("Invalid UTF-8 in field value"))?;
-        let json_value: serde_json::Value =
-            serde_json::from_str(json_str).map_err(|e| runtime_error(format!("Failed to parse field value: {}", e)))?;
-        unsafe {
-            kreuzberg_free_string(field_ptr as *mut c_char);
-        }
-        json_value_to_ruby(ruby, &json_value)
-    };
-
-    unsafe {
-        kreuzberg_config_free(config_ptr);
-    }
-    result
-}
-
-/// Merge two configs
-/// @param base_json [String] Base config JSON
-/// @param override_json [String] Override config JSON
-/// @return [String] Merged config JSON
-fn config_merge_wrapper(_ruby: &Ruby, base_json: String, override_json: String) -> Result<String, Error> {
-    let c_base =
-        std::ffi::CString::new(base_json).map_err(|e| runtime_error(format!("Invalid base config JSON: {}", e)))?;
-    let c_override = std::ffi::CString::new(override_json)
-        .map_err(|e| runtime_error(format!("Invalid override config JSON: {}", e)))?;
-
-    let base_ptr = unsafe { kreuzberg_config_from_json(c_base.as_ptr()) };
-    if base_ptr.is_null() {
-        return Err(runtime_error("Failed to parse base config from JSON"));
-    }
-
-    let override_ptr = unsafe { kreuzberg_config_from_json(c_override.as_ptr()) };
-    if override_ptr.is_null() {
-        unsafe {
-            kreuzberg_config_free(base_ptr);
-        }
-        return Err(runtime_error("Failed to parse override config from JSON"));
-    }
-
-    let merge_result = unsafe { kreuzberg_config_merge(base_ptr, override_ptr) };
-
-    let result = if merge_result == 0 {
-        Err(runtime_error("Failed to merge configs"))
-    } else {
-        let json_ptr = unsafe { kreuzberg_config_to_json(base_ptr) };
-        if json_ptr.is_null() {
-            Err(runtime_error("Failed to serialize merged config"))
-        } else {
-            let c_str = unsafe { std::ffi::CStr::from_ptr(json_ptr) };
-            let json = c_str
-                .to_str()
-                .map_err(|_| runtime_error("Invalid UTF-8 in merged config"))?
-                .to_string();
-            unsafe {
-                kreuzberg_free_string(json_ptr as *mut c_char);
-            }
-            Ok(json)
-        }
-    };
-
-    unsafe {
-        kreuzberg_config_free(base_ptr);
-        kreuzberg_config_free(override_ptr);
-    }
-    result
-}
-
-/// Get page count from result
-/// @param result_ptr [Integer] Opaque pointer to ExtractionResult (as integer)
-/// @return [Integer] Page count, or -1 on error
-fn result_page_count(_ruby: &Ruby, result_ptr: i64) -> Result<i32, Error> {
-    if result_ptr == 0 {
-        return Err(runtime_error("Invalid result pointer"));
-    }
-
-    let page_count = unsafe { kreuzberg_result_get_page_count(result_ptr as *const RustExtractionResult) };
-
-    Ok(page_count)
-}
-
-/// Get chunk count from result
-/// @param result_ptr [Integer] Opaque pointer to ExtractionResult (as integer)
-/// @return [Integer] Chunk count, or -1 on error
-fn result_chunk_count(_ruby: &Ruby, result_ptr: i64) -> Result<i32, Error> {
-    if result_ptr == 0 {
-        return Err(runtime_error("Invalid result pointer"));
-    }
-
-    let chunk_count = unsafe { kreuzberg_result_get_chunk_count(result_ptr as *const RustExtractionResult) };
-
-    Ok(chunk_count)
-}
-
-/// Get detected language from result
-/// @param result_ptr [Integer] Opaque pointer to ExtractionResult (as integer)
-/// @return [String, nil] Detected language code, or nil if not detected
-fn result_detected_language(_ruby: &Ruby, result_ptr: i64) -> Result<Value, Error> {
-    if result_ptr == 0 {
-        return Err(runtime_error("Invalid result pointer"));
-    }
-
-    let lang_ptr = unsafe { kreuzberg_result_get_detected_language(result_ptr as *const RustExtractionResult) };
-
-    if lang_ptr.is_null() {
-        return Ok(_ruby.qnil().as_value());
-    }
-
-    let c_str = unsafe { std::ffi::CStr::from_ptr(lang_ptr) };
-    let lang = c_str
-        .to_str()
-        .map_err(|_| runtime_error("Invalid UTF-8 in detected language"))?
-        .to_string();
-
-    unsafe {
-        kreuzberg_free_string(lang_ptr as *mut c_char);
-    }
-
-    Ok(_ruby.str_new(&lang).into_value_with(_ruby))
-}
-
-/// Get metadata field from result
-/// @param result_ptr [Integer] Opaque pointer to ExtractionResult (as integer)
-/// @param field_name [String] Field name (supports dot notation)
-/// @return [Object, nil] Parsed JSON value, or nil if field doesn't exist
-fn result_metadata_field(ruby: &Ruby, result_ptr: i64, field_name: String) -> Result<Value, Error> {
-    if result_ptr == 0 {
-        return Err(runtime_error("Invalid result pointer"));
-    }
-
-    let c_field =
-        std::ffi::CString::new(field_name).map_err(|e| runtime_error(format!("Invalid field name: {}", e)))?;
-
-    let field = unsafe { kreuzberg_result_get_metadata_field(result_ptr as *const RustExtractionResult, c_field.as_ptr()) };
-
-    if field.is_null != 0 {
-        return Ok(ruby.qnil().as_value());
-    }
-
-    if field.json_value.is_null() {
-        return Ok(ruby.qnil().as_value());
-    }
-
-    let c_str = unsafe { std::ffi::CStr::from_ptr(field.json_value) };
-    let json_str = c_str
-        .to_str()
-        .map_err(|_| runtime_error("Invalid UTF-8 in field value"))?;
-    let json_value: serde_json::Value =
-        serde_json::from_str(json_str).map_err(|e| runtime_error(format!("Failed to parse field value: {}", e)))?;
-
-    unsafe {
-        kreuzberg_free_string(field.json_value);
-    }
-
-    json_value_to_ruby(ruby, &json_value)
-}
-
-/// Get structured error details from FFI
-/// @return [Hash] Error details with keys: :message, :error_code, :error_type, :source_file, :source_function, :source_line, :context_info, :is_panic
-fn get_error_details_native(ruby: &Ruby) -> Result<Value, Error> {
-    let details = kreuzberg_get_error_details();
-
-    let hash = ruby.hash_new();
-
-    unsafe {
-        let message = if !details.message.is_null() {
-            let c_str = std::ffi::CStr::from_ptr(details.message);
-            let msg = c_str.to_str().unwrap_or("").to_string();
-            kreuzberg_free_string(details.message);
-            msg
-        } else {
-            String::new()
-        };
-
-        let error_type = if !details.error_type.is_null() {
-            let c_str = std::ffi::CStr::from_ptr(details.error_type);
-            let ty = c_str.to_str().unwrap_or("unknown").to_string();
-            kreuzberg_free_string(details.error_type);
-            ty
-        } else {
-            "unknown".to_string()
-        };
-
-        let source_file = if !details.source_file.is_null() {
-            let c_str = std::ffi::CStr::from_ptr(details.source_file);
-            let file = c_str.to_str().ok().map(|s| s.to_string());
-            kreuzberg_free_string(details.source_file);
-            file
-        } else {
-            None
-        };
-
-        let source_function = if !details.source_function.is_null() {
-            let c_str = std::ffi::CStr::from_ptr(details.source_function);
-            let func = c_str.to_str().ok().map(|s| s.to_string());
-            kreuzberg_free_string(details.source_function);
-            func
-        } else {
-            None
-        };
-
-        let context_info = if !details.context_info.is_null() {
-            let c_str = std::ffi::CStr::from_ptr(details.context_info);
-            let ctx = c_str.to_str().ok().map(|s| s.to_string());
-            kreuzberg_free_string(details.context_info);
-            ctx
-        } else {
-            None
-        };
-
-        hash.aset(ruby.to_symbol("message"), ruby.str_new(&message).as_value())?;
-        hash.aset(ruby.to_symbol("error_code"), details.error_code.into_value_with(ruby))?;
-        hash.aset(ruby.to_symbol("error_type"), ruby.str_new(&error_type).as_value())?;
-
-        if let Some(file) = source_file {
-            hash.aset(ruby.to_symbol("source_file"), ruby.str_new(&file).as_value())?;
-        } else {
-            hash.aset(ruby.to_symbol("source_file"), ruby.qnil().as_value())?;
-        }
-
-        if let Some(func) = source_function {
-            hash.aset(ruby.to_symbol("source_function"), ruby.str_new(&func).as_value())?;
-        } else {
-            hash.aset(ruby.to_symbol("source_function"), ruby.qnil().as_value())?;
-        }
-
-        hash.aset(ruby.to_symbol("source_line"), details.source_line.into_value_with(ruby))?;
-
-        if let Some(ctx) = context_info {
-            hash.aset(ruby.to_symbol("context_info"), ruby.str_new(&ctx).as_value())?;
-        } else {
-            hash.aset(ruby.to_symbol("context_info"), ruby.qnil().as_value())?;
-        }
-
-        hash.aset(
-            ruby.to_symbol("is_panic"),
-            (details.is_panic != 0).into_value_with(ruby),
-        )?;
-    }
-
-    Ok(hash.into_value_with(ruby))
-}
-
-/// Classify an error based on an error message string
-/// @param message [String] The error message to classify
-/// @return [Integer] Error code (0-7)
-fn classify_error_native(ruby: &Ruby, message: String) -> Result<Value, Error> {
-    let c_message =
-        std::ffi::CString::new(message).map_err(|e| runtime_error(format!("Invalid error message: {}", e)))?;
-
-    let code = unsafe { kreuzberg_classify_error(c_message.as_ptr()) };
-
-    Ok(code.into_value_with(ruby))
-}
-
-/// Get the human-readable name of an error code
-/// @param code [Integer] Numeric error code (0-7)
-/// @return [String] Human-readable error code name
-fn error_code_name_native(ruby: &Ruby, code: u32) -> Result<Value, Error> {
-    let name_ptr = kreuzberg_error_code_name(code);
-
-    if name_ptr.is_null() {
-        return Ok(ruby.str_new("unknown").as_value());
-    }
-
-    let c_str = unsafe { std::ffi::CStr::from_ptr(name_ptr) };
-    let name = c_str.to_str().unwrap_or("unknown").to_string();
-
-    Ok(ruby.str_new(&name).as_value())
-}
-
-/// Get the description of an error code
-/// @param code [Integer] Numeric error code (0-7)
-/// @return [String] Description of the error code
-fn error_code_description_native(ruby: &Ruby, code: u32) -> Result<Value, Error> {
-    let desc_ptr = kreuzberg_error_code_description(code);
-
-    if desc_ptr.is_null() {
-        return Ok(ruby.str_new("Unknown error code").as_value());
-    }
-
-    let c_str = unsafe { std::ffi::CStr::from_ptr(desc_ptr) };
-    let desc = c_str.to_str().unwrap_or("Unknown error code").to_string();
-
-    Ok(ruby.str_new(&desc).as_value())
-}
-
 /// Initialize the Kreuzberg Ruby module
 #[magnus::init]
 fn init(ruby: &Ruby) -> Result<(), Error> {
@@ -3547,66 +2742,8 @@ fn init(ruby: &Ruby) -> Result<(), Error> {
     module.define_module_function("get_extensions_for_mime", function!(get_extensions_for_mime_native, 1))?;
     module.define_module_function("validate_mime_type", function!(validate_mime_type_native, 1))?;
 
-    #[cfg(feature = "embeddings")]
-    {
-        module.define_module_function("list_embedding_presets", function!(list_embedding_presets, 0))?;
-        module.define_module_function("get_embedding_preset", function!(get_embedding_preset, 1))?;
-    }
-
-    module.define_module_function("_last_error_code_native", function!(last_error_code, 0))?;
-    module.define_module_function("_last_panic_context_json_native", function!(last_panic_context_json, 0))?;
-
-    module.define_module_function(
-        "_validate_binarization_method_native",
-        function!(validate_binarization_method, 1),
-    )?;
-    module.define_module_function("_validate_ocr_backend_native", function!(validate_ocr_backend, 1))?;
-    module.define_module_function("_validate_language_code_native", function!(validate_language_code, 1))?;
-    module.define_module_function(
-        "_validate_token_reduction_level_native",
-        function!(validate_token_reduction_level, 1),
-    )?;
-    module.define_module_function("_validate_tesseract_psm_native", function!(validate_tesseract_psm, 1))?;
-    module.define_module_function("_validate_tesseract_oem_native", function!(validate_tesseract_oem, 1))?;
-    module.define_module_function("_validate_output_format_native", function!(validate_output_format, 1))?;
-    module.define_module_function("_validate_confidence_native", function!(validate_confidence, 1))?;
-    module.define_module_function("_validate_dpi_native", function!(validate_dpi, 1))?;
-    module.define_module_function(
-        "_validate_chunking_params_native",
-        function!(validate_chunking_params, 2),
-    )?;
-    module.define_module_function(
-        "_get_valid_binarization_methods_native",
-        function!(get_valid_binarization_methods, 0),
-    )?;
-    module.define_module_function(
-        "_get_valid_language_codes_native",
-        function!(get_valid_language_codes, 0),
-    )?;
-    module.define_module_function("_get_valid_ocr_backends_native", function!(get_valid_ocr_backends, 0))?;
-    module.define_module_function(
-        "_get_valid_token_reduction_levels_native",
-        function!(get_valid_token_reduction_levels, 0),
-    )?;
-
-    module.define_module_function("_config_to_json_native", function!(config_to_json_wrapper, 1))?;
-    module.define_module_function("_config_get_field_native", function!(config_get_field_wrapper, 2))?;
-    module.define_module_function("_config_merge_native", function!(config_merge_wrapper, 2))?;
-    module.define_module_function("_result_page_count_native", function!(result_page_count, 1))?;
-    module.define_module_function("_result_chunk_count_native", function!(result_chunk_count, 1))?;
-    module.define_module_function(
-        "_result_detected_language_native",
-        function!(result_detected_language, 1),
-    )?;
-    module.define_module_function("_result_metadata_field_native", function!(result_metadata_field, 2))?;
-
-    module.define_module_function("_get_error_details_native", function!(get_error_details_native, 0))?;
-    module.define_module_function("_classify_error_native", function!(classify_error_native, 1))?;
-    module.define_module_function("_error_code_name_native", function!(error_code_name_native, 1))?;
-    module.define_module_function(
-        "_error_code_description_native",
-        function!(error_code_description_native, 1),
-    )?;
+    module.define_module_function("list_embedding_presets", function!(list_embedding_presets, 0))?;
+    module.define_module_function("get_embedding_preset", function!(get_embedding_preset, 1))?;
 
     Ok(())
 }