kreuzberg 4.0.0.pre.rc.13 → 4.0.0.pre.rc.15

data/vendor/kreuzberg-ffi/src/panic_shield.rs

@@ -1,247 +0,0 @@
-use kreuzberg::panic_context::PanicContext;
-use std::cell::RefCell;
-
-/// Structured error that includes both the error message and optional panic context.
-#[derive(Debug, Clone)]
-pub struct StructuredError {
-    /// The error message
-    pub message: String,
-    /// Optional panic context if this error originated from a panic
-    pub panic_context: Option<PanicContext>,
-    /// Error code for programmatic error handling
-    pub code: ErrorCode,
-}
-
-/// Error codes for different types of errors.
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-#[repr(i32)]
-pub enum ErrorCode {
-    /// No error
-    Success = 0,
-    /// Generic error
-    GenericError = 1,
-    /// Panic was caught
-    Panic = 2,
-    /// Invalid argument
-    InvalidArgument = 3,
-    /// IO error
-    IoError = 4,
-    /// Parsing error
-    ParsingError = 5,
-    /// OCR error
-    OcrError = 6,
-    /// Missing dependency
-    MissingDependency = 7,
-}
-
-impl StructuredError {
-    /// Creates a new StructuredError from a panic context.
-    pub fn from_panic(context: PanicContext) -> Self {
-        Self {
-            message: context.format(),
-            panic_context: Some(context),
-            code: ErrorCode::Panic,
-        }
-    }
-
-    /// Creates a new StructuredError from a regular error message.
-    pub fn from_message(message: String, code: ErrorCode) -> Self {
-        Self {
-            message,
-            panic_context: None,
-            code,
-        }
-    }
-
-    /// Returns the full error message including panic context if available.
-    pub fn full_message(&self) -> String {
-        if let Some(ref ctx) = self.panic_context {
-            format!("{} (at {}:{}:{})", self.message, ctx.file, ctx.line, ctx.function)
-        } else {
-            self.message.clone()
-        }
-    }
-}
-
-thread_local! {
-    static LAST_STRUCTURED_ERROR: RefCell<Option<StructuredError>> = const { RefCell::new(None) };
-}
-
-/// Sets the last structured error.
-pub fn set_structured_error(error: StructuredError) {
-    LAST_STRUCTURED_ERROR.with(|last| *last.borrow_mut() = Some(error));
-}
-
-/// Gets the last structured error message (for compatibility with existing code).
-pub fn get_last_error_message() -> Option<String> {
-    LAST_STRUCTURED_ERROR.with(|last| last.borrow().as_ref().map(|e| e.full_message()))
-}
-
-/// Gets the last error code.
-pub fn get_last_error_code() -> ErrorCode {
-    LAST_STRUCTURED_ERROR.with(|last| last.borrow().as_ref().map(|e| e.code).unwrap_or(ErrorCode::Success))
-}
-
-/// Gets the last panic context if the last error was a panic.
-pub fn get_last_panic_context() -> Option<PanicContext> {
-    LAST_STRUCTURED_ERROR.with(|last| last.borrow().as_ref().and_then(|e| e.panic_context.clone()))
-}
-
-/// Clears the last structured error.
-pub fn clear_structured_error() {
-    LAST_STRUCTURED_ERROR.with(|last| *last.borrow_mut() = None);
-}
-
-/// Macro to wrap FFI functions with panic catching.
-///
-/// This macro catches panics at FFI boundaries and converts them to structured errors.
-/// It captures file, line, and function information for better error reporting.
-///
-/// # Usage
-///
-/// ```rust,ignore
-/// #[no_mangle]
-/// pub extern "C" fn my_ffi_function(arg: *const c_char) -> *mut ExtractionResult {
-///     ffi_panic_guard!("my_ffi_function", {
-///         // Your FFI function body here
-///         // Return the result normally
-///     })
-/// }
-/// ```
-///
-/// For bool-returning functions:
-///
-/// ```rust,ignore
-/// #[no_mangle]
-/// pub extern "C" fn my_bool_function(arg: *const c_char) -> bool {
-///     ffi_panic_guard_bool!("my_bool_function", {
-///         // Your FFI function body here
-///         // Return true or false normally
-///     })
-/// }
-/// ```
-///
-/// The macro will:
-/// - Catch any panics that occur in the wrapped code
-/// - Create a PanicContext with file/line/function information
-/// - Store the structured error in thread-local storage
-/// - Return a null pointer (for pointer-returning functions) or false (for bool-returning functions) to indicate failure
-#[macro_export]
-macro_rules! ffi_panic_guard {
-    ($function_name:expr, $body:expr) => {{
-        match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| $body)) {
-            Ok(result) => result,
-            Err(panic_info) => {
-                let context =
-                    kreuzberg::panic_context::PanicContext::new(file!(), line!(), $function_name, panic_info.as_ref());
-                $crate::panic_shield::set_structured_error($crate::panic_shield::StructuredError::from_panic(context));
-                std::ptr::null_mut()
-            }
-        }
-    }};
-}
-
-/// Macro to wrap FFI functions that return bool with panic catching.
-///
-/// This variant of ffi_panic_guard returns false on panic (suitable for bool-returning functions).
-#[macro_export]
-macro_rules! ffi_panic_guard_bool {
-    ($function_name:expr, $body:expr) => {{
-        match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| $body)) {
-            Ok(result) => result,
-            Err(panic_info) => {
-                let context =
-                    kreuzberg::panic_context::PanicContext::new(file!(), line!(), $function_name, panic_info.as_ref());
-                $crate::panic_shield::set_structured_error($crate::panic_shield::StructuredError::from_panic(context));
-                false
-            }
-        }
-    }};
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_structured_error_from_panic() {
-        let panic_msg = "test panic".to_string();
-        let ctx = PanicContext::new("test.rs", 42, "test_fn", &panic_msg);
-        let err = StructuredError::from_panic(ctx);
-
-        assert_eq!(err.code, ErrorCode::Panic);
-        assert!(err.panic_context.is_some());
-        assert!(err.message.contains("test panic"));
-    }
-
-    #[test]
-    fn test_structured_error_from_message() {
-        let err = StructuredError::from_message("test error".to_string(), ErrorCode::GenericError);
-
-        assert_eq!(err.code, ErrorCode::GenericError);
-        assert!(err.panic_context.is_none());
-        assert_eq!(err.message, "test error");
-    }
-
-    #[test]
-    fn test_error_storage() {
-        clear_structured_error();
-        assert!(get_last_error_message().is_none());
-
-        let err = StructuredError::from_message("test".to_string(), ErrorCode::IoError);
-        set_structured_error(err);
-
-        assert_eq!(get_last_error_message(), Some("test".to_string()));
-        assert_eq!(get_last_error_code(), ErrorCode::IoError);
-
-        clear_structured_error();
-        assert!(get_last_error_message().is_none());
-    }
-
-    #[test]
-    fn test_panic_context_extraction() {
-        clear_structured_error();
-
-        let panic_msg = "panic message".to_string();
-        let ctx = PanicContext::new("file.rs", 10, "func", &panic_msg);
-        let err = StructuredError::from_panic(ctx);
-        set_structured_error(err);
-
-        let retrieved_ctx = get_last_panic_context();
-        assert!(retrieved_ctx.is_some());
-
-        let ctx = retrieved_ctx.unwrap();
-        assert_eq!(ctx.file, "file.rs");
-        assert_eq!(ctx.line, 10);
-        assert_eq!(ctx.function, "func");
-    }
-
-    #[test]
-    fn test_ffi_panic_guard_success() {
-        let result = crate::ffi_panic_guard!("test_success", { Box::into_raw(Box::new(42)) });
-        assert!(!result.is_null());
-        unsafe {
-            assert_eq!(*result, 42);
-            let _ = Box::from_raw(result);
-        }
-    }
-
-    #[test]
-    fn test_ffi_panic_guard_panic() {
-        clear_structured_error();
-
-        let result: *mut i32 = crate::ffi_panic_guard!("test_panic", {
-            panic!("intentional panic");
-            #[allow(unreachable_code)]
-            Box::into_raw(Box::new(42))
-        });
-
-        assert!(result.is_null());
-        assert!(get_last_error_message().is_some());
-        assert_eq!(get_last_error_code(), ErrorCode::Panic);
-
-        let msg = get_last_error_message().unwrap();
-        assert!(msg.contains("intentional panic"));
-        assert!(msg.contains("test_panic"));
-    }
-}

data/vendor/kreuzberg-ffi/tests.disabled/README.md

@@ -1,48 +0,0 @@
-# Disabled Integration Tests
-
-## Why These Tests Were Disabled
-
-These integration tests were attempting to test the C FFI interface from Rust using `extern "C"` declarations. However, this approach has fundamental issues:
-
-1. **Linking Problem**: Integration tests in `tests/` directory run as separate binaries. When the library is compiled as `cdylib` or `staticlib`, these crate types don't automatically link to Rust integration tests.
-
-2. **Architecture Mismatch**: These tests were trying to use `unsafe extern "C"` blocks to declare functions that are already defined in the same crate. This creates a circular dependency where the test binary expects to dynamically link against symbols that should be statically linked.
-
-3. **Wrong Testing Layer**: C FFI functions should be tested from the actual language bindings (Java, Go, etc.) that consume them, not from Rust integration tests.
-
-## Testing Strategy
-
-The FFI layer is thoroughly tested through:
-
-1. **Unit tests** in `src/lib.rs` that directly call the public Rust FFI functions
-2. **Language binding tests**:
-   - Java: `packages/java/src/test/java/`
-   - Go: `packages/go/v4/*_test.go`
-   - Python: `packages/python/tests/`
-   - TypeScript: `packages/typescript/**/*.spec.ts`
-   - Ruby: `packages/ruby/spec/`
-
-3. **E2E tests** in `e2e/` directory that test the complete integration
-
-## Error That Led to Disabling
-
-```
-error: linking with `cc` failed: exit status: 1
-rust-lld: error: undefined symbol: kreuzberg_last_error
-rust-lld: error: undefined symbol: kreuzberg_load_extraction_config_from_file
-rust-lld: error: undefined symbol: kreuzberg_free_string
-```
-
-This occurred because the integration test binaries couldn't link to the C symbols defined in the FFI crate.
-
-## Future Improvements
-
-If C FFI integration tests are needed from Rust:
-
-1. Create a separate C test harness that links against the compiled `.so`/`.dylib`
-2. Move critical tests into unit tests within `src/lib.rs`
-3. Ensure comprehensive coverage through language binding tests instead
-
-## CI Impact
-
-This change resolves CI failures in run 19607600358 on both macOS-14 and Ubuntu runners.

data/vendor/kreuzberg-ffi/tests.disabled/config_loading_tests.rs

@@ -1,299 +0,0 @@
-//! FFI config loading integration tests.
-//!
-//! Tests the FFI layer for configuration loading from files and discovery.
-
-use std::ffi::{CStr, CString};
-use std::fs;
-use std::os::raw::c_char;
-use tempfile::TempDir;
-
-unsafe extern "C" {
-    fn kreuzberg_load_extraction_config_from_file(file_path: *const c_char) -> *mut c_char;
-    fn kreuzberg_free_string(s: *mut c_char);
-    fn kreuzberg_last_error() -> *const c_char;
-}
-
-/// Helper to convert *const c_char to String
-unsafe fn c_str_to_string(ptr: *const c_char) -> Option<String> {
-    if ptr.is_null() {
-        None
-    } else {
-        unsafe { Some(CStr::from_ptr(ptr).to_string_lossy().into_owned()) }
-    }
-}
-
-/// Helper to get last error message
-unsafe fn get_last_error() -> Option<String> {
-    let error_ptr = unsafe { kreuzberg_last_error() };
-    unsafe { c_str_to_string(error_ptr) }
-}
-
-/// Test successful config loading from TOML file.
-#[test]
-fn test_load_config_from_toml_file_succeeds() {
-    let temp_dir = TempDir::new().unwrap();
-    let config_path = temp_dir.path().join("config.toml");
-
-    let toml_content = r#"
-[ocr]
-enabled = true
-backend = "tesseract"
-
-[chunking]
-max_chars = 1000
-max_overlap = 100
-"#;
-
-    fs::write(&config_path, toml_content).unwrap();
-
-    unsafe {
-        let path_cstr = CString::new(config_path.to_str().unwrap()).unwrap();
-        let result = kreuzberg_load_extraction_config_from_file(path_cstr.as_ptr());
-
-        assert!(!result.is_null(), "Result should not be null");
-
-        let json_str = c_str_to_string(result).expect("Should have valid JSON");
-        kreuzberg_free_string(result);
-
-        let json_value: serde_json::Value = serde_json::from_str(&json_str).unwrap();
-        assert!(json_value.get("ocr").is_some(), "Should have OCR config");
-        assert!(json_value.get("chunking").is_some(), "Should have chunking config");
-    }
-}
-
-/// Test successful config loading from YAML file.
-#[test]
-fn test_load_config_from_yaml_file_succeeds() {
-    let temp_dir = TempDir::new().unwrap();
-    let config_path = temp_dir.path().join("config.yaml");
-
-    let yaml_content = r#"
-ocr:
-  enabled: true
-  backend: tesseract
-chunking:
-  max_chars: 1000
-  max_overlap: 100
-"#;
-
-    fs::write(&config_path, yaml_content).unwrap();
-
-    unsafe {
-        let path_cstr = CString::new(config_path.to_str().unwrap()).unwrap();
-        let result = kreuzberg_load_extraction_config_from_file(path_cstr.as_ptr());
-
-        assert!(!result.is_null(), "Result should not be null");
-
-        let json_str = c_str_to_string(result).expect("Should have valid JSON");
-        kreuzberg_free_string(result);
-
-        let json_value: serde_json::Value = serde_json::from_str(&json_str).unwrap();
-        assert!(json_value.get("ocr").is_some(), "Should have OCR config");
-        assert!(json_value.get("chunking").is_some(), "Should have chunking config");
-    }
-}
-
-/// Test successful config loading from JSON file.
-#[test]
-fn test_load_config_from_json_file_succeeds() {
-    let temp_dir = TempDir::new().unwrap();
-    let config_path = temp_dir.path().join("config.json");
-
-    let json_content = r#"
-{
-  "ocr": {
-    "enabled": true,
-    "backend": "tesseract"
-  },
-  "chunking": {
-    "max_chars": 1000,
-    "max_overlap": 100
-  }
-}
-"#;
-
-    fs::write(&config_path, json_content).unwrap();
-
-    unsafe {
-        let path_cstr = CString::new(config_path.to_str().unwrap()).unwrap();
-        let result = kreuzberg_load_extraction_config_from_file(path_cstr.as_ptr());
-
-        assert!(!result.is_null(), "Result should not be null");
-
-        let json_str = c_str_to_string(result).expect("Should have valid JSON");
-        kreuzberg_free_string(result);
-
-        let json_value: serde_json::Value = serde_json::from_str(&json_str).unwrap();
-        assert!(json_value.get("ocr").is_some(), "Should have OCR config");
-        assert!(json_value.get("chunking").is_some(), "Should have chunking config");
-    }
-}
-
-/// Test config loading fails gracefully with invalid file path.
-#[test]
-fn test_load_config_from_invalid_path_fails_gracefully() {
-    unsafe {
-        let invalid_path = CString::new("/nonexistent/path/config.toml").unwrap();
-        let result = kreuzberg_load_extraction_config_from_file(invalid_path.as_ptr());
-
-        assert!(result.is_null(), "Result should be null for invalid path");
-
-        let error = get_last_error();
-        assert!(error.is_some(), "Should have error message");
-        let error_msg = error.unwrap();
-        assert!(
-            error_msg.contains("No such file") || error_msg.contains("not found"),
-            "Error should mention file not found: {}",
-            error_msg
-        );
-    }
-}
-
-/// Test config loading fails gracefully with null pointer.
-#[test]
-fn test_load_config_from_null_pointer_fails_gracefully() {
-    unsafe {
-        let result = kreuzberg_load_extraction_config_from_file(std::ptr::null());
-
-        assert!(result.is_null(), "Result should be null for null pointer");
-
-        let error = get_last_error();
-        assert!(error.is_some(), "Should have error message");
-        let error_msg = error.unwrap();
-        assert!(
-            error_msg.contains("null") || error_msg.contains("invalid"),
-            "Error should mention null/invalid: {}",
-            error_msg
-        );
-    }
-}
-
-/// Test config loading fails gracefully with malformed TOML.
-#[test]
-fn test_load_config_from_malformed_toml_fails_gracefully() {
-    let temp_dir = TempDir::new().unwrap();
-    let config_path = temp_dir.path().join("config.toml");
-
-    let malformed_toml = r#"
-[ocr
-enabled = true
-"#;
-
-    fs::write(&config_path, malformed_toml).unwrap();
-
-    unsafe {
-        let path_cstr = CString::new(config_path.to_str().unwrap()).unwrap();
-        let result = kreuzberg_load_extraction_config_from_file(path_cstr.as_ptr());
-
-        assert!(result.is_null(), "Result should be null for malformed TOML");
-
-        let error = get_last_error();
-        assert!(error.is_some(), "Should have error message");
-        let error_msg = error.unwrap();
-        assert!(
-            error_msg.contains("parse") || error_msg.contains("invalid") || error_msg.contains("TOML"),
-            "Error should mention parsing issue: {}",
-            error_msg
-        );
-    }
-}
-
-/// Test config loading fails gracefully with malformed JSON.
-#[test]
-fn test_load_config_from_malformed_json_fails_gracefully() {
-    let temp_dir = TempDir::new().unwrap();
-    let config_path = temp_dir.path().join("config.json");
-
-    let malformed_json = r#"
-{
-  "ocr": {
-    "enabled": true
-  }
-  "chunking": {}
-}
-"#;
-
-    fs::write(&config_path, malformed_json).unwrap();
-
-    unsafe {
-        let path_cstr = CString::new(config_path.to_str().unwrap()).unwrap();
-        let result = kreuzberg_load_extraction_config_from_file(path_cstr.as_ptr());
-
-        assert!(result.is_null(), "Result should be null for malformed JSON");
-
-        let error = get_last_error();
-        assert!(error.is_some(), "Should have error message");
-        let error_msg = error.unwrap();
-        assert!(
-            error_msg.contains("parse") || error_msg.contains("invalid") || error_msg.contains("JSON"),
-            "Error should mention parsing issue: {}",
-            error_msg
-        );
-    }
-}
-
-/// Test config loading with invalid UTF-8 in path.
-#[test]
-fn test_load_config_with_invalid_utf8_fails_gracefully() {
-    unsafe {
-        let invalid_bytes = vec![0xFF, 0xFE, 0xFD];
-        let invalid_cstr = CString::new(invalid_bytes).unwrap_or_else(|_| CString::new("").unwrap());
-
-        let result = kreuzberg_load_extraction_config_from_file(invalid_cstr.as_ptr());
-
-        assert!(result.is_null(), "Result should be null for invalid UTF-8");
-
-        let error = get_last_error();
-        assert!(error.is_some(), "Should have error message");
-    }
-}
-
-/// Test config loading with empty file.
-#[test]
-fn test_load_config_from_empty_file_uses_defaults() {
-    let temp_dir = TempDir::new().unwrap();
-    let config_path = temp_dir.path().join("config.toml");
-
-    fs::write(&config_path, "").unwrap();
-
-    unsafe {
-        let path_cstr = CString::new(config_path.to_str().unwrap()).unwrap();
-        let result = kreuzberg_load_extraction_config_from_file(path_cstr.as_ptr());
-
-        assert!(!result.is_null(), "Result should not be null for empty file");
-
-        let json_str = c_str_to_string(result).expect("Should have valid JSON");
-        kreuzberg_free_string(result);
-
-        let json_value: serde_json::Value = serde_json::from_str(&json_str).unwrap();
-        assert!(json_value.is_object(), "Should be a JSON object");
-    }
-}
-
-/// Test format detection from file extension.
-#[test]
-fn test_config_format_detection_from_extension() {
-    let temp_dir = TempDir::new().unwrap();
-
-    let yml_path = temp_dir.path().join("config.yml");
-    let yaml_content = "ocr:\n  enabled: true";
-    fs::write(&yml_path, yaml_content).unwrap();
-
-    unsafe {
-        let path_cstr = CString::new(yml_path.to_str().unwrap()).unwrap();
-        let result = kreuzberg_load_extraction_config_from_file(path_cstr.as_ptr());
-        assert!(!result.is_null(), ".yml extension should be recognized");
-        kreuzberg_free_string(result);
-    }
-
-    let json_path = temp_dir.path().join("config.json");
-    let json_content = r#"{"ocr": {"enabled": true}}"#;
-    fs::write(&json_path, json_content).unwrap();
-
-    unsafe {
-        let path_cstr = CString::new(json_path.to_str().unwrap()).unwrap();
-        let result = kreuzberg_load_extraction_config_from_file(path_cstr.as_ptr());
-        assert!(!result.is_null(), ".json extension should be recognized");
-        kreuzberg_free_string(result);
-    }
-}