kreuzberg 4.0.0.pre.rc.29 → 4.0.0.rc1

data/vendor/kreuzberg-ffi/tests.disabled/README.md

@@ -1,48 +0,0 @@
-# Disabled Integration Tests
-
-## Why These Tests Were Disabled
-
-These integration tests were attempting to test the C FFI interface from Rust using `extern "C"` declarations. However, this approach has fundamental issues:
-
-1. **Linking Problem**: Integration tests in `tests/` directory run as separate binaries. When the library is compiled as `cdylib` or `staticlib`, these crate types don't automatically link to Rust integration tests.
-
-2. **Architecture Mismatch**: These tests were trying to use `unsafe extern "C"` blocks to declare functions that are already defined in the same crate. This creates a circular dependency where the test binary expects to dynamically link against symbols that should be statically linked.
-
-3. **Wrong Testing Layer**: C FFI functions should be tested from the actual language bindings (Java, Go, etc.) that consume them, not from Rust integration tests.
-
-## Testing Strategy
-
-The FFI layer is thoroughly tested through:
-
-1. **Unit tests** in `src/lib.rs` that directly call the public Rust FFI functions
-2. **Language binding tests**:
-   - Java: `packages/java/src/test/java/`
-   - Go: `packages/go/v4/*_test.go`
-   - Python: `packages/python/tests/`
-   - TypeScript: `packages/typescript/**/*.spec.ts`
-   - Ruby: `packages/ruby/spec/`
-
-3. **E2E tests** in `e2e/` directory that test the complete integration
-
-## Error That Led to Disabling
-
-```
-error: linking with `cc` failed: exit status: 1
-rust-lld: error: undefined symbol: kreuzberg_last_error
-rust-lld: error: undefined symbol: kreuzberg_load_extraction_config_from_file
-rust-lld: error: undefined symbol: kreuzberg_free_string
-```
-
-This occurred because the integration test binaries couldn't link to the C symbols defined in the FFI crate.
-
-## Future Improvements
-
-If C FFI integration tests are needed from Rust:
-
-1. Create a separate C test harness that links against the compiled `.so`/`.dylib`
-2. Move critical tests into unit tests within `src/lib.rs`
-3. Ensure comprehensive coverage through language binding tests instead
-
-## CI Impact
-
-This change resolves CI failures in run 19607600358 on both macOS-14 and Ubuntu runners.

data/vendor/kreuzberg-ffi/tests.disabled/config_loading_tests.rs

@@ -1,299 +0,0 @@
-//! FFI config loading integration tests.
-//!
-//! Tests the FFI layer for configuration loading from files and discovery.
-
-use std::ffi::{CStr, CString};
-use std::fs;
-use std::os::raw::c_char;
-use tempfile::TempDir;
-
-unsafe extern "C" {
-    fn kreuzberg_load_extraction_config_from_file(file_path: *const c_char) -> *mut c_char;
-    fn kreuzberg_free_string(s: *mut c_char);
-    fn kreuzberg_last_error() -> *const c_char;
-}
-
-/// Helper to convert *const c_char to String
-unsafe fn c_str_to_string(ptr: *const c_char) -> Option<String> {
-    if ptr.is_null() {
-        None
-    } else {
-        unsafe { Some(CStr::from_ptr(ptr).to_string_lossy().into_owned()) }
-    }
-}
-
-/// Helper to get last error message
-unsafe fn get_last_error() -> Option<String> {
-    let error_ptr = unsafe { kreuzberg_last_error() };
-    unsafe { c_str_to_string(error_ptr) }
-}
-
-/// Test successful config loading from TOML file.
-#[test]
-fn test_load_config_from_toml_file_succeeds() {
-    let temp_dir = TempDir::new().unwrap();
-    let config_path = temp_dir.path().join("config.toml");
-
-    let toml_content = r#"
-[ocr]
-enabled = true
-backend = "tesseract"
-
-[chunking]
-max_chars = 1000
-max_overlap = 100
-"#;
-
-    fs::write(&config_path, toml_content).unwrap();
-
-    unsafe {
-        let path_cstr = CString::new(config_path.to_str().unwrap()).unwrap();
-        let result = kreuzberg_load_extraction_config_from_file(path_cstr.as_ptr());
-
-        assert!(!result.is_null(), "Result should not be null");
-
-        let json_str = c_str_to_string(result).expect("Should have valid JSON");
-        kreuzberg_free_string(result);
-
-        let json_value: serde_json::Value = serde_json::from_str(&json_str).unwrap();
-        assert!(json_value.get("ocr").is_some(), "Should have OCR config");
-        assert!(json_value.get("chunking").is_some(), "Should have chunking config");
-    }
-}
-
-/// Test successful config loading from YAML file.
-#[test]
-fn test_load_config_from_yaml_file_succeeds() {
-    let temp_dir = TempDir::new().unwrap();
-    let config_path = temp_dir.path().join("config.yaml");
-
-    let yaml_content = r#"
-ocr:
-  enabled: true
-  backend: tesseract
-chunking:
-  max_chars: 1000
-  max_overlap: 100
-"#;
-
-    fs::write(&config_path, yaml_content).unwrap();
-
-    unsafe {
-        let path_cstr = CString::new(config_path.to_str().unwrap()).unwrap();
-        let result = kreuzberg_load_extraction_config_from_file(path_cstr.as_ptr());
-
-        assert!(!result.is_null(), "Result should not be null");
-
-        let json_str = c_str_to_string(result).expect("Should have valid JSON");
-        kreuzberg_free_string(result);
-
-        let json_value: serde_json::Value = serde_json::from_str(&json_str).unwrap();
-        assert!(json_value.get("ocr").is_some(), "Should have OCR config");
-        assert!(json_value.get("chunking").is_some(), "Should have chunking config");
-    }
-}
-
-/// Test successful config loading from JSON file.
-#[test]
-fn test_load_config_from_json_file_succeeds() {
-    let temp_dir = TempDir::new().unwrap();
-    let config_path = temp_dir.path().join("config.json");
-
-    let json_content = r#"
-{
-  "ocr": {
-    "enabled": true,
-    "backend": "tesseract"
-  },
-  "chunking": {
-    "max_chars": 1000,
-    "max_overlap": 100
-  }
-}
-"#;
-
-    fs::write(&config_path, json_content).unwrap();
-
-    unsafe {
-        let path_cstr = CString::new(config_path.to_str().unwrap()).unwrap();
-        let result = kreuzberg_load_extraction_config_from_file(path_cstr.as_ptr());
-
-        assert!(!result.is_null(), "Result should not be null");
-
-        let json_str = c_str_to_string(result).expect("Should have valid JSON");
-        kreuzberg_free_string(result);
-
-        let json_value: serde_json::Value = serde_json::from_str(&json_str).unwrap();
-        assert!(json_value.get("ocr").is_some(), "Should have OCR config");
-        assert!(json_value.get("chunking").is_some(), "Should have chunking config");
-    }
-}
-
-/// Test config loading fails gracefully with invalid file path.
-#[test]
-fn test_load_config_from_invalid_path_fails_gracefully() {
-    unsafe {
-        let invalid_path = CString::new("/nonexistent/path/config.toml").unwrap();
-        let result = kreuzberg_load_extraction_config_from_file(invalid_path.as_ptr());
-
-        assert!(result.is_null(), "Result should be null for invalid path");
-
-        let error = get_last_error();
-        assert!(error.is_some(), "Should have error message");
-        let error_msg = error.unwrap();
-        assert!(
-            error_msg.contains("No such file") || error_msg.contains("not found"),
-            "Error should mention file not found: {}",
-            error_msg
-        );
-    }
-}
-
-/// Test config loading fails gracefully with null pointer.
-#[test]
-fn test_load_config_from_null_pointer_fails_gracefully() {
-    unsafe {
-        let result = kreuzberg_load_extraction_config_from_file(std::ptr::null());
-
-        assert!(result.is_null(), "Result should be null for null pointer");
-
-        let error = get_last_error();
-        assert!(error.is_some(), "Should have error message");
-        let error_msg = error.unwrap();
-        assert!(
-            error_msg.contains("null") || error_msg.contains("invalid"),
-            "Error should mention null/invalid: {}",
-            error_msg
-        );
-    }
-}
-
-/// Test config loading fails gracefully with malformed TOML.
-#[test]
-fn test_load_config_from_malformed_toml_fails_gracefully() {
-    let temp_dir = TempDir::new().unwrap();
-    let config_path = temp_dir.path().join("config.toml");
-
-    let malformed_toml = r#"
-[ocr
-enabled = true
-"#;
-
-    fs::write(&config_path, malformed_toml).unwrap();
-
-    unsafe {
-        let path_cstr = CString::new(config_path.to_str().unwrap()).unwrap();
-        let result = kreuzberg_load_extraction_config_from_file(path_cstr.as_ptr());
-
-        assert!(result.is_null(), "Result should be null for malformed TOML");
-
-        let error = get_last_error();
-        assert!(error.is_some(), "Should have error message");
-        let error_msg = error.unwrap();
-        assert!(
-            error_msg.contains("parse") || error_msg.contains("invalid") || error_msg.contains("TOML"),
-            "Error should mention parsing issue: {}",
-            error_msg
-        );
-    }
-}
-
-/// Test config loading fails gracefully with malformed JSON.
-#[test]
-fn test_load_config_from_malformed_json_fails_gracefully() {
-    let temp_dir = TempDir::new().unwrap();
-    let config_path = temp_dir.path().join("config.json");
-
-    let malformed_json = r#"
-{
-  "ocr": {
-    "enabled": true
-  }
-  "chunking": {}
-}
-"#;
-
-    fs::write(&config_path, malformed_json).unwrap();
-
-    unsafe {
-        let path_cstr = CString::new(config_path.to_str().unwrap()).unwrap();
-        let result = kreuzberg_load_extraction_config_from_file(path_cstr.as_ptr());
-
-        assert!(result.is_null(), "Result should be null for malformed JSON");
-
-        let error = get_last_error();
-        assert!(error.is_some(), "Should have error message");
-        let error_msg = error.unwrap();
-        assert!(
-            error_msg.contains("parse") || error_msg.contains("invalid") || error_msg.contains("JSON"),
-            "Error should mention parsing issue: {}",
-            error_msg
-        );
-    }
-}
-
-/// Test config loading with invalid UTF-8 in path.
-#[test]
-fn test_load_config_with_invalid_utf8_fails_gracefully() {
-    unsafe {
-        let invalid_bytes = vec![0xFF, 0xFE, 0xFD];
-        let invalid_cstr = CString::new(invalid_bytes).unwrap_or_else(|_| CString::new("").unwrap());
-
-        let result = kreuzberg_load_extraction_config_from_file(invalid_cstr.as_ptr());
-
-        assert!(result.is_null(), "Result should be null for invalid UTF-8");
-
-        let error = get_last_error();
-        assert!(error.is_some(), "Should have error message");
-    }
-}
-
-/// Test config loading with empty file.
-#[test]
-fn test_load_config_from_empty_file_uses_defaults() {
-    let temp_dir = TempDir::new().unwrap();
-    let config_path = temp_dir.path().join("config.toml");
-
-    fs::write(&config_path, "").unwrap();
-
-    unsafe {
-        let path_cstr = CString::new(config_path.to_str().unwrap()).unwrap();
-        let result = kreuzberg_load_extraction_config_from_file(path_cstr.as_ptr());
-
-        assert!(!result.is_null(), "Result should not be null for empty file");
-
-        let json_str = c_str_to_string(result).expect("Should have valid JSON");
-        kreuzberg_free_string(result);
-
-        let json_value: serde_json::Value = serde_json::from_str(&json_str).unwrap();
-        assert!(json_value.is_object(), "Should be a JSON object");
-    }
-}
-
-/// Test format detection from file extension.
-#[test]
-fn test_config_format_detection_from_extension() {
-    let temp_dir = TempDir::new().unwrap();
-
-    let yml_path = temp_dir.path().join("config.yml");
-    let yaml_content = "ocr:\n  enabled: true";
-    fs::write(&yml_path, yaml_content).unwrap();
-
-    unsafe {
-        let path_cstr = CString::new(yml_path.to_str().unwrap()).unwrap();
-        let result = kreuzberg_load_extraction_config_from_file(path_cstr.as_ptr());
-        assert!(!result.is_null(), ".yml extension should be recognized");
-        kreuzberg_free_string(result);
-    }
-
-    let json_path = temp_dir.path().join("config.json");
-    let json_content = r#"{"ocr": {"enabled": true}}"#;
-    fs::write(&json_path, json_content).unwrap();
-
-    unsafe {
-        let path_cstr = CString::new(json_path.to_str().unwrap()).unwrap();
-        let result = kreuzberg_load_extraction_config_from_file(path_cstr.as_ptr());
-        assert!(!result.is_null(), ".json extension should be recognized");
-        kreuzberg_free_string(result);
-    }
-}

data/vendor/kreuzberg-ffi/tests.disabled/config_tests.rs

@@ -1,346 +0,0 @@
-use std::ffi::{CStr, CString};
-use std::fs;
-use std::os::raw::c_char;
-use std::ptr;
-use tempfile::TempDir;
-
-unsafe extern "C" {
-    fn kreuzberg_config_from_file(path: *const c_char) -> *mut std::ffi::c_void;
-
-    fn kreuzberg_config_discover() -> *mut std::ffi::c_void;
-
-    fn kreuzberg_last_error() -> *const c_char;
-
-    fn kreuzberg_free_config(config: *mut std::ffi::c_void);
-}
-
-#[test]
-fn test_config_from_file_toml() {
-    unsafe {
-        let temp_dir = TempDir::new().unwrap();
-        let config_path = temp_dir.path().join("kreuzberg.toml");
-
-        let config_content = r#"
-[ocr]
-enabled = true
-backend = "tesseract"
-
-[chunking]
-enabled = false
-        "#;
-
-        fs::write(&config_path, config_content).unwrap();
-
-        let path_str = CString::new(config_path.to_str().unwrap()).unwrap();
-        let config_ptr = kreuzberg_config_from_file(path_str.as_ptr());
-
-        assert!(!config_ptr.is_null(), "Config should be loaded successfully");
-
-        kreuzberg_free_config(config_ptr);
-    }
-}
-
-#[test]
-fn test_config_from_file_yaml() {
-    unsafe {
-        let temp_dir = TempDir::new().unwrap();
-        let config_path = temp_dir.path().join("kreuzberg.yaml");
-
-        let config_content = r#"
-ocr:
-  enabled: true
-  backend: tesseract
-
-chunking:
-  enabled: false
-        "#;
-
-        fs::write(&config_path, config_content).unwrap();
-
-        let path_str = CString::new(config_path.to_str().unwrap()).unwrap();
-        let config_ptr = kreuzberg_config_from_file(path_str.as_ptr());
-
-        assert!(!config_ptr.is_null(), "Config should be loaded successfully");
-
-        kreuzberg_free_config(config_ptr);
-    }
-}
-
-#[test]
-fn test_config_from_file_json() {
-    unsafe {
-        let temp_dir = TempDir::new().unwrap();
-        let config_path = temp_dir.path().join("kreuzberg.json");
-
-        let config_content = r#"
-{
-  "ocr": {
-    "enabled": true,
-    "backend": "tesseract"
-  },
-  "chunking": {
-    "enabled": false
-  }
-}
-        "#;
-
-        fs::write(&config_path, config_content).unwrap();
-
-        let path_str = CString::new(config_path.to_str().unwrap()).unwrap();
-        let config_ptr = kreuzberg_config_from_file(path_str.as_ptr());
-
-        assert!(!config_ptr.is_null(), "Config should be loaded successfully");
-
-        kreuzberg_free_config(config_ptr);
-    }
-}
-
-#[test]
-fn test_config_from_file_null_path() {
-    unsafe {
-        let config_ptr = kreuzberg_config_from_file(ptr::null());
-
-        assert!(config_ptr.is_null(), "Should return NULL for NULL path");
-
-        let error = kreuzberg_last_error();
-        assert!(!error.is_null());
-        let error_str = CStr::from_ptr(error).to_str().unwrap();
-        assert!(error_str.contains("NULL"), "Error should mention NULL: {}", error_str);
-    }
-}
-
-#[test]
-fn test_config_from_file_nonexistent() {
-    unsafe {
-        let path = CString::new("/nonexistent/path/kreuzberg.toml").unwrap();
-        let config_ptr = kreuzberg_config_from_file(path.as_ptr());
-
-        assert!(config_ptr.is_null(), "Should return NULL for nonexistent file");
-
-        let error = kreuzberg_last_error();
-        assert!(!error.is_null());
-        let error_str = CStr::from_ptr(error).to_str().unwrap();
-        assert!(
-            error_str.contains("IO") || error_str.contains("not found") || error_str.contains("No such"),
-            "Error should indicate file not found: {}",
-            error_str
-        );
-    }
-}
-
-#[test]
-fn test_config_from_file_invalid_toml() {
-    unsafe {
-        let temp_dir = TempDir::new().unwrap();
-        let config_path = temp_dir.path().join("invalid.toml");
-
-        let config_content = r#"
-[ocr
-enabled = true  # Missing closing bracket
-        "#;
-
-        fs::write(&config_path, config_content).unwrap();
-
-        let path_str = CString::new(config_path.to_str().unwrap()).unwrap();
-        let config_ptr = kreuzberg_config_from_file(path_str.as_ptr());
-
-        assert!(config_ptr.is_null(), "Should return NULL for invalid TOML");
-
-        let error = kreuzberg_last_error();
-        assert!(!error.is_null());
-    }
-}
-
-#[test]
-fn test_config_from_file_invalid_json() {
-    unsafe {
-        let temp_dir = TempDir::new().unwrap();
-        let config_path = temp_dir.path().join("invalid.json");
-
-        let config_content = r#"
-{
-  "ocr": {
-    "enabled": true,
-  }  // Trailing comma is invalid in strict JSON
-}
-        "#;
-
-        fs::write(&config_path, config_content).unwrap();
-
-        let path_str = CString::new(config_path.to_str().unwrap()).unwrap();
-        let config_ptr = kreuzberg_config_from_file(path_str.as_ptr());
-
-        assert!(config_ptr.is_null(), "Should return NULL for invalid JSON");
-
-        let error = kreuzberg_last_error();
-        assert!(!error.is_null());
-    }
-}
-
-#[test]
-fn test_config_from_file_no_extension() {
-    unsafe {
-        let temp_dir = TempDir::new().unwrap();
-        let config_path = temp_dir.path().join("kreuzberg");
-
-        fs::write(&config_path, "some content").unwrap();
-
-        let path_str = CString::new(config_path.to_str().unwrap()).unwrap();
-        let config_ptr = kreuzberg_config_from_file(path_str.as_ptr());
-
-        assert!(config_ptr.is_null(), "Should return NULL for file without extension");
-
-        let error = kreuzberg_last_error();
-        assert!(!error.is_null());
-        let error_str = CStr::from_ptr(error).to_str().unwrap();
-        assert!(
-            error_str.contains("extension") || error_str.contains("format"),
-            "Error should mention extension: {}",
-            error_str
-        );
-    }
-}
-
-#[test]
-fn test_config_from_file_invalid_utf8_path() {
-    unsafe {
-        let invalid_path = b"/tmp/test\xFF\xFEinvalid.toml\0";
-
-        let config_ptr = kreuzberg_config_from_file(invalid_path.as_ptr() as *const c_char);
-
-        assert!(config_ptr.is_null(), "Should return NULL for invalid UTF-8 path");
-
-        let error = kreuzberg_last_error();
-        assert!(!error.is_null());
-        let error_str = CStr::from_ptr(error).to_str().unwrap();
-        assert!(error_str.contains("UTF-8"));
-    }
-}
-
-#[test]
-fn test_config_discover_not_found() {
-    unsafe {
-        let temp_dir = TempDir::new().unwrap();
-        let original_dir = std::env::current_dir().unwrap();
-
-        std::env::set_current_dir(&temp_dir).unwrap();
-
-        let config_ptr = kreuzberg_config_discover();
-
-        assert!(config_ptr.is_null(), "Should return NULL when no config found");
-
-        let error = kreuzberg_last_error();
-        if !error.is_null() {
-            let error_str = CStr::from_ptr(error).to_str().unwrap();
-            assert!(
-                error_str.is_empty() || error_str.contains("not found") || error_str.contains("IO"),
-                "Error should be empty or indicate not found: {}",
-                error_str
-            );
-        }
-
-        std::env::set_current_dir(original_dir).unwrap();
-    }
-}
-
-#[test]
-fn test_config_discover_toml() {
-    unsafe {
-        let temp_dir = TempDir::new().unwrap();
-        let config_path = temp_dir.path().join("kreuzberg.toml");
-
-        let config_content = r#"
-[ocr]
-enabled = true
-        "#;
-
-        fs::write(&config_path, config_content).unwrap();
-
-        let original_dir = std::env::current_dir().unwrap();
-        std::env::set_current_dir(&temp_dir).unwrap();
-
-        let config_ptr = kreuzberg_config_discover();
-
-        assert!(!config_ptr.is_null(), "Should discover config in current directory");
-
-        kreuzberg_free_config(config_ptr);
-
-        std::env::set_current_dir(original_dir).unwrap();
-    }
-}
-
-#[test]
-fn test_config_discover_yaml() {
-    unsafe {
-        let temp_dir = TempDir::new().unwrap();
-        let config_path = temp_dir.path().join("kreuzberg.yaml");
-
-        let config_content = r#"
-ocr:
-  enabled: true
-        "#;
-
-        fs::write(&config_path, config_content).unwrap();
-
-        let original_dir = std::env::current_dir().unwrap();
-        std::env::set_current_dir(&temp_dir).unwrap();
-
-        let config_ptr = kreuzberg_config_discover();
-
-        assert!(!config_ptr.is_null(), "Should discover YAML config");
-
-        kreuzberg_free_config(config_ptr);
-
-        std::env::set_current_dir(original_dir).unwrap();
-    }
-}
-
-#[test]
-fn test_config_discover_parent_directory() {
-    unsafe {
-        let temp_dir = TempDir::new().unwrap();
-        let config_path = temp_dir.path().join("kreuzberg.toml");
-
-        let config_content = r#"
-[ocr]
-enabled = true
-        "#;
-
-        fs::write(&config_path, config_content).unwrap();
-
-        let subdir = temp_dir.path().join("subdir");
-        fs::create_dir(&subdir).unwrap();
-
-        let original_dir = std::env::current_dir().unwrap();
-        std::env::set_current_dir(&subdir).unwrap();
-
-        let config_ptr = kreuzberg_config_discover();
-
-        assert!(!config_ptr.is_null(), "Should discover config in parent directory");
-
-        kreuzberg_free_config(config_ptr);
-
-        std::env::set_current_dir(original_dir).unwrap();
-    }
-}
-
-#[test]
-fn test_config_discover_preference_order() {
-    unsafe {
-        let temp_dir = TempDir::new().unwrap();
-
-        fs::write(temp_dir.path().join("kreuzberg.toml"), "[ocr]\nenabled = true").unwrap();
-        fs::write(temp_dir.path().join("kreuzberg.yaml"), "ocr:\n  enabled: false").unwrap();
-
-        let original_dir = std::env::current_dir().unwrap();
-        std::env::set_current_dir(&temp_dir).unwrap();
-
-        let config_ptr = kreuzberg_config_discover();
-
-        assert!(!config_ptr.is_null(), "Should discover a config file");
-
-        kreuzberg_free_config(config_ptr);
-
-        std::env::set_current_dir(original_dir).unwrap();
-    }
-}