kreuzberg 4.0.0.rc2 → 4.0.1

data/vendor/kreuzberg-ffi/tests.disabled/README.md

@@ -0,0 +1,48 @@
+# Disabled Integration Tests
+
+## Why These Tests Were Disabled
+
+These integration tests were attempting to test the C FFI interface from Rust using `extern "C"` declarations. However, this approach has fundamental issues:
+
+1. **Linking Problem**: Integration tests in `tests/` directory run as separate binaries. When the library is compiled as `cdylib` or `staticlib`, these crate types don't automatically link to Rust integration tests.
+
+2. **Architecture Mismatch**: These tests were trying to use `unsafe extern "C"` blocks to declare functions that are already defined in the same crate. This creates a circular dependency where the test binary expects to dynamically link against symbols that should be statically linked.
+
+3. **Wrong Testing Layer**: C FFI functions should be tested from the actual language bindings (Java, Go, etc.) that consume them, not from Rust integration tests.
+
+## Testing Strategy
+
+The FFI layer is thoroughly tested through:
+
+1. **Unit tests** in `src/lib.rs` that directly call the public Rust FFI functions
+2. **Language binding tests**:
+   - Java: `packages/java/src/test/java/`
+   - Go: `packages/go/v4/*_test.go`
+   - Python: `packages/python/tests/`
+   - TypeScript: `packages/typescript/**/*.spec.ts`
+   - Ruby: `packages/ruby/spec/`
+
+3. **E2E tests** in `e2e/` directory that test the complete integration
+
+## Error That Led to Disabling
+
+```
+error: linking with `cc` failed: exit status: 1
+rust-lld: error: undefined symbol: kreuzberg_last_error
+rust-lld: error: undefined symbol: kreuzberg_load_extraction_config_from_file
+rust-lld: error: undefined symbol: kreuzberg_free_string
+```
+
+This occurred because the integration test binaries couldn't link to the C symbols defined in the FFI crate.
+
+## Future Improvements
+
+If C FFI integration tests are needed from Rust:
+
+1. Create a separate C test harness that links against the compiled `.so`/`.dylib`
+2. Move critical tests into unit tests within `src/lib.rs`
+3. Ensure comprehensive coverage through language binding tests instead
+
+## CI Impact
+
+This change resolves CI failures in run 19607600358 on both macOS-14 and Ubuntu runners.

data/vendor/kreuzberg-ffi/tests.disabled/config_loading_tests.rs

@@ -0,0 +1,299 @@
+//! FFI config loading integration tests.
+//!
+//! Tests the FFI layer for configuration loading from files and discovery.
+
+use std::ffi::{CStr, CString};
+use std::fs;
+use std::os::raw::c_char;
+use tempfile::TempDir;
+
+unsafe extern "C" {
+    fn kreuzberg_load_extraction_config_from_file(file_path: *const c_char) -> *mut c_char;
+    fn kreuzberg_free_string(s: *mut c_char);
+    fn kreuzberg_last_error() -> *const c_char;
+}
+
+/// Helper to convert *const c_char to String
+unsafe fn c_str_to_string(ptr: *const c_char) -> Option<String> {
+    if ptr.is_null() {
+        None
+    } else {
+        unsafe { Some(CStr::from_ptr(ptr).to_string_lossy().into_owned()) }
+    }
+}
+
+/// Helper to get last error message
+unsafe fn get_last_error() -> Option<String> {
+    let error_ptr = unsafe { kreuzberg_last_error() };
+    unsafe { c_str_to_string(error_ptr) }
+}
+
+/// Test successful config loading from TOML file.
+#[test]
+fn test_load_config_from_toml_file_succeeds() {
+    let temp_dir = TempDir::new().unwrap();
+    let config_path = temp_dir.path().join("config.toml");
+
+    let toml_content = r#"
+[ocr]
+enabled = true
+backend = "tesseract"
+
+[chunking]
+max_chars = 1000
+max_overlap = 100
+"#;
+
+    fs::write(&config_path, toml_content).unwrap();
+
+    unsafe {
+        let path_cstr = CString::new(config_path.to_str().unwrap()).unwrap();
+        let result = kreuzberg_load_extraction_config_from_file(path_cstr.as_ptr());
+
+        assert!(!result.is_null(), "Result should not be null");
+
+        let json_str = c_str_to_string(result).expect("Should have valid JSON");
+        kreuzberg_free_string(result);
+
+        let json_value: serde_json::Value = serde_json::from_str(&json_str).unwrap();
+        assert!(json_value.get("ocr").is_some(), "Should have OCR config");
+        assert!(json_value.get("chunking").is_some(), "Should have chunking config");
+    }
+}
+
+/// Test successful config loading from YAML file.
+#[test]
+fn test_load_config_from_yaml_file_succeeds() {
+    let temp_dir = TempDir::new().unwrap();
+    let config_path = temp_dir.path().join("config.yaml");
+
+    let yaml_content = r#"
+ocr:
+  enabled: true
+  backend: tesseract
+chunking:
+  max_chars: 1000
+  max_overlap: 100
+"#;
+
+    fs::write(&config_path, yaml_content).unwrap();
+
+    unsafe {
+        let path_cstr = CString::new(config_path.to_str().unwrap()).unwrap();
+        let result = kreuzberg_load_extraction_config_from_file(path_cstr.as_ptr());
+
+        assert!(!result.is_null(), "Result should not be null");
+
+        let json_str = c_str_to_string(result).expect("Should have valid JSON");
+        kreuzberg_free_string(result);
+
+        let json_value: serde_json::Value = serde_json::from_str(&json_str).unwrap();
+        assert!(json_value.get("ocr").is_some(), "Should have OCR config");
+        assert!(json_value.get("chunking").is_some(), "Should have chunking config");
+    }
+}
+
+/// Test successful config loading from JSON file.
+#[test]
+fn test_load_config_from_json_file_succeeds() {
+    let temp_dir = TempDir::new().unwrap();
+    let config_path = temp_dir.path().join("config.json");
+
+    let json_content = r#"
+{
+  "ocr": {
+    "enabled": true,
+    "backend": "tesseract"
+  },
+  "chunking": {
+    "max_chars": 1000,
+    "max_overlap": 100
+  }
+}
+"#;
+
+    fs::write(&config_path, json_content).unwrap();
+
+    unsafe {
+        let path_cstr = CString::new(config_path.to_str().unwrap()).unwrap();
+        let result = kreuzberg_load_extraction_config_from_file(path_cstr.as_ptr());
+
+        assert!(!result.is_null(), "Result should not be null");
+
+        let json_str = c_str_to_string(result).expect("Should have valid JSON");
+        kreuzberg_free_string(result);
+
+        let json_value: serde_json::Value = serde_json::from_str(&json_str).unwrap();
+        assert!(json_value.get("ocr").is_some(), "Should have OCR config");
+        assert!(json_value.get("chunking").is_some(), "Should have chunking config");
+    }
+}
+
+/// Test config loading fails gracefully with invalid file path.
+#[test]
+fn test_load_config_from_invalid_path_fails_gracefully() {
+    unsafe {
+        let invalid_path = CString::new("/nonexistent/path/config.toml").unwrap();
+        let result = kreuzberg_load_extraction_config_from_file(invalid_path.as_ptr());
+
+        assert!(result.is_null(), "Result should be null for invalid path");
+
+        let error = get_last_error();
+        assert!(error.is_some(), "Should have error message");
+        let error_msg = error.unwrap();
+        assert!(
+            error_msg.contains("No such file") || error_msg.contains("not found"),
+            "Error should mention file not found: {}",
+            error_msg
+        );
+    }
+}
+
+/// Test config loading fails gracefully with null pointer.
+#[test]
+fn test_load_config_from_null_pointer_fails_gracefully() {
+    unsafe {
+        let result = kreuzberg_load_extraction_config_from_file(std::ptr::null());
+
+        assert!(result.is_null(), "Result should be null for null pointer");
+
+        let error = get_last_error();
+        assert!(error.is_some(), "Should have error message");
+        let error_msg = error.unwrap();
+        assert!(
+            error_msg.contains("null") || error_msg.contains("invalid"),
+            "Error should mention null/invalid: {}",
+            error_msg
+        );
+    }
+}
+
+/// Test config loading fails gracefully with malformed TOML.
+#[test]
+fn test_load_config_from_malformed_toml_fails_gracefully() {
+    let temp_dir = TempDir::new().unwrap();
+    let config_path = temp_dir.path().join("config.toml");
+
+    let malformed_toml = r#"
+[ocr
+enabled = true
+"#;
+
+    fs::write(&config_path, malformed_toml).unwrap();
+
+    unsafe {
+        let path_cstr = CString::new(config_path.to_str().unwrap()).unwrap();
+        let result = kreuzberg_load_extraction_config_from_file(path_cstr.as_ptr());
+
+        assert!(result.is_null(), "Result should be null for malformed TOML");
+
+        let error = get_last_error();
+        assert!(error.is_some(), "Should have error message");
+        let error_msg = error.unwrap();
+        assert!(
+            error_msg.contains("parse") || error_msg.contains("invalid") || error_msg.contains("TOML"),
+            "Error should mention parsing issue: {}",
+            error_msg
+        );
+    }
+}
+
+/// Test config loading fails gracefully with malformed JSON.
+#[test]
+fn test_load_config_from_malformed_json_fails_gracefully() {
+    let temp_dir = TempDir::new().unwrap();
+    let config_path = temp_dir.path().join("config.json");
+
+    let malformed_json = r#"
+{
+  "ocr": {
+    "enabled": true
+  }
+  "chunking": {}
+}
+"#;
+
+    fs::write(&config_path, malformed_json).unwrap();
+
+    unsafe {
+        let path_cstr = CString::new(config_path.to_str().unwrap()).unwrap();
+        let result = kreuzberg_load_extraction_config_from_file(path_cstr.as_ptr());
+
+        assert!(result.is_null(), "Result should be null for malformed JSON");
+
+        let error = get_last_error();
+        assert!(error.is_some(), "Should have error message");
+        let error_msg = error.unwrap();
+        assert!(
+            error_msg.contains("parse") || error_msg.contains("invalid") || error_msg.contains("JSON"),
+            "Error should mention parsing issue: {}",
+            error_msg
+        );
+    }
+}
+
+/// Test config loading with invalid UTF-8 in path.
+#[test]
+fn test_load_config_with_invalid_utf8_fails_gracefully() {
+    unsafe {
+        let invalid_bytes = vec![0xFF, 0xFE, 0xFD];
+        let invalid_cstr = CString::new(invalid_bytes).unwrap_or_else(|_| CString::new("").unwrap());
+
+        let result = kreuzberg_load_extraction_config_from_file(invalid_cstr.as_ptr());
+
+        assert!(result.is_null(), "Result should be null for invalid UTF-8");
+
+        let error = get_last_error();
+        assert!(error.is_some(), "Should have error message");
+    }
+}
+
+/// Test config loading with empty file.
+#[test]
+fn test_load_config_from_empty_file_uses_defaults() {
+    let temp_dir = TempDir::new().unwrap();
+    let config_path = temp_dir.path().join("config.toml");
+
+    fs::write(&config_path, "").unwrap();
+
+    unsafe {
+        let path_cstr = CString::new(config_path.to_str().unwrap()).unwrap();
+        let result = kreuzberg_load_extraction_config_from_file(path_cstr.as_ptr());
+
+        assert!(!result.is_null(), "Result should not be null for empty file");
+
+        let json_str = c_str_to_string(result).expect("Should have valid JSON");
+        kreuzberg_free_string(result);
+
+        let json_value: serde_json::Value = serde_json::from_str(&json_str).unwrap();
+        assert!(json_value.is_object(), "Should be a JSON object");
+    }
+}
+
+/// Test format detection from file extension.
+#[test]
+fn test_config_format_detection_from_extension() {
+    let temp_dir = TempDir::new().unwrap();
+
+    let yml_path = temp_dir.path().join("config.yml");
+    let yaml_content = "ocr:\n  enabled: true";
+    fs::write(&yml_path, yaml_content).unwrap();
+
+    unsafe {
+        let path_cstr = CString::new(yml_path.to_str().unwrap()).unwrap();
+        let result = kreuzberg_load_extraction_config_from_file(path_cstr.as_ptr());
+        assert!(!result.is_null(), ".yml extension should be recognized");
+        kreuzberg_free_string(result);
+    }
+
+    let json_path = temp_dir.path().join("config.json");
+    let json_content = r#"{"ocr": {"enabled": true}}"#;
+    fs::write(&json_path, json_content).unwrap();
+
+    unsafe {
+        let path_cstr = CString::new(json_path.to_str().unwrap()).unwrap();
+        let result = kreuzberg_load_extraction_config_from_file(path_cstr.as_ptr());
+        assert!(!result.is_null(), ".json extension should be recognized");
+        kreuzberg_free_string(result);
+    }
+}

data/vendor/kreuzberg-ffi/tests.disabled/config_tests.rs

@@ -0,0 +1,346 @@
+use std::ffi::{CStr, CString};
+use std::fs;
+use std::os::raw::c_char;
+use std::ptr;
+use tempfile::TempDir;
+
+unsafe extern "C" {
+    fn kreuzberg_config_from_file(path: *const c_char) -> *mut std::ffi::c_void;
+
+    fn kreuzberg_config_discover() -> *mut std::ffi::c_void;
+
+    fn kreuzberg_last_error() -> *const c_char;
+
+    fn kreuzberg_free_config(config: *mut std::ffi::c_void);
+}
+
+#[test]
+fn test_config_from_file_toml() {
+    unsafe {
+        let temp_dir = TempDir::new().unwrap();
+        let config_path = temp_dir.path().join("kreuzberg.toml");
+
+        let config_content = r#"
+[ocr]
+enabled = true
+backend = "tesseract"
+
+[chunking]
+enabled = false
+        "#;
+
+        fs::write(&config_path, config_content).unwrap();
+
+        let path_str = CString::new(config_path.to_str().unwrap()).unwrap();
+        let config_ptr = kreuzberg_config_from_file(path_str.as_ptr());
+
+        assert!(!config_ptr.is_null(), "Config should be loaded successfully");
+
+        kreuzberg_free_config(config_ptr);
+    }
+}
+
+#[test]
+fn test_config_from_file_yaml() {
+    unsafe {
+        let temp_dir = TempDir::new().unwrap();
+        let config_path = temp_dir.path().join("kreuzberg.yaml");
+
+        let config_content = r#"
+ocr:
+  enabled: true
+  backend: tesseract
+
+chunking:
+  enabled: false
+        "#;
+
+        fs::write(&config_path, config_content).unwrap();
+
+        let path_str = CString::new(config_path.to_str().unwrap()).unwrap();
+        let config_ptr = kreuzberg_config_from_file(path_str.as_ptr());
+
+        assert!(!config_ptr.is_null(), "Config should be loaded successfully");
+
+        kreuzberg_free_config(config_ptr);
+    }
+}
+
+#[test]
+fn test_config_from_file_json() {
+    unsafe {
+        let temp_dir = TempDir::new().unwrap();
+        let config_path = temp_dir.path().join("kreuzberg.json");
+
+        let config_content = r#"
+{
+  "ocr": {
+    "enabled": true,
+    "backend": "tesseract"
+  },
+  "chunking": {
+    "enabled": false
+  }
+}
+        "#;
+
+        fs::write(&config_path, config_content).unwrap();
+
+        let path_str = CString::new(config_path.to_str().unwrap()).unwrap();
+        let config_ptr = kreuzberg_config_from_file(path_str.as_ptr());
+
+        assert!(!config_ptr.is_null(), "Config should be loaded successfully");
+
+        kreuzberg_free_config(config_ptr);
+    }
+}
+
+#[test]
+fn test_config_from_file_null_path() {
+    unsafe {
+        let config_ptr = kreuzberg_config_from_file(ptr::null());
+
+        assert!(config_ptr.is_null(), "Should return NULL for NULL path");
+
+        let error = kreuzberg_last_error();
+        assert!(!error.is_null());
+        let error_str = CStr::from_ptr(error).to_str().unwrap();
+        assert!(error_str.contains("NULL"), "Error should mention NULL: {}", error_str);
+    }
+}
+
+#[test]
+fn test_config_from_file_nonexistent() {
+    unsafe {
+        let path = CString::new("/nonexistent/path/kreuzberg.toml").unwrap();
+        let config_ptr = kreuzberg_config_from_file(path.as_ptr());
+
+        assert!(config_ptr.is_null(), "Should return NULL for nonexistent file");
+
+        let error = kreuzberg_last_error();
+        assert!(!error.is_null());
+        let error_str = CStr::from_ptr(error).to_str().unwrap();
+        assert!(
+            error_str.contains("IO") || error_str.contains("not found") || error_str.contains("No such"),
+            "Error should indicate file not found: {}",
+            error_str
+        );
+    }
+}
+
+#[test]
+fn test_config_from_file_invalid_toml() {
+    unsafe {
+        let temp_dir = TempDir::new().unwrap();
+        let config_path = temp_dir.path().join("invalid.toml");
+
+        let config_content = r#"
+[ocr
+enabled = true  # Missing closing bracket
+        "#;
+
+        fs::write(&config_path, config_content).unwrap();
+
+        let path_str = CString::new(config_path.to_str().unwrap()).unwrap();
+        let config_ptr = kreuzberg_config_from_file(path_str.as_ptr());
+
+        assert!(config_ptr.is_null(), "Should return NULL for invalid TOML");
+
+        let error = kreuzberg_last_error();
+        assert!(!error.is_null());
+    }
+}
+
+#[test]
+fn test_config_from_file_invalid_json() {
+    unsafe {
+        let temp_dir = TempDir::new().unwrap();
+        let config_path = temp_dir.path().join("invalid.json");
+
+        let config_content = r#"
+{
+  "ocr": {
+    "enabled": true,
+  }  // Trailing comma is invalid in strict JSON
+}
+        "#;
+
+        fs::write(&config_path, config_content).unwrap();
+
+        let path_str = CString::new(config_path.to_str().unwrap()).unwrap();
+        let config_ptr = kreuzberg_config_from_file(path_str.as_ptr());
+
+        assert!(config_ptr.is_null(), "Should return NULL for invalid JSON");
+
+        let error = kreuzberg_last_error();
+        assert!(!error.is_null());
+    }
+}
+
+#[test]
+fn test_config_from_file_no_extension() {
+    unsafe {
+        let temp_dir = TempDir::new().unwrap();
+        let config_path = temp_dir.path().join("kreuzberg");
+
+        fs::write(&config_path, "some content").unwrap();
+
+        let path_str = CString::new(config_path.to_str().unwrap()).unwrap();
+        let config_ptr = kreuzberg_config_from_file(path_str.as_ptr());
+
+        assert!(config_ptr.is_null(), "Should return NULL for file without extension");
+
+        let error = kreuzberg_last_error();
+        assert!(!error.is_null());
+        let error_str = CStr::from_ptr(error).to_str().unwrap();
+        assert!(
+            error_str.contains("extension") || error_str.contains("format"),
+            "Error should mention extension: {}",
+            error_str
+        );
+    }
+}
+
+#[test]
+fn test_config_from_file_invalid_utf8_path() {
+    unsafe {
+        let invalid_path = b"/tmp/test\xFF\xFEinvalid.toml\0";
+
+        let config_ptr = kreuzberg_config_from_file(invalid_path.as_ptr() as *const c_char);
+
+        assert!(config_ptr.is_null(), "Should return NULL for invalid UTF-8 path");
+
+        let error = kreuzberg_last_error();
+        assert!(!error.is_null());
+        let error_str = CStr::from_ptr(error).to_str().unwrap();
+        assert!(error_str.contains("UTF-8"));
+    }
+}
+
+#[test]
+fn test_config_discover_not_found() {
+    unsafe {
+        let temp_dir = TempDir::new().unwrap();
+        let original_dir = std::env::current_dir().unwrap();
+
+        std::env::set_current_dir(&temp_dir).unwrap();
+
+        let config_ptr = kreuzberg_config_discover();
+
+        assert!(config_ptr.is_null(), "Should return NULL when no config found");
+
+        let error = kreuzberg_last_error();
+        if !error.is_null() {
+            let error_str = CStr::from_ptr(error).to_str().unwrap();
+            assert!(
+                error_str.is_empty() || error_str.contains("not found") || error_str.contains("IO"),
+                "Error should be empty or indicate not found: {}",
+                error_str
+            );
+        }
+
+        std::env::set_current_dir(original_dir).unwrap();
+    }
+}
+
+#[test]
+fn test_config_discover_toml() {
+    unsafe {
+        let temp_dir = TempDir::new().unwrap();
+        let config_path = temp_dir.path().join("kreuzberg.toml");
+
+        let config_content = r#"
+[ocr]
+enabled = true
+        "#;
+
+        fs::write(&config_path, config_content).unwrap();
+
+        let original_dir = std::env::current_dir().unwrap();
+        std::env::set_current_dir(&temp_dir).unwrap();
+
+        let config_ptr = kreuzberg_config_discover();
+
+        assert!(!config_ptr.is_null(), "Should discover config in current directory");
+
+        kreuzberg_free_config(config_ptr);
+
+        std::env::set_current_dir(original_dir).unwrap();
+    }
+}
+
+#[test]
+fn test_config_discover_yaml() {
+    unsafe {
+        let temp_dir = TempDir::new().unwrap();
+        let config_path = temp_dir.path().join("kreuzberg.yaml");
+
+        let config_content = r#"
+ocr:
+  enabled: true
+        "#;
+
+        fs::write(&config_path, config_content).unwrap();
+
+        let original_dir = std::env::current_dir().unwrap();
+        std::env::set_current_dir(&temp_dir).unwrap();
+
+        let config_ptr = kreuzberg_config_discover();
+
+        assert!(!config_ptr.is_null(), "Should discover YAML config");
+
+        kreuzberg_free_config(config_ptr);
+
+        std::env::set_current_dir(original_dir).unwrap();
+    }
+}
+
+#[test]
+fn test_config_discover_parent_directory() {
+    unsafe {
+        let temp_dir = TempDir::new().unwrap();
+        let config_path = temp_dir.path().join("kreuzberg.toml");
+
+        let config_content = r#"
+[ocr]
+enabled = true
+        "#;
+
+        fs::write(&config_path, config_content).unwrap();
+
+        let subdir = temp_dir.path().join("subdir");
+        fs::create_dir(&subdir).unwrap();
+
+        let original_dir = std::env::current_dir().unwrap();
+        std::env::set_current_dir(&subdir).unwrap();
+
+        let config_ptr = kreuzberg_config_discover();
+
+        assert!(!config_ptr.is_null(), "Should discover config in parent directory");
+
+        kreuzberg_free_config(config_ptr);
+
+        std::env::set_current_dir(original_dir).unwrap();
+    }
+}
+
+#[test]
+fn test_config_discover_preference_order() {
+    unsafe {
+        let temp_dir = TempDir::new().unwrap();
+
+        fs::write(temp_dir.path().join("kreuzberg.toml"), "[ocr]\nenabled = true").unwrap();
+        fs::write(temp_dir.path().join("kreuzberg.yaml"), "ocr:\n  enabled: false").unwrap();
+
+        let original_dir = std::env::current_dir().unwrap();
+        std::env::set_current_dir(&temp_dir).unwrap();
+
+        let config_ptr = kreuzberg_config_discover();
+
+        assert!(!config_ptr.is_null(), "Should discover a config file");
+
+        kreuzberg_free_config(config_ptr);
+
+        std::env::set_current_dir(original_dir).unwrap();
+    }
+}