kreuzberg 4.0.0.pre.rc.29 → 4.0.0.rc1

data/vendor/kreuzberg-ffi/benches/result_view_benchmark.rs

@@ -1,227 +0,0 @@
-use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main};
-use kreuzberg::types::{Chunk, ChunkMetadata, ExtractionResult, Metadata, PageStructure, PageUnitType};
-use kreuzberg_ffi::{CExtractionResultView, kreuzberg_get_result_view};
-use std::ffi::CString;
-use std::hint;
-use std::mem;
-
-/// Create a test extraction result with configurable complexity
-fn create_test_result(content_size: usize, chunk_count: usize) -> ExtractionResult {
-    let mut metadata = Metadata {
-        title: Some("Benchmark Test Document".to_string()),
-        language: Some("en".to_string()),
-        created_at: Some("2025-01-01T00:00:00Z".to_string()),
-        subject: Some("Performance Testing".to_string()),
-        ..Default::default()
-    };
-
-    let page_structure = PageStructure {
-        total_count: 100,
-        unit_type: PageUnitType::Page,
-        boundaries: None,
-        pages: None,
-    };
-    metadata.pages = Some(page_structure);
-
-    let content = format!(
-        "{}{}{}",
-        "Lorem ipsum dolor sit amet, consectetur adipiscing elit. ",
-        "Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. ".repeat(content_size / 120),
-        "Final sentence."
-    );
-
-    let chunks = if chunk_count > 0 {
-        let chunk_size = content.len() / chunk_count;
-        Some(
-            (0..chunk_count)
-                .map(|i| {
-                    let start = i * chunk_size;
-                    let end = if i == chunk_count - 1 {
-                        content.len()
-                    } else {
-                        (i + 1) * chunk_size
-                    };
-                    Chunk {
-                        content: content[start..end].to_string(),
-                        embedding: None,
-                        metadata: ChunkMetadata {
-                            byte_start: start,
-                            byte_end: end,
-                            token_count: Some((end - start) / 4),
-                            chunk_index: i,
-                            total_chunks: chunk_count,
-                            first_page: Some(1 + (i / 10)),
-                            last_page: Some(1 + (i / 10)),
-                        },
-                    }
-                })
-                .collect(),
-        )
-    } else {
-        None
-    };
-
-    ExtractionResult {
-        content,
-        mime_type: "application/pdf".to_string(),
-        metadata,
-        tables: vec![],
-        detected_languages: Some(vec!["en".to_string(), "de".to_string()]),
-        chunks,
-        images: None,
-        pages: None,
-    }
-}
-
-/// Benchmark: Zero-copy result view creation
-fn bench_zero_copy_view_creation(c: &mut Criterion) {
-    let mut group = c.benchmark_group("result_view_creation");
-
-    for size in [1_000, 10_000, 100_000, 1_000_000].iter() {
-        group.throughput(Throughput::Bytes(*size as u64));
-        group.bench_with_input(BenchmarkId::new("zero_copy", size), size, |b, &size| {
-            let result = create_test_result(size, 10);
-            let result_ptr = &result as *const ExtractionResult;
-
-            b.iter(|| {
-                let mut view: CExtractionResultView = unsafe { mem::zeroed() };
-                unsafe {
-                    kreuzberg_get_result_view(hint::black_box(result_ptr), &mut view);
-                }
-                hint::black_box(view);
-            });
-        });
-    }
-
-    group.finish();
-}
-
-/// Benchmark: Copy-based approach (simulated via CString allocation)
-fn bench_copy_based_approach(c: &mut Criterion) {
-    let mut group = c.benchmark_group("result_copy_based");
-
-    for size in [1_000, 10_000, 100_000, 1_000_000].iter() {
-        group.throughput(Throughput::Bytes(*size as u64));
-        group.bench_with_input(BenchmarkId::new("copy_with_cstring", size), size, |b, &size| {
-            let result = create_test_result(size, 10);
-
-            b.iter(|| {
-                let content_cstr = CString::new(result.content.as_str()).unwrap();
-                let mime_cstr = CString::new(result.mime_type.as_str()).unwrap();
-                let language_cstr = result
-                    .metadata
-                    .language
-                    .as_ref()
-                    .map(|s| CString::new(s.as_str()).unwrap());
-                let title_cstr = result
-                    .metadata
-                    .title
-                    .as_ref()
-                    .map(|s| CString::new(s.as_str()).unwrap());
-
-                hint::black_box(content_cstr);
-                hint::black_box(mime_cstr);
-                hint::black_box(language_cstr);
-                hint::black_box(title_cstr);
-            });
-        });
-    }
-
-    group.finish();
-}
-
-/// Benchmark: Field access from zero-copy view
-fn bench_zero_copy_field_access(c: &mut Criterion) {
-    let mut group = c.benchmark_group("result_view_field_access");
-
-    let result = create_test_result(100_000, 50);
-    let result_ptr = &result as *const ExtractionResult;
-    let mut view: CExtractionResultView = unsafe { mem::zeroed() };
-    unsafe {
-        kreuzberg_get_result_view(result_ptr, &mut view);
-    }
-
-    group.bench_function("access_content_length", |b| {
-        b.iter(|| {
-            hint::black_box(view.content_len);
-        });
-    });
-
-    group.bench_function("access_all_counts", |b| {
-        b.iter(|| {
-            hint::black_box(view.table_count);
-            hint::black_box(view.chunk_count);
-            hint::black_box(view.detected_language_count);
-            hint::black_box(view.image_count);
-            hint::black_box(view.page_count);
-        });
-    });
-
-    group.bench_function("read_content_slice", |b| {
-        b.iter(|| {
-            let content_slice = unsafe { std::slice::from_raw_parts(view.content_ptr, view.content_len) };
-            hint::black_box(content_slice);
-        });
-    });
-
-    group.finish();
-}
-
-/// Benchmark: Multiple views from same result (zero overhead)
-fn bench_multiple_views(c: &mut Criterion) {
-    let result = create_test_result(50_000, 20);
-    let result_ptr = &result as *const ExtractionResult;
-
-    c.bench_function("create_multiple_views", |b| {
-        b.iter(|| {
-            for _ in 0..10 {
-                let mut view: CExtractionResultView = unsafe { mem::zeroed() };
-                unsafe {
-                    kreuzberg_get_result_view(hint::black_box(result_ptr), &mut view);
-                }
-                hint::black_box(view);
-            }
-        });
-    });
-}
-
-/// Benchmark: Zero-copy vs JSON serialization overhead
-fn bench_vs_json_serialization(c: &mut Criterion) {
-    let mut group = c.benchmark_group("result_vs_json");
-
-    for size in [10_000, 100_000].iter() {
-        group.throughput(Throughput::Bytes(*size as u64));
-
-        let result = create_test_result(*size, 10);
-        let result_ptr = &result as *const ExtractionResult;
-
-        group.bench_with_input(BenchmarkId::new("zero_copy", size), size, |b, _| {
-            b.iter(|| {
-                let mut view: CExtractionResultView = unsafe { mem::zeroed() };
-                unsafe {
-                    kreuzberg_get_result_view(hint::black_box(result_ptr), &mut view);
-                }
-                hint::black_box(view);
-            });
-        });
-
-        group.bench_with_input(BenchmarkId::new("json_serialize", size), size, |b, _| {
-            b.iter(|| {
-                let json = serde_json::to_string(&result.metadata).unwrap();
-                hint::black_box(json);
-            });
-        });
-    }
-
-    group.finish();
-}
-
-criterion_group!(
-    benches,
-    bench_zero_copy_view_creation,
-    bench_copy_based_approach,
-    bench_zero_copy_field_access,
-    bench_multiple_views,
-    bench_vs_json_serialization
-);
-criterion_main!(benches);

data/vendor/kreuzberg-ffi/build.rs

@@ -1,168 +0,0 @@
-use std::env;
-use std::fs;
-use std::path::{Path, PathBuf};
-
-fn main() {
-    if let Err(e) = run() {
-        eprintln!("Build script error: {}", e);
-        std::process::exit(1);
-    }
-}
-
-fn run() -> Result<(), String> {
-    let crate_dir = env::var("CARGO_MANIFEST_DIR").map_err(|_| "CARGO_MANIFEST_DIR not set".to_string())?;
-
-    let config =
-        cbindgen::Config::from_file("cbindgen.toml").map_err(|e| format!("Failed to load cbindgen config: {}", e))?;
-
-    cbindgen::generate_with_config(&crate_dir, config)
-        .map_err(|e| format!("Failed to generate C bindings: {}", e))?
-        .write_to_file("kreuzberg.h");
-
-    let pc_template = std::fs::read_to_string("kreuzberg-ffi.pc.in")
-        .map_err(|e| format!("Failed to read pkg-config template: {}", e))?;
-
-    let version = env::var("CARGO_PKG_VERSION").map_err(|_| "CARGO_PKG_VERSION not set".to_string())?;
-
-    let repo_root = Path::new(&crate_dir).parent().and_then(|p| p.parent()).ok_or_else(|| {
-        "CARGO_MANIFEST_DIR did not have expected depth (expected crates/kreuzberg-ffi/...)".to_string()
-    })?;
-
-    let dev_prefix = repo_root.to_string_lossy().replace('\\', "/");
-
-    let target_os = env::var("CARGO_CFG_TARGET_OS").unwrap_or_else(|_| "unknown".to_string());
-    let target_env = env::var("CARGO_CFG_TARGET_ENV").unwrap_or_else(|_| "gnu".to_string());
-
-    let libs_private = match target_os.as_str() {
-        "linux" => "-lpthread -ldl -lm",
-        "macos" => "-framework CoreFoundation -framework Security -lpthread",
-        "windows" => match target_env.as_str() {
-            "msvc" => "-lws2_32 -luserenv -lbcrypt",
-            "gnu" => "-lpthread -lws2_32 -luserenv -lbcrypt -static-libgcc -static-libstdc++",
-            _ => "-lws2_32 -luserenv -lbcrypt",
-        },
-        _ => "",
-    };
-
-    let out_dir = PathBuf::from(env::var("OUT_DIR").map_err(|_| "OUT_DIR not set".to_string())?);
-    let profile_dir = out_dir
-        .ancestors()
-        .nth(3)
-        .ok_or_else(|| "OUT_DIR did not have expected depth (expected target/{debug,release}/build/...)".to_string())?;
-
-    copy_pdfium_to_profile_dir(profile_dir)?;
-
-    let dev_libdir = profile_dir.to_string_lossy().replace('\\', "/");
-    let dev_includedir = format!("{}/crates/kreuzberg-ffi", dev_prefix);
-    let dev_pc = format!(
-        r#"prefix={}
-exec_prefix=${{prefix}}
-libdir={}
-includedir={}
-
-Name: kreuzberg-ffi
-Description: C FFI bindings for Kreuzberg document intelligence library
-Version: {}
-URL: https://kreuzberg.dev
-Libs: -L${{libdir}} -lkreuzberg_ffi
-Libs.private: {}
-Cflags: -I${{includedir}}
-"#,
-        dev_prefix, dev_libdir, dev_includedir, version, libs_private
-    );
-    std::fs::write("kreuzberg-ffi.pc", dev_pc).map_err(|e| format!("Failed to write development pkg-config: {}", e))?;
-
-    let install_pc = pc_template
-        .replace("@PREFIX@", "/usr/local")
-        .replace("@VERSION@", &version)
-        .replace("@LIBS_PRIVATE@", libs_private);
-    std::fs::write("kreuzberg-ffi-install.pc", install_pc)
-        .map_err(|e| format!("Failed to write installation pkg-config: {}", e))?;
-
-    #[cfg(target_os = "macos")]
-    {
-        println!("cargo:rustc-link-arg=-rpath");
-        println!("cargo:rustc-link-arg=@loader_path");
-
-        println!("cargo:rustc-link-arg=-rpath");
-        println!("cargo:rustc-link-arg=@executable_path/../target/release");
-    }
-
-    println!("cargo:rerun-if-changed=cbindgen.toml");
-    println!("cargo:rerun-if-changed=src/lib.rs");
-    println!("cargo:rerun-if-changed=kreuzberg-ffi.pc.in");
-
-    Ok(())
-}
-
-/// Copy bundled PDFium library from kreuzberg build output to profile directory.
-/// This enables Java/other language bindings to find libpdfium.dylib/so/dll at runtime.
-fn copy_pdfium_to_profile_dir(profile_dir: &Path) -> Result<(), String> {
-    let build_dir = profile_dir.join("build");
-
-    if let Ok(entries) = fs::read_dir(&build_dir) {
-        for entry in entries.flatten() {
-            let path = entry.path();
-            if path.is_dir()
-                && path
-                    .file_name()
-                    .is_some_and(|n| n.to_string_lossy().starts_with("kreuzberg-"))
-            {
-                let out_dir = path.join("out");
-                if out_dir.exists() {
-                    if copy_pdfium_from_dir(&out_dir, profile_dir).is_err() {
-                        continue;
-                    } else {
-                        return Ok(());
-                    }
-                }
-            }
-        }
-    }
-
-    eprintln!("Warning: bundled PDFium library not found in build output. Some features may not work.");
-    eprintln!("If PDFium is needed, ensure the 'bundled-pdfium' feature is enabled.");
-
-    Ok(())
-}
-
-/// Copy PDFium library files from source directory to destination.
-fn copy_pdfium_from_dir(src_dir: &Path, dest_dir: &Path) -> Result<(), String> {
-    let entries = fs::read_dir(src_dir).map_err(|e| format!("Failed to read {}: {}", src_dir.display(), e))?;
-
-    for entry in entries.flatten() {
-        let path = entry.path();
-        let file_name = path.file_name().ok_or("No file name")?;
-        let file_name_str = file_name.to_string_lossy();
-
-        if file_name_str.starts_with("libpdfium") || file_name_str.starts_with("pdfium") {
-            let dest_file = dest_dir.join(file_name);
-
-            if dest_file.exists() {
-                eprintln!(
-                    "PDFium library already exists at {}, skipping copy",
-                    dest_file.display()
-                );
-                return Ok(());
-            }
-
-            match fs::copy(&path, &dest_file) {
-                Ok(bytes_copied) => {
-                    eprintln!(
-                        "Copied {} ({} bytes) to {}",
-                        path.display(),
-                        bytes_copied,
-                        dest_file.display()
-                    );
-                    return Ok(());
-                }
-                Err(e) => {
-                    eprintln!("Failed to copy {} to {}: {}", path.display(), dest_file.display(), e);
-                    return Err(format!("Failed to copy PDFium: {}", e));
-                }
-            }
-        }
-    }
-
-    Err(format!("PDFium library not found in {}", src_dir.display()))
-}

data/vendor/kreuzberg-ffi/cbindgen.toml

@@ -1,37 +0,0 @@
-language = "C"
-include_guard = "KREUZBERG_FFI_H"
-pragma_once = true
-header = "/* Auto-generated C bindings for Kreuzberg */"
-autogen_warning = "/* Warning, this file is autogenerated by cbindgen. Don't modify this manually. */"
-documentation = true
-line_length = 100
-after_includes = """
-/**
- * Opaque type for extraction configuration.
- * This is an opaque pointer type - callers should not access its internals.
- */
-typedef struct ExtractionConfig ExtractionConfig;
-
-/**
- * Opaque type for extraction result.
- * This is an opaque pointer type - callers should not access its internals.
- * Use the kreuzberg_result_* accessor functions to extract data.
- */
-typedef struct ExtractionResult ExtractionResult;
-"""
-
-[export.rename]
-ExtractionResult = "ExtractionResult"
-
-[export]
-include = [
-    "kreuzberg_extract_file_sync",
-    "kreuzberg_free_string",
-    "kreuzberg_free_result",
-    "kreuzberg_last_error",
-    "kreuzberg_version",
-]
-
-[parse]
-parse_deps = false
-include = []

data/vendor/kreuzberg-ffi/kreuzberg-ffi.pc.in

@@ -1,12 +0,0 @@
-prefix=@PREFIX@
-exec_prefix=${prefix}
-libdir=${exec_prefix}/lib
-includedir=${prefix}/include
-
-Name: kreuzberg-ffi
-Description: C FFI bindings for Kreuzberg document intelligence library
-Version: @VERSION@
-URL: https://kreuzberg.dev
-Libs: -L${libdir} -lkreuzberg_ffi
-Libs.private: @LIBS_PRIVATE@
-Cflags: -I${includedir}