kreuzberg 4.0.0.pre.rc.6 → 4.0.0.rc1

data/vendor/kreuzberg/src/panic_context.rs

@@ -1,154 +0,0 @@
-use std::any::Any;
-use std::time::{SystemTime, UNIX_EPOCH};
-
-/// Context information captured when a panic occurs.
-///
-/// This struct stores detailed information about where and when a panic happened,
-/// enabling better error reporting across FFI boundaries.
-#[derive(Debug, Clone)]
-pub struct PanicContext {
-    /// Source file where the panic occurred
-    pub file: &'static str,
-    /// Line number where the panic occurred
-    pub line: u32,
-    /// Function name where the panic occurred
-    pub function: &'static str,
-    /// Panic message extracted from the panic payload
-    pub message: String,
-    /// Timestamp when the panic was captured
-    pub timestamp: SystemTime,
-}
-
-impl PanicContext {
-    /// Creates a new PanicContext with the given parameters.
-    ///
-    /// # Arguments
-    ///
-    /// * `file` - Source file path
-    /// * `line` - Line number
-    /// * `function` - Function name
-    /// * `panic_info` - The panic payload to extract message from
-    pub fn new(file: &'static str, line: u32, function: &'static str, panic_info: &dyn Any) -> Self {
-        let timestamp = std::panic::catch_unwind(SystemTime::now).unwrap_or(UNIX_EPOCH);
-
-        Self {
-            file,
-            line,
-            function,
-            message: extract_panic_message(panic_info),
-            timestamp,
-        }
-    }
-
-    /// Formats the panic context as a human-readable string.
-    pub fn format(&self) -> String {
-        format!(
-            "Panic at {}:{}:{} - {}",
-            self.file, self.line, self.function, self.message
-        )
-    }
-}
-
-/// Maximum panic message length to prevent DoS attacks
-const MAX_PANIC_MESSAGE_LEN: usize = 4096;
-
-/// Extracts a human-readable message from a panic payload.
-///
-/// Attempts to downcast the panic payload to common types (String, &str)
-/// to extract a meaningful error message.
-///
-/// Message is truncated to 4KB to prevent DoS attacks via extremely large panic messages.
-///
-/// # Arguments
-///
-/// * `panic_info` - The panic payload from catch_unwind
-///
-/// # Returns
-///
-/// A string representation of the panic message (truncated if necessary)
-pub fn extract_panic_message(panic_info: &dyn Any) -> String {
-    let msg = if let Some(s) = panic_info.downcast_ref::<String>() {
-        s.clone()
-    } else if let Some(s) = panic_info.downcast_ref::<&str>() {
-        (*s).to_string()
-    } else {
-        "Unknown panic payload".to_string()
-    };
-
-    if msg.len() > MAX_PANIC_MESSAGE_LEN {
-        let truncate_at = msg.floor_char_boundary(MAX_PANIC_MESSAGE_LEN);
-        format!("{}... [truncated]", &msg[..truncate_at])
-    } else {
-        msg
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_extract_panic_message_string() {
-        let panic_msg = "test panic".to_string();
-        let msg = extract_panic_message(&panic_msg);
-        assert_eq!(msg, "test panic");
-    }
-
-    #[test]
-    fn test_extract_panic_message_str() {
-        let panic_msg: &str = "test panic";
-        let msg = extract_panic_message(&panic_msg);
-        assert_eq!(msg, "test panic");
-    }
-
-    #[test]
-    fn test_extract_panic_message_unknown() {
-        let panic_msg = 42i32;
-        let msg = extract_panic_message(&panic_msg);
-        assert_eq!(msg, "Unknown panic payload");
-    }
-
-    #[test]
-    fn test_panic_context_format() {
-        let panic_msg = "test error".to_string();
-        let ctx = PanicContext::new("test.rs", 42, "test_function", &panic_msg);
-
-        let formatted = ctx.format();
-        assert!(formatted.contains("test.rs"));
-        assert!(formatted.contains("42"));
-        assert!(formatted.contains("test_function"));
-        assert!(formatted.contains("test error"));
-    }
-
-    #[test]
-    fn test_panic_message_truncation() {
-        let long_msg = "x".repeat(5000);
-        let msg = extract_panic_message(&long_msg);
-        assert!(msg.len() <= MAX_PANIC_MESSAGE_LEN + 20);
-        assert!(msg.ends_with("... [truncated]"));
-    }
-
-    #[test]
-    fn test_panic_message_truncation_utf8_boundary() {
-        let mut msg = "x".repeat(4093);
-        msg.push('🦀');
-        msg.push_str("yyy");
-
-        let truncated = extract_panic_message(&msg);
-
-        assert!(truncated.ends_with("... [truncated]"));
-
-        assert!(std::str::from_utf8(truncated.as_bytes()).is_ok());
-
-        assert!(!truncated.contains("🦀"));
-        assert!(!truncated.contains("yyy"));
-    }
-
-    #[test]
-    fn test_panic_message_no_truncation_needed() {
-        let short_msg = "short".to_string();
-        let msg = extract_panic_message(&short_msg);
-        assert_eq!(msg, "short");
-        assert!(!msg.contains("[truncated]"));
-    }
-}

data/vendor/kreuzberg/tests/api_extract_multipart.rs

@@ -1,52 +0,0 @@
-#![cfg(feature = "api")]
-//! Integration test for the `/extract` API handler using multipart uploads.
-
-use axum::{
-    body::{Body, to_bytes},
-    http::{Request, StatusCode},
-};
-use kreuzberg::{
-    ExtractionConfig,
-    api::{ApiSizeLimits, create_router_with_limits},
-};
-use serde_json::Value;
-use tower::ServiceExt;
-
-#[tokio::test]
-async fn test_extract_accepts_single_file_multipart() {
-    let router = create_router_with_limits(ExtractionConfig::default(), ApiSizeLimits::from_mb(5, 5));
-
-    let boundary = "X-BOUNDARY";
-    let body = format!(
-        "--{boundary}\r\n\
-Content-Disposition: form-data; name=\"files\"; filename=\"test.txt\"\r\n\
-Content-Type: text/plain\r\n\
-\r\n\
-Hello world\r\n\
---{boundary}--\r\n"
-    );
-    let body_bytes = body.into_bytes();
-
-    let request = Request::builder()
-        .method("POST")
-        .uri("/extract")
-        .header("content-type", format!("multipart/form-data; boundary={boundary}"))
-        .header("content-length", body_bytes.len())
-        .body(Body::from(body_bytes))
-        .expect("Failed to build request");
-
-    let response = router.oneshot(request).await.expect("Request failed");
-    assert_eq!(response.status(), StatusCode::OK);
-
-    let bytes = to_bytes(response.into_body(), 1_000_000)
-        .await
-        .expect("Failed to read body");
-    let value: Value = serde_json::from_slice(&bytes).expect("Response JSON parse failed");
-    let content = value
-        .get(0)
-        .and_then(|v| v.get("content"))
-        .and_then(Value::as_str)
-        .expect("Response should include extracted content");
-
-    assert_eq!(content.trim_end_matches('\n'), "Hello world");
-}

data/vendor/kreuzberg/tests/bibtex_parity_test.rs

@@ -1,421 +0,0 @@
-#![cfg(feature = "office")]
-//! Comprehensive test for BibTeX extractor parity with Pandoc
-
-use kreuzberg::core::config::ExtractionConfig;
-use kreuzberg::extractors::BibtexExtractor;
-use kreuzberg::plugins::DocumentExtractor;
-
-mod helpers;
-use helpers::get_test_file_path;
-
-#[tokio::test]
-async fn test_all_entry_types() {
-    let extractor = BibtexExtractor::new();
-
-    let test_cases = vec![
-        (
-            "@article{test, author={John Doe}, title={Test}, journal={Journal}, year={2023}}",
-            "article",
-        ),
-        (
-            "@book{test, author={John Doe}, title={Test}, publisher={Publisher}, year={2023}}",
-            "book",
-        ),
-        (
-            "@inproceedings{test, author={John Doe}, title={Test}, booktitle={Conference}, year={2023}}",
-            "inproceedings",
-        ),
-        (
-            "@phdthesis{test, author={John Doe}, title={Test}, school={University}, year={2023}}",
-            "phdthesis",
-        ),
-        (
-            "@mastersthesis{test, author={John Doe}, title={Test}, school={University}, year={2023}}",
-            "mastersthesis",
-        ),
-        (
-            "@techreport{test, author={John Doe}, title={Test}, institution={Institute}, year={2023}}",
-            "techreport",
-        ),
-        ("@manual{test, title={Test Manual}, year={2023}}", "manual"),
-        ("@misc{test, author={John Doe}, title={Test}, year={2023}}", "misc"),
-        (
-            "@unpublished{test, author={John Doe}, title={Test}, note={Unpublished}, year={2023}}",
-            "unpublished",
-        ),
-        (
-            "@incollection{test, author={John Doe}, title={Test}, booktitle={Book}, publisher={Pub}, year={2023}}",
-            "incollection",
-        ),
-        (
-            "@inbook{test, author={John Doe}, title={Test}, chapter={5}, publisher={Pub}, year={2023}}",
-            "inbook",
-        ),
-        (
-            "@proceedings{test, title={Conference Proceedings}, year={2023}}",
-            "proceedings",
-        ),
-        ("@booklet{test, title={Booklet}, year={2023}}", "booklet"),
-    ];
-
-    for (bibtex_content, expected_type) in test_cases {
-        let config = ExtractionConfig::default();
-        let result = extractor
-            .extract_bytes(bibtex_content.as_bytes(), "application/x-bibtex", &config)
-            .await;
-
-        assert!(result.is_ok(), "Failed to parse {} entry", expected_type);
-        let result = result.unwrap();
-
-        if let Some(entry_types) = result.metadata.additional.get("entry_types") {
-            assert!(entry_types.as_object().is_some(), "Entry types should be an object");
-            println!("Entry type '{}' extracted successfully", expected_type);
-        }
-    }
-}
-
-#[tokio::test]
-async fn test_all_common_fields() {
-    let extractor = BibtexExtractor::new();
-
-    let bibtex_content = r#"
-@article{comprehensive,
-    author = {Smith, John and Doe, Jane},
-    title = {Comprehensive Test},
-    journal = {Test Journal},
-    year = {2023},
-    volume = {42},
-    number = {3},
-    pages = {123--145},
-    month = {June},
-    doi = {10.1234/test.001},
-    url = {https://example.com},
-    issn = {1234-5678},
-    isbn = {978-0-12-345678-9},
-    abstract = {This is an abstract},
-    keywords = {test, bibtex},
-    note = {Additional notes},
-    publisher = {Test Publisher},
-    address = {Test City},
-    edition = {2nd},
-    editor = {Editor Name},
-    series = {Test Series},
-    organization = {Test Org},
-    institution = {Test Institute},
-    school = {Test School},
-    howpublished = {Online},
-    type = {Research Article},
-    chapter = {5},
-    booktitle = {Book Title}
-}
-"#;
-
-    let config = ExtractionConfig::default();
-    let result = extractor
-        .extract_bytes(bibtex_content.as_bytes(), "application/x-bibtex", &config)
-        .await;
-
-    assert!(result.is_ok());
-    let result = result.unwrap();
-
-    let content = &result.content;
-
-    let expected_fields = vec![
-        "author",
-        "title",
-        "journal",
-        "year",
-        "volume",
-        "number",
-        "pages",
-        "month",
-        "doi",
-        "url",
-        "issn",
-        "isbn",
-        "abstract",
-        "keywords",
-        "note",
-        "publisher",
-        "address",
-        "edition",
-        "editor",
-        "series",
-        "organization",
-        "institution",
-        "school",
-        "howpublished",
-        "type",
-        "chapter",
-        "booktitle",
-    ];
-
-    let num_fields = expected_fields.len();
-    for field in expected_fields {
-        assert!(content.contains(field), "Field '{}' should be present in output", field);
-    }
-
-    println!("All {} fields were extracted", num_fields);
-}
-
-#[tokio::test]
-async fn test_author_parsing() {
-    let extractor = BibtexExtractor::new();
-
-    let test_cases = vec![
-        ("author = {John Doe}", vec!["John Doe"]),
-        ("author = {John Doe and Jane Smith}", vec!["John Doe", "Jane Smith"]),
-        ("author = {Smith, John and Doe, Jane}", vec!["Smith, John", "Doe, Jane"]),
-        (
-            "author = {John Doe and Jane Smith and Bob Jones}",
-            vec!["John Doe", "Jane Smith", "Bob Jones"],
-        ),
-        ("author = {van der Berg, Hans}", vec!["van der Berg, Hans"]),
-        ("author = {Smith, Jr., John}", vec!["Smith, Jr., John"]),
-    ];
-
-    for (author_field, expected_authors) in test_cases {
-        let bibtex = format!("@article{{test, {}, title={{Test}}, year={{2023}}}}", author_field);
-
-        let config = ExtractionConfig::default();
-        let result = extractor
-            .extract_bytes(bibtex.as_bytes(), "application/x-bibtex", &config)
-            .await;
-
-        assert!(result.is_ok());
-        let result = result.unwrap();
-
-        if let Some(authors) = result.metadata.additional.get("authors") {
-            let authors_array = authors.as_array().expect("Authors should be an array");
-
-            for expected_author in &expected_authors {
-                let found = authors_array
-                    .iter()
-                    .any(|a| a.as_str().map(|s| s.contains(expected_author)).unwrap_or(false));
-                assert!(
-                    found,
-                    "Expected author '{}' not found in {:?}",
-                    expected_author, authors_array
-                );
-            }
-        }
-    }
-}
-
-#[tokio::test]
-async fn test_special_characters() {
-    let extractor = BibtexExtractor::new();
-
-    let bibtex_content = r#"
-@article{special,
-    author = {M{\"u}ller, Hans and Sch{\"o}n, Anna and Garc{\'\i}a, Jos{\'e}},
-    title = {Special Characters in {BibTeX}: {\"O}berblick},
-    journal = {Test Journal},
-    year = {2022}
-}
-"#;
-
-    let config = ExtractionConfig::default();
-    let result = extractor
-        .extract_bytes(bibtex_content.as_bytes(), "application/x-bibtex", &config)
-        .await;
-
-    assert!(result.is_ok());
-    let result = result.unwrap();
-
-    assert_eq!(
-        result.metadata.additional.get("entry_count"),
-        Some(&serde_json::json!(1))
-    );
-
-    if let Some(authors) = result.metadata.additional.get("authors") {
-        let authors_array = authors.as_array().expect("Authors should be an array");
-        assert!(authors_array.len() >= 3, "Should have 3 authors");
-    }
-}
-
-#[tokio::test]
-async fn test_year_range_extraction() {
-    let extractor = BibtexExtractor::new();
-
-    let bibtex_content = r#"
-@article{old, author={A}, title={Old}, year={1990}}
-@article{mid, author={B}, title={Mid}, year={2005}}
-@article{new, author={C}, title={New}, year={2023}}
-"#;
-
-    let config = ExtractionConfig::default();
-    let result = extractor
-        .extract_bytes(bibtex_content.as_bytes(), "application/x-bibtex", &config)
-        .await;
-
-    assert!(result.is_ok());
-    let result = result.unwrap();
-
-    if let Some(year_range) = result.metadata.additional.get("year_range") {
-        assert_eq!(year_range.get("min"), Some(&serde_json::json!(1990)));
-        assert_eq!(year_range.get("max"), Some(&serde_json::json!(2023)));
-
-        if let Some(years) = year_range.get("years") {
-            let years_array = years.as_array().expect("Years should be an array");
-            assert_eq!(years_array.len(), 3, "Should have 3 unique years");
-        }
-    } else {
-        panic!("Year range not extracted");
-    }
-}
-
-#[tokio::test]
-async fn test_citation_keys_extraction() {
-    let extractor = BibtexExtractor::new();
-
-    let bibtex_content = r#"
-@article{key1, author={A}, title={T1}, year={2023}}
-@book{key2, author={B}, title={T2}, year={2023}}
-@inproceedings{key3, author={C}, title={T3}, year={2023}}
-"#;
-
-    let config = ExtractionConfig::default();
-    let result = extractor
-        .extract_bytes(bibtex_content.as_bytes(), "application/x-bibtex", &config)
-        .await;
-
-    assert!(result.is_ok());
-    let result = result.unwrap();
-
-    if let Some(citation_keys) = result.metadata.additional.get("citation_keys") {
-        let keys_array = citation_keys.as_array().expect("Citation keys should be an array");
-        assert_eq!(keys_array.len(), 3);
-
-        let expected_keys = vec!["key1", "key2", "key3"];
-        for expected_key in expected_keys {
-            let found = keys_array.iter().any(|k| k.as_str() == Some(expected_key));
-            assert!(found, "Citation key '{}' not found", expected_key);
-        }
-    } else {
-        panic!("Citation keys not extracted");
-    }
-}
-
-#[tokio::test]
-async fn test_entry_type_distribution() {
-    let extractor = BibtexExtractor::new();
-
-    let bibtex_content = r#"
-@article{a1, author={A}, title={T1}, year={2023}}
-@article{a2, author={B}, title={T2}, year={2023}}
-@book{b1, author={C}, title={T3}, year={2023}}
-@inproceedings{c1, author={D}, title={T4}, year={2023}}
-@inproceedings{c2, author={E}, title={T5}, year={2023}}
-@inproceedings{c3, author={F}, title={T6}, year={2023}}
-"#;
-
-    let config = ExtractionConfig::default();
-    let result = extractor
-        .extract_bytes(bibtex_content.as_bytes(), "application/x-bibtex", &config)
-        .await;
-
-    assert!(result.is_ok());
-    let result = result.unwrap();
-
-    if let Some(entry_types) = result.metadata.additional.get("entry_types") {
-        let types_obj = entry_types.as_object().expect("Entry types should be an object");
-
-        assert_eq!(types_obj.get("article"), Some(&serde_json::json!(2)));
-        assert_eq!(types_obj.get("book"), Some(&serde_json::json!(1)));
-        assert_eq!(types_obj.get("inproceedings"), Some(&serde_json::json!(3)));
-    } else {
-        panic!("Entry types not extracted");
-    }
-}
-
-#[tokio::test]
-async fn test_unicode_support() {
-    let extractor = BibtexExtractor::new();
-
-    let bibtex_content = r#"
-@article{unicode,
-    author = {Müller, Hans and Søren, Kierkegård},
-    title = {Unicode in BibTeX: A Global Perspective},
-    journal = {International Journal},
-    year = {2023}
-}
-"#;
-
-    let config = ExtractionConfig::default();
-    let result = extractor
-        .extract_bytes(bibtex_content.as_bytes(), "application/x-bibtex", &config)
-        .await;
-
-    assert!(result.is_ok());
-    let result = result.unwrap();
-
-    assert_eq!(
-        result.metadata.additional.get("entry_count"),
-        Some(&serde_json::json!(1))
-    );
-}
-
-#[tokio::test]
-async fn test_empty_fields() {
-    let extractor = BibtexExtractor::new();
-
-    let bibtex_content = r#"
-@article{empty,
-    author = {Smith, John},
-    title = {Test},
-    journal = {},
-    year = {2023},
-    volume = {}
-}
-"#;
-
-    let config = ExtractionConfig::default();
-    let result = extractor
-        .extract_bytes(bibtex_content.as_bytes(), "application/x-bibtex", &config)
-        .await;
-
-    assert!(result.is_ok());
-    let result = result.unwrap();
-    assert_eq!(
-        result.metadata.additional.get("entry_count"),
-        Some(&serde_json::json!(1))
-    );
-}
-
-#[tokio::test]
-async fn test_comprehensive_file() {
-    let extractor = BibtexExtractor::new();
-
-    let fixture_path = get_test_file_path("bibtex/comprehensive.bib");
-    let bibtex_content = std::fs::read(&fixture_path)
-        .unwrap_or_else(|err| panic!("Failed to read test file at {}: {}", fixture_path.display(), err));
-
-    let config = ExtractionConfig::default();
-    let result = extractor
-        .extract_bytes(&bibtex_content, "application/x-bibtex", &config)
-        .await;
-
-    assert!(result.is_ok());
-    let result = result.unwrap();
-
-    assert_eq!(
-        result.metadata.additional.get("entry_count"),
-        Some(&serde_json::json!(20))
-    );
-
-    if let Some(entry_types) = result.metadata.additional.get("entry_types") {
-        let types_obj = entry_types.as_object().expect("Entry types should be an object");
-        assert!(types_obj.len() >= 10, "Should have at least 10 different entry types");
-    }
-
-    if let Some(authors) = result.metadata.additional.get("authors") {
-        let authors_array = authors.as_array().expect("Authors should be an array");
-        assert!(authors_array.len() > 10, "Should have many unique authors");
-    }
-
-    if let Some(year_range) = result.metadata.additional.get("year_range") {
-        assert!(year_range.get("min").is_some());
-        assert!(year_range.get("max").is_some());
-    }
-}