kreuzberg 4.0.0.rc1 → 4.0.0.rc2

data/vendor/kreuzberg/src/extraction/pandoc/version.rs

@@ -1,162 +0,0 @@
-use crate::error::{KreuzbergError, Result};
-use once_cell::sync::OnceCell;
-use regex::Regex;
-use tokio::process::Command;
-
-static PANDOC_VALIDATED: OnceCell<bool> = OnceCell::new();
-
-/// Validate that Pandoc version 2 or above is installed and available
-pub async fn validate_pandoc_version() -> Result<()> {
-    if PANDOC_VALIDATED.get().is_some() {
-        return Ok(());
-    }
-
-    let output = Command::new("pandoc").arg("--version").output().await.map_err(|e| {
-        KreuzbergError::MissingDependency(format!(
-            "Pandoc version 2 or above is required but not found in PATH: {}",
-            e
-        ))
-    })?;
-
-    if !output.status.success() {
-        return Err(KreuzbergError::MissingDependency(
-            "Pandoc version 2 or above is required but command failed".to_string(),
-        ));
-    }
-
-    let stdout = String::from_utf8_lossy(&output.stdout);
-
-    let version = extract_version(&stdout).ok_or_else(|| {
-        KreuzbergError::MissingDependency(format!("Could not parse Pandoc version from output: {}", stdout))
-    })?;
-
-    if version.major < 2 {
-        return Err(KreuzbergError::MissingDependency(format!(
-            "Pandoc version 2 or above is required, found version {}.{}.{}",
-            version.major, version.minor, version.patch
-        )));
-    }
-
-    let _ = PANDOC_VALIDATED.set(true);
-
-    Ok(())
-}
-
-#[derive(Debug, Clone)]
-struct Version {
-    major: u32,
-    minor: u32,
-    patch: u32,
-}
-
-fn extract_version(output: &str) -> Option<Version> {
-    let patterns = [
-        r"pandoc(?:\.exe)?(?:\s+|\s+v|\s+version\s+)(\d+)\.(\d+)(?:\.(\d+))?",
-        r"pandoc\s+\(version\s+(\d+)\.(\d+)(?:\.(\d+))?\)",
-        r"pandoc-(\d+)\.(\d+)(?:\.(\d+))?",
-        r"^(\d+)\.(\d+)(?:\.(\d+)(?:\.(\d+))?)?",
-        r"(?:^|\s)(\d+)\.(\d+)(?:\.(\d+))?(?:\s|$)",
-    ];
-
-    for pattern in &patterns {
-        if let Ok(re) = Regex::new(pattern)
-            && let Some(caps) = re.captures(output)
-        {
-            let major = caps.get(1)?.as_str().parse().ok()?;
-            let minor = caps.get(2)?.as_str().parse().ok()?;
-            let patch = caps.get(3).and_then(|m| m.as_str().parse().ok()).unwrap_or(0);
-
-            return Some(Version { major, minor, patch });
-        }
-    }
-
-    for line in output.lines() {
-        for token in line.split_whitespace() {
-            if let Some(version) = parse_version_token(token) {
-                return Some(version);
-            }
-        }
-    }
-
-    None
-}
-
-fn parse_version_token(token: &str) -> Option<Version> {
-    let parts: Vec<&str> = token.split('.').collect();
-    if parts.len() >= 2
-        && let (Ok(major), Ok(minor)) = (parts[0].parse(), parts[1].parse())
-    {
-        let patch = parts.get(2).and_then(|p| p.parse().ok()).unwrap_or(0);
-        return Some(Version { major, minor, patch });
-    }
-    None
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_extract_version_standard_format() {
-        let output = "pandoc 3.1.2";
-        let version = extract_version(output).unwrap();
-        assert_eq!(version.major, 3);
-        assert_eq!(version.minor, 1);
-        assert_eq!(version.patch, 2);
-    }
-
-    #[test]
-    fn test_extract_version_with_parens() {
-        let output = "pandoc (version 2.19.2)";
-        let version = extract_version(output).unwrap();
-        assert_eq!(version.major, 2);
-        assert_eq!(version.minor, 19);
-    }
-
-    #[test]
-    fn test_extract_version_with_exe() {
-        let output = "pandoc.exe 3.0";
-        let version = extract_version(output).unwrap();
-        assert_eq!(version.major, 3);
-        assert_eq!(version.minor, 0);
-    }
-
-    #[test]
-    fn test_extract_version_multiline() {
-        let output = "pandoc 3.1.2\nCopyright (C) 2006-2023 John MacFarlane";
-        let version = extract_version(output).unwrap();
-        assert_eq!(version.major, 3);
-        assert_eq!(version.minor, 1);
-    }
-
-    #[test]
-    fn test_extract_version_no_patch() {
-        let output = "pandoc 2.5";
-        let version = extract_version(output).unwrap();
-        assert_eq!(version.major, 2);
-        assert_eq!(version.minor, 5);
-        assert_eq!(version.patch, 0);
-    }
-
-    #[test]
-    fn test_parse_version_token() {
-        let version = parse_version_token("2.19.2").unwrap();
-        assert_eq!(version.major, 2);
-        assert_eq!(version.minor, 19);
-        assert_eq!(version.patch, 2);
-    }
-
-    #[test]
-    fn test_parse_version_token_no_patch() {
-        let version = parse_version_token("3.1").unwrap();
-        assert_eq!(version.major, 3);
-        assert_eq!(version.minor, 1);
-        assert_eq!(version.patch, 0);
-    }
-
-    #[test]
-    fn test_parse_version_token_invalid() {
-        let version = parse_version_token("abc");
-        assert!(version.is_none());
-    }
-}

data/vendor/kreuzberg/src/extractors/pandoc.rs

@@ -1,201 +0,0 @@
-//! Pandoc-based extractors for various document formats.
-//!
-//! Supports: DOCX, ODT, EPUB, LaTeX, RST, RTF, and many more formats via Pandoc.
-
-use crate::Result;
-use crate::core::config::ExtractionConfig;
-use crate::extraction::pandoc::extract_bytes_from_mime;
-use crate::plugins::{DocumentExtractor, Plugin};
-use crate::types::{ExtractionResult, Metadata};
-use async_trait::async_trait;
-
-/// Generic Pandoc extractor for all Pandoc-supported formats.
-///
-/// This extractor handles all document formats supported by Pandoc, including:
-/// - Microsoft Word (DOCX)
-/// - OpenDocument Text (ODT)
-/// - EPUB
-/// - LaTeX
-/// - reStructuredText (RST)
-/// - RTF
-/// - And many more
-pub struct PandocExtractor;
-
-impl PandocExtractor {
-    /// Create a new Pandoc extractor.
-    pub fn new() -> Self {
-        Self
-    }
-}
-
-impl Default for PandocExtractor {
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
-impl Plugin for PandocExtractor {
-    fn name(&self) -> &str {
-        "pandoc-extractor"
-    }
-
-    fn version(&self) -> String {
-        env!("CARGO_PKG_VERSION").to_string()
-    }
-
-    fn initialize(&self) -> Result<()> {
-        Ok(())
-    }
-
-    fn shutdown(&self) -> Result<()> {
-        Ok(())
-    }
-
-    fn description(&self) -> &str {
-        "Extracts content from Pandoc-supported formats (DOCX, ODT, EPUB, LaTeX, RST, RTF, etc.)"
-    }
-
-    fn author(&self) -> &str {
-        "Kreuzberg Team"
-    }
-}
-
-#[async_trait]
-impl DocumentExtractor for PandocExtractor {
-    async fn extract_bytes(
-        &self,
-        content: &[u8],
-        mime_type: &str,
-        _config: &ExtractionConfig,
-    ) -> Result<ExtractionResult> {
-        let pandoc_result = extract_bytes_from_mime(content, mime_type).await?;
-
-        let mut additional = std::collections::HashMap::new();
-        for (key, value) in pandoc_result.metadata {
-            additional.insert(key, value);
-        }
-
-        Ok(ExtractionResult {
-            content: pandoc_result.content,
-            mime_type: mime_type.to_string(),
-            metadata: Metadata {
-                additional,
-                ..Default::default()
-            },
-            tables: vec![],
-            detected_languages: None,
-            chunks: None,
-            images: None,
-        })
-    }
-
-    fn supported_mime_types(&self) -> &[&str] {
-        &[
-            "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
-            "application/vnd.oasis.opendocument.text",
-            "application/epub+zip",
-            "application/x-latex",
-            "text/x-tex",
-            "text/x-rst",
-            "text/prs.fallenstein.rst",
-            "application/rtf",
-            "text/rtf",
-            "application/x-typst",
-            "application/x-ipynb+json",
-            "application/x-fictionbook+xml",
-            "text/x-org",
-            "text/x-commonmark",
-            "text/x-gfm",
-            "text/x-multimarkdown",
-            "text/x-markdown-extra",
-            "application/docbook+xml",
-            "application/x-jats+xml",
-            "application/x-opml+xml",
-        ]
-    }
-
-    fn priority(&self) -> i32 {
-        40
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::extraction::pandoc::validate_pandoc_version;
-
-    #[tokio::test]
-    async fn test_pandoc_extractor_plugin_interface() {
-        let extractor = PandocExtractor::new();
-        assert_eq!(extractor.name(), "pandoc-extractor");
-        assert_eq!(extractor.version(), env!("CARGO_PKG_VERSION"));
-        assert_eq!(extractor.priority(), 40);
-        assert!(!extractor.supported_mime_types().is_empty());
-    }
-
-    #[tokio::test]
-    async fn test_pandoc_extractor_supports_docx() {
-        let extractor = PandocExtractor::new();
-        assert!(
-            extractor
-                .supported_mime_types()
-                .contains(&"application/vnd.openxmlformats-officedocument.wordprocessingml.document")
-        );
-    }
-
-    #[tokio::test]
-    async fn test_pandoc_extractor_supports_odt() {
-        let extractor = PandocExtractor::new();
-        assert!(
-            extractor
-                .supported_mime_types()
-                .contains(&"application/vnd.oasis.opendocument.text")
-        );
-    }
-
-    #[tokio::test]
-    async fn test_pandoc_extractor_supports_epub() {
-        let extractor = PandocExtractor::new();
-        assert!(extractor.supported_mime_types().contains(&"application/epub+zip"));
-    }
-
-    #[tokio::test]
-    async fn test_pandoc_extractor_supports_latex() {
-        let extractor = PandocExtractor::new();
-        assert!(extractor.supported_mime_types().contains(&"application/x-latex"));
-    }
-
-    #[tokio::test]
-    async fn test_pandoc_extractor_supports_rst() {
-        let extractor = PandocExtractor::new();
-        assert!(extractor.supported_mime_types().contains(&"text/x-rst"));
-    }
-
-    #[tokio::test]
-    async fn test_pandoc_extractor_markdown() {
-        if validate_pandoc_version().await.is_err() {
-            return;
-        }
-
-        let extractor = PandocExtractor::new();
-        let markdown = b"# Hello World\n\nThis is a test.";
-        let config = ExtractionConfig::default();
-
-        let result = extractor.extract_bytes(markdown, "text/x-rst", &config).await;
-
-        let _ = result;
-    }
-
-    #[tokio::test]
-    async fn test_pandoc_extractor_default() {
-        let extractor = PandocExtractor;
-        assert_eq!(extractor.name(), "pandoc-extractor");
-    }
-
-    #[tokio::test]
-    async fn test_pandoc_extractor_initialize_shutdown() {
-        let extractor = PandocExtractor::new();
-        assert!(extractor.initialize().is_ok());
-        assert!(extractor.shutdown().is_ok());
-    }
-}

data/vendor/kreuzberg/tests/chunking_offset_demo.rs

@@ -1,92 +0,0 @@
-#[cfg(feature = "chunking")]
-#[test]
-fn demonstrate_correct_offset_calculation() {
-    use kreuzberg::chunking::{ChunkerType, ChunkingConfig, chunk_text};
-
-    println!("\n=== Demonstrating Correct Chunking Offset Calculation ===\n");
-
-    let config_with_overlap = ChunkingConfig {
-        max_characters: 20,
-        overlap: 5,
-        trim: false,
-        chunker_type: ChunkerType::Text,
-    };
-
-    let text = "AAAAA BBBBB CCCCC DDDDD EEEEE FFFFF";
-    println!("Text: \"{}\"", text);
-    println!(
-        "Max characters: {}, Overlap: {}\n",
-        config_with_overlap.max_characters, config_with_overlap.overlap
-    );
-
-    let result = chunk_text(text, &config_with_overlap).unwrap();
-
-    println!("WITH OVERLAP (5 chars):");
-    for (i, chunk) in result.chunks.iter().enumerate() {
-        println!(
-            "  Chunk {}: [{:3} - {:3}] = \"{}\"",
-            i,
-            chunk.metadata.char_start,
-            chunk.metadata.char_end,
-            chunk.content.replace('\n', "\\n")
-        );
-    }
-
-    println!("\nOverlap verification:");
-    for i in 0..result.chunks.len() - 1 {
-        let current = &result.chunks[i];
-        let next = &result.chunks[i + 1];
-        let overlap_size = current.metadata.char_end - next.metadata.char_start;
-        println!(
-            "  Chunks {} and {}: overlap = {} chars (next starts at {} while current ends at {})",
-            i,
-            i + 1,
-            overlap_size,
-            next.metadata.char_start,
-            current.metadata.char_end
-        );
-        assert!(
-            overlap_size > 0 && overlap_size <= config_with_overlap.overlap + 10,
-            "Overlap should exist and be reasonable"
-        );
-    }
-
-    println!("\n\n=== Without Overlap ===\n");
-    let config_no_overlap = ChunkingConfig {
-        max_characters: 20,
-        overlap: 0,
-        trim: false,
-        chunker_type: ChunkerType::Text,
-    };
-
-    let result_no_overlap = chunk_text(text, &config_no_overlap).unwrap();
-
-    println!("WITHOUT OVERLAP:");
-    for (i, chunk) in result_no_overlap.chunks.iter().enumerate() {
-        println!(
-            "  Chunk {}: [{:3} - {:3}] = \"{}\"",
-            i,
-            chunk.metadata.char_start,
-            chunk.metadata.char_end,
-            chunk.content.replace('\n', "\\n")
-        );
-    }
-
-    println!("\nAdjacency verification:");
-    for i in 0..result_no_overlap.chunks.len() - 1 {
-        let current = &result_no_overlap.chunks[i];
-        let next = &result_no_overlap.chunks[i + 1];
-        let gap = next.metadata.char_start as i32 - current.metadata.char_end as i32;
-        println!(
-            "  Chunks {} and {}: gap = {} (next starts at {}, current ends at {})",
-            i,
-            i + 1,
-            gap,
-            next.metadata.char_start,
-            current.metadata.char_end
-        );
-        assert!(gap >= 0, "Should have no overlap (gap >= 0)");
-    }
-
-    println!("\n✓ All offset calculations are correct!");
-}