kreuzberg 4.0.8 → 4.1.0

data/vendor/kreuzberg/src/extraction/archive/sevenz.rs

@@ -0,0 +1,98 @@
+//! 7Z archive extraction.
+//!
+//! Provides functions for extracting metadata and text content from 7Z archives.
+
+use super::{ArchiveEntry, ArchiveMetadata, TEXT_EXTENSIONS};
+use crate::error::{KreuzbergError, Result};
+use sevenz_rust2::{ArchiveReader, Password};
+use std::collections::HashMap;
+use std::io::Cursor;
+
+/// Extract metadata from a 7z archive.
+///
+/// # Arguments
+///
+/// * `bytes` - The 7z archive bytes
+///
+/// # Returns
+///
+/// Returns `ArchiveMetadata` containing:
+/// - Format: "7Z"
+/// - File list with paths, sizes, and directory flags
+/// - Total file count
+/// - Total uncompressed size
+///
+/// # Errors
+///
+/// Returns an error if the 7z archive cannot be read or parsed.
+pub fn extract_7z_metadata(bytes: &[u8]) -> Result<ArchiveMetadata> {
+    let cursor = Cursor::new(bytes);
+    let archive = ArchiveReader::new(cursor, Password::empty())
+        .map_err(|e| KreuzbergError::parsing(format!("Failed to read 7z archive: {}", e)))?;
+
+    let mut file_list = Vec::new();
+    let mut total_size = 0u64;
+
+    for entry in &archive.archive().files {
+        let path = entry.name().to_string();
+        let size = entry.size();
+        let is_dir = entry.is_directory();
+
+        if !is_dir {
+            total_size += size;
+        }
+
+        file_list.push(ArchiveEntry { path, size, is_dir });
+    }
+
+    let file_count = file_list.len();
+
+    Ok(ArchiveMetadata {
+        format: "7Z".to_string(),
+        file_list,
+        file_count,
+        total_size,
+    })
+}
+
+/// Extract text content from files within a 7z archive.
+///
+/// Only extracts files with common text extensions: .txt, .md, .json, .xml, .html, .csv, .log, .yaml, .toml
+///
+/// # Arguments
+///
+/// * `bytes` - The 7z archive bytes
+///
+/// # Returns
+///
+/// Returns a `HashMap` mapping file paths to their text content.
+/// Binary files and files with non-text extensions are excluded.
+///
+/// # Errors
+///
+/// Returns an error if the 7z archive cannot be read or parsed.
+pub fn extract_7z_text_content(bytes: &[u8]) -> Result<HashMap<String, String>> {
+    let cursor = Cursor::new(bytes);
+    let mut archive = ArchiveReader::new(cursor, Password::empty())
+        .map_err(|e| KreuzbergError::parsing(format!("Failed to read 7z archive: {}", e)))?;
+
+    let mut contents = HashMap::new();
+
+    archive
+        .for_each_entries(|entry, reader| {
+            let path = entry.name().to_string();
+
+            if !entry.is_directory() && TEXT_EXTENSIONS.iter().any(|ext| path.to_lowercase().ends_with(ext)) {
+                let mut content = Vec::new();
+                if let Ok(_) = reader.read_to_end(&mut content)
+                    && let Ok(text) = String::from_utf8(content)
+                {
+                    contents.insert(path, text);
+                }
+            }
+            Ok(true)
+        })
+        .map_err(|e| KreuzbergError::parsing(format!("Failed to read 7z entries: {}", e)))?;
+
+    Ok(contents)
+}

data/vendor/kreuzberg/src/extraction/archive/tar.rs

@@ -0,0 +1,118 @@
+//! TAR archive extraction.
+//!
+//! Provides functions for extracting metadata and text content from TAR archives.
+//! Supports plain TAR as well as compressed variants (TAR.GZ, TAR.BZ2).
+
+use super::{ArchiveEntry, ArchiveMetadata, TEXT_EXTENSIONS};
+use crate::error::{KreuzbergError, Result};
+use std::collections::HashMap;
+use std::io::{Cursor, Read};
+use tar::Archive as TarArchive;
+
+/// Extract metadata from a TAR archive.
+///
+/// # Arguments
+///
+/// * `bytes` - The TAR archive bytes (can be compressed with gzip or bzip2)
+///
+/// # Returns
+///
+/// Returns `ArchiveMetadata` containing:
+/// - Format: "TAR"
+/// - File list with paths, sizes, and directory flags
+/// - Total file count
+/// - Total uncompressed size
+///
+/// # Errors
+///
+/// Returns an error if the TAR archive cannot be read or parsed.
+pub fn extract_tar_metadata(bytes: &[u8]) -> Result<ArchiveMetadata> {
+    let cursor = Cursor::new(bytes);
+    let mut archive = TarArchive::new(cursor);
+
+    let estimated_entries = bytes.len().saturating_div(512).max(16);
+    let mut file_list = Vec::with_capacity(estimated_entries);
+    let mut total_size = 0u64;
+    let mut file_count = 0;
+
+    let entries = archive
+        .entries()
+        .map_err(|e| KreuzbergError::parsing(format!("Failed to read TAR archive: {}", e)))?;
+
+    for entry_result in entries {
+        let entry = entry_result.map_err(|e| KreuzbergError::parsing(format!("Failed to read TAR entry: {}", e)))?;
+
+        let path = entry
+            .path()
+            .map_err(|e| KreuzbergError::parsing(format!("Failed to read TAR entry path: {}", e)))?
+            .to_string_lossy()
+            .to_string();
+
+        let size = entry.size();
+        let is_dir = entry.header().entry_type().is_dir();
+
+        if !is_dir {
+            total_size += size;
+        }
+
+        file_count += 1;
+        file_list.push(ArchiveEntry { path, size, is_dir });
+    }
+
+    Ok(ArchiveMetadata {
+        format: "TAR".to_string(),
+        file_list,
+        file_count,
+        total_size,
+    })
+}
+
+/// Extract text content from files within a TAR archive.
+///
+/// Only extracts files with common text extensions: .txt, .md, .json, .xml, .html, .csv, .log, .yaml, .toml
+///
+/// # Arguments
+///
+/// * `bytes` - The TAR archive bytes (can be compressed with gzip or bzip2)
+///
+/// # Returns
+///
+/// Returns a `HashMap` mapping file paths to their text content.
+/// Binary files and files with non-text extensions are excluded.
+///
+/// # Errors
+///
+/// Returns an error if the TAR archive cannot be read or parsed.
+pub fn extract_tar_text_content(bytes: &[u8]) -> Result<HashMap<String, String>> {
+    let cursor = Cursor::new(bytes);
+    let mut archive = TarArchive::new(cursor);
+
+    let estimated_text_files = bytes.len().saturating_div(1024 * 10).min(100);
+    let mut contents = HashMap::with_capacity(estimated_text_files.max(2));
+
+    let entries = archive
+        .entries()
+        .map_err(|e| KreuzbergError::parsing(format!("Failed to read TAR archive: {}", e)))?;
+
+    for entry_result in entries {
+        let mut entry =
+            entry_result.map_err(|e| KreuzbergError::parsing(format!("Failed to read TAR entry: {}", e)))?;
+
+        let path = entry
+            .path()
+            .map_err(|e| KreuzbergError::parsing(format!("Failed to read TAR entry path: {}", e)))?
+            .to_string_lossy()
+            .to_string();
+
+        if !entry.header().entry_type().is_dir() && TEXT_EXTENSIONS.iter().any(|ext| path.to_lowercase().ends_with(ext))
+        {
+            let estimated_size = (entry.size().min(10 * 1024 * 1024)) as usize;
+            let mut content = String::with_capacity(estimated_size);
+            if entry.read_to_string(&mut content).is_ok() {
+                contents.insert(path, content);
+            }
+        }
+    }
+
+    Ok(contents)
+}

data/vendor/kreuzberg/src/extraction/archive/zip.rs

@@ -0,0 +1,101 @@
+//! ZIP archive extraction.
+//!
+//! Provides functions for extracting metadata and text content from ZIP archives.
+
+use super::{ArchiveEntry, ArchiveMetadata, TEXT_EXTENSIONS};
+use crate::error::{KreuzbergError, Result};
+use std::collections::HashMap;
+use std::io::{Cursor, Read};
+use zip::ZipArchive;
+
+/// Extract metadata from a ZIP archive.
+///
+/// # Arguments
+///
+/// * `bytes` - The ZIP archive bytes
+///
+/// # Returns
+///
+/// Returns `ArchiveMetadata` containing:
+/// - Format: "ZIP"
+/// - File list with paths, sizes, and directory flags
+/// - Total file count
+/// - Total uncompressed size
+///
+/// # Errors
+///
+/// Returns an error if the ZIP archive cannot be read or parsed.
+pub fn extract_zip_metadata(bytes: &[u8]) -> Result<ArchiveMetadata> {
+    let cursor = Cursor::new(bytes);
+    let mut archive =
+        ZipArchive::new(cursor).map_err(|e| KreuzbergError::parsing(format!("Failed to read ZIP archive: {}", e)))?;
+
+    let mut file_list = Vec::with_capacity(archive.len());
+    let mut total_size = 0u64;
+
+    for i in 0..archive.len() {
+        let file = archive
+            .by_index(i)
+            .map_err(|e| KreuzbergError::parsing(format!("Failed to read ZIP entry: {}", e)))?;
+
+        let path = file.name().to_string();
+        let size = file.size();
+        let is_dir = file.is_dir();
+
+        if !is_dir {
+            total_size += size;
+        }
+
+        file_list.push(ArchiveEntry { path, size, is_dir });
+    }
+
+    Ok(ArchiveMetadata {
+        format: "ZIP".to_string(),
+        file_list,
+        file_count: archive.len(),
+        total_size,
+    })
+}
+
+/// Extract text content from files within a ZIP archive.
+///
+/// Only extracts files with common text extensions: .txt, .md, .json, .xml, .html, .csv, .log, .yaml, .toml
+///
+/// # Arguments
+///
+/// * `bytes` - The ZIP archive bytes
+///
+/// # Returns
+///
+/// Returns a `HashMap` mapping file paths to their text content.
+/// Binary files and files with non-text extensions are excluded.
+///
+/// # Errors
+///
+/// Returns an error if the ZIP archive cannot be read or parsed.
+pub fn extract_zip_text_content(bytes: &[u8]) -> Result<HashMap<String, String>> {
+    let cursor = Cursor::new(bytes);
+    let mut archive =
+        ZipArchive::new(cursor).map_err(|e| KreuzbergError::parsing(format!("Failed to read ZIP archive: {}", e)))?;
+
+    let estimated_text_files = archive.len().saturating_mul(3).saturating_div(10).max(2);
+    let mut contents = HashMap::with_capacity(estimated_text_files);
+
+    for i in 0..archive.len() {
+        let mut file = archive
+            .by_index(i)
+            .map_err(|e| KreuzbergError::parsing(format!("Failed to read ZIP entry: {}", e)))?;
+
+        let path = file.name().to_string();
+
+        if !file.is_dir() && TEXT_EXTENSIONS.iter().any(|ext| path.to_lowercase().ends_with(ext)) {
+            let estimated_size = (file.size() as usize).min(10 * 1024 * 1024);
+            let mut content = String::with_capacity(estimated_size);
+            if file.read_to_string(&mut content).is_ok() {
+                contents.insert(path, content);
+            }
+        }
+    }
+
+    Ok(contents)
+}