kreuzberg 4.0.0.rc1 → 4.0.0.rc2

data/vendor/kreuzberg/src/extraction/pandoc/subprocess.rs

@@ -1,1188 +0,0 @@
-use crate::error::{KreuzbergError, Result};
-#[cfg(feature = "quality")]
-use crate::text::normalize_spaces;
-use serde_json::Value;
-use std::collections::HashMap;
-use std::path::{Path, PathBuf};
-use tokio::fs;
-use tokio::process::Command;
-use tokio::time::{Duration, timeout};
-
-/// Default timeout for Pandoc operations (120 seconds)
-const PANDOC_TIMEOUT_SECONDS: u64 = 120;
-
-/// RAII guard for automatic temporary file cleanup
-struct TempFile {
-    path: PathBuf,
-}
-
-impl TempFile {
-    fn new(path: PathBuf) -> Self {
-        Self { path }
-    }
-}
-
-impl Drop for TempFile {
-    fn drop(&mut self) {
-        let path = self.path.clone();
-        tokio::spawn(async move {
-            let _ = fs::remove_file(&path).await;
-        });
-    }
-}
-
-/// Extract content from file using Pandoc (convert to markdown)
-#[allow(dead_code)]
-pub async fn extract_content(path: &Path, from_format: &str) -> Result<String> {
-    let child = Command::new("pandoc")
-        .arg(path)
-        .arg(format!("--from={}", from_format))
-        .arg("--to=markdown")
-        .arg("--standalone")
-        .arg("--wrap=preserve")
-        .arg("--quiet")
-        .stdout(std::process::Stdio::piped())
-        .stderr(std::process::Stdio::piped())
-        .spawn()
-        .map_err(|e| {
-            // Failed to execute pandoc - this is an IO error (command not found, etc.) ~keep
-            std::io::Error::other(format!("Failed to execute pandoc: {}", e))
-        })?;
-
-    let output = match timeout(Duration::from_secs(PANDOC_TIMEOUT_SECONDS), child.wait_with_output()).await {
-        Ok(Ok(output)) => output,
-        Ok(Err(e)) => return Err(std::io::Error::other(format!("Failed to wait for pandoc: {}", e)).into()),
-        Err(_) => {
-            // Timeout - child was already consumed by wait_with_output(), process will be killed on drop ~keep
-            return Err(KreuzbergError::parsing(format!(
-                "Pandoc content extraction timed out after {} seconds",
-                PANDOC_TIMEOUT_SECONDS
-            )));
-        }
-    };
-
-    if !output.status.success() {
-        let stderr = String::from_utf8_lossy(&output.stderr);
-
-        // Subprocess error analysis - wrap only if format/parsing error detected ~keep
-        let stderr_lower = stderr.to_lowercase();
-        if stderr_lower.contains("format")
-            || stderr_lower.contains("unsupported")
-            || stderr_lower.contains("error:")
-            || stderr_lower.contains("failed")
-        {
-            return Err(KreuzbergError::parsing(format!(
-                "Pandoc format/parsing error: {}",
-                stderr
-            )));
-        }
-
-        // True system error - bubble up as IO error ~keep
-        return Err(std::io::Error::other(format!("Pandoc system error: {}", stderr)).into());
-    }
-
-    let content = String::from_utf8(output.stdout)
-        .map_err(|e| KreuzbergError::parsing(format!("Failed to decode pandoc output: {}", e)))?;
-
-    #[cfg(feature = "quality")]
-    {
-        Ok(normalize_spaces(&content))
-    }
-    #[cfg(not(feature = "quality"))]
-    {
-        Ok(content)
-    }
-}
-
-/// Extract metadata from file using Pandoc JSON output
-#[allow(dead_code)]
-pub async fn extract_metadata(path: &Path, from_format: &str) -> Result<HashMap<String, Value>> {
-    let child = Command::new("pandoc")
-        .arg(path)
-        .arg(format!("--from={}", from_format))
-        .arg("--to=json")
-        .arg("--standalone")
-        .arg("--quiet")
-        .stdout(std::process::Stdio::piped())
-        .stderr(std::process::Stdio::piped())
-        .spawn()
-        .map_err(|e| {
-            // Failed to execute pandoc - this is an IO error (command not found, etc.) ~keep
-            std::io::Error::other(format!("Failed to execute pandoc: {}", e))
-        })?;
-
-    let output = match timeout(Duration::from_secs(PANDOC_TIMEOUT_SECONDS), child.wait_with_output()).await {
-        Ok(Ok(output)) => output,
-        Ok(Err(e)) => return Err(std::io::Error::other(format!("Failed to wait for pandoc: {}", e)).into()),
-        Err(_) => {
-            // Timeout - child was already consumed by wait_with_output(), process will be killed on drop ~keep
-            return Err(KreuzbergError::parsing(format!(
-                "Pandoc metadata extraction timed out after {} seconds",
-                PANDOC_TIMEOUT_SECONDS
-            )));
-        }
-    };
-
-    if !output.status.success() {
-        let stderr = String::from_utf8_lossy(&output.stderr);
-
-        // Subprocess error analysis - wrap only if format/parsing error detected ~keep
-        let stderr_lower = stderr.to_lowercase();
-        if stderr_lower.contains("format")
-            || stderr_lower.contains("unsupported")
-            || stderr_lower.contains("error:")
-            || stderr_lower.contains("failed")
-        {
-            return Err(KreuzbergError::parsing(format!(
-                "Pandoc metadata extraction format/parsing error: {}",
-                stderr
-            )));
-        }
-
-        // True system error - bubble up as IO error ~keep
-        return Err(std::io::Error::other(format!("Pandoc metadata extraction system error: {}", stderr)).into());
-    }
-
-    let json_content = String::from_utf8(output.stdout)
-        .map_err(|e| KreuzbergError::parsing(format!("Failed to decode pandoc JSON output: {}", e)))?;
-
-    let json_data: Value = serde_json::from_str(&json_content)
-        .map_err(|e| KreuzbergError::parsing(format!("Failed to parse pandoc JSON: {}", e)))?;
-
-    extract_metadata_from_json(&json_data)
-}
-
-/// Valid metadata field names (must match Python's _VALID_METADATA_KEYS)
-const VALID_METADATA_KEYS: &[&str] = &[
-    "abstract",
-    "authors",
-    "categories",
-    "character_count",
-    "citations",
-    "code_blocks",
-    "comments",
-    "content",
-    "copyright",
-    "created_at",
-    "created_by",
-    "description",
-    "fonts",
-    "headers",
-    "height",
-    "identifier",
-    "keywords",
-    "languages",
-    "license",
-    "line_count",
-    "links",
-    "modified_at",
-    "modified_by",
-    "organization",
-    "parse_error",
-    "publisher",
-    "references",
-    "sheet_count",
-    "sheet_names",
-    "status",
-    "subject",
-    "subtitle",
-    "summary",
-    "title",
-    "total_cells",
-    "version",
-    "warning",
-    "width",
-    "word_count",
-    "email_from",
-    "email_to",
-    "email_cc",
-    "email_bcc",
-    "date",
-    "attachments",
-    "table_count",
-    "tables_summary",
-    "quality_score",
-    "image_preprocessing",
-    "source_format",
-    "converted_via",
-    "error",
-    "error_context",
-    "json_schema",
-    "notes",
-    "note",
-    "name",
-    "body",
-    "text",
-    "message",
-    "attributes",
-    "token_reduction",
-    "processing_errors",
-    "extraction_error",
-    "element_count",
-    "unique_elements",
-];
-
-/// Extract metadata from Pandoc JSON AST
-pub(crate) fn extract_metadata_from_json(json: &Value) -> Result<HashMap<String, Value>> {
-    let mut metadata = HashMap::new();
-
-    if let Some(meta) = json.get("meta").and_then(|m| m.as_object()) {
-        for (key, value) in meta {
-            let pandoc_key = get_pandoc_key(key);
-            if !VALID_METADATA_KEYS.contains(&pandoc_key.as_str()) {
-                continue;
-            }
-            if let Some(extracted) = extract_meta_value(value) {
-                metadata.insert(pandoc_key, extracted);
-            }
-        }
-    }
-
-    if let Some(blocks) = json.get("blocks").and_then(|b| b.as_array()) {
-        let mut citations = Vec::new();
-        extract_citations_from_blocks(blocks, &mut citations);
-
-        if !citations.is_empty() {
-            if let Some(existing) = metadata.get_mut("citations") {
-                if let Some(arr) = existing.as_array_mut() {
-                    for cite in citations {
-                        if !arr.contains(&Value::String(cite.clone())) {
-                            arr.push(Value::String(cite));
-                        }
-                    }
-                }
-            } else {
-                metadata.insert(
-                    "citations".to_string(),
-                    Value::Array(citations.into_iter().map(Value::String).collect()),
-                );
-            }
-        }
-    }
-
-    if let Some(citations) = json.get("citations").and_then(|c| c.as_array()) {
-        let cite_ids: Vec<String> = citations
-            .iter()
-            .filter_map(|c| c.get("citationId").and_then(|id| id.as_str()).map(String::from))
-            .collect();
-
-        if !cite_ids.is_empty() {
-            metadata.insert(
-                "citations".to_string(),
-                Value::Array(cite_ids.into_iter().map(Value::String).collect()),
-            );
-        }
-    }
-
-    Ok(metadata)
-}
-
-/// Extract markdown content from Pandoc JSON AST
-///
-/// Converts the JSON AST blocks back to markdown format, similar to what
-/// `pandoc --to=markdown` would produce. This allows us to extract both
-/// content and metadata from a single JSON extraction.
-pub(crate) fn extract_content_from_json(json: &Value) -> Result<String> {
-    let mut content = String::new();
-
-    if let Some(meta) = json.get("meta").and_then(|m| m.as_object())
-        && let Some(title_node) = meta.get("title")
-        && let Some(title_value) = extract_meta_value(title_node)
-        && let Some(title_str) = title_value.as_str()
-    {
-        content.push_str(&format!("# {}\n\n", title_str));
-    }
-
-    if let Some(blocks) = json.get("blocks").and_then(|b| b.as_array()) {
-        for block in blocks {
-            if let Some(text) = extract_block_text(block) {
-                if !content.is_empty() && !content.ends_with("\n\n") {
-                    content.push_str("\n\n");
-                }
-                content.push_str(&text);
-            }
-        }
-    }
-
-    Ok(content)
-}
-
-/// Extract text from a Pandoc JSON AST block
-fn extract_block_text(block: &Value) -> Option<String> {
-    let obj = block.as_object()?;
-    let block_type = obj.get("t")?.as_str()?;
-    let content = obj.get("c");
-
-    match block_type {
-        "Para" | "Plain" => {
-            if let Some(inlines) = content.and_then(|c| c.as_array()) {
-                return extract_inlines(inlines).and_then(|v| v.as_str().map(String::from));
-            }
-        }
-        "Header" => {
-            if let Some(arr) = content.and_then(|c| c.as_array())
-                && arr.len() >= 3
-                && let Some(level) = arr[0].as_u64()
-                && let Some(inlines) = arr[2].as_array()
-            {
-                let header_text = extract_inlines(inlines).and_then(|v| v.as_str().map(String::from))?;
-                let prefix = "#".repeat(level as usize);
-                return Some(format!("{} {}", prefix, header_text));
-            }
-        }
-        "CodeBlock" => {
-            if let Some(arr) = content.and_then(|c| c.as_array())
-                && arr.len() >= 2
-                && let Some(code) = arr[1].as_str()
-            {
-                return Some(format!("```\n{}\n```", code));
-            }
-        }
-        "BlockQuote" => {
-            if let Some(blocks) = content.and_then(|c| c.as_array()) {
-                let mut quote_text = String::new();
-                for inner_block in blocks {
-                    if let Some(text) = extract_block_text(inner_block) {
-                        quote_text.push_str("> ");
-                        quote_text.push_str(&text);
-                        quote_text.push('\n');
-                    }
-                }
-                return Some(quote_text.trim_end().to_string());
-            }
-        }
-        "BulletList" => {
-            if let Some(items) = content.and_then(|c| c.as_array()) {
-                let mut list_text = String::new();
-                for item in items {
-                    if let Some(item_blocks) = item.as_array() {
-                        for block in item_blocks {
-                            if let Some(text) = extract_block_text(block) {
-                                list_text.push_str("- ");
-                                list_text.push_str(&text);
-                                list_text.push('\n');
-                            }
-                        }
-                    }
-                }
-                return Some(list_text.trim_end().to_string());
-            }
-        }
-        "OrderedList" => {
-            if let Some(arr) = content.and_then(|c| c.as_array())
-                && arr.len() >= 2
-                && let Some(items) = arr[1].as_array()
-            {
-                let mut list_text = String::new();
-                for (idx, item) in items.iter().enumerate() {
-                    if let Some(item_blocks) = item.as_array() {
-                        for block in item_blocks {
-                            if let Some(text) = extract_block_text(block) {
-                                list_text.push_str(&format!("{}. {}\n", idx + 1, text));
-                            }
-                        }
-                    }
-                }
-                return Some(list_text.trim_end().to_string());
-            }
-        }
-        "HorizontalRule" => {
-            return Some("---".to_string());
-        }
-        _ => {}
-    }
-
-    None
-}
-
-/// Map Pandoc metadata keys to standard keys
-fn get_pandoc_key(key: &str) -> String {
-    match key {
-        "abstract" => "summary".to_string(),
-        "date" => "created_at".to_string(),
-        "contributors" | "author" => "authors".to_string(),
-        "institute" => "organization".to_string(),
-        _ => key.to_string(),
-    }
-}
-
-/// Extract value from Pandoc metadata node
-fn extract_meta_value(node: &Value) -> Option<Value> {
-    if let Some(obj) = node.as_object() {
-        let node_type = obj.get("t")?.as_str()?;
-        let content = obj.get("c");
-
-        match node_type {
-            "MetaString" => {
-                if let Some(s) = content.and_then(|c| c.as_str()) {
-                    return Some(Value::String(s.to_string()));
-                }
-            }
-            "MetaInlines" => {
-                if let Some(inlines) = content.and_then(|c| c.as_array()) {
-                    return extract_inlines(inlines);
-                }
-            }
-            "MetaList" => {
-                if let Some(list) = content.and_then(|c| c.as_array()) {
-                    let mut values = Vec::new();
-                    for item in list {
-                        if let Some(val) = extract_meta_value(item) {
-                            if let Some(arr) = val.as_array() {
-                                values.extend_from_slice(arr);
-                            } else {
-                                values.push(val);
-                            }
-                        }
-                    }
-                    if !values.is_empty() {
-                        return Some(Value::Array(values));
-                    }
-                }
-            }
-            "MetaBlocks" => {
-                if let Some(blocks) = content.and_then(|c| c.as_array()) {
-                    let mut texts = Vec::new();
-                    for block in blocks {
-                        if let Some(block_obj) = block.as_object()
-                            && block_obj.get("t")?.as_str()? == "Para"
-                            && let Some(para_content) = block_obj.get("c").and_then(|c| c.as_array())
-                            && let Some(text) = extract_inlines(para_content)
-                            && let Some(s) = text.as_str()
-                        {
-                            texts.push(s.to_string());
-                        }
-                    }
-                    if !texts.is_empty() {
-                        return Some(Value::String(texts.join(" ")));
-                    }
-                }
-            }
-            "MetaMap" => {
-                if let Some(map) = content.and_then(|c| c.as_object()) {
-                    let mut result = serde_json::Map::new();
-                    for (k, v) in map {
-                        if let Some(val) = extract_meta_value(v) {
-                            result.insert(k.clone(), val);
-                        }
-                    }
-                    if !result.is_empty() {
-                        return Some(Value::Object(result));
-                    }
-                }
-            }
-            _ => {}
-        }
-    }
-
-    None
-}
-
-/// Extract inline text from Pandoc inline nodes
-fn extract_inlines(inlines: &[Value]) -> Option<Value> {
-    let mut texts = Vec::new();
-
-    for inline in inlines {
-        if let Some(text) = extract_inline_text(inline) {
-            texts.push(text);
-        }
-    }
-
-    let result = texts.join("");
-    if result.is_empty() {
-        None
-    } else {
-        Some(Value::String(result))
-    }
-}
-
-/// Extract text from a single inline node
-fn extract_inline_text(node: &Value) -> Option<String> {
-    if let Some(obj) = node.as_object() {
-        let node_type = obj.get("t")?.as_str()?;
-
-        match node_type {
-            "Str" => {
-                return obj.get("c")?.as_str().map(String::from);
-            }
-            "Space" => {
-                return Some(" ".to_string());
-            }
-            "Emph" | "Strong" | "Strikeout" | "Superscript" | "Subscript" | "SmallCaps" => {
-                if let Some(content) = obj.get("c").and_then(|c| c.as_array()) {
-                    return extract_inlines(content).and_then(|v| v.as_str().map(String::from));
-                }
-            }
-            "Code" => {
-                if let Some(arr) = obj.get("c").and_then(|c| c.as_array())
-                    && arr.len() == 2
-                {
-                    return arr[1].as_str().map(String::from);
-                }
-            }
-            "Link" | "Image" => {
-                if let Some(arr) = obj.get("c").and_then(|c| c.as_array())
-                    && arr.len() == 3
-                    && let Some(inlines) = arr[1].as_array()
-                {
-                    return extract_inlines(inlines).and_then(|v| v.as_str().map(String::from));
-                }
-            }
-            "Quoted" => {
-                if let Some(arr) = obj.get("c").and_then(|c| c.as_array())
-                    && arr.len() == 2
-                    && let Some(inlines) = arr[1].as_array()
-                {
-                    return extract_inlines(inlines).and_then(|v| v.as_str().map(String::from));
-                }
-            }
-            "Cite" => {
-                if let Some(arr) = obj.get("c").and_then(|c| c.as_array())
-                    && arr.len() == 2
-                    && let Some(inlines) = arr[1].as_array()
-                {
-                    return extract_inlines(inlines).and_then(|v| v.as_str().map(String::from));
-                }
-            }
-            "Math" => {
-                if let Some(arr) = obj.get("c").and_then(|c| c.as_array())
-                    && arr.len() == 2
-                {
-                    return arr[1].as_str().map(String::from);
-                }
-            }
-            "LineBreak" | "SoftBreak" => {
-                return Some("\n".to_string());
-            }
-            _ => {}
-        }
-    }
-
-    None
-}
-
-/// Extract citations from block nodes
-fn extract_citations_from_blocks(blocks: &[Value], citations: &mut Vec<String>) {
-    for block in blocks {
-        if let Some(obj) = block.as_object() {
-            let block_type = obj.get("t").and_then(|t| t.as_str());
-
-            if block_type == Some("Cite")
-                && let Some(arr) = obj.get("c").and_then(|c| c.as_array())
-                && let Some(cite_list) = arr.first().and_then(|c| c.as_array())
-            {
-                for cite in cite_list {
-                    if let Some(cite_id) = cite.get("citationId").and_then(|id| id.as_str()) {
-                        citations.push(cite_id.to_string());
-                    }
-                }
-            }
-
-            if let Some(content) = obj.get("c") {
-                if let Some(nested_blocks) = content.as_array() {
-                    extract_citations_from_blocks(nested_blocks, citations);
-                } else if let Some(nested_obj) = content.as_object() {
-                    for value in nested_obj.values() {
-                        if let Some(arr) = value.as_array() {
-                            extract_citations_from_blocks(arr, citations);
-                        }
-                    }
-                }
-            }
-        }
-    }
-}
-
-/// Wrapper functions for backwards compatibility
-pub async fn extract_with_pandoc(path: &Path, from_format: &str) -> Result<(String, HashMap<String, Value>)> {
-    let child = Command::new("pandoc")
-        .arg(path)
-        .arg(format!("--from={}", from_format))
-        .arg("--to=json")
-        .arg("--standalone")
-        .arg("--quiet")
-        .stdout(std::process::Stdio::piped())
-        .stderr(std::process::Stdio::piped())
-        .spawn()
-        .map_err(|e| {
-            // Failed to execute pandoc - this is an IO error (command not found, etc.) ~keep
-            std::io::Error::other(format!("Failed to execute pandoc: {}", e))
-        })?;
-
-    let output = match timeout(Duration::from_secs(PANDOC_TIMEOUT_SECONDS), child.wait_with_output()).await {
-        Ok(Ok(output)) => output,
-        Ok(Err(e)) => return Err(std::io::Error::other(format!("Failed to wait for pandoc: {}", e)).into()),
-        Err(_) => {
-            // Timeout - child was already consumed by wait_with_output(), process will be killed on drop ~keep
-            return Err(KreuzbergError::parsing(format!(
-                "Pandoc extraction timed out after {} seconds",
-                PANDOC_TIMEOUT_SECONDS
-            )));
-        }
-    };
-
-    if !output.status.success() {
-        let stderr = String::from_utf8_lossy(&output.stderr);
-
-        // Subprocess error analysis - wrap only if format/parsing error detected ~keep
-        let stderr_lower = stderr.to_lowercase();
-        if stderr_lower.contains("format")
-            || stderr_lower.contains("unsupported")
-            || stderr_lower.contains("error:")
-            || stderr_lower.contains("failed")
-        {
-            return Err(KreuzbergError::parsing(format!(
-                "Pandoc format/parsing error: {}",
-                stderr
-            )));
-        }
-
-        // True system error - bubble up as IO error ~keep
-        return Err(std::io::Error::other(format!("Pandoc system error: {}", stderr)).into());
-    }
-
-    let json_content = String::from_utf8(output.stdout)
-        .map_err(|e| KreuzbergError::parsing(format!("Failed to decode pandoc JSON output: {}", e)))?;
-
-    let json_data: Value = serde_json::from_str(&json_content)
-        .map_err(|e| KreuzbergError::parsing(format!("Failed to parse pandoc JSON: {}", e)))?;
-
-    let content = extract_content_from_json(&json_data)?;
-    let metadata = extract_metadata_from_json(&json_data)?;
-
-    #[cfg(feature = "quality")]
-    {
-        Ok((normalize_spaces(&content), metadata))
-    }
-    #[cfg(not(feature = "quality"))]
-    {
-        Ok((content, metadata))
-    }
-}
-
-pub async fn extract_with_pandoc_from_bytes(
-    bytes: &[u8],
-    from_format: &str,
-    extension: &str,
-) -> Result<(String, HashMap<String, Value>)> {
-    let temp_dir = std::env::temp_dir();
-    let temp_file_path = temp_dir.join(format!(
-        "pandoc_temp_{}_{}.{}",
-        std::process::id(),
-        uuid::Uuid::new_v4(),
-        extension
-    ));
-
-    // RAII guard ensures cleanup on all paths including panic ~keep
-    let _temp_guard = TempFile::new(temp_file_path.clone());
-
-    fs::write(&temp_file_path, bytes).await?;
-
-    extract_with_pandoc(&temp_file_path, from_format).await
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use serde_json::json;
-
-    #[test]
-    fn test_get_pandoc_key() {
-        assert_eq!(get_pandoc_key("abstract"), "summary");
-        assert_eq!(get_pandoc_key("date"), "created_at");
-        assert_eq!(get_pandoc_key("author"), "authors");
-        assert_eq!(get_pandoc_key("contributors"), "authors");
-        assert_eq!(get_pandoc_key("institute"), "organization");
-        assert_eq!(get_pandoc_key("title"), "title");
-    }
-
-    #[test]
-    fn test_extract_meta_value_string() {
-        let node = json!({
-            "t": "MetaString",
-            "c": "Test Title"
-        });
-
-        let result = extract_meta_value(&node).unwrap();
-        assert_eq!(result, Value::String("Test Title".to_string()));
-    }
-
-    #[test]
-    fn test_extract_meta_value_inlines() {
-        let node = json!({
-            "t": "MetaInlines",
-            "c": [
-                {"t": "Str", "c": "Hello"},
-                {"t": "Space"},
-                {"t": "Str", "c": "World"}
-            ]
-        });
-
-        let result = extract_meta_value(&node).unwrap();
-        assert_eq!(result, Value::String("Hello World".to_string()));
-    }
-
-    #[test]
-    fn test_extract_meta_value_list() {
-        let node = json!({
-            "t": "MetaList",
-            "c": [
-                {"t": "MetaString", "c": "Author1"},
-                {"t": "MetaString", "c": "Author2"}
-            ]
-        });
-
-        let result = extract_meta_value(&node).unwrap();
-        assert_eq!(
-            result,
-            Value::Array(vec![
-                Value::String("Author1".to_string()),
-                Value::String("Author2".to_string())
-            ])
-        );
-    }
-
-    #[test]
-    fn test_extract_inline_text_str() {
-        let node = json!({"t": "Str", "c": "Hello"});
-        let result = extract_inline_text(&node).unwrap();
-        assert_eq!(result, "Hello");
-    }
-
-    #[test]
-    fn test_extract_inline_text_space() {
-        let node = json!({"t": "Space"});
-        let result = extract_inline_text(&node).unwrap();
-        assert_eq!(result, " ");
-    }
-
-    #[test]
-    fn test_extract_inline_text_emph() {
-        let node = json!({
-            "t": "Emph",
-            "c": [
-                {"t": "Str", "c": "emphasized"}
-            ]
-        });
-        let result = extract_inline_text(&node).unwrap();
-        assert_eq!(result, "emphasized");
-    }
-
-    #[test]
-    fn test_extract_inline_text_code() {
-        let node = json!({
-            "t": "Code",
-            "c": [["", [], []], "code_snippet"]
-        });
-        let result = extract_inline_text(&node).unwrap();
-        assert_eq!(result, "code_snippet");
-    }
-
-    #[test]
-    fn test_extract_inlines() {
-        let inlines = vec![
-            json!({"t": "Str", "c": "Hello"}),
-            json!({"t": "Space"}),
-            json!({"t": "Emph", "c": [{"t": "Str", "c": "World"}]}),
-        ];
-
-        let result = extract_inlines(&inlines).unwrap();
-        assert_eq!(result, Value::String("Hello World".to_string()));
-    }
-
-    #[test]
-    fn test_extract_citations_from_blocks() {
-        let blocks = vec![json!({
-            "t": "Cite",
-            "c": [
-                [
-                    {"citationId": "cite1"},
-                    {"citationId": "cite2"}
-                ],
-                []
-            ]
-        })];
-
-        let mut citations = Vec::new();
-        extract_citations_from_blocks(&blocks, &mut citations);
-
-        assert_eq!(citations, vec!["cite1", "cite2"]);
-    }
-
-    #[test]
-    fn test_extract_metadata_from_json() {
-        let json = json!({
-            "meta": {
-                "title": {"t": "MetaString", "c": "Test Document"},
-                "author": {"t": "MetaList", "c": [
-                    {"t": "MetaString", "c": "Author One"}
-                ]},
-                "date": {"t": "MetaString", "c": "2024-01-01"}
-            },
-            "blocks": []
-        });
-
-        let metadata = extract_metadata_from_json(&json).unwrap();
-
-        assert_eq!(
-            metadata.get("title").unwrap(),
-            &Value::String("Test Document".to_string())
-        );
-        assert_eq!(
-            metadata.get("authors").unwrap(),
-            &Value::Array(vec![Value::String("Author One".to_string())])
-        );
-        assert_eq!(
-            metadata.get("created_at").unwrap(),
-            &Value::String("2024-01-01".to_string())
-        );
-    }
-
-    #[test]
-    fn test_metadata_field_filtering() {
-        let json = json!({
-            "meta": {
-                "title": {"t": "MetaString", "c": "Valid Title"},
-                "invalid_field": {"t": "MetaString", "c": "Should be filtered"},
-                "random_key": {"t": "MetaString", "c": "Not in valid keys"},
-                "author": {"t": "MetaString", "c": "Valid Author"}
-            },
-            "blocks": []
-        });
-
-        let metadata = extract_metadata_from_json(&json).unwrap();
-
-        assert!(metadata.contains_key("title"));
-        assert!(metadata.contains_key("authors"));
-
-        assert!(!metadata.contains_key("invalid_field"));
-        assert!(!metadata.contains_key("random_key"));
-    }
-
-    #[test]
-    fn test_extract_meta_value_meta_blocks() {
-        let node = json!({
-            "t": "MetaBlocks",
-            "c": [
-                {
-                    "t": "Para",
-                    "c": [
-                        {"t": "Str", "c": "First"},
-                        {"t": "Space"},
-                        {"t": "Str", "c": "paragraph"}
-                    ]
-                },
-                {
-                    "t": "Para",
-                    "c": [
-                        {"t": "Str", "c": "Second"},
-                        {"t": "Space"},
-                        {"t": "Str", "c": "paragraph"}
-                    ]
-                }
-            ]
-        });
-
-        let result = extract_meta_value(&node).unwrap();
-        assert_eq!(result, Value::String("First paragraph Second paragraph".to_string()));
-    }
-
-    #[test]
-    fn test_extract_meta_value_meta_map() {
-        let node = json!({
-            "t": "MetaMap",
-            "c": {
-                "key1": {"t": "MetaString", "c": "value1"},
-                "key2": {"t": "MetaString", "c": "value2"}
-            }
-        });
-
-        let result = extract_meta_value(&node).unwrap();
-        let obj = result.as_object().unwrap();
-        assert_eq!(obj.get("key1").unwrap(), &Value::String("value1".to_string()));
-        assert_eq!(obj.get("key2").unwrap(), &Value::String("value2".to_string()));
-    }
-
-    #[test]
-    fn test_extract_inline_text_strong() {
-        let node = json!({
-            "t": "Strong",
-            "c": [
-                {"t": "Str", "c": "bold"}
-            ]
-        });
-        let result = extract_inline_text(&node).unwrap();
-        assert_eq!(result, "bold");
-    }
-
-    #[test]
-    fn test_extract_inline_text_link() {
-        let node = json!({
-            "t": "Link",
-            "c": [
-                ["", [], []],
-                [{"t": "Str", "c": "link text"}],
-                ["https://example.com", ""]
-            ]
-        });
-        let result = extract_inline_text(&node).unwrap();
-        assert_eq!(result, "link text");
-    }
-
-    #[test]
-    fn test_extract_inline_text_image() {
-        let node = json!({
-            "t": "Image",
-            "c": [
-                ["", [], []],
-                [{"t": "Str", "c": "alt text"}],
-                ["image.png", ""]
-            ]
-        });
-        let result = extract_inline_text(&node).unwrap();
-        assert_eq!(result, "alt text");
-    }
-
-    #[test]
-    fn test_extract_inline_text_quoted() {
-        let node = json!({
-            "t": "Quoted",
-            "c": [
-                {"t": "DoubleQuote"},
-                [{"t": "Str", "c": "quoted text"}]
-            ]
-        });
-        let result = extract_inline_text(&node).unwrap();
-        assert_eq!(result, "quoted text");
-    }
-
-    #[test]
-    fn test_extract_inline_text_cite() {
-        let node = json!({
-            "t": "Cite",
-            "c": [
-                [{"citationId": "cite1"}],
-                [{"t": "Str", "c": "citation text"}]
-            ]
-        });
-        let result = extract_inline_text(&node).unwrap();
-        assert_eq!(result, "citation text");
-    }
-
-    #[test]
-    fn test_extract_inline_text_math() {
-        let node = json!({
-            "t": "Math",
-            "c": [
-                {"t": "InlineMath"},
-                "x^2 + y^2"
-            ]
-        });
-        let result = extract_inline_text(&node).unwrap();
-        assert_eq!(result, "x^2 + y^2");
-    }
-
-    #[test]
-    fn test_extract_inline_text_linebreak() {
-        let node = json!({"t": "LineBreak"});
-        let result = extract_inline_text(&node).unwrap();
-        assert_eq!(result, "\n");
-    }
-
-    #[test]
-    fn test_extract_inline_text_softbreak() {
-        let node = json!({"t": "SoftBreak"});
-        let result = extract_inline_text(&node).unwrap();
-        assert_eq!(result, "\n");
-    }
-
-    #[test]
-    fn test_extract_inline_text_strikeout() {
-        let node = json!({
-            "t": "Strikeout",
-            "c": [{"t": "Str", "c": "deleted"}]
-        });
-        let result = extract_inline_text(&node).unwrap();
-        assert_eq!(result, "deleted");
-    }
-
-    #[test]
-    fn test_extract_inline_text_superscript() {
-        let node = json!({
-            "t": "Superscript",
-            "c": [{"t": "Str", "c": "2"}]
-        });
-        let result = extract_inline_text(&node).unwrap();
-        assert_eq!(result, "2");
-    }
-
-    #[test]
-    fn test_extract_inline_text_subscript() {
-        let node = json!({
-            "t": "Subscript",
-            "c": [{"t": "Str", "c": "i"}]
-        });
-        let result = extract_inline_text(&node).unwrap();
-        assert_eq!(result, "i");
-    }
-
-    #[test]
-    fn test_extract_inline_text_smallcaps() {
-        let node = json!({
-            "t": "SmallCaps",
-            "c": [{"t": "Str", "c": "small"}]
-        });
-        let result = extract_inline_text(&node).unwrap();
-        assert_eq!(result, "small");
-    }
-
-    #[test]
-    fn test_extract_inline_text_unknown_type() {
-        let node = json!({
-            "t": "UnknownType",
-            "c": "should be ignored"
-        });
-        let result = extract_inline_text(&node);
-        assert!(result.is_none());
-    }
-
-    #[test]
-    fn test_extract_citations_from_nested_blocks() {
-        let blocks = vec![json!({
-            "t": "BulletList",
-            "c": [
-                [
-                    {
-                        "t": "Plain",
-                        "c": [
-                            {"t": "Str", "c": "text"}
-                        ]
-                    }
-                ]
-            ]
-        })];
-
-        let mut citations = Vec::new();
-        extract_citations_from_blocks(&blocks, &mut citations);
-
-        assert!(citations.is_empty());
-    }
-
-    #[test]
-    fn test_extract_metadata_from_json_with_citations() {
-        let json = json!({
-            "meta": {
-                "title": {"t": "MetaString", "c": "Paper"}
-            },
-            "citations": [
-                {"citationId": "cite1"},
-                {"citationId": "cite2"}
-            ],
-            "blocks": []
-        });
-
-        let metadata = extract_metadata_from_json(&json).unwrap();
-
-        assert!(metadata.contains_key("citations"));
-        let citations = metadata.get("citations").unwrap().as_array().unwrap();
-        assert_eq!(citations.len(), 2);
-        assert_eq!(citations[0], Value::String("cite1".to_string()));
-        assert_eq!(citations[1], Value::String("cite2".to_string()));
-    }
-
-    #[test]
-    fn test_extract_metadata_from_json_empty_meta() {
-        let json = json!({
-            "meta": {},
-            "blocks": []
-        });
-
-        let metadata = extract_metadata_from_json(&json).unwrap();
-        assert!(metadata.is_empty());
-    }
-
-    #[test]
-    fn test_extract_meta_value_empty_list() {
-        let node = json!({
-            "t": "MetaList",
-            "c": []
-        });
-
-        let result = extract_meta_value(&node);
-        assert!(result.is_none());
-    }
-
-    #[test]
-    fn test_extract_meta_value_empty_map() {
-        let node = json!({
-            "t": "MetaMap",
-            "c": {}
-        });
-
-        let result = extract_meta_value(&node);
-        assert!(result.is_none());
-    }
-
-    #[test]
-    fn test_extract_inlines_empty() {
-        let inlines = vec![];
-        let result = extract_inlines(&inlines);
-        assert!(result.is_none());
-    }
-
-    #[test]
-    fn test_valid_metadata_keys_contains_standard_fields() {
-        assert!(VALID_METADATA_KEYS.contains(&"title"));
-        assert!(VALID_METADATA_KEYS.contains(&"authors"));
-        assert!(VALID_METADATA_KEYS.contains(&"date"));
-        assert!(VALID_METADATA_KEYS.contains(&"keywords"));
-        assert!(VALID_METADATA_KEYS.contains(&"abstract"));
-        assert!(VALID_METADATA_KEYS.contains(&"citations"));
-    }
-
-    #[test]
-    fn test_get_pandoc_key_unmapped() {
-        assert_eq!(get_pandoc_key("title"), "title");
-        assert_eq!(get_pandoc_key("keywords"), "keywords");
-        assert_eq!(get_pandoc_key("custom_field"), "custom_field");
-    }
-
-    #[tokio::test]
-    async fn test_tempfile_raii_cleanup() {
-        use crate::extraction::pandoc::version::validate_pandoc_version;
-
-        if validate_pandoc_version().await.is_err() {
-            return;
-        }
-
-        let temp_path = std::env::temp_dir().join(format!("test_raii_{}.md", uuid::Uuid::new_v4()));
-
-        {
-            let _guard = TempFile::new(temp_path.clone());
-            fs::write(&temp_path, b"test content").await.unwrap();
-            assert!(temp_path.exists());
-        }
-
-        tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;
-
-        assert!(!temp_path.exists());
-    }
-
-    #[tokio::test]
-    async fn test_extract_content_timeout_kills_process() {
-        use crate::extraction::pandoc::version::validate_pandoc_version;
-
-        if validate_pandoc_version().await.is_err() {
-            return;
-        }
-
-        let temp_dir = std::env::temp_dir();
-        let test_file = temp_dir.join(format!("test_timeout_{}.md", uuid::Uuid::new_v4()));
-        fs::write(&test_file, b"# Test\n\nContent").await.unwrap();
-
-        let result = extract_content(&test_file, "markdown").await;
-        assert!(result.is_ok());
-
-        let _ = fs::remove_file(&test_file).await;
-    }
-}