RubyGems - kreuzberg - Versions diffs - 4.0.7 → 4.1.0 - Mend

kreuzberg 4.0.7 → 4.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (308) hide show

checksums.yaml +4 -4
data/Gemfile.lock +24 -16
data/README.md +4 -1
data/ext/kreuzberg_rb/native/Cargo.lock +94 -98
data/ext/kreuzberg_rb/native/Cargo.toml +4 -2
data/ext/kreuzberg_rb/native/src/batch.rs +139 -0
data/ext/kreuzberg_rb/native/src/config/mod.rs +10 -0
data/ext/kreuzberg_rb/native/src/config/types.rs +1058 -0
data/ext/kreuzberg_rb/native/src/error_handling.rs +125 -0
data/ext/kreuzberg_rb/native/src/extraction.rs +79 -0
data/ext/kreuzberg_rb/native/src/gc_guarded_value.rs +35 -0
data/ext/kreuzberg_rb/native/src/helpers.rs +176 -0
data/ext/kreuzberg_rb/native/src/lib.rs +342 -3622
data/ext/kreuzberg_rb/native/src/metadata.rs +34 -0
data/ext/kreuzberg_rb/native/src/plugins/mod.rs +92 -0
data/ext/kreuzberg_rb/native/src/plugins/ocr_backend.rs +159 -0
data/ext/kreuzberg_rb/native/src/plugins/post_processor.rs +126 -0
data/ext/kreuzberg_rb/native/src/plugins/validator.rs +99 -0
data/ext/kreuzberg_rb/native/src/result.rs +326 -0
data/ext/kreuzberg_rb/native/src/validation.rs +4 -0
data/lib/kreuzberg/config.rb +66 -0
data/lib/kreuzberg/result.rb +107 -2
data/lib/kreuzberg/types.rb +104 -0
data/lib/kreuzberg/version.rb +1 -1
data/lib/kreuzberg.rb +0 -4
data/sig/kreuzberg.rbs +105 -1
data/vendor/Cargo.toml +3 -3
data/vendor/kreuzberg/Cargo.toml +4 -3
data/vendor/kreuzberg/README.md +1 -1
data/vendor/kreuzberg/src/api/config.rs +69 -0
data/vendor/kreuzberg/src/api/handlers.rs +99 -2
data/vendor/kreuzberg/src/api/mod.rs +14 -7
data/vendor/kreuzberg/src/api/router.rs +214 -0
data/vendor/kreuzberg/src/api/startup.rs +243 -0
data/vendor/kreuzberg/src/api/types.rs +78 -0
data/vendor/kreuzberg/src/cache/cleanup.rs +277 -0
data/vendor/kreuzberg/src/cache/core.rs +428 -0
data/vendor/kreuzberg/src/cache/mod.rs +21 -843
data/vendor/kreuzberg/src/cache/utilities.rs +156 -0
data/vendor/kreuzberg/src/chunking/boundaries.rs +301 -0
data/vendor/kreuzberg/src/chunking/builder.rs +294 -0
data/vendor/kreuzberg/src/chunking/config.rs +52 -0
data/vendor/kreuzberg/src/chunking/core.rs +1017 -0
data/vendor/kreuzberg/src/chunking/mod.rs +14 -2211
data/vendor/kreuzberg/src/chunking/processor.rs +10 -0
data/vendor/kreuzberg/src/chunking/validation.rs +686 -0
data/vendor/kreuzberg/src/core/config/extraction/core.rs +169 -0
data/vendor/kreuzberg/src/core/config/extraction/env.rs +179 -0
data/vendor/kreuzberg/src/core/config/extraction/loaders.rs +204 -0
data/vendor/kreuzberg/src/core/config/extraction/mod.rs +42 -0
data/vendor/kreuzberg/src/core/config/extraction/types.rs +93 -0
data/vendor/kreuzberg/src/core/config/formats.rs +135 -0
data/vendor/kreuzberg/src/core/config/mod.rs +20 -0
data/vendor/kreuzberg/src/core/config/ocr.rs +73 -0
data/vendor/kreuzberg/src/core/config/page.rs +57 -0
data/vendor/kreuzberg/src/core/config/pdf.rs +111 -0
data/vendor/kreuzberg/src/core/config/processing.rs +312 -0
data/vendor/kreuzberg/src/core/config_validation/dependencies.rs +187 -0
data/vendor/kreuzberg/src/core/config_validation/mod.rs +386 -0
data/vendor/kreuzberg/src/core/config_validation/sections.rs +401 -0
data/vendor/kreuzberg/src/core/extractor/batch.rs +246 -0
data/vendor/kreuzberg/src/core/extractor/bytes.rs +116 -0
data/vendor/kreuzberg/src/core/extractor/file.rs +240 -0
data/vendor/kreuzberg/src/core/extractor/helpers.rs +71 -0
data/vendor/kreuzberg/src/core/extractor/legacy.rs +62 -0
data/vendor/kreuzberg/src/core/extractor/mod.rs +490 -0
data/vendor/kreuzberg/src/core/extractor/sync.rs +208 -0
data/vendor/kreuzberg/src/core/mod.rs +4 -1
data/vendor/kreuzberg/src/core/pipeline/cache.rs +60 -0
data/vendor/kreuzberg/src/core/pipeline/execution.rs +89 -0
data/vendor/kreuzberg/src/core/pipeline/features.rs +108 -0
data/vendor/kreuzberg/src/core/pipeline/format.rs +392 -0
data/vendor/kreuzberg/src/core/pipeline/initialization.rs +67 -0
data/vendor/kreuzberg/src/core/pipeline/mod.rs +135 -0
data/vendor/kreuzberg/src/core/pipeline/tests.rs +975 -0
data/vendor/kreuzberg/src/core/server_config/env.rs +90 -0
data/vendor/kreuzberg/src/core/server_config/loader.rs +202 -0
data/vendor/kreuzberg/src/core/server_config/mod.rs +380 -0
data/vendor/kreuzberg/src/core/server_config/tests/basic_tests.rs +124 -0
data/vendor/kreuzberg/src/core/server_config/tests/env_tests.rs +216 -0
data/vendor/kreuzberg/src/core/server_config/tests/file_loading_tests.rs +341 -0
data/vendor/kreuzberg/src/core/server_config/tests/mod.rs +5 -0
data/vendor/kreuzberg/src/core/server_config/validation.rs +17 -0
data/vendor/kreuzberg/src/embeddings.rs +136 -13
data/vendor/kreuzberg/src/extraction/{archive.rs → archive/mod.rs} +45 -239
data/vendor/kreuzberg/src/extraction/archive/sevenz.rs +98 -0
data/vendor/kreuzberg/src/extraction/archive/tar.rs +118 -0
data/vendor/kreuzberg/src/extraction/archive/zip.rs +101 -0
data/vendor/kreuzberg/src/extraction/html/converter.rs +592 -0
data/vendor/kreuzberg/src/extraction/html/image_handling.rs +95 -0
data/vendor/kreuzberg/src/extraction/html/mod.rs +53 -0
data/vendor/kreuzberg/src/extraction/html/processor.rs +659 -0
data/vendor/kreuzberg/src/extraction/html/stack_management.rs +103 -0
data/vendor/kreuzberg/src/extraction/html/types.rs +28 -0
data/vendor/kreuzberg/src/extraction/mod.rs +6 -2
data/vendor/kreuzberg/src/extraction/pptx/container.rs +159 -0
data/vendor/kreuzberg/src/extraction/pptx/content_builder.rs +168 -0
data/vendor/kreuzberg/src/extraction/pptx/elements.rs +132 -0
data/vendor/kreuzberg/src/extraction/pptx/image_handling.rs +57 -0
data/vendor/kreuzberg/src/extraction/pptx/metadata.rs +160 -0
data/vendor/kreuzberg/src/extraction/pptx/mod.rs +558 -0
data/vendor/kreuzberg/src/extraction/pptx/parser.rs +379 -0
data/vendor/kreuzberg/src/extraction/transform/content.rs +205 -0
data/vendor/kreuzberg/src/extraction/transform/elements.rs +211 -0
data/vendor/kreuzberg/src/extraction/transform/mod.rs +480 -0
data/vendor/kreuzberg/src/extraction/transform/types.rs +27 -0
data/vendor/kreuzberg/src/extractors/archive.rs +2 -0
data/vendor/kreuzberg/src/extractors/bibtex.rs +2 -0
data/vendor/kreuzberg/src/extractors/djot_format/attributes.rs +134 -0
data/vendor/kreuzberg/src/extractors/djot_format/conversion.rs +223 -0
data/vendor/kreuzberg/src/extractors/djot_format/extractor.rs +172 -0
data/vendor/kreuzberg/src/extractors/djot_format/mod.rs +24 -0
data/vendor/kreuzberg/src/extractors/djot_format/parsing/block_handlers.rs +271 -0
data/vendor/kreuzberg/src/extractors/djot_format/parsing/content_extraction.rs +257 -0
data/vendor/kreuzberg/src/extractors/djot_format/parsing/event_handlers.rs +101 -0
data/vendor/kreuzberg/src/extractors/djot_format/parsing/inline_handlers.rs +201 -0
data/vendor/kreuzberg/src/extractors/djot_format/parsing/mod.rs +16 -0
data/vendor/kreuzberg/src/extractors/djot_format/parsing/state.rs +78 -0
data/vendor/kreuzberg/src/extractors/djot_format/parsing/table_extraction.rs +68 -0
data/vendor/kreuzberg/src/extractors/djot_format/parsing/text_extraction.rs +61 -0
data/vendor/kreuzberg/src/extractors/djot_format/rendering.rs +452 -0
data/vendor/kreuzberg/src/extractors/docbook.rs +2 -0
data/vendor/kreuzberg/src/extractors/docx.rs +12 -1
data/vendor/kreuzberg/src/extractors/email.rs +2 -0
data/vendor/kreuzberg/src/extractors/epub/content.rs +333 -0
data/vendor/kreuzberg/src/extractors/epub/metadata.rs +137 -0
data/vendor/kreuzberg/src/extractors/epub/mod.rs +186 -0
data/vendor/kreuzberg/src/extractors/epub/parsing.rs +86 -0
data/vendor/kreuzberg/src/extractors/excel.rs +4 -0
data/vendor/kreuzberg/src/extractors/fictionbook.rs +2 -0
data/vendor/kreuzberg/src/extractors/frontmatter_utils.rs +466 -0
data/vendor/kreuzberg/src/extractors/html.rs +80 -8
data/vendor/kreuzberg/src/extractors/image.rs +8 -1
data/vendor/kreuzberg/src/extractors/jats/elements.rs +350 -0
data/vendor/kreuzberg/src/extractors/jats/metadata.rs +21 -0
data/vendor/kreuzberg/src/extractors/{jats.rs → jats/mod.rs} +10 -412
data/vendor/kreuzberg/src/extractors/jats/parser.rs +52 -0
data/vendor/kreuzberg/src/extractors/jupyter.rs +2 -0
data/vendor/kreuzberg/src/extractors/latex/commands.rs +93 -0
data/vendor/kreuzberg/src/extractors/latex/environments.rs +157 -0
data/vendor/kreuzberg/src/extractors/latex/metadata.rs +27 -0
data/vendor/kreuzberg/src/extractors/latex/mod.rs +146 -0
data/vendor/kreuzberg/src/extractors/latex/parser.rs +231 -0
data/vendor/kreuzberg/src/extractors/latex/utilities.rs +126 -0
data/vendor/kreuzberg/src/extractors/markdown.rs +39 -162
data/vendor/kreuzberg/src/extractors/mod.rs +9 -1
data/vendor/kreuzberg/src/extractors/odt.rs +2 -0
data/vendor/kreuzberg/src/extractors/opml/core.rs +165 -0
data/vendor/kreuzberg/src/extractors/opml/mod.rs +31 -0
data/vendor/kreuzberg/src/extractors/opml/parser.rs +479 -0
data/vendor/kreuzberg/src/extractors/orgmode.rs +2 -0
data/vendor/kreuzberg/src/extractors/pdf/extraction.rs +106 -0
data/vendor/kreuzberg/src/extractors/{pdf.rs → pdf/mod.rs} +25 -324
data/vendor/kreuzberg/src/extractors/pdf/ocr.rs +214 -0
data/vendor/kreuzberg/src/extractors/pdf/pages.rs +51 -0
data/vendor/kreuzberg/src/extractors/pptx.rs +9 -2
data/vendor/kreuzberg/src/extractors/rst.rs +2 -0
data/vendor/kreuzberg/src/extractors/rtf/encoding.rs +116 -0
data/vendor/kreuzberg/src/extractors/rtf/formatting.rs +24 -0
data/vendor/kreuzberg/src/extractors/rtf/images.rs +72 -0
data/vendor/kreuzberg/src/extractors/rtf/metadata.rs +216 -0
data/vendor/kreuzberg/src/extractors/rtf/mod.rs +142 -0
data/vendor/kreuzberg/src/extractors/rtf/parser.rs +259 -0
data/vendor/kreuzberg/src/extractors/rtf/tables.rs +83 -0
data/vendor/kreuzberg/src/extractors/structured.rs +2 -0
data/vendor/kreuzberg/src/extractors/text.rs +4 -0
data/vendor/kreuzberg/src/extractors/typst.rs +2 -0
data/vendor/kreuzberg/src/extractors/xml.rs +2 -0
data/vendor/kreuzberg/src/keywords/processor.rs +14 -0
data/vendor/kreuzberg/src/language_detection/processor.rs +10 -0
data/vendor/kreuzberg/src/lib.rs +2 -2
data/vendor/kreuzberg/src/mcp/errors.rs +312 -0
data/vendor/kreuzberg/src/mcp/format.rs +211 -0
data/vendor/kreuzberg/src/mcp/mod.rs +9 -3
data/vendor/kreuzberg/src/mcp/params.rs +196 -0
data/vendor/kreuzberg/src/mcp/server.rs +39 -1438
data/vendor/kreuzberg/src/mcp/tools/cache.rs +179 -0
data/vendor/kreuzberg/src/mcp/tools/extraction.rs +403 -0
data/vendor/kreuzberg/src/mcp/tools/mime.rs +150 -0
data/vendor/kreuzberg/src/mcp/tools/mod.rs +11 -0
data/vendor/kreuzberg/src/ocr/backends/easyocr.rs +96 -0
data/vendor/kreuzberg/src/ocr/backends/mod.rs +7 -0
data/vendor/kreuzberg/src/ocr/backends/paddleocr.rs +27 -0
data/vendor/kreuzberg/src/ocr/backends/tesseract.rs +134 -0
data/vendor/kreuzberg/src/ocr/hocr.rs +60 -16
data/vendor/kreuzberg/src/ocr/language_registry.rs +11 -235
data/vendor/kreuzberg/src/ocr/mod.rs +1 -0
data/vendor/kreuzberg/src/ocr/processor/config.rs +203 -0
data/vendor/kreuzberg/src/ocr/processor/execution.rs +494 -0
data/vendor/kreuzberg/src/ocr/processor/mod.rs +265 -0
data/vendor/kreuzberg/src/ocr/processor/validation.rs +145 -0
data/vendor/kreuzberg/src/ocr/tesseract_backend.rs +41 -24
data/vendor/kreuzberg/src/pdf/bindings.rs +21 -8
data/vendor/kreuzberg/src/pdf/hierarchy/bounding_box.rs +289 -0
data/vendor/kreuzberg/src/pdf/hierarchy/clustering.rs +199 -0
data/vendor/kreuzberg/src/pdf/{hierarchy.rs → hierarchy/extraction.rs} +6 -346
data/vendor/kreuzberg/src/pdf/hierarchy/mod.rs +18 -0
data/vendor/kreuzberg/src/plugins/extractor/mod.rs +319 -0
data/vendor/kreuzberg/src/plugins/extractor/registry.rs +434 -0
data/vendor/kreuzberg/src/plugins/extractor/trait.rs +391 -0
data/vendor/kreuzberg/src/plugins/mod.rs +13 -0
data/vendor/kreuzberg/src/plugins/ocr.rs +11 -0
data/vendor/kreuzberg/src/plugins/processor/mod.rs +365 -0
data/vendor/kreuzberg/src/plugins/processor/registry.rs +37 -0
data/vendor/kreuzberg/src/plugins/processor/trait.rs +284 -0
data/vendor/kreuzberg/src/plugins/registry/extractor.rs +416 -0
data/vendor/kreuzberg/src/plugins/registry/mod.rs +116 -0
data/vendor/kreuzberg/src/plugins/registry/ocr.rs +293 -0
data/vendor/kreuzberg/src/plugins/registry/processor.rs +304 -0
data/vendor/kreuzberg/src/plugins/registry/validator.rs +238 -0
data/vendor/kreuzberg/src/plugins/validator/mod.rs +424 -0
data/vendor/kreuzberg/src/plugins/validator/registry.rs +355 -0
data/vendor/kreuzberg/src/plugins/validator/trait.rs +276 -0
data/vendor/kreuzberg/src/stopwords/languages/asian.rs +40 -0
data/vendor/kreuzberg/src/stopwords/languages/germanic.rs +36 -0
data/vendor/kreuzberg/src/stopwords/languages/mod.rs +10 -0
data/vendor/kreuzberg/src/stopwords/languages/other.rs +44 -0
data/vendor/kreuzberg/src/stopwords/languages/romance.rs +36 -0
data/vendor/kreuzberg/src/stopwords/languages/slavic.rs +36 -0
data/vendor/kreuzberg/src/stopwords/mod.rs +7 -33
data/vendor/kreuzberg/src/text/quality.rs +1 -1
data/vendor/kreuzberg/src/text/quality_processor.rs +10 -0
data/vendor/kreuzberg/src/text/token_reduction/core/analysis.rs +238 -0
data/vendor/kreuzberg/src/text/token_reduction/core/mod.rs +8 -0
data/vendor/kreuzberg/src/text/token_reduction/core/punctuation.rs +54 -0
data/vendor/kreuzberg/src/text/token_reduction/core/reducer.rs +384 -0
data/vendor/kreuzberg/src/text/token_reduction/core/sentence_selection.rs +68 -0
data/vendor/kreuzberg/src/text/token_reduction/core/word_filtering.rs +156 -0
data/vendor/kreuzberg/src/text/token_reduction/filters/general.rs +377 -0
data/vendor/kreuzberg/src/text/token_reduction/filters/html.rs +51 -0
data/vendor/kreuzberg/src/text/token_reduction/filters/markdown.rs +285 -0
data/vendor/kreuzberg/src/text/token_reduction/filters.rs +131 -246
data/vendor/kreuzberg/src/types/djot.rs +209 -0
data/vendor/kreuzberg/src/types/extraction.rs +301 -0
data/vendor/kreuzberg/src/types/formats.rs +443 -0
data/vendor/kreuzberg/src/types/metadata.rs +560 -0
data/vendor/kreuzberg/src/types/mod.rs +281 -0
data/vendor/kreuzberg/src/types/page.rs +182 -0
data/vendor/kreuzberg/src/types/serde_helpers.rs +132 -0
data/vendor/kreuzberg/src/types/tables.rs +39 -0
data/vendor/kreuzberg/src/utils/quality/heuristics.rs +58 -0
data/vendor/kreuzberg/src/utils/{quality.rs → quality/mod.rs} +168 -489
data/vendor/kreuzberg/src/utils/quality/patterns.rs +117 -0
data/vendor/kreuzberg/src/utils/quality/scoring.rs +178 -0
data/vendor/kreuzberg/src/utils/string_pool/buffer_pool.rs +325 -0
data/vendor/kreuzberg/src/utils/string_pool/interned.rs +102 -0
data/vendor/kreuzberg/src/utils/string_pool/language_pool.rs +119 -0
data/vendor/kreuzberg/src/utils/string_pool/mime_pool.rs +235 -0
data/vendor/kreuzberg/src/utils/string_pool/mod.rs +41 -0
data/vendor/kreuzberg/tests/api_chunk.rs +313 -0
data/vendor/kreuzberg/tests/api_embed.rs +6 -9
data/vendor/kreuzberg/tests/batch_orchestration.rs +1 -0
data/vendor/kreuzberg/tests/concurrency_stress.rs +7 -0
data/vendor/kreuzberg/tests/core_integration.rs +1 -0
data/vendor/kreuzberg/tests/docx_metadata_extraction_test.rs +130 -0
data/vendor/kreuzberg/tests/epub_native_extractor_tests.rs +5 -14
data/vendor/kreuzberg/tests/format_integration.rs +2 -0
data/vendor/kreuzberg/tests/helpers/mod.rs +1 -0
data/vendor/kreuzberg/tests/html_table_test.rs +11 -11
data/vendor/kreuzberg/tests/ocr_configuration.rs +16 -0
data/vendor/kreuzberg/tests/ocr_errors.rs +18 -0
data/vendor/kreuzberg/tests/ocr_quality.rs +9 -0
data/vendor/kreuzberg/tests/ocr_stress.rs +1 -0
data/vendor/kreuzberg/tests/pipeline_integration.rs +50 -0
data/vendor/kreuzberg/tests/plugin_ocr_backend_test.rs +13 -0
data/vendor/kreuzberg/tests/plugin_system.rs +12 -0
data/vendor/kreuzberg/tests/registry_integration_tests.rs +2 -0
data/vendor/kreuzberg-ffi/Cargo.toml +2 -1
data/vendor/kreuzberg-ffi/benches/result_view_benchmark.rs +2 -0
data/vendor/kreuzberg-ffi/kreuzberg.h +347 -178
data/vendor/kreuzberg-ffi/src/config/html.rs +318 -0
data/vendor/kreuzberg-ffi/src/config/loader.rs +154 -0
data/vendor/kreuzberg-ffi/src/config/merge.rs +104 -0
data/vendor/kreuzberg-ffi/src/config/mod.rs +385 -0
data/vendor/kreuzberg-ffi/src/config/parse.rs +91 -0
data/vendor/kreuzberg-ffi/src/config/serialize.rs +118 -0
data/vendor/kreuzberg-ffi/src/config_builder.rs +598 -0
data/vendor/kreuzberg-ffi/src/error.rs +46 -14
data/vendor/kreuzberg-ffi/src/helpers.rs +26 -353
data/vendor/kreuzberg-ffi/src/html_options.rs +421 -0
data/vendor/kreuzberg-ffi/src/lib.rs +16 -0
data/vendor/kreuzberg-ffi/src/panic_shield.rs +11 -0
data/vendor/kreuzberg-ffi/src/plugins/ocr_backend.rs +2 -0
data/vendor/kreuzberg-ffi/src/result.rs +148 -122
data/vendor/kreuzberg-ffi/src/result_view.rs +4 -0
data/vendor/kreuzberg-tesseract/Cargo.toml +3 -3
metadata +200 -28
data/vendor/kreuzberg/src/api/server.rs +0 -518
data/vendor/kreuzberg/src/core/config.rs +0 -1914
data/vendor/kreuzberg/src/core/config_validation.rs +0 -949
data/vendor/kreuzberg/src/core/extractor.rs +0 -1200
data/vendor/kreuzberg/src/core/pipeline.rs +0 -1223
data/vendor/kreuzberg/src/core/server_config.rs +0 -1220
data/vendor/kreuzberg/src/extraction/html.rs +0 -1830
data/vendor/kreuzberg/src/extraction/pptx.rs +0 -3102
data/vendor/kreuzberg/src/extractors/epub.rs +0 -696
data/vendor/kreuzberg/src/extractors/latex.rs +0 -653
data/vendor/kreuzberg/src/extractors/opml.rs +0 -635
data/vendor/kreuzberg/src/extractors/rtf.rs +0 -809
data/vendor/kreuzberg/src/ocr/processor.rs +0 -858
data/vendor/kreuzberg/src/plugins/extractor.rs +0 -1042
data/vendor/kreuzberg/src/plugins/processor.rs +0 -650
data/vendor/kreuzberg/src/plugins/registry.rs +0 -1339
data/vendor/kreuzberg/src/plugins/validator.rs +0 -967
data/vendor/kreuzberg/src/text/token_reduction/core.rs +0 -832
data/vendor/kreuzberg/src/types.rs +0 -1713
data/vendor/kreuzberg/src/utils/string_pool.rs +0 -762
data/vendor/kreuzberg-ffi/src/config.rs +0 -1341

data/vendor/kreuzberg/src/extractors/opml/parser.rs ADDED Viewed

@@ -0,0 +1,479 @@
+//! OPML parsing and content extraction.
+//!
+//! This module handles XML parsing, metadata extraction from the `<head>` section,
+//! and recursive processing of `<outline>` elements in the `<body>` section.
+use crate::Result;
+use std::collections::HashMap;
+#[cfg(feature = "office")]
+use roxmltree::Node;
+/// Extract OPML content and metadata from raw bytes.
+///
+/// Parses the XML document structure, extracts metadata from the `<head>` section,
+/// and processes the outline hierarchy in the `<body>` section.
+///
+/// # Returns
+///
+/// A tuple containing:
+/// - Extracted content as a String (outline hierarchy with indentation)
+/// - Metadata HashMap with key-value pairs from the head section
+#[cfg(feature = "office")]
+pub(crate) fn extract_content_and_metadata(content: &[u8]) -> Result<(String, HashMap<String, serde_json::Value>)> {
+    let doc = roxmltree::Document::parse(
+        std::str::from_utf8(content)
+            .map_err(|e| crate::KreuzbergError::Other(format!("Invalid UTF-8 in OPML: {}", e)))?,
+    )
+    .map_err(|e| crate::KreuzbergError::Other(format!("Failed to parse OPML: {}", e)))?;
+    let mut extracted_content = String::new();
+    let mut metadata = HashMap::new();
+    if let Some(opml) = doc.root().children().find(|n| n.tag_name().name() == "opml") {
+        if let Some(head) = opml.children().find(|n| n.tag_name().name() == "head") {
+            extract_metadata_from_head(head, &mut metadata);
+        }
+        if let Some(body) = opml.children().find(|n| n.tag_name().name() == "body") {
+            if let Some(title) = metadata.get("title").and_then(|v| v.as_str()) {
+                extracted_content.push_str(title);
+                extracted_content.push('\n');
+                extracted_content.push('\n');
+            }
+            for outline in body.children().filter(|n| n.tag_name().name() == "outline") {
+                process_outline(outline, 0, &mut extracted_content);
+            }
+        }
+    }
+    Ok((extracted_content.trim().to_string(), metadata))
+}
+/// Extract metadata from the OPML `<head>` section.
+///
+/// Extracts standard OPML metadata fields:
+/// - title: The document title
+/// - dateCreated: Creation date
+/// - dateModified: Last modification date
+/// - ownerName: Document owner's name
+/// - ownerEmail: Document owner's email
+#[cfg(feature = "office")]
+fn extract_metadata_from_head(head: Node, metadata: &mut HashMap<String, serde_json::Value>) {
+    for child in head.children().filter(|n| n.is_element()) {
+        let tag = child.tag_name().name();
+        let text = child.text().unwrap_or("").trim();
+        if text.is_empty() {
+            continue;
+        }
+        match tag {
+            "title" => {
+                metadata.insert("title".to_string(), serde_json::json!(text));
+            }
+            "dateCreated" => {
+                metadata.insert("dateCreated".to_string(), serde_json::json!(text));
+            }
+            "dateModified" => {
+                metadata.insert("dateModified".to_string(), serde_json::json!(text));
+            }
+            "ownerName" => {
+                metadata.insert("ownerName".to_string(), serde_json::json!(text));
+            }
+            "ownerEmail" => {
+                metadata.insert("ownerEmail".to_string(), serde_json::json!(text));
+            }
+            _ => {}
+        }
+    }
+}
+/// Process outline elements recursively.
+///
+/// Extracts text content from outline hierarchy while preserving nesting depth
+/// through indentation. URL attributes are excluded from the main content.
+///
+/// # Arguments
+///
+/// * `node` - The outline node to process
+/// * `depth` - Current nesting depth (for indentation)
+/// * `output` - Output string buffer to append content to
+#[cfg(feature = "office")]
+pub(crate) fn process_outline(node: Node, depth: usize, output: &mut String) {
+    let text = node.attribute("text").unwrap_or("").trim();
+    if !text.is_empty() {
+        let indent = "  ".repeat(depth);
+        output.push_str(&indent);
+        output.push_str(text);
+        output.push('\n');
+    }
+    for child in node.children().filter(|n| n.tag_name().name() == "outline") {
+        process_outline(child, depth + 1, output);
+    }
+}
+#[cfg(all(test, feature = "office"))]
+mod tests {
+    use super::*;
+    #[test]
+    fn test_simple_outline_parsing() {
+        let opml = br#"<?xml version="1.0"?>
+<opml version="2.0">
+  <head>
+    <title>Test</title>
+  </head>
+  <body>
+    <outline text="Item 1" />
+    <outline text="Item 2" />
+  </body>
+</opml>"#;
+        let (content, metadata) = extract_content_and_metadata(opml).expect("Should parse simple OPML");
+        assert!(content.contains("Item 1"), "Should extract first item");
+        assert!(content.contains("Item 2"), "Should extract second item");
+        assert_eq!(
+            metadata.get("title").and_then(|v| v.as_str()),
+            Some("Test"),
+            "Should extract title"
+        );
+    }
+    #[test]
+    fn test_nested_hierarchy() {
+        let opml = br#"<?xml version="1.0"?>
+<opml version="2.0">
+  <head>
+    <title>Hierarchy Test</title>
+  </head>
+  <body>
+    <outline text="Category">
+      <outline text="Subcategory">
+        <outline text="Item" />
+      </outline>
+    </outline>
+  </body>
+</opml>"#;
+        let (content, _) = extract_content_and_metadata(opml).expect("Should parse nested OPML");
+        assert!(content.contains("Category"), "Should contain top level");
+        assert!(content.contains("Subcategory"), "Should contain nested level");
+        assert!(content.contains("Item"), "Should contain deep item");
+        assert!(content.contains("  "), "Should have indentation for nested items");
+    }
+    #[test]
+    fn test_rss_feeds() {
+        let opml = br#"<?xml version="1.0"?>
+<opml version="2.0">
+  <head>
+    <title>Feeds</title>
+  </head>
+  <body>
+    <outline text="Tech">
+      <outline text="Hacker News" type="rss" xmlUrl="https://news.ycombinator.com/rss" htmlUrl="https://news.ycombinator.com/" />
+      <outline text="TechCrunch" type="rss" xmlUrl="https://techcrunch.com/feed/" />
+    </outline>
+  </body>
+</opml>"#;
+        let (content, _) = extract_content_and_metadata(opml).expect("Should parse RSS OPML");
+        assert!(content.contains("Hacker News"), "Should extract feed title");
+        assert!(
+            !content.contains("https://"),
+            "Should NOT extract feed URLs (text-only extraction)"
+        );
+        assert!(content.contains("TechCrunch"), "Should extract multiple feeds");
+    }
+    #[test]
+    fn test_metadata_extraction() {
+        let opml = br#"<?xml version="1.0"?>
+<opml version="2.0">
+  <head>
+    <title>My Feeds</title>
+    <dateCreated>Mon, 06 Nov 2023 00:00:00 GMT</dateCreated>
+    <dateModified>Fri, 01 Dec 2023 12:00:00 GMT</dateModified>
+    <ownerName>John Doe</ownerName>
+    <ownerEmail>john@example.com</ownerEmail>
+  </head>
+  <body>
+    <outline text="Item" />
+  </body>
+</opml>"#;
+        let (_content, metadata) = extract_content_and_metadata(opml).expect("Should extract metadata");
+        assert_eq!(metadata.get("title").and_then(|v| v.as_str()), Some("My Feeds"));
+        assert_eq!(metadata.get("ownerName").and_then(|v| v.as_str()), Some("John Doe"));
+        assert_eq!(
+            metadata.get("ownerEmail").and_then(|v| v.as_str()),
+            Some("john@example.com")
+        );
+        assert!(metadata.contains_key("dateCreated"));
+        assert!(metadata.contains_key("dateModified"));
+    }
+    #[test]
+    fn test_with_special_characters() {
+        let opml = br#"<?xml version="1.0"?>
+<opml version="2.0">
+  <head>
+    <title>Test &amp; Special</title>
+  </head>
+  <body>
+    <outline text="Business &amp; Startups" />
+    <outline text="Science &lt;Advanced&gt;" />
+  </body>
+</opml>"#;
+        let (content, metadata) = extract_content_and_metadata(opml).expect("Should handle special characters");
+        assert!(
+            content.contains("Business") && content.contains("Startups"),
+            "Should decode HTML entities"
+        );
+        let title = metadata.get("title").and_then(|v| v.as_str()).unwrap_or("");
+        assert!(!title.is_empty(), "Should extract title");
+    }
+    #[test]
+    fn test_empty_body() {
+        let opml = br#"<?xml version="1.0"?>
+<opml version="2.0">
+  <head>
+    <title>Empty</title>
+  </head>
+  <body>
+  </body>
+</opml>"#;
+        let (_content, metadata) = extract_content_and_metadata(opml).expect("Should handle empty body");
+        assert_eq!(metadata.get("title").and_then(|v| v.as_str()), Some("Empty"));
+    }
+    #[test]
+    fn test_malformed_missing_closing_tag() {
+        let opml = br#"<?xml version="1.0"?>
+<opml version="2.0">
+  <head>
+    <title>Broken</title>
+  </head>
+  <body>
+    <outline text="Unclosed"
+  </body>
+</opml>"#;
+        let result = extract_content_and_metadata(opml);
+        assert!(result.is_err(), "Should fail to parse OPML with missing closing tags");
+    }
+    #[test]
+    fn test_malformed_invalid_nesting() {
+        let opml = br#"<?xml version="1.0"?>
+<opml version="2.0">
+  <head>
+    <title>Invalid Nesting</title>
+  <body>
+    <outline text="Item" />
+  </body>
+</opml>"#;
+        let result = extract_content_and_metadata(opml);
+        assert!(result.is_err(), "Should fail to parse OPML with invalid nesting");
+    }
+    #[test]
+    fn test_empty_outline_elements() {
+        let opml = br#"<?xml version="1.0"?>
+<opml version="2.0">
+  <head>
+    <title>Empty Outlines</title>
+  </head>
+  <body>
+    <outline text="" />
+    <outline />
+    <outline text="Valid Item">
+      <outline text="" />
+      <outline text="Another Valid" />
+    </outline>
+  </body>
+</opml>"#;
+        let (content, metadata) = extract_content_and_metadata(opml).expect("Should handle empty outline elements");
+        assert!(content.contains("Valid Item"), "Should extract valid items");
+        assert!(content.contains("Another Valid"), "Should extract nested valid items");
+        let empty_count = content.matches("\n\n").count();
+        assert!(empty_count < 3, "Should skip empty outline elements");
+        assert_eq!(metadata.get("title").and_then(|v| v.as_str()), Some("Empty Outlines"));
+    }
+    #[test]
+    fn test_deeply_nested_empty_nodes() {
+        let opml = br#"<?xml version="1.0"?>
+<opml version="2.0">
+  <head>
+    <title>Deep Nesting</title>
+  </head>
+  <body>
+    <outline text="Level 1">
+      <outline text="">
+        <outline text="">
+          <outline text="Deep Item">
+            <outline text="" />
+          </outline>
+        </outline>
+      </outline>
+      <outline text="Level 1 Sibling" />
+    </outline>
+  </body>
+</opml>"#;
+        let (content, _) = extract_content_and_metadata(opml).expect("Should handle deeply nested structures");
+        assert!(content.contains("Level 1"), "Should extract top-level item");
+        assert!(content.contains("Deep Item"), "Should extract deeply nested item");
+        assert!(content.contains("Level 1 Sibling"), "Should extract sibling items");
+    }
+    #[test]
+    fn test_outline_with_missing_text_attribute() {
+        let opml = br#"<?xml version="1.0"?>
+<opml version="2.0">
+  <head>
+    <title>Missing Attributes</title>
+  </head>
+  <body>
+    <outline type="folder" />
+    <outline text="Valid Item" type="rss" />
+    <outline type="rss" xmlUrl="https://example.com/feed" />
+  </body>
+</opml>"#;
+        let (content, metadata) =
+            extract_content_and_metadata(opml).expect("Should handle outline with missing text attribute");
+        assert!(content.contains("Valid Item"), "Should extract item with text");
+        assert!(!content.contains("https://"), "Should not extract URLs");
+        assert_eq!(
+            metadata.get("title").and_then(|v| v.as_str()),
+            Some("Missing Attributes")
+        );
+    }
+    #[test]
+    fn test_whitespace_only_text_attribute() {
+        let opml = br#"<?xml version="1.0"?>
+<opml version="2.0">
+  <head>
+    <title>Whitespace Test</title>
+  </head>
+  <body>
+    <outline text="   " />
+    <outline text="
+" />
+    <outline text="Real Content" />
+  </body>
+</opml>"#;
+        let (content, _) = extract_content_and_metadata(opml).expect("Should handle whitespace-only text");
+        assert!(
+            content.contains("Real Content"),
+            "Should extract non-whitespace content"
+        );
+        let trimmed = content.trim();
+        assert!(trimmed.contains("Whitespace Test"), "Should have title");
+        assert!(trimmed.contains("Real Content"), "Should have real content");
+    }
+    #[test]
+    fn test_html_entity_in_nested_structure() {
+        let opml = br#"<?xml version="1.0"?>
+<opml version="2.0">
+  <head>
+    <title>Entities &amp; Nesting</title>
+  </head>
+  <body>
+    <outline text="News &amp; Updates">
+      <outline text="Tech &lt; Science" />
+      <outline text="Health &gt; Wealth" />
+    </outline>
+  </body>
+</opml>"#;
+        let (content, metadata) = extract_content_and_metadata(opml).expect("Should handle HTML entities");
+        assert!(
+            content.contains("News") && content.contains("Updates"),
+            "Should decode &amp; entity"
+        );
+        assert!(content.contains("Tech"), "Should handle &lt; entity");
+        assert!(content.contains("Science"), "Should decode entity properly");
+        let title = metadata.get("title").and_then(|v| v.as_str()).unwrap_or("");
+        assert!(
+            title.contains("&") && title.contains("Nesting"),
+            "Title should have decoded entity"
+        );
+    }
+    #[test]
+    fn test_single_outline_no_children() {
+        let opml = br#"<?xml version="1.0"?>
+<opml version="2.0">
+  <head>
+    <title>Single</title>
+  </head>
+  <body>
+    <outline text="Only Item" />
+  </body>
+</opml>"#;
+        let (content, metadata) = extract_content_and_metadata(opml).expect("Should handle single outline");
+        assert!(content.contains("Only Item"), "Should extract single item");
+        assert_eq!(metadata.get("title").and_then(|v| v.as_str()), Some("Single"));
+    }
+    #[test]
+    fn test_head_without_body() {
+        let opml = br#"<?xml version="1.0"?>
+<opml version="2.0">
+  <head>
+    <title>No Body</title>
+  </head>
+</opml>"#;
+        let (content, metadata) = extract_content_and_metadata(opml).expect("Should handle OPML without body");
+        assert_eq!(metadata.get("title").and_then(|v| v.as_str()), Some("No Body"));
+        assert!(content.is_empty() || content.trim() == "No Body");
+    }
+    #[test]
+    fn test_body_without_head() {
+        let opml = br#"<?xml version="1.0"?>
+<opml version="2.0">
+  <body>
+    <outline text="Item" />
+  </body>
+</opml>"#;
+        let (content, metadata) = extract_content_and_metadata(opml).expect("Should handle OPML without head");
+        assert!(content.contains("Item"), "Should extract body content");
+        assert!(metadata.is_empty(), "Should have no metadata without head");
+    }
+}

data/vendor/kreuzberg/src/extractors/orgmode.rs CHANGED Viewed

@@ -304,7 +304,9 @@ impl DocumentExtractor for OrgModeExtractor {
             detected_languages: None,
             chunks: None,
             images: None,
+            djot_content: None,
             pages: None,
+            elements: None,
         })
     }

data/vendor/kreuzberg/src/extractors/pdf/extraction.rs ADDED Viewed

@@ -0,0 +1,106 @@
+//! Core PDF extraction functionality.
+//!
+//! Handles document loading, text extraction, metadata parsing, and table detection.
+use crate::Result;
+use crate::core::config::ExtractionConfig;
+use crate::types::PageContent;
+#[cfg(feature = "pdf")]
+use crate::types::Table;
+#[cfg(feature = "pdf")]
+use pdfium_render::prelude::*;
+#[cfg(feature = "pdf")]
+pub(crate) type PdfExtractionPhaseResult = (
+    crate::pdf::metadata::PdfExtractionMetadata,
+    String,
+    Vec<Table>,
+    Option<Vec<PageContent>>,
+);
+/// Extract text, metadata, and tables from a PDF document using a single shared instance.
+///
+/// This method consolidates all PDF extraction phases (text, metadata, tables) into a single
+/// operation using a single PdfDocument instance. This avoids redundant document parsing
+/// and pdfium initialization overhead.
+///
+/// # Performance
+///
+/// By reusing a single document instance across all extraction phases, we eliminate:
+/// - Duplicate document parsing overhead (25-40ms saved)
+/// - Redundant pdfium bindings initialization
+/// - Multiple page tree traversals
+///
+/// Expected improvement: 20-30% faster PDF processing.
+///
+/// # Returns
+///
+/// A tuple containing:
+/// - PDF metadata (title, authors, dates, page structure, etc.)
+/// - Native extracted text (or empty if using OCR)
+/// - Extracted tables (if OCR feature enabled)
+/// - Per-page content (if page extraction configured)
+#[cfg(feature = "pdf")]
+pub(crate) fn extract_all_from_document(
+    document: &PdfDocument,
+    config: &ExtractionConfig,
+) -> Result<PdfExtractionPhaseResult> {
+    let (native_text, _boundaries, page_contents, pdf_metadata) =
+        crate::pdf::text::extract_text_and_metadata_from_pdf_document(document, Some(config))?;
+    let tables = extract_tables_from_document(document, &pdf_metadata)?;
+    Ok((pdf_metadata, native_text, tables, page_contents))
+}
+/// Extract tables from PDF document using native text positions.
+///
+/// This function converts PDF character positions to HocrWord format,
+/// then uses the existing table reconstruction logic to detect tables.
+///
+/// Uses the shared PdfDocument reference (wrapped in Arc<RwLock<>> for thread-safety).
+#[cfg(all(feature = "pdf", feature = "ocr"))]
+fn extract_tables_from_document(
+    document: &PdfDocument,
+    _metadata: &crate::pdf::metadata::PdfExtractionMetadata,
+) -> Result<Vec<Table>> {
+    use crate::ocr::table::{reconstruct_table, table_to_markdown};
+    use crate::pdf::table::extract_words_from_page;
+    let mut all_tables = Vec::new();
+    for (page_index, page) in document.pages().iter().enumerate() {
+        let words = extract_words_from_page(&page, 0.0)?;
+        if words.is_empty() {
+            continue;
+        }
+        let column_threshold = 50;
+        let row_threshold_ratio = 0.5;
+        let table_cells = reconstruct_table(&words, column_threshold, row_threshold_ratio);
+        if !table_cells.is_empty() {
+            let markdown = table_to_markdown(&table_cells);
+            all_tables.push(Table {
+                cells: table_cells,
+                markdown,
+                page_number: page_index + 1,
+            });
+        }
+    }
+    Ok(all_tables)
+}
+/// Fallback for when OCR feature is not enabled - returns empty tables.
+#[cfg(all(feature = "pdf", not(feature = "ocr")))]
+fn extract_tables_from_document(
+    _document: &PdfDocument,
+    _metadata: &crate::pdf::metadata::PdfExtractionMetadata,
+) -> Result<Vec<crate::types::Table>> {
+    Ok(vec![])
+}