kreuzberg 4.0.0.rc1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (265) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +8 -0
  3. data/.rspec +3 -0
  4. data/.rubocop.yaml +534 -0
  5. data/Gemfile +9 -0
  6. data/Gemfile.lock +157 -0
  7. data/README.md +421 -0
  8. data/Rakefile +25 -0
  9. data/Steepfile +47 -0
  10. data/examples/async_patterns.rb +340 -0
  11. data/ext/kreuzberg_rb/extconf.rb +35 -0
  12. data/ext/kreuzberg_rb/native/Cargo.toml +36 -0
  13. data/ext/kreuzberg_rb/native/README.md +425 -0
  14. data/ext/kreuzberg_rb/native/build.rs +17 -0
  15. data/ext/kreuzberg_rb/native/include/ieeefp.h +11 -0
  16. data/ext/kreuzberg_rb/native/include/msvc_compat/strings.h +14 -0
  17. data/ext/kreuzberg_rb/native/include/strings.h +20 -0
  18. data/ext/kreuzberg_rb/native/include/unistd.h +47 -0
  19. data/ext/kreuzberg_rb/native/src/lib.rs +2939 -0
  20. data/extconf.rb +28 -0
  21. data/kreuzberg.gemspec +105 -0
  22. data/lib/kreuzberg/api_proxy.rb +142 -0
  23. data/lib/kreuzberg/cache_api.rb +45 -0
  24. data/lib/kreuzberg/cli.rb +55 -0
  25. data/lib/kreuzberg/cli_proxy.rb +127 -0
  26. data/lib/kreuzberg/config.rb +684 -0
  27. data/lib/kreuzberg/errors.rb +50 -0
  28. data/lib/kreuzberg/extraction_api.rb +84 -0
  29. data/lib/kreuzberg/mcp_proxy.rb +186 -0
  30. data/lib/kreuzberg/ocr_backend_protocol.rb +113 -0
  31. data/lib/kreuzberg/post_processor_protocol.rb +86 -0
  32. data/lib/kreuzberg/result.rb +216 -0
  33. data/lib/kreuzberg/setup_lib_path.rb +79 -0
  34. data/lib/kreuzberg/validator_protocol.rb +89 -0
  35. data/lib/kreuzberg/version.rb +5 -0
  36. data/lib/kreuzberg.rb +82 -0
  37. data/pkg/kreuzberg-4.0.0.rc1.gem +0 -0
  38. data/sig/kreuzberg/internal.rbs +184 -0
  39. data/sig/kreuzberg.rbs +468 -0
  40. data/spec/binding/cache_spec.rb +227 -0
  41. data/spec/binding/cli_proxy_spec.rb +87 -0
  42. data/spec/binding/cli_spec.rb +54 -0
  43. data/spec/binding/config_spec.rb +345 -0
  44. data/spec/binding/config_validation_spec.rb +283 -0
  45. data/spec/binding/error_handling_spec.rb +213 -0
  46. data/spec/binding/errors_spec.rb +66 -0
  47. data/spec/binding/plugins/ocr_backend_spec.rb +307 -0
  48. data/spec/binding/plugins/postprocessor_spec.rb +269 -0
  49. data/spec/binding/plugins/validator_spec.rb +274 -0
  50. data/spec/examples.txt +104 -0
  51. data/spec/fixtures/config.toml +39 -0
  52. data/spec/fixtures/config.yaml +42 -0
  53. data/spec/fixtures/invalid_config.toml +4 -0
  54. data/spec/smoke/package_spec.rb +178 -0
  55. data/spec/spec_helper.rb +42 -0
  56. data/vendor/kreuzberg/Cargo.toml +134 -0
  57. data/vendor/kreuzberg/README.md +175 -0
  58. data/vendor/kreuzberg/build.rs +460 -0
  59. data/vendor/kreuzberg/src/api/error.rs +81 -0
  60. data/vendor/kreuzberg/src/api/handlers.rs +199 -0
  61. data/vendor/kreuzberg/src/api/mod.rs +79 -0
  62. data/vendor/kreuzberg/src/api/server.rs +353 -0
  63. data/vendor/kreuzberg/src/api/types.rs +170 -0
  64. data/vendor/kreuzberg/src/bin/profile_extract.rs +455 -0
  65. data/vendor/kreuzberg/src/cache/mod.rs +1143 -0
  66. data/vendor/kreuzberg/src/chunking/mod.rs +677 -0
  67. data/vendor/kreuzberg/src/core/batch_mode.rs +35 -0
  68. data/vendor/kreuzberg/src/core/config.rs +1032 -0
  69. data/vendor/kreuzberg/src/core/extractor.rs +903 -0
  70. data/vendor/kreuzberg/src/core/io.rs +327 -0
  71. data/vendor/kreuzberg/src/core/mime.rs +615 -0
  72. data/vendor/kreuzberg/src/core/mod.rs +42 -0
  73. data/vendor/kreuzberg/src/core/pipeline.rs +906 -0
  74. data/vendor/kreuzberg/src/embeddings.rs +323 -0
  75. data/vendor/kreuzberg/src/error.rs +431 -0
  76. data/vendor/kreuzberg/src/extraction/archive.rs +954 -0
  77. data/vendor/kreuzberg/src/extraction/docx.rs +40 -0
  78. data/vendor/kreuzberg/src/extraction/email.rs +854 -0
  79. data/vendor/kreuzberg/src/extraction/excel.rs +688 -0
  80. data/vendor/kreuzberg/src/extraction/html.rs +553 -0
  81. data/vendor/kreuzberg/src/extraction/image.rs +368 -0
  82. data/vendor/kreuzberg/src/extraction/libreoffice.rs +564 -0
  83. data/vendor/kreuzberg/src/extraction/mod.rs +77 -0
  84. data/vendor/kreuzberg/src/extraction/office_metadata/app_properties.rs +398 -0
  85. data/vendor/kreuzberg/src/extraction/office_metadata/core_properties.rs +247 -0
  86. data/vendor/kreuzberg/src/extraction/office_metadata/custom_properties.rs +240 -0
  87. data/vendor/kreuzberg/src/extraction/office_metadata/mod.rs +128 -0
  88. data/vendor/kreuzberg/src/extraction/pandoc/batch.rs +275 -0
  89. data/vendor/kreuzberg/src/extraction/pandoc/mime_types.rs +178 -0
  90. data/vendor/kreuzberg/src/extraction/pandoc/mod.rs +491 -0
  91. data/vendor/kreuzberg/src/extraction/pandoc/server.rs +496 -0
  92. data/vendor/kreuzberg/src/extraction/pandoc/subprocess.rs +1188 -0
  93. data/vendor/kreuzberg/src/extraction/pandoc/version.rs +162 -0
  94. data/vendor/kreuzberg/src/extraction/pptx.rs +3000 -0
  95. data/vendor/kreuzberg/src/extraction/structured.rs +490 -0
  96. data/vendor/kreuzberg/src/extraction/table.rs +328 -0
  97. data/vendor/kreuzberg/src/extraction/text.rs +269 -0
  98. data/vendor/kreuzberg/src/extraction/xml.rs +333 -0
  99. data/vendor/kreuzberg/src/extractors/archive.rs +425 -0
  100. data/vendor/kreuzberg/src/extractors/docx.rs +479 -0
  101. data/vendor/kreuzberg/src/extractors/email.rs +129 -0
  102. data/vendor/kreuzberg/src/extractors/excel.rs +344 -0
  103. data/vendor/kreuzberg/src/extractors/html.rs +410 -0
  104. data/vendor/kreuzberg/src/extractors/image.rs +195 -0
  105. data/vendor/kreuzberg/src/extractors/mod.rs +268 -0
  106. data/vendor/kreuzberg/src/extractors/pandoc.rs +201 -0
  107. data/vendor/kreuzberg/src/extractors/pdf.rs +496 -0
  108. data/vendor/kreuzberg/src/extractors/pptx.rs +234 -0
  109. data/vendor/kreuzberg/src/extractors/structured.rs +126 -0
  110. data/vendor/kreuzberg/src/extractors/text.rs +242 -0
  111. data/vendor/kreuzberg/src/extractors/xml.rs +128 -0
  112. data/vendor/kreuzberg/src/image/dpi.rs +164 -0
  113. data/vendor/kreuzberg/src/image/mod.rs +6 -0
  114. data/vendor/kreuzberg/src/image/preprocessing.rs +417 -0
  115. data/vendor/kreuzberg/src/image/resize.rs +89 -0
  116. data/vendor/kreuzberg/src/keywords/config.rs +154 -0
  117. data/vendor/kreuzberg/src/keywords/mod.rs +237 -0
  118. data/vendor/kreuzberg/src/keywords/processor.rs +267 -0
  119. data/vendor/kreuzberg/src/keywords/rake.rs +294 -0
  120. data/vendor/kreuzberg/src/keywords/types.rs +68 -0
  121. data/vendor/kreuzberg/src/keywords/yake.rs +163 -0
  122. data/vendor/kreuzberg/src/language_detection/mod.rs +942 -0
  123. data/vendor/kreuzberg/src/lib.rs +102 -0
  124. data/vendor/kreuzberg/src/mcp/mod.rs +32 -0
  125. data/vendor/kreuzberg/src/mcp/server.rs +1966 -0
  126. data/vendor/kreuzberg/src/ocr/cache.rs +469 -0
  127. data/vendor/kreuzberg/src/ocr/error.rs +37 -0
  128. data/vendor/kreuzberg/src/ocr/hocr.rs +216 -0
  129. data/vendor/kreuzberg/src/ocr/mod.rs +58 -0
  130. data/vendor/kreuzberg/src/ocr/processor.rs +847 -0
  131. data/vendor/kreuzberg/src/ocr/table/mod.rs +4 -0
  132. data/vendor/kreuzberg/src/ocr/table/tsv_parser.rs +144 -0
  133. data/vendor/kreuzberg/src/ocr/tesseract_backend.rs +450 -0
  134. data/vendor/kreuzberg/src/ocr/types.rs +393 -0
  135. data/vendor/kreuzberg/src/ocr/utils.rs +47 -0
  136. data/vendor/kreuzberg/src/ocr/validation.rs +206 -0
  137. data/vendor/kreuzberg/src/pdf/error.rs +122 -0
  138. data/vendor/kreuzberg/src/pdf/images.rs +139 -0
  139. data/vendor/kreuzberg/src/pdf/metadata.rs +346 -0
  140. data/vendor/kreuzberg/src/pdf/mod.rs +50 -0
  141. data/vendor/kreuzberg/src/pdf/rendering.rs +369 -0
  142. data/vendor/kreuzberg/src/pdf/table.rs +420 -0
  143. data/vendor/kreuzberg/src/pdf/text.rs +161 -0
  144. data/vendor/kreuzberg/src/plugins/extractor.rs +1010 -0
  145. data/vendor/kreuzberg/src/plugins/mod.rs +209 -0
  146. data/vendor/kreuzberg/src/plugins/ocr.rs +629 -0
  147. data/vendor/kreuzberg/src/plugins/processor.rs +641 -0
  148. data/vendor/kreuzberg/src/plugins/registry.rs +1324 -0
  149. data/vendor/kreuzberg/src/plugins/traits.rs +258 -0
  150. data/vendor/kreuzberg/src/plugins/validator.rs +955 -0
  151. data/vendor/kreuzberg/src/stopwords/mod.rs +1470 -0
  152. data/vendor/kreuzberg/src/text/mod.rs +19 -0
  153. data/vendor/kreuzberg/src/text/quality.rs +697 -0
  154. data/vendor/kreuzberg/src/text/string_utils.rs +217 -0
  155. data/vendor/kreuzberg/src/text/token_reduction/cjk_utils.rs +164 -0
  156. data/vendor/kreuzberg/src/text/token_reduction/config.rs +100 -0
  157. data/vendor/kreuzberg/src/text/token_reduction/core.rs +796 -0
  158. data/vendor/kreuzberg/src/text/token_reduction/filters.rs +902 -0
  159. data/vendor/kreuzberg/src/text/token_reduction/mod.rs +160 -0
  160. data/vendor/kreuzberg/src/text/token_reduction/semantic.rs +619 -0
  161. data/vendor/kreuzberg/src/text/token_reduction/simd_text.rs +147 -0
  162. data/vendor/kreuzberg/src/types.rs +873 -0
  163. data/vendor/kreuzberg/src/utils/mod.rs +17 -0
  164. data/vendor/kreuzberg/src/utils/quality.rs +959 -0
  165. data/vendor/kreuzberg/src/utils/string_utils.rs +381 -0
  166. data/vendor/kreuzberg/stopwords/af_stopwords.json +53 -0
  167. data/vendor/kreuzberg/stopwords/ar_stopwords.json +482 -0
  168. data/vendor/kreuzberg/stopwords/bg_stopwords.json +261 -0
  169. data/vendor/kreuzberg/stopwords/bn_stopwords.json +400 -0
  170. data/vendor/kreuzberg/stopwords/br_stopwords.json +1205 -0
  171. data/vendor/kreuzberg/stopwords/ca_stopwords.json +280 -0
  172. data/vendor/kreuzberg/stopwords/cs_stopwords.json +425 -0
  173. data/vendor/kreuzberg/stopwords/da_stopwords.json +172 -0
  174. data/vendor/kreuzberg/stopwords/de_stopwords.json +622 -0
  175. data/vendor/kreuzberg/stopwords/el_stopwords.json +849 -0
  176. data/vendor/kreuzberg/stopwords/en_stopwords.json +1300 -0
  177. data/vendor/kreuzberg/stopwords/eo_stopwords.json +175 -0
  178. data/vendor/kreuzberg/stopwords/es_stopwords.json +734 -0
  179. data/vendor/kreuzberg/stopwords/et_stopwords.json +37 -0
  180. data/vendor/kreuzberg/stopwords/eu_stopwords.json +100 -0
  181. data/vendor/kreuzberg/stopwords/fa_stopwords.json +801 -0
  182. data/vendor/kreuzberg/stopwords/fi_stopwords.json +849 -0
  183. data/vendor/kreuzberg/stopwords/fr_stopwords.json +693 -0
  184. data/vendor/kreuzberg/stopwords/ga_stopwords.json +111 -0
  185. data/vendor/kreuzberg/stopwords/gl_stopwords.json +162 -0
  186. data/vendor/kreuzberg/stopwords/gu_stopwords.json +226 -0
  187. data/vendor/kreuzberg/stopwords/ha_stopwords.json +41 -0
  188. data/vendor/kreuzberg/stopwords/he_stopwords.json +196 -0
  189. data/vendor/kreuzberg/stopwords/hi_stopwords.json +227 -0
  190. data/vendor/kreuzberg/stopwords/hr_stopwords.json +181 -0
  191. data/vendor/kreuzberg/stopwords/hu_stopwords.json +791 -0
  192. data/vendor/kreuzberg/stopwords/hy_stopwords.json +47 -0
  193. data/vendor/kreuzberg/stopwords/id_stopwords.json +760 -0
  194. data/vendor/kreuzberg/stopwords/it_stopwords.json +634 -0
  195. data/vendor/kreuzberg/stopwords/ja_stopwords.json +136 -0
  196. data/vendor/kreuzberg/stopwords/kn_stopwords.json +84 -0
  197. data/vendor/kreuzberg/stopwords/ko_stopwords.json +681 -0
  198. data/vendor/kreuzberg/stopwords/ku_stopwords.json +64 -0
  199. data/vendor/kreuzberg/stopwords/la_stopwords.json +51 -0
  200. data/vendor/kreuzberg/stopwords/lt_stopwords.json +476 -0
  201. data/vendor/kreuzberg/stopwords/lv_stopwords.json +163 -0
  202. data/vendor/kreuzberg/stopwords/ml_stopwords.json +1 -0
  203. data/vendor/kreuzberg/stopwords/mr_stopwords.json +101 -0
  204. data/vendor/kreuzberg/stopwords/ms_stopwords.json +477 -0
  205. data/vendor/kreuzberg/stopwords/ne_stopwords.json +490 -0
  206. data/vendor/kreuzberg/stopwords/nl_stopwords.json +415 -0
  207. data/vendor/kreuzberg/stopwords/no_stopwords.json +223 -0
  208. data/vendor/kreuzberg/stopwords/pl_stopwords.json +331 -0
  209. data/vendor/kreuzberg/stopwords/pt_stopwords.json +562 -0
  210. data/vendor/kreuzberg/stopwords/ro_stopwords.json +436 -0
  211. data/vendor/kreuzberg/stopwords/ru_stopwords.json +561 -0
  212. data/vendor/kreuzberg/stopwords/si_stopwords.json +193 -0
  213. data/vendor/kreuzberg/stopwords/sk_stopwords.json +420 -0
  214. data/vendor/kreuzberg/stopwords/sl_stopwords.json +448 -0
  215. data/vendor/kreuzberg/stopwords/so_stopwords.json +32 -0
  216. data/vendor/kreuzberg/stopwords/st_stopwords.json +33 -0
  217. data/vendor/kreuzberg/stopwords/sv_stopwords.json +420 -0
  218. data/vendor/kreuzberg/stopwords/sw_stopwords.json +76 -0
  219. data/vendor/kreuzberg/stopwords/ta_stopwords.json +129 -0
  220. data/vendor/kreuzberg/stopwords/te_stopwords.json +54 -0
  221. data/vendor/kreuzberg/stopwords/th_stopwords.json +118 -0
  222. data/vendor/kreuzberg/stopwords/tl_stopwords.json +149 -0
  223. data/vendor/kreuzberg/stopwords/tr_stopwords.json +506 -0
  224. data/vendor/kreuzberg/stopwords/uk_stopwords.json +75 -0
  225. data/vendor/kreuzberg/stopwords/ur_stopwords.json +519 -0
  226. data/vendor/kreuzberg/stopwords/vi_stopwords.json +647 -0
  227. data/vendor/kreuzberg/stopwords/yo_stopwords.json +62 -0
  228. data/vendor/kreuzberg/stopwords/zh_stopwords.json +796 -0
  229. data/vendor/kreuzberg/stopwords/zu_stopwords.json +31 -0
  230. data/vendor/kreuzberg/tests/api_tests.rs +966 -0
  231. data/vendor/kreuzberg/tests/archive_integration.rs +543 -0
  232. data/vendor/kreuzberg/tests/batch_orchestration.rs +542 -0
  233. data/vendor/kreuzberg/tests/batch_processing.rs +304 -0
  234. data/vendor/kreuzberg/tests/chunking_offset_demo.rs +92 -0
  235. data/vendor/kreuzberg/tests/concurrency_stress.rs +509 -0
  236. data/vendor/kreuzberg/tests/config_features.rs +580 -0
  237. data/vendor/kreuzberg/tests/config_loading_tests.rs +439 -0
  238. data/vendor/kreuzberg/tests/core_integration.rs +493 -0
  239. data/vendor/kreuzberg/tests/csv_integration.rs +424 -0
  240. data/vendor/kreuzberg/tests/docx_metadata_extraction_test.rs +124 -0
  241. data/vendor/kreuzberg/tests/email_integration.rs +325 -0
  242. data/vendor/kreuzberg/tests/error_handling.rs +393 -0
  243. data/vendor/kreuzberg/tests/format_integration.rs +159 -0
  244. data/vendor/kreuzberg/tests/helpers/mod.rs +142 -0
  245. data/vendor/kreuzberg/tests/image_integration.rs +253 -0
  246. data/vendor/kreuzberg/tests/keywords_integration.rs +479 -0
  247. data/vendor/kreuzberg/tests/keywords_quality.rs +509 -0
  248. data/vendor/kreuzberg/tests/mime_detection.rs +428 -0
  249. data/vendor/kreuzberg/tests/ocr_configuration.rs +510 -0
  250. data/vendor/kreuzberg/tests/ocr_errors.rs +676 -0
  251. data/vendor/kreuzberg/tests/ocr_quality.rs +627 -0
  252. data/vendor/kreuzberg/tests/ocr_stress.rs +469 -0
  253. data/vendor/kreuzberg/tests/pandoc_integration.rs +503 -0
  254. data/vendor/kreuzberg/tests/pdf_integration.rs +43 -0
  255. data/vendor/kreuzberg/tests/pipeline_integration.rs +1412 -0
  256. data/vendor/kreuzberg/tests/plugin_ocr_backend_test.rs +771 -0
  257. data/vendor/kreuzberg/tests/plugin_postprocessor_test.rs +561 -0
  258. data/vendor/kreuzberg/tests/plugin_system.rs +921 -0
  259. data/vendor/kreuzberg/tests/plugin_validator_test.rs +783 -0
  260. data/vendor/kreuzberg/tests/registry_integration_tests.rs +607 -0
  261. data/vendor/kreuzberg/tests/security_validation.rs +404 -0
  262. data/vendor/kreuzberg/tests/stopwords_integration_test.rs +888 -0
  263. data/vendor/kreuzberg/tests/test_fastembed.rs +609 -0
  264. data/vendor/kreuzberg/tests/xlsx_metadata_extraction_test.rs +87 -0
  265. metadata +471 -0
@@ -0,0 +1,688 @@
1
+ //! Excel and spreadsheet extraction functions.
2
+ //!
3
+ //! This module provides Excel file parsing using the `calamine` library.
4
+ //! Supports both modern Office Open XML formats (.xlsx, .xlsm, .xlam, .xltm, .xlsb)
5
+ //! and legacy binary formats (.xls, .xla), as well as OpenDocument spreadsheets (.ods).
6
+ //!
7
+ //! # Features
8
+ //!
9
+ //! - **Multiple formats**: XLSX, XLSM, XLS, XLSB, ODS
10
+ //! - **Sheet extraction**: Reads all sheets from workbook
11
+ //! - **Markdown conversion**: Converts spreadsheet data to Markdown tables
12
+ //! - **Office metadata**: Extracts core properties, custom properties (when `office` feature enabled)
13
+ //! - **Error handling**: Distinguishes between format errors and true I/O errors
14
+ //!
15
+ //! # Example
16
+ //!
17
+ //! ```rust
18
+ //! use kreuzberg::extraction::excel::read_excel_file;
19
+ //!
20
+ //! # fn example() -> kreuzberg::Result<()> {
21
+ //! let workbook = read_excel_file("data.xlsx")?;
22
+ //!
23
+ //! println!("Sheet count: {}", workbook.sheets.len());
24
+ //! for sheet in &workbook.sheets {
25
+ //! println!("Sheet: {}", sheet.name);
26
+ //! }
27
+ //! # Ok(())
28
+ //! # }
29
+ //! ```
30
+ use calamine::{Data, Range, Reader, open_workbook_auto};
31
+ use std::collections::HashMap;
32
+ use std::fmt::Write as FmtWrite;
33
+ use std::io::Cursor;
34
+ use std::path::Path;
35
+
36
+ use crate::error::{KreuzbergError, Result};
37
+ use crate::types::{ExcelSheet, ExcelWorkbook};
38
+
39
+ #[cfg(feature = "office")]
40
+ use crate::extraction::office_metadata::{
41
+ extract_core_properties, extract_custom_properties, extract_xlsx_app_properties,
42
+ };
43
+ #[cfg(feature = "office")]
44
+ use serde_json::Value;
45
+
46
+ pub fn read_excel_file(file_path: &str) -> Result<ExcelWorkbook> {
47
+ #[cfg(feature = "office")]
48
+ let office_metadata = if file_path.to_lowercase().ends_with(".xlsx")
49
+ || file_path.to_lowercase().ends_with(".xlsm")
50
+ || file_path.to_lowercase().ends_with(".xlam")
51
+ || file_path.to_lowercase().ends_with(".xltm")
52
+ {
53
+ extract_xlsx_office_metadata_from_file(file_path).ok()
54
+ } else {
55
+ None
56
+ };
57
+
58
+ #[cfg(not(feature = "office"))]
59
+ let office_metadata: Option<HashMap<String, String>> = None;
60
+
61
+ // We analyze the error and only wrap format errors, letting real IO errors bubble up ~keep
62
+ let workbook = match open_workbook_auto(Path::new(file_path)) {
63
+ Ok(wb) => wb,
64
+ Err(calamine::Error::Io(io_err)) => {
65
+ if io_err.kind() == std::io::ErrorKind::InvalidData {
66
+ return Err(KreuzbergError::parsing(format!(
67
+ "Cannot detect Excel file format: {}",
68
+ io_err
69
+ )));
70
+ }
71
+ // Real IO error - bubble up unchanged ~keep
72
+ return Err(io_err.into());
73
+ }
74
+ Err(e) => return Err(KreuzbergError::parsing(format!("Failed to parse Excel file: {}", e))),
75
+ };
76
+
77
+ process_workbook(workbook, office_metadata)
78
+ }
79
+
80
+ pub fn read_excel_bytes(data: &[u8], file_extension: &str) -> Result<ExcelWorkbook> {
81
+ #[cfg(feature = "office")]
82
+ let office_metadata = match file_extension.to_lowercase().as_str() {
83
+ ".xlsx" | ".xlsm" | ".xlam" | ".xltm" => extract_xlsx_office_metadata_from_bytes(data).ok(),
84
+ _ => None,
85
+ };
86
+
87
+ #[cfg(not(feature = "office"))]
88
+ let office_metadata: Option<HashMap<String, String>> = None;
89
+
90
+ let cursor = Cursor::new(data);
91
+
92
+ match file_extension.to_lowercase().as_str() {
93
+ ".xlsx" | ".xlsm" | ".xlam" | ".xltm" => {
94
+ let workbook = calamine::Xlsx::new(cursor)
95
+ .map_err(|e| KreuzbergError::parsing(format!("Failed to parse XLSX: {}", e)))?;
96
+ process_workbook(workbook, office_metadata)
97
+ }
98
+ ".xls" | ".xla" => {
99
+ let workbook = calamine::Xls::new(cursor)
100
+ .map_err(|e| KreuzbergError::parsing(format!("Failed to parse XLS: {}", e)))?;
101
+ process_workbook(workbook, office_metadata)
102
+ }
103
+ ".xlsb" => {
104
+ let workbook = calamine::Xlsb::new(cursor)
105
+ .map_err(|e| KreuzbergError::parsing(format!("Failed to parse XLSB: {}", e)))?;
106
+ process_workbook(workbook, office_metadata)
107
+ }
108
+ ".ods" => {
109
+ let workbook = calamine::Ods::new(cursor)
110
+ .map_err(|e| KreuzbergError::parsing(format!("Failed to parse ODS: {}", e)))?;
111
+ process_workbook(workbook, office_metadata)
112
+ }
113
+ _ => Err(KreuzbergError::parsing(format!(
114
+ "Unsupported file extension: {}",
115
+ file_extension
116
+ ))),
117
+ }
118
+ }
119
+
120
+ fn process_workbook<RS, R>(mut workbook: R, office_metadata: Option<HashMap<String, String>>) -> Result<ExcelWorkbook>
121
+ where
122
+ RS: std::io::Read + std::io::Seek,
123
+ R: Reader<RS>,
124
+ {
125
+ let sheet_names = workbook.sheet_names();
126
+
127
+ let mut sheets = Vec::with_capacity(sheet_names.len());
128
+
129
+ for name in &sheet_names {
130
+ if let Ok(range) = workbook.worksheet_range(name) {
131
+ sheets.push(process_sheet(name, &range));
132
+ }
133
+ }
134
+
135
+ let metadata = extract_metadata(&workbook, &sheet_names, office_metadata);
136
+
137
+ Ok(ExcelWorkbook { sheets, metadata })
138
+ }
139
+
140
+ #[inline]
141
+ fn process_sheet(name: &str, range: &Range<Data>) -> ExcelSheet {
142
+ let (rows, cols) = range.get_size();
143
+ let cell_count = range.used_cells().count();
144
+
145
+ let estimated_capacity = 50 + (cols * 20) + (rows * cols * 12);
146
+
147
+ let markdown = if rows == 0 || cols == 0 {
148
+ format!("## {}\n\n*Empty sheet*", name)
149
+ } else {
150
+ generate_markdown_from_range_optimized(name, range, estimated_capacity)
151
+ };
152
+
153
+ ExcelSheet {
154
+ name: name.to_owned(),
155
+ markdown,
156
+ row_count: rows,
157
+ col_count: cols,
158
+ cell_count,
159
+ }
160
+ }
161
+
162
+ fn generate_markdown_from_range_optimized(sheet_name: &str, range: &Range<Data>, capacity: usize) -> String {
163
+ let mut result = String::with_capacity(capacity);
164
+
165
+ write!(result, "## {}\n\n", sheet_name).unwrap();
166
+
167
+ let rows: Vec<_> = range.rows().collect();
168
+ if rows.is_empty() {
169
+ result.push_str("*No data*");
170
+ return result;
171
+ }
172
+
173
+ let header = &rows[0];
174
+ let header_len = header.len();
175
+
176
+ result.push_str("| ");
177
+ for (i, cell) in header.iter().enumerate() {
178
+ if i > 0 {
179
+ result.push_str(" | ");
180
+ }
181
+ format_cell_value_into(&mut result, cell);
182
+ }
183
+ result.push_str(" |\n");
184
+
185
+ result.push_str("| ");
186
+ for i in 0..header_len {
187
+ if i > 0 {
188
+ result.push_str(" | ");
189
+ }
190
+ result.push_str("---");
191
+ }
192
+ result.push_str(" |\n");
193
+
194
+ for row in rows.iter().skip(1) {
195
+ result.push_str("| ");
196
+ for i in 0..header_len {
197
+ if i > 0 {
198
+ result.push_str(" | ");
199
+ }
200
+ if let Some(cell) = row.get(i) {
201
+ format_cell_value_into(&mut result, cell);
202
+ }
203
+ }
204
+ result.push_str(" |\n");
205
+ }
206
+
207
+ result
208
+ }
209
+
210
+ #[inline]
211
+ fn format_cell_value_into(buffer: &mut String, data: &Data) {
212
+ match data {
213
+ Data::Empty => {}
214
+ Data::String(s) => {
215
+ if s.contains('|') || s.contains('\\') {
216
+ escape_markdown_into(buffer, s);
217
+ } else {
218
+ buffer.push_str(s);
219
+ }
220
+ }
221
+ Data::Float(f) => {
222
+ if f.fract() == 0.0 {
223
+ write!(buffer, "{:.1}", f).unwrap();
224
+ } else {
225
+ write!(buffer, "{}", f).unwrap();
226
+ }
227
+ }
228
+ Data::Int(i) => {
229
+ write!(buffer, "{}", i).unwrap();
230
+ }
231
+ Data::Bool(b) => {
232
+ buffer.push_str(if *b { "true" } else { "false" });
233
+ }
234
+ Data::DateTime(dt) => {
235
+ if let Some(datetime) = dt.as_datetime() {
236
+ write!(buffer, "{}", datetime.format("%Y-%m-%d %H:%M:%S")).unwrap();
237
+ } else {
238
+ write!(buffer, "{:?}", dt).unwrap();
239
+ }
240
+ }
241
+ Data::Error(e) => {
242
+ write!(buffer, "#ERR: {:?}", e).unwrap();
243
+ }
244
+ Data::DateTimeIso(s) => {
245
+ buffer.push_str(s);
246
+ }
247
+ Data::DurationIso(s) => {
248
+ buffer.push_str("DURATION: ");
249
+ buffer.push_str(s);
250
+ }
251
+ }
252
+ }
253
+
254
+ #[inline]
255
+ fn escape_markdown_into(buffer: &mut String, s: &str) {
256
+ for ch in s.chars() {
257
+ match ch {
258
+ '|' => buffer.push_str("\\|"),
259
+ '\\' => buffer.push_str("\\\\"),
260
+ _ => buffer.push(ch),
261
+ }
262
+ }
263
+ }
264
+
265
+ fn extract_metadata<RS, R>(
266
+ workbook: &R,
267
+ sheet_names: &[String],
268
+ office_metadata: Option<HashMap<String, String>>,
269
+ ) -> HashMap<String, String>
270
+ where
271
+ RS: std::io::Read + std::io::Seek,
272
+ R: Reader<RS>,
273
+ {
274
+ let mut metadata = HashMap::with_capacity(4);
275
+
276
+ let sheet_count = sheet_names.len();
277
+ metadata.insert("sheet_count".to_owned(), sheet_count.to_string());
278
+
279
+ let sheet_names_str = if sheet_count <= 5 {
280
+ sheet_names.join(", ")
281
+ } else {
282
+ let mut result = String::with_capacity(100);
283
+ for (i, name) in sheet_names.iter().take(5).enumerate() {
284
+ if i > 0 {
285
+ result.push_str(", ");
286
+ }
287
+ result.push_str(name);
288
+ }
289
+ write!(result, ", ... ({} total)", sheet_count).unwrap();
290
+ result
291
+ };
292
+ metadata.insert("sheet_names".to_owned(), sheet_names_str);
293
+
294
+ let _workbook_metadata = workbook.metadata();
295
+
296
+ if let Some(office_meta) = office_metadata {
297
+ for (key, value) in office_meta {
298
+ metadata.insert(key, value);
299
+ }
300
+ }
301
+
302
+ metadata
303
+ }
304
+
305
+ pub fn excel_to_markdown(workbook: &ExcelWorkbook) -> String {
306
+ let total_capacity: usize = workbook.sheets.iter().map(|sheet| sheet.markdown.len() + 2).sum();
307
+
308
+ let mut result = String::with_capacity(total_capacity);
309
+
310
+ for (i, sheet) in workbook.sheets.iter().enumerate() {
311
+ if i > 0 {
312
+ result.push_str("\n\n");
313
+ }
314
+ let sheet_content = sheet.markdown.trim_end();
315
+ result.push_str(sheet_content);
316
+ }
317
+
318
+ result
319
+ }
320
+
321
+ #[cfg(feature = "office")]
322
+ fn extract_xlsx_office_metadata_from_file(file_path: &str) -> Result<HashMap<String, String>> {
323
+ use std::fs::File;
324
+ use zip::ZipArchive;
325
+
326
+ // OSError/RuntimeError must bubble up - system errors need user reports ~keep
327
+ let file = File::open(file_path)?;
328
+
329
+ let mut archive =
330
+ ZipArchive::new(file).map_err(|e| KreuzbergError::parsing(format!("Failed to open ZIP archive: {}", e)))?;
331
+
332
+ extract_xlsx_office_metadata_from_archive(&mut archive)
333
+ }
334
+
335
+ #[cfg(feature = "office")]
336
+ fn extract_xlsx_office_metadata_from_bytes(data: &[u8]) -> Result<HashMap<String, String>> {
337
+ use zip::ZipArchive;
338
+
339
+ let cursor = Cursor::new(data);
340
+ let mut archive =
341
+ ZipArchive::new(cursor).map_err(|e| KreuzbergError::parsing(format!("Failed to open ZIP archive: {}", e)))?;
342
+
343
+ extract_xlsx_office_metadata_from_archive(&mut archive)
344
+ }
345
+
346
+ #[cfg(feature = "office")]
347
+ fn extract_xlsx_office_metadata_from_archive<R: std::io::Read + std::io::Seek>(
348
+ archive: &mut zip::ZipArchive<R>,
349
+ ) -> Result<HashMap<String, String>> {
350
+ let mut metadata = HashMap::new();
351
+
352
+ if let Ok(core) = extract_core_properties(archive) {
353
+ if let Some(title) = core.title {
354
+ metadata.insert("title".to_string(), title);
355
+ }
356
+ if let Some(creator) = core.creator {
357
+ metadata.insert("creator".to_string(), creator.clone());
358
+ metadata.insert("created_by".to_string(), creator);
359
+ }
360
+ if let Some(subject) = core.subject {
361
+ metadata.insert("subject".to_string(), subject);
362
+ }
363
+ if let Some(keywords) = core.keywords {
364
+ metadata.insert("keywords".to_string(), keywords);
365
+ }
366
+ if let Some(description) = core.description {
367
+ metadata.insert("description".to_string(), description);
368
+ }
369
+ if let Some(modified_by) = core.last_modified_by {
370
+ metadata.insert("modified_by".to_string(), modified_by);
371
+ }
372
+ if let Some(created) = core.created {
373
+ metadata.insert("created_at".to_string(), created);
374
+ }
375
+ if let Some(modified) = core.modified {
376
+ metadata.insert("modified_at".to_string(), modified);
377
+ }
378
+ if let Some(revision) = core.revision {
379
+ metadata.insert("revision".to_string(), revision);
380
+ }
381
+ if let Some(category) = core.category {
382
+ metadata.insert("category".to_string(), category);
383
+ }
384
+ if let Some(content_status) = core.content_status {
385
+ metadata.insert("content_status".to_string(), content_status);
386
+ }
387
+ if let Some(language) = core.language {
388
+ metadata.insert("language".to_string(), language);
389
+ }
390
+ }
391
+
392
+ if let Ok(app) = extract_xlsx_app_properties(archive) {
393
+ if !app.worksheet_names.is_empty() {
394
+ metadata.insert("worksheet_names".to_string(), app.worksheet_names.join(", "));
395
+ }
396
+ if let Some(company) = app.company {
397
+ metadata.insert("organization".to_string(), company);
398
+ }
399
+ if let Some(application) = app.application {
400
+ metadata.insert("application".to_string(), application);
401
+ }
402
+ if let Some(app_version) = app.app_version {
403
+ metadata.insert("application_version".to_string(), app_version);
404
+ }
405
+ }
406
+
407
+ if let Ok(custom) = extract_custom_properties(archive) {
408
+ for (key, value) in custom {
409
+ let value_str = match value {
410
+ Value::String(s) => s,
411
+ Value::Number(n) => n.to_string(),
412
+ Value::Bool(b) => b.to_string(),
413
+ Value::Null => "null".to_string(),
414
+ Value::Array(_) | Value::Object(_) => value.to_string(),
415
+ };
416
+ metadata.insert(format!("custom_{}", key), value_str);
417
+ }
418
+ }
419
+
420
+ Ok(metadata)
421
+ }
422
+
423
+ #[cfg(test)]
424
+ mod tests {
425
+ use super::*;
426
+
427
+ #[test]
428
+ fn test_format_cell_value_into() {
429
+ let mut buffer = String::with_capacity(100);
430
+
431
+ format_cell_value_into(&mut buffer, &Data::Empty);
432
+ assert_eq!(buffer, "");
433
+
434
+ buffer.clear();
435
+ format_cell_value_into(&mut buffer, &Data::String("test".to_owned()));
436
+ assert_eq!(buffer, "test");
437
+
438
+ buffer.clear();
439
+ format_cell_value_into(&mut buffer, &Data::Float(42.0));
440
+ assert_eq!(buffer, "42.0");
441
+
442
+ buffer.clear();
443
+ format_cell_value_into(&mut buffer, &Data::Float(std::f64::consts::PI));
444
+ assert_eq!(buffer, "3.141592653589793");
445
+
446
+ buffer.clear();
447
+ format_cell_value_into(&mut buffer, &Data::Int(100));
448
+ assert_eq!(buffer, "100");
449
+
450
+ buffer.clear();
451
+ format_cell_value_into(&mut buffer, &Data::Bool(true));
452
+ assert_eq!(buffer, "true");
453
+ }
454
+
455
+ #[test]
456
+ fn test_escape_markdown_into() {
457
+ let mut buffer = String::with_capacity(50);
458
+
459
+ escape_markdown_into(&mut buffer, "normal text");
460
+ assert_eq!(buffer, "normal text");
461
+
462
+ buffer.clear();
463
+ escape_markdown_into(&mut buffer, "text|with|pipes");
464
+ assert_eq!(buffer, "text\\|with\\|pipes");
465
+
466
+ buffer.clear();
467
+ escape_markdown_into(&mut buffer, "back\\slash");
468
+ assert_eq!(buffer, "back\\\\slash");
469
+ }
470
+
471
+ #[test]
472
+ fn test_capacity_optimization() {
473
+ let mut buffer = String::with_capacity(100);
474
+ format_cell_value_into(&mut buffer, &Data::String("test".to_owned()));
475
+
476
+ assert!(buffer.capacity() >= 100);
477
+ }
478
+
479
+ #[test]
480
+ fn test_format_cell_value_datetime() {
481
+ use calamine::{ExcelDateTime, ExcelDateTimeType};
482
+ let mut buffer = String::new();
483
+
484
+ let dt = Data::DateTime(ExcelDateTime::new(49353.5, ExcelDateTimeType::DateTime, false));
485
+ format_cell_value_into(&mut buffer, &dt);
486
+ assert!(!buffer.is_empty());
487
+ }
488
+
489
+ #[test]
490
+ fn test_format_cell_value_error() {
491
+ use calamine::CellErrorType;
492
+ let mut buffer = String::new();
493
+
494
+ format_cell_value_into(&mut buffer, &Data::Error(CellErrorType::Div0));
495
+ assert!(buffer.contains("#ERR"));
496
+ }
497
+
498
+ #[test]
499
+ fn test_format_cell_value_datetime_iso() {
500
+ let mut buffer = String::new();
501
+ format_cell_value_into(&mut buffer, &Data::DateTimeIso("2024-01-01T10:30:00".to_owned()));
502
+ assert_eq!(buffer, "2024-01-01T10:30:00");
503
+ }
504
+
505
+ #[test]
506
+ fn test_format_cell_value_duration_iso() {
507
+ let mut buffer = String::new();
508
+ format_cell_value_into(&mut buffer, &Data::DurationIso("PT1H30M".to_owned()));
509
+ assert_eq!(buffer, "DURATION: PT1H30M");
510
+ }
511
+
512
+ #[test]
513
+ fn test_escape_markdown_combined() {
514
+ let mut buffer = String::new();
515
+ escape_markdown_into(&mut buffer, "text|with|pipes\\and\\slashes");
516
+ assert_eq!(buffer, "text\\|with\\|pipes\\\\and\\\\slashes");
517
+ }
518
+
519
+ #[test]
520
+ fn test_escape_markdown_no_special_chars() {
521
+ let mut buffer = String::new();
522
+ escape_markdown_into(&mut buffer, "plain text");
523
+ assert_eq!(buffer, "plain text");
524
+ }
525
+
526
+ #[test]
527
+ fn test_process_sheet_empty() {
528
+ let range: Range<Data> = Range::empty();
529
+ let sheet = process_sheet("EmptySheet", &range);
530
+
531
+ assert_eq!(sheet.name, "EmptySheet");
532
+ assert_eq!(sheet.row_count, 0);
533
+ assert_eq!(sheet.col_count, 0);
534
+ assert_eq!(sheet.cell_count, 0);
535
+ assert!(sheet.markdown.contains("Empty sheet"));
536
+ }
537
+
538
+ #[test]
539
+ fn test_process_sheet_single_cell() {
540
+ let mut range: Range<Data> = Range::new((0, 0), (0, 0));
541
+ range.set_value((0, 0), Data::String("Single Cell".to_owned()));
542
+
543
+ let sheet = process_sheet("Sheet1", &range);
544
+
545
+ assert_eq!(sheet.name, "Sheet1");
546
+ assert_eq!(sheet.row_count, 1);
547
+ assert_eq!(sheet.col_count, 1);
548
+ assert_eq!(sheet.cell_count, 1);
549
+ assert!(sheet.markdown.contains("Single Cell"));
550
+ }
551
+
552
+ #[test]
553
+ fn test_process_sheet_with_data() {
554
+ let mut range: Range<Data> = Range::new((0, 0), (2, 1));
555
+ range.set_value((0, 0), Data::String("Name".to_owned()));
556
+ range.set_value((0, 1), Data::String("Age".to_owned()));
557
+ range.set_value((1, 0), Data::String("Alice".to_owned()));
558
+ range.set_value((1, 1), Data::Int(30));
559
+ range.set_value((2, 0), Data::String("Bob".to_owned()));
560
+ range.set_value((2, 1), Data::Int(25));
561
+
562
+ let sheet = process_sheet("People", &range);
563
+
564
+ assert_eq!(sheet.name, "People");
565
+ assert_eq!(sheet.row_count, 3);
566
+ assert_eq!(sheet.col_count, 2);
567
+ assert!(sheet.markdown.contains("Name"));
568
+ assert!(sheet.markdown.contains("Age"));
569
+ assert!(sheet.markdown.contains("Alice"));
570
+ assert!(sheet.markdown.contains("30"));
571
+ }
572
+
573
+ #[test]
574
+ fn test_generate_markdown_empty_range() {
575
+ let range: Range<Data> = Range::new((0, 0), (0, 0));
576
+ let markdown = generate_markdown_from_range_optimized("Test", &range, 100);
577
+
578
+ assert!(markdown.contains("## Test"));
579
+ assert!(markdown.contains("|"));
580
+ }
581
+
582
+ #[test]
583
+ fn test_generate_markdown_with_headers() {
584
+ let mut range: Range<Data> = Range::new((0, 0), (1, 2));
585
+ range.set_value((0, 0), Data::String("Col1".to_owned()));
586
+ range.set_value((0, 1), Data::String("Col2".to_owned()));
587
+ range.set_value((0, 2), Data::String("Col3".to_owned()));
588
+ range.set_value((1, 0), Data::String("A".to_owned()));
589
+ range.set_value((1, 1), Data::String("B".to_owned()));
590
+ range.set_value((1, 2), Data::String("C".to_owned()));
591
+
592
+ let markdown = generate_markdown_from_range_optimized("Sheet1", &range, 200);
593
+
594
+ assert!(markdown.contains("## Sheet1"));
595
+ assert!(markdown.contains("Col1"));
596
+ assert!(markdown.contains("Col2"));
597
+ assert!(markdown.contains("Col3"));
598
+ assert!(markdown.contains("---"));
599
+ assert!(markdown.contains("A"));
600
+ assert!(markdown.contains("B"));
601
+ assert!(markdown.contains("C"));
602
+ }
603
+
604
+ #[test]
605
+ fn test_generate_markdown_sparse_data() {
606
+ let mut range: Range<Data> = Range::new((0, 0), (2, 2));
607
+ range.set_value((0, 0), Data::String("A".to_owned()));
608
+ range.set_value((0, 1), Data::String("B".to_owned()));
609
+ range.set_value((0, 2), Data::String("C".to_owned()));
610
+ range.set_value((1, 0), Data::String("X".to_owned()));
611
+ range.set_value((1, 2), Data::String("Z".to_owned()));
612
+
613
+ let markdown = generate_markdown_from_range_optimized("Sparse", &range, 200);
614
+
615
+ assert!(markdown.contains("X"));
616
+ assert!(markdown.contains("Z"));
617
+ let lines: Vec<&str> = markdown.lines().collect();
618
+ assert!(lines.iter().any(|line| line.contains("| |") || line.contains("| |")));
619
+ }
620
+
621
+ #[test]
622
+ fn test_format_cell_value_float_integer() {
623
+ let mut buffer = String::new();
624
+ format_cell_value_into(&mut buffer, &Data::Float(100.0));
625
+ assert_eq!(buffer, "100.0");
626
+ }
627
+
628
+ #[test]
629
+ fn test_format_cell_value_float_decimal() {
630
+ let mut buffer = String::new();
631
+ format_cell_value_into(&mut buffer, &Data::Float(12.3456));
632
+ assert_eq!(buffer, "12.3456");
633
+ }
634
+
635
+ #[test]
636
+ fn test_format_cell_value_bool_false() {
637
+ let mut buffer = String::new();
638
+ format_cell_value_into(&mut buffer, &Data::Bool(false));
639
+ assert_eq!(buffer, "false");
640
+ }
641
+
642
+ #[test]
643
+ fn test_format_cell_value_string_with_pipe() {
644
+ let mut buffer = String::new();
645
+ format_cell_value_into(&mut buffer, &Data::String("value|with|pipes".to_owned()));
646
+ assert_eq!(buffer, "value\\|with\\|pipes");
647
+ }
648
+
649
+ #[test]
650
+ fn test_format_cell_value_string_with_backslash() {
651
+ let mut buffer = String::new();
652
+ format_cell_value_into(&mut buffer, &Data::String("path\\to\\file".to_owned()));
653
+ assert_eq!(buffer, "path\\\\to\\\\file");
654
+ }
655
+
656
+ #[test]
657
+ fn test_markdown_table_structure() {
658
+ let mut range: Range<Data> = Range::new((0, 0), (2, 1));
659
+ range.set_value((0, 0), Data::String("H1".to_owned()));
660
+ range.set_value((0, 1), Data::String("H2".to_owned()));
661
+ range.set_value((1, 0), Data::String("A".to_owned()));
662
+ range.set_value((1, 1), Data::String("B".to_owned()));
663
+
664
+ let markdown = generate_markdown_from_range_optimized("Test", &range, 100);
665
+
666
+ let lines: Vec<&str> = markdown.lines().collect();
667
+ assert!(lines[0].contains("## Test"));
668
+ assert!(lines[2].starts_with("| "));
669
+ assert!(lines[3].contains("---"));
670
+ assert!(lines[4].starts_with("| "));
671
+ }
672
+
673
+ #[test]
674
+ fn test_process_sheet_metadata() {
675
+ let mut range: Range<Data> = Range::new((0, 0), (9, 4));
676
+ for row in 0..10 {
677
+ for col in 0..5 {
678
+ range.set_value((row, col), Data::String(format!("R{}C{}", row, col)));
679
+ }
680
+ }
681
+
682
+ let sheet = process_sheet("Data", &range);
683
+
684
+ assert_eq!(sheet.row_count, 10);
685
+ assert_eq!(sheet.col_count, 5);
686
+ assert_eq!(sheet.cell_count, 50);
687
+ }
688
+ }