kreuzberg 4.0.0.rc1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (265) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +8 -0
  3. data/.rspec +3 -0
  4. data/.rubocop.yaml +534 -0
  5. data/Gemfile +9 -0
  6. data/Gemfile.lock +157 -0
  7. data/README.md +421 -0
  8. data/Rakefile +25 -0
  9. data/Steepfile +47 -0
  10. data/examples/async_patterns.rb +340 -0
  11. data/ext/kreuzberg_rb/extconf.rb +35 -0
  12. data/ext/kreuzberg_rb/native/Cargo.toml +36 -0
  13. data/ext/kreuzberg_rb/native/README.md +425 -0
  14. data/ext/kreuzberg_rb/native/build.rs +17 -0
  15. data/ext/kreuzberg_rb/native/include/ieeefp.h +11 -0
  16. data/ext/kreuzberg_rb/native/include/msvc_compat/strings.h +14 -0
  17. data/ext/kreuzberg_rb/native/include/strings.h +20 -0
  18. data/ext/kreuzberg_rb/native/include/unistd.h +47 -0
  19. data/ext/kreuzberg_rb/native/src/lib.rs +2939 -0
  20. data/extconf.rb +28 -0
  21. data/kreuzberg.gemspec +105 -0
  22. data/lib/kreuzberg/api_proxy.rb +142 -0
  23. data/lib/kreuzberg/cache_api.rb +45 -0
  24. data/lib/kreuzberg/cli.rb +55 -0
  25. data/lib/kreuzberg/cli_proxy.rb +127 -0
  26. data/lib/kreuzberg/config.rb +684 -0
  27. data/lib/kreuzberg/errors.rb +50 -0
  28. data/lib/kreuzberg/extraction_api.rb +84 -0
  29. data/lib/kreuzberg/mcp_proxy.rb +186 -0
  30. data/lib/kreuzberg/ocr_backend_protocol.rb +113 -0
  31. data/lib/kreuzberg/post_processor_protocol.rb +86 -0
  32. data/lib/kreuzberg/result.rb +216 -0
  33. data/lib/kreuzberg/setup_lib_path.rb +79 -0
  34. data/lib/kreuzberg/validator_protocol.rb +89 -0
  35. data/lib/kreuzberg/version.rb +5 -0
  36. data/lib/kreuzberg.rb +82 -0
  37. data/pkg/kreuzberg-4.0.0.rc1.gem +0 -0
  38. data/sig/kreuzberg/internal.rbs +184 -0
  39. data/sig/kreuzberg.rbs +468 -0
  40. data/spec/binding/cache_spec.rb +227 -0
  41. data/spec/binding/cli_proxy_spec.rb +87 -0
  42. data/spec/binding/cli_spec.rb +54 -0
  43. data/spec/binding/config_spec.rb +345 -0
  44. data/spec/binding/config_validation_spec.rb +283 -0
  45. data/spec/binding/error_handling_spec.rb +213 -0
  46. data/spec/binding/errors_spec.rb +66 -0
  47. data/spec/binding/plugins/ocr_backend_spec.rb +307 -0
  48. data/spec/binding/plugins/postprocessor_spec.rb +269 -0
  49. data/spec/binding/plugins/validator_spec.rb +274 -0
  50. data/spec/examples.txt +104 -0
  51. data/spec/fixtures/config.toml +39 -0
  52. data/spec/fixtures/config.yaml +42 -0
  53. data/spec/fixtures/invalid_config.toml +4 -0
  54. data/spec/smoke/package_spec.rb +178 -0
  55. data/spec/spec_helper.rb +42 -0
  56. data/vendor/kreuzberg/Cargo.toml +134 -0
  57. data/vendor/kreuzberg/README.md +175 -0
  58. data/vendor/kreuzberg/build.rs +460 -0
  59. data/vendor/kreuzberg/src/api/error.rs +81 -0
  60. data/vendor/kreuzberg/src/api/handlers.rs +199 -0
  61. data/vendor/kreuzberg/src/api/mod.rs +79 -0
  62. data/vendor/kreuzberg/src/api/server.rs +353 -0
  63. data/vendor/kreuzberg/src/api/types.rs +170 -0
  64. data/vendor/kreuzberg/src/bin/profile_extract.rs +455 -0
  65. data/vendor/kreuzberg/src/cache/mod.rs +1143 -0
  66. data/vendor/kreuzberg/src/chunking/mod.rs +677 -0
  67. data/vendor/kreuzberg/src/core/batch_mode.rs +35 -0
  68. data/vendor/kreuzberg/src/core/config.rs +1032 -0
  69. data/vendor/kreuzberg/src/core/extractor.rs +903 -0
  70. data/vendor/kreuzberg/src/core/io.rs +327 -0
  71. data/vendor/kreuzberg/src/core/mime.rs +615 -0
  72. data/vendor/kreuzberg/src/core/mod.rs +42 -0
  73. data/vendor/kreuzberg/src/core/pipeline.rs +906 -0
  74. data/vendor/kreuzberg/src/embeddings.rs +323 -0
  75. data/vendor/kreuzberg/src/error.rs +431 -0
  76. data/vendor/kreuzberg/src/extraction/archive.rs +954 -0
  77. data/vendor/kreuzberg/src/extraction/docx.rs +40 -0
  78. data/vendor/kreuzberg/src/extraction/email.rs +854 -0
  79. data/vendor/kreuzberg/src/extraction/excel.rs +688 -0
  80. data/vendor/kreuzberg/src/extraction/html.rs +553 -0
  81. data/vendor/kreuzberg/src/extraction/image.rs +368 -0
  82. data/vendor/kreuzberg/src/extraction/libreoffice.rs +564 -0
  83. data/vendor/kreuzberg/src/extraction/mod.rs +77 -0
  84. data/vendor/kreuzberg/src/extraction/office_metadata/app_properties.rs +398 -0
  85. data/vendor/kreuzberg/src/extraction/office_metadata/core_properties.rs +247 -0
  86. data/vendor/kreuzberg/src/extraction/office_metadata/custom_properties.rs +240 -0
  87. data/vendor/kreuzberg/src/extraction/office_metadata/mod.rs +128 -0
  88. data/vendor/kreuzberg/src/extraction/pandoc/batch.rs +275 -0
  89. data/vendor/kreuzberg/src/extraction/pandoc/mime_types.rs +178 -0
  90. data/vendor/kreuzberg/src/extraction/pandoc/mod.rs +491 -0
  91. data/vendor/kreuzberg/src/extraction/pandoc/server.rs +496 -0
  92. data/vendor/kreuzberg/src/extraction/pandoc/subprocess.rs +1188 -0
  93. data/vendor/kreuzberg/src/extraction/pandoc/version.rs +162 -0
  94. data/vendor/kreuzberg/src/extraction/pptx.rs +3000 -0
  95. data/vendor/kreuzberg/src/extraction/structured.rs +490 -0
  96. data/vendor/kreuzberg/src/extraction/table.rs +328 -0
  97. data/vendor/kreuzberg/src/extraction/text.rs +269 -0
  98. data/vendor/kreuzberg/src/extraction/xml.rs +333 -0
  99. data/vendor/kreuzberg/src/extractors/archive.rs +425 -0
  100. data/vendor/kreuzberg/src/extractors/docx.rs +479 -0
  101. data/vendor/kreuzberg/src/extractors/email.rs +129 -0
  102. data/vendor/kreuzberg/src/extractors/excel.rs +344 -0
  103. data/vendor/kreuzberg/src/extractors/html.rs +410 -0
  104. data/vendor/kreuzberg/src/extractors/image.rs +195 -0
  105. data/vendor/kreuzberg/src/extractors/mod.rs +268 -0
  106. data/vendor/kreuzberg/src/extractors/pandoc.rs +201 -0
  107. data/vendor/kreuzberg/src/extractors/pdf.rs +496 -0
  108. data/vendor/kreuzberg/src/extractors/pptx.rs +234 -0
  109. data/vendor/kreuzberg/src/extractors/structured.rs +126 -0
  110. data/vendor/kreuzberg/src/extractors/text.rs +242 -0
  111. data/vendor/kreuzberg/src/extractors/xml.rs +128 -0
  112. data/vendor/kreuzberg/src/image/dpi.rs +164 -0
  113. data/vendor/kreuzberg/src/image/mod.rs +6 -0
  114. data/vendor/kreuzberg/src/image/preprocessing.rs +417 -0
  115. data/vendor/kreuzberg/src/image/resize.rs +89 -0
  116. data/vendor/kreuzberg/src/keywords/config.rs +154 -0
  117. data/vendor/kreuzberg/src/keywords/mod.rs +237 -0
  118. data/vendor/kreuzberg/src/keywords/processor.rs +267 -0
  119. data/vendor/kreuzberg/src/keywords/rake.rs +294 -0
  120. data/vendor/kreuzberg/src/keywords/types.rs +68 -0
  121. data/vendor/kreuzberg/src/keywords/yake.rs +163 -0
  122. data/vendor/kreuzberg/src/language_detection/mod.rs +942 -0
  123. data/vendor/kreuzberg/src/lib.rs +102 -0
  124. data/vendor/kreuzberg/src/mcp/mod.rs +32 -0
  125. data/vendor/kreuzberg/src/mcp/server.rs +1966 -0
  126. data/vendor/kreuzberg/src/ocr/cache.rs +469 -0
  127. data/vendor/kreuzberg/src/ocr/error.rs +37 -0
  128. data/vendor/kreuzberg/src/ocr/hocr.rs +216 -0
  129. data/vendor/kreuzberg/src/ocr/mod.rs +58 -0
  130. data/vendor/kreuzberg/src/ocr/processor.rs +847 -0
  131. data/vendor/kreuzberg/src/ocr/table/mod.rs +4 -0
  132. data/vendor/kreuzberg/src/ocr/table/tsv_parser.rs +144 -0
  133. data/vendor/kreuzberg/src/ocr/tesseract_backend.rs +450 -0
  134. data/vendor/kreuzberg/src/ocr/types.rs +393 -0
  135. data/vendor/kreuzberg/src/ocr/utils.rs +47 -0
  136. data/vendor/kreuzberg/src/ocr/validation.rs +206 -0
  137. data/vendor/kreuzberg/src/pdf/error.rs +122 -0
  138. data/vendor/kreuzberg/src/pdf/images.rs +139 -0
  139. data/vendor/kreuzberg/src/pdf/metadata.rs +346 -0
  140. data/vendor/kreuzberg/src/pdf/mod.rs +50 -0
  141. data/vendor/kreuzberg/src/pdf/rendering.rs +369 -0
  142. data/vendor/kreuzberg/src/pdf/table.rs +420 -0
  143. data/vendor/kreuzberg/src/pdf/text.rs +161 -0
  144. data/vendor/kreuzberg/src/plugins/extractor.rs +1010 -0
  145. data/vendor/kreuzberg/src/plugins/mod.rs +209 -0
  146. data/vendor/kreuzberg/src/plugins/ocr.rs +629 -0
  147. data/vendor/kreuzberg/src/plugins/processor.rs +641 -0
  148. data/vendor/kreuzberg/src/plugins/registry.rs +1324 -0
  149. data/vendor/kreuzberg/src/plugins/traits.rs +258 -0
  150. data/vendor/kreuzberg/src/plugins/validator.rs +955 -0
  151. data/vendor/kreuzberg/src/stopwords/mod.rs +1470 -0
  152. data/vendor/kreuzberg/src/text/mod.rs +19 -0
  153. data/vendor/kreuzberg/src/text/quality.rs +697 -0
  154. data/vendor/kreuzberg/src/text/string_utils.rs +217 -0
  155. data/vendor/kreuzberg/src/text/token_reduction/cjk_utils.rs +164 -0
  156. data/vendor/kreuzberg/src/text/token_reduction/config.rs +100 -0
  157. data/vendor/kreuzberg/src/text/token_reduction/core.rs +796 -0
  158. data/vendor/kreuzberg/src/text/token_reduction/filters.rs +902 -0
  159. data/vendor/kreuzberg/src/text/token_reduction/mod.rs +160 -0
  160. data/vendor/kreuzberg/src/text/token_reduction/semantic.rs +619 -0
  161. data/vendor/kreuzberg/src/text/token_reduction/simd_text.rs +147 -0
  162. data/vendor/kreuzberg/src/types.rs +873 -0
  163. data/vendor/kreuzberg/src/utils/mod.rs +17 -0
  164. data/vendor/kreuzberg/src/utils/quality.rs +959 -0
  165. data/vendor/kreuzberg/src/utils/string_utils.rs +381 -0
  166. data/vendor/kreuzberg/stopwords/af_stopwords.json +53 -0
  167. data/vendor/kreuzberg/stopwords/ar_stopwords.json +482 -0
  168. data/vendor/kreuzberg/stopwords/bg_stopwords.json +261 -0
  169. data/vendor/kreuzberg/stopwords/bn_stopwords.json +400 -0
  170. data/vendor/kreuzberg/stopwords/br_stopwords.json +1205 -0
  171. data/vendor/kreuzberg/stopwords/ca_stopwords.json +280 -0
  172. data/vendor/kreuzberg/stopwords/cs_stopwords.json +425 -0
  173. data/vendor/kreuzberg/stopwords/da_stopwords.json +172 -0
  174. data/vendor/kreuzberg/stopwords/de_stopwords.json +622 -0
  175. data/vendor/kreuzberg/stopwords/el_stopwords.json +849 -0
  176. data/vendor/kreuzberg/stopwords/en_stopwords.json +1300 -0
  177. data/vendor/kreuzberg/stopwords/eo_stopwords.json +175 -0
  178. data/vendor/kreuzberg/stopwords/es_stopwords.json +734 -0
  179. data/vendor/kreuzberg/stopwords/et_stopwords.json +37 -0
  180. data/vendor/kreuzberg/stopwords/eu_stopwords.json +100 -0
  181. data/vendor/kreuzberg/stopwords/fa_stopwords.json +801 -0
  182. data/vendor/kreuzberg/stopwords/fi_stopwords.json +849 -0
  183. data/vendor/kreuzberg/stopwords/fr_stopwords.json +693 -0
  184. data/vendor/kreuzberg/stopwords/ga_stopwords.json +111 -0
  185. data/vendor/kreuzberg/stopwords/gl_stopwords.json +162 -0
  186. data/vendor/kreuzberg/stopwords/gu_stopwords.json +226 -0
  187. data/vendor/kreuzberg/stopwords/ha_stopwords.json +41 -0
  188. data/vendor/kreuzberg/stopwords/he_stopwords.json +196 -0
  189. data/vendor/kreuzberg/stopwords/hi_stopwords.json +227 -0
  190. data/vendor/kreuzberg/stopwords/hr_stopwords.json +181 -0
  191. data/vendor/kreuzberg/stopwords/hu_stopwords.json +791 -0
  192. data/vendor/kreuzberg/stopwords/hy_stopwords.json +47 -0
  193. data/vendor/kreuzberg/stopwords/id_stopwords.json +760 -0
  194. data/vendor/kreuzberg/stopwords/it_stopwords.json +634 -0
  195. data/vendor/kreuzberg/stopwords/ja_stopwords.json +136 -0
  196. data/vendor/kreuzberg/stopwords/kn_stopwords.json +84 -0
  197. data/vendor/kreuzberg/stopwords/ko_stopwords.json +681 -0
  198. data/vendor/kreuzberg/stopwords/ku_stopwords.json +64 -0
  199. data/vendor/kreuzberg/stopwords/la_stopwords.json +51 -0
  200. data/vendor/kreuzberg/stopwords/lt_stopwords.json +476 -0
  201. data/vendor/kreuzberg/stopwords/lv_stopwords.json +163 -0
  202. data/vendor/kreuzberg/stopwords/ml_stopwords.json +1 -0
  203. data/vendor/kreuzberg/stopwords/mr_stopwords.json +101 -0
  204. data/vendor/kreuzberg/stopwords/ms_stopwords.json +477 -0
  205. data/vendor/kreuzberg/stopwords/ne_stopwords.json +490 -0
  206. data/vendor/kreuzberg/stopwords/nl_stopwords.json +415 -0
  207. data/vendor/kreuzberg/stopwords/no_stopwords.json +223 -0
  208. data/vendor/kreuzberg/stopwords/pl_stopwords.json +331 -0
  209. data/vendor/kreuzberg/stopwords/pt_stopwords.json +562 -0
  210. data/vendor/kreuzberg/stopwords/ro_stopwords.json +436 -0
  211. data/vendor/kreuzberg/stopwords/ru_stopwords.json +561 -0
  212. data/vendor/kreuzberg/stopwords/si_stopwords.json +193 -0
  213. data/vendor/kreuzberg/stopwords/sk_stopwords.json +420 -0
  214. data/vendor/kreuzberg/stopwords/sl_stopwords.json +448 -0
  215. data/vendor/kreuzberg/stopwords/so_stopwords.json +32 -0
  216. data/vendor/kreuzberg/stopwords/st_stopwords.json +33 -0
  217. data/vendor/kreuzberg/stopwords/sv_stopwords.json +420 -0
  218. data/vendor/kreuzberg/stopwords/sw_stopwords.json +76 -0
  219. data/vendor/kreuzberg/stopwords/ta_stopwords.json +129 -0
  220. data/vendor/kreuzberg/stopwords/te_stopwords.json +54 -0
  221. data/vendor/kreuzberg/stopwords/th_stopwords.json +118 -0
  222. data/vendor/kreuzberg/stopwords/tl_stopwords.json +149 -0
  223. data/vendor/kreuzberg/stopwords/tr_stopwords.json +506 -0
  224. data/vendor/kreuzberg/stopwords/uk_stopwords.json +75 -0
  225. data/vendor/kreuzberg/stopwords/ur_stopwords.json +519 -0
  226. data/vendor/kreuzberg/stopwords/vi_stopwords.json +647 -0
  227. data/vendor/kreuzberg/stopwords/yo_stopwords.json +62 -0
  228. data/vendor/kreuzberg/stopwords/zh_stopwords.json +796 -0
  229. data/vendor/kreuzberg/stopwords/zu_stopwords.json +31 -0
  230. data/vendor/kreuzberg/tests/api_tests.rs +966 -0
  231. data/vendor/kreuzberg/tests/archive_integration.rs +543 -0
  232. data/vendor/kreuzberg/tests/batch_orchestration.rs +542 -0
  233. data/vendor/kreuzberg/tests/batch_processing.rs +304 -0
  234. data/vendor/kreuzberg/tests/chunking_offset_demo.rs +92 -0
  235. data/vendor/kreuzberg/tests/concurrency_stress.rs +509 -0
  236. data/vendor/kreuzberg/tests/config_features.rs +580 -0
  237. data/vendor/kreuzberg/tests/config_loading_tests.rs +439 -0
  238. data/vendor/kreuzberg/tests/core_integration.rs +493 -0
  239. data/vendor/kreuzberg/tests/csv_integration.rs +424 -0
  240. data/vendor/kreuzberg/tests/docx_metadata_extraction_test.rs +124 -0
  241. data/vendor/kreuzberg/tests/email_integration.rs +325 -0
  242. data/vendor/kreuzberg/tests/error_handling.rs +393 -0
  243. data/vendor/kreuzberg/tests/format_integration.rs +159 -0
  244. data/vendor/kreuzberg/tests/helpers/mod.rs +142 -0
  245. data/vendor/kreuzberg/tests/image_integration.rs +253 -0
  246. data/vendor/kreuzberg/tests/keywords_integration.rs +479 -0
  247. data/vendor/kreuzberg/tests/keywords_quality.rs +509 -0
  248. data/vendor/kreuzberg/tests/mime_detection.rs +428 -0
  249. data/vendor/kreuzberg/tests/ocr_configuration.rs +510 -0
  250. data/vendor/kreuzberg/tests/ocr_errors.rs +676 -0
  251. data/vendor/kreuzberg/tests/ocr_quality.rs +627 -0
  252. data/vendor/kreuzberg/tests/ocr_stress.rs +469 -0
  253. data/vendor/kreuzberg/tests/pandoc_integration.rs +503 -0
  254. data/vendor/kreuzberg/tests/pdf_integration.rs +43 -0
  255. data/vendor/kreuzberg/tests/pipeline_integration.rs +1412 -0
  256. data/vendor/kreuzberg/tests/plugin_ocr_backend_test.rs +771 -0
  257. data/vendor/kreuzberg/tests/plugin_postprocessor_test.rs +561 -0
  258. data/vendor/kreuzberg/tests/plugin_system.rs +921 -0
  259. data/vendor/kreuzberg/tests/plugin_validator_test.rs +783 -0
  260. data/vendor/kreuzberg/tests/registry_integration_tests.rs +607 -0
  261. data/vendor/kreuzberg/tests/security_validation.rs +404 -0
  262. data/vendor/kreuzberg/tests/stopwords_integration_test.rs +888 -0
  263. data/vendor/kreuzberg/tests/test_fastembed.rs +609 -0
  264. data/vendor/kreuzberg/tests/xlsx_metadata_extraction_test.rs +87 -0
  265. metadata +471 -0
@@ -0,0 +1,2939 @@
1
+ //! Kreuzberg Ruby Bindings (Magnus)
2
+ //!
3
+ //! High-performance document intelligence framework bindings for Ruby.
4
+ //! Provides extraction, OCR, chunking, and language detection for 30+ file formats.
5
+
6
+ use html_to_markdown_rs::options::{
7
+ CodeBlockStyle, ConversionOptions, HeadingStyle, HighlightStyle, ListIndentType, NewlineStyle, PreprocessingPreset,
8
+ WhitespaceMode,
9
+ };
10
+ use kreuzberg::keywords::{
11
+ KeywordAlgorithm as RustKeywordAlgorithm, KeywordConfig as RustKeywordConfig, RakeParams as RustRakeParams,
12
+ YakeParams as RustYakeParams,
13
+ };
14
+ use kreuzberg::types::TesseractConfig as RustTesseractConfig;
15
+ use kreuzberg::{
16
+ ChunkingConfig, EmbeddingConfig, ExtractionConfig, ExtractionResult as RustExtractionResult, ImageExtractionConfig,
17
+ ImagePreprocessingConfig, KreuzbergError, LanguageDetectionConfig, OcrConfig, PdfConfig, PostProcessorConfig,
18
+ TokenReductionConfig,
19
+ };
20
+ use magnus::exception::ExceptionClass;
21
+ use magnus::r_hash::ForEach;
22
+ use magnus::value::ReprValue;
23
+ use magnus::{Error, IntoValue, RArray, RHash, Ruby, Symbol, TryConvert, Value, function, scan_args::scan_args};
24
+ use std::fs;
25
+ use std::path::{Path, PathBuf};
26
+
27
+ /// Keeps Ruby values alive across plugin registrations by informing the GC.
28
+ struct GcGuardedValue {
29
+ value: Value,
30
+ }
31
+
32
+ impl GcGuardedValue {
33
+ fn new(value: Value) -> Self {
34
+ let ruby = Ruby::get().expect("Ruby not initialized");
35
+ ruby.gc_register_address(&value);
36
+ Self { value }
37
+ }
38
+
39
+ fn value(&self) -> Value {
40
+ self.value
41
+ }
42
+ }
43
+
44
+ impl Drop for GcGuardedValue {
45
+ fn drop(&mut self) {
46
+ if let Ok(ruby) = Ruby::get() {
47
+ ruby.gc_unregister_address(&self.value);
48
+ }
49
+ }
50
+ }
51
+
52
+ /// Convert Kreuzberg errors to Ruby exceptions
53
+ fn kreuzberg_error(err: KreuzbergError) -> Error {
54
+ let ruby = Ruby::get().expect("Ruby not initialized");
55
+
56
+ let fetch_error_class = |name: &str| -> Option<ExceptionClass> {
57
+ ruby.eval::<ExceptionClass>(&format!("Kreuzberg::Errors::{}", name))
58
+ .ok()
59
+ };
60
+
61
+ match err {
62
+ KreuzbergError::Validation { message, .. } => {
63
+ if let Some(class) = fetch_error_class("ValidationError") {
64
+ Error::new(class, message)
65
+ } else {
66
+ Error::new(ruby.exception_arg_error(), message)
67
+ }
68
+ }
69
+ KreuzbergError::Parsing { message, .. } => {
70
+ if let Some(class) = fetch_error_class("ParsingError") {
71
+ Error::new(class, message)
72
+ } else {
73
+ Error::new(ruby.exception_runtime_error(), format!("ParsingError: {}", message))
74
+ }
75
+ }
76
+ KreuzbergError::Ocr { message, .. } => {
77
+ if let Some(class) = fetch_error_class("OCRError") {
78
+ Error::new(class, message)
79
+ } else {
80
+ Error::new(ruby.exception_runtime_error(), format!("OCRError: {}", message))
81
+ }
82
+ }
83
+ KreuzbergError::MissingDependency(message) => {
84
+ if let Some(class) = fetch_error_class("MissingDependencyError") {
85
+ Error::new(class, message)
86
+ } else {
87
+ Error::new(
88
+ ruby.exception_runtime_error(),
89
+ format!("MissingDependencyError: {}", message),
90
+ )
91
+ }
92
+ }
93
+ KreuzbergError::Plugin { message, plugin_name } => {
94
+ if let Some(class) = fetch_error_class("PluginError") {
95
+ Error::new(class, format!("{}: {}", plugin_name, message))
96
+ } else {
97
+ Error::new(
98
+ ruby.exception_runtime_error(),
99
+ format!("Plugin error in '{}': {}", plugin_name, message),
100
+ )
101
+ }
102
+ }
103
+ KreuzbergError::Io(err) => {
104
+ if let Some(class) = fetch_error_class("IOError") {
105
+ Error::new(class, err.to_string())
106
+ } else {
107
+ Error::new(ruby.exception_runtime_error(), format!("IO error: {}", err))
108
+ }
109
+ }
110
+ KreuzbergError::UnsupportedFormat(message) => {
111
+ if let Some(class) = fetch_error_class("UnsupportedFormatError") {
112
+ Error::new(class, message)
113
+ } else {
114
+ Error::new(
115
+ ruby.exception_runtime_error(),
116
+ format!("UnsupportedFormatError: {}", message),
117
+ )
118
+ }
119
+ }
120
+ other => Error::new(ruby.exception_runtime_error(), other.to_string()),
121
+ }
122
+ }
123
+
124
+ fn runtime_error(message: impl Into<String>) -> Error {
125
+ let ruby = Ruby::get().expect("Ruby not initialized");
126
+ Error::new(ruby.exception_runtime_error(), message.into())
127
+ }
128
+
129
+ /// Convert Ruby Symbol or String to Rust String
130
+ fn symbol_to_string(value: Value) -> Result<String, Error> {
131
+ if let Some(symbol) = Symbol::from_value(value) {
132
+ Ok(symbol.name()?.to_string())
133
+ } else {
134
+ String::try_convert(value)
135
+ }
136
+ }
137
+
138
+ /// Get keyword argument from hash (supports both symbol and string keys)
139
+ fn get_kw(ruby: &Ruby, hash: RHash, name: &str) -> Option<Value> {
140
+ hash.get(name).or_else(|| {
141
+ let sym = ruby.intern(name);
142
+ hash.get(sym)
143
+ })
144
+ }
145
+
146
+ fn set_hash_entry(_ruby: &Ruby, hash: &RHash, key: &str, value: Value) -> Result<(), Error> {
147
+ hash.aset(key, value)?;
148
+ Ok(())
149
+ }
150
+
151
+ fn ocr_config_to_ruby_hash(ruby: &Ruby, config: &kreuzberg::OcrConfig) -> Result<RHash, Error> {
152
+ let value =
153
+ serde_json::to_value(config).map_err(|e| runtime_error(format!("Failed to serialize OCR config: {}", e)))?;
154
+ let ruby_value = json_value_to_ruby(ruby, &value)?;
155
+ RHash::try_convert(ruby_value).map_err(|_| runtime_error("OCR config must return a Hash"))
156
+ }
157
+
158
+ fn cache_root_dir() -> Result<PathBuf, Error> {
159
+ std::env::current_dir()
160
+ .map(|dir| dir.join(".kreuzberg"))
161
+ .map_err(|e| runtime_error(format!("Failed to get current directory: {}", e)))
162
+ }
163
+
164
+ fn cache_directories(root: &Path) -> Result<Vec<PathBuf>, Error> {
165
+ if !root.exists() {
166
+ return Ok(vec![]);
167
+ }
168
+
169
+ let mut dirs = vec![root.to_path_buf()];
170
+ let entries = fs::read_dir(root).map_err(|e| runtime_error(format!("Failed to read cache root: {}", e)))?;
171
+
172
+ for entry in entries {
173
+ let entry = entry.map_err(|e| runtime_error(format!("Failed to read cache directory entry: {}", e)))?;
174
+ if entry
175
+ .file_type()
176
+ .map_err(|e| runtime_error(format!("Failed to determine cache entry type: {}", e)))?
177
+ .is_dir()
178
+ {
179
+ dirs.push(entry.path());
180
+ }
181
+ }
182
+
183
+ Ok(dirs)
184
+ }
185
+
186
+ fn json_value_to_ruby(ruby: &Ruby, value: &serde_json::Value) -> Result<Value, Error> {
187
+ Ok(match value {
188
+ serde_json::Value::Null => ruby.qnil().as_value(),
189
+ serde_json::Value::Bool(b) => {
190
+ if *b {
191
+ ruby.qtrue().as_value()
192
+ } else {
193
+ ruby.qfalse().as_value()
194
+ }
195
+ }
196
+ serde_json::Value::Number(num) => {
197
+ if let Some(i) = num.as_i64() {
198
+ ruby.integer_from_i64(i).into_value_with(ruby)
199
+ } else if let Some(u) = num.as_u64() {
200
+ ruby.integer_from_u64(u).into_value_with(ruby)
201
+ } else if let Some(f) = num.as_f64() {
202
+ ruby.float_from_f64(f).into_value_with(ruby)
203
+ } else {
204
+ ruby.qnil().as_value()
205
+ }
206
+ }
207
+ serde_json::Value::String(s) => ruby.str_new(s).into_value_with(ruby),
208
+ serde_json::Value::Array(items) => {
209
+ let ary = ruby.ary_new();
210
+ for item in items {
211
+ ary.push(json_value_to_ruby(ruby, item)?)?;
212
+ }
213
+ ary.into_value_with(ruby)
214
+ }
215
+ serde_json::Value::Object(map) => {
216
+ let hash = ruby.hash_new();
217
+ for (key, val) in map {
218
+ let key_value = ruby.str_new(key).into_value_with(ruby);
219
+ let val_value = json_value_to_ruby(ruby, val)?;
220
+ hash.aset(key_value, val_value)?;
221
+ }
222
+ hash.into_value_with(ruby)
223
+ }
224
+ })
225
+ }
226
+
227
+ fn ruby_key_to_string(value: Value) -> Result<String, Error> {
228
+ if let Ok(sym) = Symbol::try_convert(value) {
229
+ Ok(sym.name()?.to_string())
230
+ } else {
231
+ String::try_convert(value)
232
+ }
233
+ }
234
+
235
+ fn ruby_value_to_json(value: Value) -> Result<serde_json::Value, Error> {
236
+ let ruby = Ruby::get().expect("Ruby not initialized");
237
+
238
+ if value.is_nil() {
239
+ return Ok(serde_json::Value::Null);
240
+ }
241
+
242
+ if value.equal(ruby.qtrue())? {
243
+ return Ok(serde_json::Value::Bool(true));
244
+ }
245
+
246
+ if value.equal(ruby.qfalse())? {
247
+ return Ok(serde_json::Value::Bool(false));
248
+ }
249
+
250
+ if let Ok(integer) = i64::try_convert(value) {
251
+ return Ok(serde_json::Value::Number(integer.into()));
252
+ }
253
+
254
+ if let Ok(unsigned) = u64::try_convert(value) {
255
+ return Ok(serde_json::Value::Number(serde_json::Number::from(unsigned)));
256
+ }
257
+
258
+ if let Ok(float) = f64::try_convert(value) {
259
+ if let Some(num) = serde_json::Number::from_f64(float) {
260
+ return Ok(serde_json::Value::Number(num));
261
+ }
262
+ }
263
+
264
+ if let Ok(sym) = Symbol::try_convert(value) {
265
+ return Ok(serde_json::Value::String(sym.name()?.to_string()));
266
+ }
267
+
268
+ if let Ok(string) = String::try_convert(value) {
269
+ return Ok(serde_json::Value::String(string));
270
+ }
271
+
272
+ if let Ok(array) = RArray::try_convert(value) {
273
+ let mut values = Vec::with_capacity(array.len());
274
+ for item in array.into_iter() {
275
+ values.push(ruby_value_to_json(item)?);
276
+ }
277
+ return Ok(serde_json::Value::Array(values));
278
+ }
279
+
280
+ if let Ok(hash) = RHash::try_convert(value) {
281
+ let mut map = serde_json::Map::new();
282
+ hash.foreach(|key: Value, val: Value| {
283
+ let key_string = ruby_key_to_string(key)?;
284
+ let json_value = ruby_value_to_json(val)?;
285
+ map.insert(key_string, json_value);
286
+ Ok(ForEach::Continue)
287
+ })?;
288
+
289
+ return Ok(serde_json::Value::Object(map));
290
+ }
291
+
292
+ Err(runtime_error("Unsupported Ruby value for JSON conversion"))
293
+ }
294
+
295
+ /// Parse OcrConfig from Ruby Hash
296
+ fn parse_ocr_config(ruby: &Ruby, hash: RHash) -> Result<OcrConfig, Error> {
297
+ let backend = if let Some(val) = get_kw(ruby, hash, "backend") {
298
+ symbol_to_string(val)?
299
+ } else {
300
+ "tesseract".to_string()
301
+ };
302
+
303
+ let language = if let Some(val) = get_kw(ruby, hash, "language") {
304
+ symbol_to_string(val)?
305
+ } else {
306
+ "eng".to_string()
307
+ };
308
+
309
+ let mut config = OcrConfig {
310
+ backend,
311
+ language,
312
+ tesseract_config: None,
313
+ };
314
+
315
+ if let Some(val) = get_kw(ruby, hash, "tesseract_config")
316
+ && !val.is_nil()
317
+ {
318
+ let tc_json = ruby_value_to_json(val)?;
319
+ let parsed: RustTesseractConfig =
320
+ serde_json::from_value(tc_json).map_err(|e| runtime_error(format!("Invalid tesseract_config: {}", e)))?;
321
+ config.tesseract_config = Some(parsed);
322
+ }
323
+
324
+ Ok(config)
325
+ }
326
+
327
+ /// Parse ChunkingConfig from Ruby Hash
328
+ fn parse_chunking_config(ruby: &Ruby, hash: RHash) -> Result<ChunkingConfig, Error> {
329
+ let max_chars = if let Some(val) = get_kw(ruby, hash, "max_chars") {
330
+ usize::try_convert(val)?
331
+ } else {
332
+ 1000
333
+ };
334
+
335
+ let max_overlap = if let Some(val) = get_kw(ruby, hash, "max_overlap") {
336
+ usize::try_convert(val)?
337
+ } else {
338
+ 200
339
+ };
340
+
341
+ let preset = if let Some(val) = get_kw(ruby, hash, "preset")
342
+ && !val.is_nil()
343
+ {
344
+ Some(symbol_to_string(val)?)
345
+ } else {
346
+ None
347
+ };
348
+
349
+ let embedding = if let Some(val) = get_kw(ruby, hash, "embedding")
350
+ && !val.is_nil()
351
+ {
352
+ let json_value = ruby_value_to_json(val)?;
353
+ let parsed: EmbeddingConfig = serde_json::from_value(json_value)
354
+ .map_err(|e| runtime_error(format!("Invalid chunking.embedding: {}", e)))?;
355
+ Some(parsed)
356
+ } else {
357
+ None
358
+ };
359
+
360
+ let config = ChunkingConfig {
361
+ max_chars,
362
+ max_overlap,
363
+ embedding,
364
+ preset,
365
+ };
366
+
367
+ Ok(config)
368
+ }
369
+
370
+ /// Parse LanguageDetectionConfig from Ruby Hash
371
+ fn parse_language_detection_config(ruby: &Ruby, hash: RHash) -> Result<LanguageDetectionConfig, Error> {
372
+ let enabled = if let Some(val) = get_kw(ruby, hash, "enabled") {
373
+ bool::try_convert(val)?
374
+ } else {
375
+ true
376
+ };
377
+
378
+ let min_confidence = if let Some(val) = get_kw(ruby, hash, "min_confidence") {
379
+ f64::try_convert(val)?
380
+ } else {
381
+ 0.8
382
+ };
383
+
384
+ let detect_multiple = if let Some(val) = get_kw(ruby, hash, "detect_multiple") {
385
+ bool::try_convert(val)?
386
+ } else {
387
+ false
388
+ };
389
+
390
+ let config = LanguageDetectionConfig {
391
+ enabled,
392
+ min_confidence,
393
+ detect_multiple,
394
+ };
395
+
396
+ Ok(config)
397
+ }
398
+
399
+ /// Parse PdfConfig from Ruby Hash
400
+ fn parse_pdf_config(ruby: &Ruby, hash: RHash) -> Result<PdfConfig, Error> {
401
+ let extract_images = if let Some(val) = get_kw(ruby, hash, "extract_images") {
402
+ bool::try_convert(val)?
403
+ } else {
404
+ false
405
+ };
406
+
407
+ let passwords = if let Some(val) = get_kw(ruby, hash, "passwords") {
408
+ if !val.is_nil() {
409
+ let arr = RArray::try_convert(val)?;
410
+ Some(arr.to_vec::<String>()?)
411
+ } else {
412
+ None
413
+ }
414
+ } else {
415
+ None
416
+ };
417
+
418
+ let extract_metadata = if let Some(val) = get_kw(ruby, hash, "extract_metadata") {
419
+ bool::try_convert(val)?
420
+ } else {
421
+ true
422
+ };
423
+
424
+ let config = PdfConfig {
425
+ extract_images,
426
+ passwords,
427
+ extract_metadata,
428
+ };
429
+
430
+ Ok(config)
431
+ }
432
+
433
+ /// Parse ImageExtractionConfig from Ruby Hash
434
+ fn parse_image_extraction_config(ruby: &Ruby, hash: RHash) -> Result<ImageExtractionConfig, Error> {
435
+ let extract_images = if let Some(val) = get_kw(ruby, hash, "extract_images") {
436
+ bool::try_convert(val)?
437
+ } else {
438
+ true
439
+ };
440
+
441
+ let target_dpi = if let Some(val) = get_kw(ruby, hash, "target_dpi") {
442
+ i32::try_convert(val)?
443
+ } else {
444
+ 300
445
+ };
446
+
447
+ let max_image_dimension = if let Some(val) = get_kw(ruby, hash, "max_image_dimension") {
448
+ i32::try_convert(val)?
449
+ } else {
450
+ 4096
451
+ };
452
+
453
+ let auto_adjust_dpi = if let Some(val) = get_kw(ruby, hash, "auto_adjust_dpi") {
454
+ bool::try_convert(val)?
455
+ } else {
456
+ true
457
+ };
458
+
459
+ let min_dpi = if let Some(val) = get_kw(ruby, hash, "min_dpi") {
460
+ i32::try_convert(val)?
461
+ } else {
462
+ 72
463
+ };
464
+
465
+ let max_dpi = if let Some(val) = get_kw(ruby, hash, "max_dpi") {
466
+ i32::try_convert(val)?
467
+ } else {
468
+ 600
469
+ };
470
+
471
+ let config = ImageExtractionConfig {
472
+ extract_images,
473
+ target_dpi,
474
+ max_image_dimension,
475
+ auto_adjust_dpi,
476
+ min_dpi,
477
+ max_dpi,
478
+ };
479
+
480
+ Ok(config)
481
+ }
482
+
483
+ /// Parse ImagePreprocessingConfig from Ruby Hash
484
+ ///
485
+ /// Note: Currently not used in ExtractionConfig but provided for completeness.
486
+ /// ImagePreprocessingConfig is typically used in OCR operations.
487
+ #[allow(dead_code)]
488
+ fn parse_image_preprocessing_config(ruby: &Ruby, hash: RHash) -> Result<ImagePreprocessingConfig, Error> {
489
+ let target_dpi = if let Some(val) = get_kw(ruby, hash, "target_dpi") {
490
+ i32::try_convert(val)?
491
+ } else {
492
+ 300
493
+ };
494
+
495
+ let auto_rotate = if let Some(val) = get_kw(ruby, hash, "auto_rotate") {
496
+ bool::try_convert(val)?
497
+ } else {
498
+ true
499
+ };
500
+
501
+ let deskew = if let Some(val) = get_kw(ruby, hash, "deskew") {
502
+ bool::try_convert(val)?
503
+ } else {
504
+ true
505
+ };
506
+
507
+ let denoise = if let Some(val) = get_kw(ruby, hash, "denoise") {
508
+ bool::try_convert(val)?
509
+ } else {
510
+ false
511
+ };
512
+
513
+ let contrast_enhance = if let Some(val) = get_kw(ruby, hash, "contrast_enhance") {
514
+ bool::try_convert(val)?
515
+ } else {
516
+ false
517
+ };
518
+
519
+ let binarization_method = if let Some(val) = get_kw(ruby, hash, "binarization_method") {
520
+ symbol_to_string(val)?
521
+ } else {
522
+ "otsu".to_string()
523
+ };
524
+
525
+ let invert_colors = if let Some(val) = get_kw(ruby, hash, "invert_colors") {
526
+ bool::try_convert(val)?
527
+ } else {
528
+ false
529
+ };
530
+
531
+ let config = ImagePreprocessingConfig {
532
+ target_dpi,
533
+ auto_rotate,
534
+ deskew,
535
+ denoise,
536
+ contrast_enhance,
537
+ binarization_method,
538
+ invert_colors,
539
+ };
540
+
541
+ Ok(config)
542
+ }
543
+
544
+ /// Parse PostProcessorConfig from Ruby Hash
545
+ fn parse_postprocessor_config(ruby: &Ruby, hash: RHash) -> Result<PostProcessorConfig, Error> {
546
+ let enabled = if let Some(val) = get_kw(ruby, hash, "enabled") {
547
+ bool::try_convert(val)?
548
+ } else {
549
+ true
550
+ };
551
+
552
+ let enabled_processors = if let Some(val) = get_kw(ruby, hash, "enabled_processors")
553
+ && !val.is_nil()
554
+ {
555
+ let arr = RArray::try_convert(val)?;
556
+ Some(arr.to_vec::<String>()?)
557
+ } else {
558
+ None
559
+ };
560
+
561
+ let disabled_processors = if let Some(val) = get_kw(ruby, hash, "disabled_processors")
562
+ && !val.is_nil()
563
+ {
564
+ let arr = RArray::try_convert(val)?;
565
+ Some(arr.to_vec::<String>()?)
566
+ } else {
567
+ None
568
+ };
569
+
570
+ let config = PostProcessorConfig {
571
+ enabled,
572
+ enabled_processors,
573
+ disabled_processors,
574
+ };
575
+
576
+ Ok(config)
577
+ }
578
+
579
+ /// Parse TokenReductionConfig from Ruby Hash
580
+ fn parse_token_reduction_config(ruby: &Ruby, hash: RHash) -> Result<TokenReductionConfig, Error> {
581
+ let mode = if let Some(val) = get_kw(ruby, hash, "mode") {
582
+ symbol_to_string(val)?
583
+ } else {
584
+ "off".to_string()
585
+ };
586
+
587
+ let preserve_important_words = if let Some(val) = get_kw(ruby, hash, "preserve_important_words") {
588
+ bool::try_convert(val)?
589
+ } else {
590
+ true
591
+ };
592
+
593
+ let config = TokenReductionConfig {
594
+ mode,
595
+ preserve_important_words,
596
+ };
597
+
598
+ Ok(config)
599
+ }
600
+
601
+ fn parse_keyword_config(ruby: &Ruby, hash: RHash) -> Result<RustKeywordConfig, Error> {
602
+ let mut config = RustKeywordConfig::default();
603
+
604
+ if let Some(val) = get_kw(ruby, hash, "algorithm") {
605
+ let algo = symbol_to_string(val)?;
606
+ config.algorithm = match algo.to_lowercase().as_str() {
607
+ "yake" => RustKeywordAlgorithm::Yake,
608
+ "rake" => RustKeywordAlgorithm::Rake,
609
+ other => {
610
+ return Err(runtime_error(format!(
611
+ "Invalid keywords.algorithm '{}', expected 'yake' or 'rake'",
612
+ other
613
+ )));
614
+ }
615
+ };
616
+ }
617
+
618
+ if let Some(val) = get_kw(ruby, hash, "max_keywords") {
619
+ config.max_keywords = usize::try_convert(val)?;
620
+ }
621
+
622
+ if let Some(val) = get_kw(ruby, hash, "min_score") {
623
+ config.min_score = f64::try_convert(val)? as f32;
624
+ }
625
+
626
+ if let Some(val) = get_kw(ruby, hash, "ngram_range") {
627
+ let ary = RArray::try_convert(val)?;
628
+ if ary.len() == 2 {
629
+ let values = ary.to_vec::<i64>()?;
630
+ config.ngram_range = (values[0] as usize, values[1] as usize);
631
+ } else {
632
+ return Err(runtime_error("keywords.ngram_range must have exactly two values"));
633
+ }
634
+ }
635
+
636
+ if let Some(val) = get_kw(ruby, hash, "language") {
637
+ if !val.is_nil() {
638
+ config.language = Some(symbol_to_string(val)?);
639
+ }
640
+ }
641
+
642
+ if let Some(val) = get_kw(ruby, hash, "yake_params")
643
+ && !val.is_nil()
644
+ {
645
+ let yake_hash = RHash::try_convert(val)?;
646
+ let window = if let Some(window_val) = get_kw(ruby, yake_hash, "window_size") {
647
+ usize::try_convert(window_val)?
648
+ } else {
649
+ 2
650
+ };
651
+ config.yake_params = Some(RustYakeParams { window_size: window });
652
+ }
653
+
654
+ if let Some(val) = get_kw(ruby, hash, "rake_params")
655
+ && !val.is_nil()
656
+ {
657
+ let rake_hash = RHash::try_convert(val)?;
658
+ let mut params = RustRakeParams::default();
659
+ if let Some(val) = get_kw(ruby, rake_hash, "min_word_length") {
660
+ params.min_word_length = usize::try_convert(val)?;
661
+ }
662
+ if let Some(val) = get_kw(ruby, rake_hash, "max_words_per_phrase") {
663
+ params.max_words_per_phrase = usize::try_convert(val)?;
664
+ }
665
+ config.rake_params = Some(params);
666
+ }
667
+
668
+ Ok(config)
669
+ }
670
+
671
+ fn parse_html_options(ruby: &Ruby, hash: RHash) -> Result<ConversionOptions, Error> {
672
+ let mut options = ConversionOptions::default();
673
+
674
+ if let Some(val) = get_kw(ruby, hash, "heading_style") {
675
+ let style = symbol_to_string(val)?;
676
+ options.heading_style = match style.to_lowercase().as_str() {
677
+ "atx" => HeadingStyle::Atx,
678
+ "underlined" => HeadingStyle::Underlined,
679
+ "atx_closed" | "atx-closed" => HeadingStyle::AtxClosed,
680
+ other => return Err(runtime_error(format!("Invalid html_options.heading_style '{}'", other))),
681
+ };
682
+ }
683
+
684
+ if let Some(val) = get_kw(ruby, hash, "list_indent_type") {
685
+ let val_str = symbol_to_string(val)?;
686
+ options.list_indent_type = match val_str.to_lowercase().as_str() {
687
+ "spaces" => ListIndentType::Spaces,
688
+ "tabs" => ListIndentType::Tabs,
689
+ other => {
690
+ return Err(runtime_error(format!(
691
+ "Invalid html_options.list_indent_type '{}'",
692
+ other
693
+ )));
694
+ }
695
+ };
696
+ }
697
+
698
+ if let Some(val) = get_kw(ruby, hash, "list_indent_width") {
699
+ options.list_indent_width = usize::try_convert(val)?;
700
+ }
701
+
702
+ if let Some(val) = get_kw(ruby, hash, "bullets") {
703
+ options.bullets = String::try_convert(val)?;
704
+ }
705
+
706
+ if let Some(val) = get_kw(ruby, hash, "strong_em_symbol") {
707
+ let symbol = String::try_convert(val)?;
708
+ let mut chars = symbol.chars();
709
+ options.strong_em_symbol = chars
710
+ .next()
711
+ .ok_or_else(|| runtime_error("html_options.strong_em_symbol must not be empty"))?;
712
+ }
713
+
714
+ if let Some(val) = get_kw(ruby, hash, "escape_asterisks") {
715
+ options.escape_asterisks = bool::try_convert(val)?;
716
+ }
717
+ if let Some(val) = get_kw(ruby, hash, "escape_underscores") {
718
+ options.escape_underscores = bool::try_convert(val)?;
719
+ }
720
+ if let Some(val) = get_kw(ruby, hash, "escape_misc") {
721
+ options.escape_misc = bool::try_convert(val)?;
722
+ }
723
+ if let Some(val) = get_kw(ruby, hash, "escape_ascii") {
724
+ options.escape_ascii = bool::try_convert(val)?;
725
+ }
726
+
727
+ if let Some(val) = get_kw(ruby, hash, "code_language") {
728
+ options.code_language = String::try_convert(val)?;
729
+ }
730
+
731
+ if let Some(val) = get_kw(ruby, hash, "autolinks") {
732
+ options.autolinks = bool::try_convert(val)?;
733
+ }
734
+
735
+ if let Some(val) = get_kw(ruby, hash, "default_title") {
736
+ options.default_title = bool::try_convert(val)?;
737
+ }
738
+
739
+ if let Some(val) = get_kw(ruby, hash, "br_in_tables") {
740
+ options.br_in_tables = bool::try_convert(val)?;
741
+ }
742
+
743
+ if let Some(val) = get_kw(ruby, hash, "hocr_spatial_tables") {
744
+ options.hocr_spatial_tables = bool::try_convert(val)?;
745
+ }
746
+
747
+ if let Some(val) = get_kw(ruby, hash, "highlight_style") {
748
+ let style = symbol_to_string(val)?;
749
+ options.highlight_style = match style.to_lowercase().as_str() {
750
+ "double_equal" | "double-equal" => HighlightStyle::DoubleEqual,
751
+ "html" => HighlightStyle::Html,
752
+ "bold" => HighlightStyle::Bold,
753
+ "none" => HighlightStyle::None,
754
+ other => {
755
+ return Err(runtime_error(format!(
756
+ "Invalid html_options.highlight_style '{}'",
757
+ other
758
+ )));
759
+ }
760
+ };
761
+ }
762
+
763
+ if let Some(val) = get_kw(ruby, hash, "extract_metadata") {
764
+ options.extract_metadata = bool::try_convert(val)?;
765
+ }
766
+
767
+ if let Some(val) = get_kw(ruby, hash, "whitespace_mode") {
768
+ let mode = symbol_to_string(val)?;
769
+ options.whitespace_mode = match mode.to_lowercase().as_str() {
770
+ "normalized" => WhitespaceMode::Normalized,
771
+ "strict" => WhitespaceMode::Strict,
772
+ other => {
773
+ return Err(runtime_error(format!(
774
+ "Invalid html_options.whitespace_mode '{}'",
775
+ other
776
+ )));
777
+ }
778
+ };
779
+ }
780
+
781
+ if let Some(val) = get_kw(ruby, hash, "strip_newlines") {
782
+ options.strip_newlines = bool::try_convert(val)?;
783
+ }
784
+
785
+ if let Some(val) = get_kw(ruby, hash, "wrap") {
786
+ options.wrap = bool::try_convert(val)?;
787
+ }
788
+
789
+ if let Some(val) = get_kw(ruby, hash, "wrap_width") {
790
+ options.wrap_width = usize::try_convert(val)?;
791
+ }
792
+
793
+ if let Some(val) = get_kw(ruby, hash, "convert_as_inline") {
794
+ options.convert_as_inline = bool::try_convert(val)?;
795
+ }
796
+
797
+ if let Some(val) = get_kw(ruby, hash, "sub_symbol") {
798
+ options.sub_symbol = String::try_convert(val)?;
799
+ }
800
+
801
+ if let Some(val) = get_kw(ruby, hash, "sup_symbol") {
802
+ options.sup_symbol = String::try_convert(val)?;
803
+ }
804
+
805
+ if let Some(val) = get_kw(ruby, hash, "newline_style") {
806
+ let style = symbol_to_string(val)?;
807
+ options.newline_style = match style.to_lowercase().as_str() {
808
+ "spaces" => NewlineStyle::Spaces,
809
+ "backslash" => NewlineStyle::Backslash,
810
+ other => return Err(runtime_error(format!("Invalid html_options.newline_style '{}'", other))),
811
+ };
812
+ }
813
+
814
+ if let Some(val) = get_kw(ruby, hash, "code_block_style") {
815
+ let style = symbol_to_string(val)?;
816
+ options.code_block_style = match style.to_lowercase().as_str() {
817
+ "indented" => CodeBlockStyle::Indented,
818
+ "backticks" => CodeBlockStyle::Backticks,
819
+ "tildes" => CodeBlockStyle::Tildes,
820
+ other => {
821
+ return Err(runtime_error(format!(
822
+ "Invalid html_options.code_block_style '{}'",
823
+ other
824
+ )));
825
+ }
826
+ };
827
+ }
828
+
829
+ if let Some(val) = get_kw(ruby, hash, "keep_inline_images_in") {
830
+ let arr = RArray::try_convert(val)?;
831
+ options.keep_inline_images_in = arr.to_vec::<String>()?;
832
+ }
833
+
834
+ if let Some(val) = get_kw(ruby, hash, "encoding") {
835
+ options.encoding = String::try_convert(val)?;
836
+ }
837
+
838
+ if let Some(val) = get_kw(ruby, hash, "debug") {
839
+ options.debug = bool::try_convert(val)?;
840
+ }
841
+
842
+ if let Some(val) = get_kw(ruby, hash, "strip_tags") {
843
+ let arr = RArray::try_convert(val)?;
844
+ options.strip_tags = arr.to_vec::<String>()?;
845
+ }
846
+
847
+ if let Some(val) = get_kw(ruby, hash, "preserve_tags") {
848
+ let arr = RArray::try_convert(val)?;
849
+ options.preserve_tags = arr.to_vec::<String>()?;
850
+ }
851
+
852
+ if let Some(val) = get_kw(ruby, hash, "preprocessing")
853
+ && !val.is_nil()
854
+ {
855
+ let pre_hash = RHash::try_convert(val)?;
856
+ let mut preprocessing = options.preprocessing.clone();
857
+ if let Some(v) = get_kw(ruby, pre_hash, "enabled") {
858
+ preprocessing.enabled = bool::try_convert(v)?;
859
+ }
860
+ if let Some(v) = get_kw(ruby, pre_hash, "preset") {
861
+ let preset = symbol_to_string(v)?;
862
+ preprocessing.preset = match preset.to_lowercase().as_str() {
863
+ "minimal" => PreprocessingPreset::Minimal,
864
+ "standard" => PreprocessingPreset::Standard,
865
+ "aggressive" => PreprocessingPreset::Aggressive,
866
+ other => {
867
+ return Err(runtime_error(format!(
868
+ "Invalid html_options.preprocessing.preset '{}'",
869
+ other
870
+ )));
871
+ }
872
+ };
873
+ }
874
+ if let Some(v) = get_kw(ruby, pre_hash, "remove_navigation") {
875
+ preprocessing.remove_navigation = bool::try_convert(v)?;
876
+ }
877
+ if let Some(v) = get_kw(ruby, pre_hash, "remove_forms") {
878
+ preprocessing.remove_forms = bool::try_convert(v)?;
879
+ }
880
+ options.preprocessing = preprocessing;
881
+ }
882
+
883
+ Ok(options)
884
+ }
885
+
886
+ fn keyword_algorithm_to_str(algo: RustKeywordAlgorithm) -> &'static str {
887
+ match algo {
888
+ RustKeywordAlgorithm::Yake => "yake",
889
+ RustKeywordAlgorithm::Rake => "rake",
890
+ }
891
+ }
892
+
893
+ fn keyword_config_to_ruby_hash(ruby: &Ruby, config: &RustKeywordConfig) -> Result<RHash, Error> {
894
+ let hash = ruby.hash_new();
895
+ hash.aset("algorithm", keyword_algorithm_to_str(config.algorithm))?;
896
+ hash.aset("max_keywords", config.max_keywords as i64)?;
897
+ hash.aset("min_score", config.min_score)?;
898
+ hash.aset("language", config.language.clone().unwrap_or_default())?;
899
+
900
+ let range_array = ruby.ary_new();
901
+ range_array.push(config.ngram_range.0 as i64)?;
902
+ range_array.push(config.ngram_range.1 as i64)?;
903
+ hash.aset("ngram_range", range_array)?;
904
+
905
+ if let Some(yake) = &config.yake_params {
906
+ let yake_hash = ruby.hash_new();
907
+ yake_hash.aset("window_size", yake.window_size as i64)?;
908
+ hash.aset("yake_params", yake_hash)?;
909
+ }
910
+
911
+ if let Some(rake) = &config.rake_params {
912
+ let rake_hash = ruby.hash_new();
913
+ rake_hash.aset("min_word_length", rake.min_word_length as i64)?;
914
+ rake_hash.aset("max_words_per_phrase", rake.max_words_per_phrase as i64)?;
915
+ hash.aset("rake_params", rake_hash)?;
916
+ }
917
+
918
+ Ok(hash)
919
+ }
920
+
921
+ fn html_options_to_ruby_hash(ruby: &Ruby, options: &ConversionOptions) -> Result<RHash, Error> {
922
+ let hash = ruby.hash_new();
923
+ hash.aset(
924
+ "heading_style",
925
+ match options.heading_style {
926
+ HeadingStyle::Atx => "atx",
927
+ HeadingStyle::Underlined => "underlined",
928
+ HeadingStyle::AtxClosed => "atx_closed",
929
+ },
930
+ )?;
931
+ hash.aset(
932
+ "list_indent_type",
933
+ match options.list_indent_type {
934
+ ListIndentType::Spaces => "spaces",
935
+ ListIndentType::Tabs => "tabs",
936
+ },
937
+ )?;
938
+ hash.aset("list_indent_width", options.list_indent_width as i64)?;
939
+ hash.aset("bullets", options.bullets.clone())?;
940
+ hash.aset("strong_em_symbol", options.strong_em_symbol.to_string())?;
941
+ hash.aset("escape_asterisks", options.escape_asterisks)?;
942
+ hash.aset("escape_underscores", options.escape_underscores)?;
943
+ hash.aset("escape_misc", options.escape_misc)?;
944
+ hash.aset("escape_ascii", options.escape_ascii)?;
945
+ hash.aset("code_language", options.code_language.clone())?;
946
+ hash.aset("autolinks", options.autolinks)?;
947
+ hash.aset("default_title", options.default_title)?;
948
+ hash.aset("br_in_tables", options.br_in_tables)?;
949
+ hash.aset("hocr_spatial_tables", options.hocr_spatial_tables)?;
950
+ hash.aset(
951
+ "highlight_style",
952
+ match options.highlight_style {
953
+ HighlightStyle::DoubleEqual => "double_equal",
954
+ HighlightStyle::Html => "html",
955
+ HighlightStyle::Bold => "bold",
956
+ HighlightStyle::None => "none",
957
+ },
958
+ )?;
959
+ hash.aset("extract_metadata", options.extract_metadata)?;
960
+ hash.aset(
961
+ "whitespace_mode",
962
+ match options.whitespace_mode {
963
+ WhitespaceMode::Normalized => "normalized",
964
+ WhitespaceMode::Strict => "strict",
965
+ },
966
+ )?;
967
+ hash.aset("strip_newlines", options.strip_newlines)?;
968
+ hash.aset("wrap", options.wrap)?;
969
+ hash.aset("wrap_width", options.wrap_width as i64)?;
970
+ hash.aset("convert_as_inline", options.convert_as_inline)?;
971
+ hash.aset("sub_symbol", options.sub_symbol.clone())?;
972
+ hash.aset("sup_symbol", options.sup_symbol.clone())?;
973
+ hash.aset(
974
+ "newline_style",
975
+ match options.newline_style {
976
+ NewlineStyle::Spaces => "spaces",
977
+ NewlineStyle::Backslash => "backslash",
978
+ },
979
+ )?;
980
+ hash.aset(
981
+ "code_block_style",
982
+ match options.code_block_style {
983
+ CodeBlockStyle::Indented => "indented",
984
+ CodeBlockStyle::Backticks => "backticks",
985
+ CodeBlockStyle::Tildes => "tildes",
986
+ },
987
+ )?;
988
+
989
+ let keep_inline = ruby.ary_new();
990
+ for tag in &options.keep_inline_images_in {
991
+ keep_inline.push(tag.as_str())?;
992
+ }
993
+ hash.aset("keep_inline_images_in", keep_inline)?;
994
+
995
+ hash.aset("encoding", options.encoding.clone())?;
996
+ hash.aset("debug", options.debug)?;
997
+
998
+ let strip_tags = ruby.ary_new();
999
+ for tag in &options.strip_tags {
1000
+ strip_tags.push(tag.as_str())?;
1001
+ }
1002
+ hash.aset("strip_tags", strip_tags)?;
1003
+
1004
+ let preserve_tags = ruby.ary_new();
1005
+ for tag in &options.preserve_tags {
1006
+ preserve_tags.push(tag.as_str())?;
1007
+ }
1008
+ hash.aset("preserve_tags", preserve_tags)?;
1009
+
1010
+ let pre_hash = ruby.hash_new();
1011
+ pre_hash.aset("enabled", options.preprocessing.enabled)?;
1012
+ pre_hash.aset(
1013
+ "preset",
1014
+ match options.preprocessing.preset {
1015
+ PreprocessingPreset::Minimal => "minimal",
1016
+ PreprocessingPreset::Standard => "standard",
1017
+ PreprocessingPreset::Aggressive => "aggressive",
1018
+ },
1019
+ )?;
1020
+ pre_hash.aset("remove_navigation", options.preprocessing.remove_navigation)?;
1021
+ pre_hash.aset("remove_forms", options.preprocessing.remove_forms)?;
1022
+ hash.aset("preprocessing", pre_hash)?;
1023
+
1024
+ Ok(hash)
1025
+ }
1026
+ /// Parse ExtractionConfig from Ruby Hash
1027
+ fn parse_extraction_config(ruby: &Ruby, opts: Option<RHash>) -> Result<ExtractionConfig, Error> {
1028
+ let mut config = ExtractionConfig::default();
1029
+
1030
+ if let Some(hash) = opts {
1031
+ if let Some(val) = get_kw(ruby, hash, "use_cache") {
1032
+ config.use_cache = bool::try_convert(val)?;
1033
+ }
1034
+
1035
+ if let Some(val) = get_kw(ruby, hash, "enable_quality_processing") {
1036
+ config.enable_quality_processing = bool::try_convert(val)?;
1037
+ }
1038
+
1039
+ if let Some(val) = get_kw(ruby, hash, "force_ocr") {
1040
+ config.force_ocr = bool::try_convert(val)?;
1041
+ }
1042
+
1043
+ if let Some(val) = get_kw(ruby, hash, "ocr")
1044
+ && !val.is_nil()
1045
+ {
1046
+ let ocr_hash = RHash::try_convert(val)?;
1047
+ config.ocr = Some(parse_ocr_config(ruby, ocr_hash)?);
1048
+ }
1049
+
1050
+ if let Some(val) = get_kw(ruby, hash, "chunking")
1051
+ && !val.is_nil()
1052
+ {
1053
+ let chunking_hash = RHash::try_convert(val)?;
1054
+ config.chunking = Some(parse_chunking_config(ruby, chunking_hash)?);
1055
+ }
1056
+
1057
+ if let Some(val) = get_kw(ruby, hash, "language_detection")
1058
+ && !val.is_nil()
1059
+ {
1060
+ let lang_hash = RHash::try_convert(val)?;
1061
+ config.language_detection = Some(parse_language_detection_config(ruby, lang_hash)?);
1062
+ }
1063
+
1064
+ if let Some(val) = get_kw(ruby, hash, "pdf_options")
1065
+ && !val.is_nil()
1066
+ {
1067
+ let pdf_hash = RHash::try_convert(val)?;
1068
+ config.pdf_options = Some(parse_pdf_config(ruby, pdf_hash)?);
1069
+ }
1070
+
1071
+ if let Some(val) = get_kw(ruby, hash, "images")
1072
+ && !val.is_nil()
1073
+ {
1074
+ let images_hash = RHash::try_convert(val)?;
1075
+ config.images = Some(parse_image_extraction_config(ruby, images_hash)?);
1076
+ }
1077
+
1078
+ if let Some(val) = get_kw(ruby, hash, "postprocessor")
1079
+ && !val.is_nil()
1080
+ {
1081
+ let postprocessor_hash = RHash::try_convert(val)?;
1082
+ config.postprocessor = Some(parse_postprocessor_config(ruby, postprocessor_hash)?);
1083
+ }
1084
+
1085
+ if let Some(val) = get_kw(ruby, hash, "token_reduction")
1086
+ && !val.is_nil()
1087
+ {
1088
+ let token_reduction_hash = RHash::try_convert(val)?;
1089
+ config.token_reduction = Some(parse_token_reduction_config(ruby, token_reduction_hash)?);
1090
+ }
1091
+
1092
+ if let Some(val) = get_kw(ruby, hash, "keywords")
1093
+ && !val.is_nil()
1094
+ {
1095
+ let keywords_hash = RHash::try_convert(val)?;
1096
+ config.keywords = Some(parse_keyword_config(ruby, keywords_hash)?);
1097
+ }
1098
+
1099
+ if let Some(val) = get_kw(ruby, hash, "html_options")
1100
+ && !val.is_nil()
1101
+ {
1102
+ let html_hash = RHash::try_convert(val)?;
1103
+ config.html_options = Some(parse_html_options(ruby, html_hash)?);
1104
+ }
1105
+
1106
+ if let Some(val) = get_kw(ruby, hash, "max_concurrent_extractions") {
1107
+ let value = usize::try_convert(val)?;
1108
+ config.max_concurrent_extractions = Some(value);
1109
+ }
1110
+ }
1111
+
1112
+ Ok(config)
1113
+ }
1114
+
1115
+ /// Convert ExtractionConfig to Ruby Hash for Config::Extraction.
1116
+ ///
1117
+ /// This function converts a Rust ExtractionConfig into a Ruby hash that can be passed
1118
+ /// to Kreuzberg::Config::Extraction.new(**hash).
1119
+ fn extraction_config_to_ruby_hash(ruby: &Ruby, config: ExtractionConfig) -> Result<RHash, Error> {
1120
+ let hash = ruby.hash_new();
1121
+
1122
+ set_hash_entry(
1123
+ ruby,
1124
+ &hash,
1125
+ "use_cache",
1126
+ if config.use_cache {
1127
+ ruby.qtrue().as_value()
1128
+ } else {
1129
+ ruby.qfalse().as_value()
1130
+ },
1131
+ )?;
1132
+ set_hash_entry(
1133
+ ruby,
1134
+ &hash,
1135
+ "enable_quality_processing",
1136
+ if config.enable_quality_processing {
1137
+ ruby.qtrue().as_value()
1138
+ } else {
1139
+ ruby.qfalse().as_value()
1140
+ },
1141
+ )?;
1142
+ set_hash_entry(
1143
+ ruby,
1144
+ &hash,
1145
+ "force_ocr",
1146
+ if config.force_ocr {
1147
+ ruby.qtrue().as_value()
1148
+ } else {
1149
+ ruby.qfalse().as_value()
1150
+ },
1151
+ )?;
1152
+
1153
+ if let Some(ocr) = config.ocr {
1154
+ let ocr_hash = ruby.hash_new();
1155
+ set_hash_entry(
1156
+ ruby,
1157
+ &ocr_hash,
1158
+ "backend",
1159
+ ruby.str_new(&ocr.backend).into_value_with(ruby),
1160
+ )?;
1161
+ set_hash_entry(
1162
+ ruby,
1163
+ &ocr_hash,
1164
+ "language",
1165
+ ruby.str_new(&ocr.language).into_value_with(ruby),
1166
+ )?;
1167
+ if let Some(tesseract_config) = ocr.tesseract_config {
1168
+ let tc_json = serde_json::to_value(&tesseract_config)
1169
+ .map_err(|e| runtime_error(format!("Failed to serialize tesseract_config: {}", e)))?;
1170
+ let tc_ruby = json_value_to_ruby(ruby, &tc_json)?;
1171
+ set_hash_entry(ruby, &ocr_hash, "tesseract_config", tc_ruby)?;
1172
+ }
1173
+ set_hash_entry(ruby, &hash, "ocr", ocr_hash.into_value_with(ruby))?;
1174
+ }
1175
+
1176
+ if let Some(chunking) = config.chunking {
1177
+ let chunking_hash = ruby.hash_new();
1178
+ set_hash_entry(
1179
+ ruby,
1180
+ &chunking_hash,
1181
+ "max_chars",
1182
+ ruby.integer_from_i64(chunking.max_chars as i64).into_value_with(ruby),
1183
+ )?;
1184
+ set_hash_entry(
1185
+ ruby,
1186
+ &chunking_hash,
1187
+ "max_overlap",
1188
+ ruby.integer_from_i64(chunking.max_overlap as i64).into_value_with(ruby),
1189
+ )?;
1190
+ if let Some(preset) = chunking.preset {
1191
+ set_hash_entry(
1192
+ ruby,
1193
+ &chunking_hash,
1194
+ "preset",
1195
+ ruby.str_new(&preset).into_value_with(ruby),
1196
+ )?;
1197
+ }
1198
+ if let Some(embedding) = chunking.embedding {
1199
+ let embedding_json = serde_json::to_value(&embedding)
1200
+ .map_err(|e| runtime_error(format!("Failed to serialize embedding config: {}", e)))?;
1201
+ let embedding_value = json_value_to_ruby(ruby, &embedding_json)?;
1202
+ set_hash_entry(ruby, &chunking_hash, "embedding", embedding_value)?;
1203
+ }
1204
+ set_hash_entry(ruby, &hash, "chunking", chunking_hash.into_value_with(ruby))?;
1205
+ }
1206
+
1207
+ if let Some(lang_detection) = config.language_detection {
1208
+ let lang_hash = ruby.hash_new();
1209
+ set_hash_entry(
1210
+ ruby,
1211
+ &lang_hash,
1212
+ "enabled",
1213
+ if lang_detection.enabled {
1214
+ ruby.qtrue().as_value()
1215
+ } else {
1216
+ ruby.qfalse().as_value()
1217
+ },
1218
+ )?;
1219
+ set_hash_entry(
1220
+ ruby,
1221
+ &lang_hash,
1222
+ "min_confidence",
1223
+ ruby.float_from_f64(lang_detection.min_confidence).into_value_with(ruby),
1224
+ )?;
1225
+ set_hash_entry(
1226
+ ruby,
1227
+ &lang_hash,
1228
+ "detect_multiple",
1229
+ if lang_detection.detect_multiple {
1230
+ ruby.qtrue().as_value()
1231
+ } else {
1232
+ ruby.qfalse().as_value()
1233
+ },
1234
+ )?;
1235
+ set_hash_entry(ruby, &hash, "language_detection", lang_hash.into_value_with(ruby))?;
1236
+ }
1237
+
1238
+ if let Some(pdf_options) = config.pdf_options {
1239
+ let pdf_hash = ruby.hash_new();
1240
+ set_hash_entry(
1241
+ ruby,
1242
+ &pdf_hash,
1243
+ "extract_images",
1244
+ if pdf_options.extract_images {
1245
+ ruby.qtrue().as_value()
1246
+ } else {
1247
+ ruby.qfalse().as_value()
1248
+ },
1249
+ )?;
1250
+ if let Some(passwords) = pdf_options.passwords {
1251
+ let passwords_array = ruby.ary_from_vec(passwords);
1252
+ set_hash_entry(ruby, &pdf_hash, "passwords", passwords_array.into_value_with(ruby))?;
1253
+ }
1254
+ set_hash_entry(
1255
+ ruby,
1256
+ &pdf_hash,
1257
+ "extract_metadata",
1258
+ if pdf_options.extract_metadata {
1259
+ ruby.qtrue().as_value()
1260
+ } else {
1261
+ ruby.qfalse().as_value()
1262
+ },
1263
+ )?;
1264
+ set_hash_entry(ruby, &hash, "pdf_options", pdf_hash.into_value_with(ruby))?;
1265
+ }
1266
+
1267
+ if let Some(images) = config.images {
1268
+ let images_hash = ruby.hash_new();
1269
+ set_hash_entry(
1270
+ ruby,
1271
+ &images_hash,
1272
+ "extract_images",
1273
+ if images.extract_images {
1274
+ ruby.qtrue().as_value()
1275
+ } else {
1276
+ ruby.qfalse().as_value()
1277
+ },
1278
+ )?;
1279
+ set_hash_entry(
1280
+ ruby,
1281
+ &images_hash,
1282
+ "target_dpi",
1283
+ ruby.integer_from_i64(images.target_dpi as i64).into_value_with(ruby),
1284
+ )?;
1285
+ set_hash_entry(
1286
+ ruby,
1287
+ &images_hash,
1288
+ "max_image_dimension",
1289
+ ruby.integer_from_i64(images.max_image_dimension as i64)
1290
+ .into_value_with(ruby),
1291
+ )?;
1292
+ set_hash_entry(
1293
+ ruby,
1294
+ &images_hash,
1295
+ "auto_adjust_dpi",
1296
+ if images.auto_adjust_dpi {
1297
+ ruby.qtrue().as_value()
1298
+ } else {
1299
+ ruby.qfalse().as_value()
1300
+ },
1301
+ )?;
1302
+ set_hash_entry(
1303
+ ruby,
1304
+ &images_hash,
1305
+ "min_dpi",
1306
+ ruby.integer_from_i64(images.min_dpi as i64).into_value_with(ruby),
1307
+ )?;
1308
+ set_hash_entry(
1309
+ ruby,
1310
+ &images_hash,
1311
+ "max_dpi",
1312
+ ruby.integer_from_i64(images.max_dpi as i64).into_value_with(ruby),
1313
+ )?;
1314
+ set_hash_entry(ruby, &hash, "image_extraction", images_hash.into_value_with(ruby))?;
1315
+ }
1316
+
1317
+ if let Some(postprocessor) = config.postprocessor {
1318
+ let pp_hash = ruby.hash_new();
1319
+ set_hash_entry(
1320
+ ruby,
1321
+ &pp_hash,
1322
+ "enabled",
1323
+ if postprocessor.enabled {
1324
+ ruby.qtrue().as_value()
1325
+ } else {
1326
+ ruby.qfalse().as_value()
1327
+ },
1328
+ )?;
1329
+ if let Some(enabled_processors) = postprocessor.enabled_processors {
1330
+ let enabled_array = ruby.ary_from_vec(enabled_processors);
1331
+ set_hash_entry(
1332
+ ruby,
1333
+ &pp_hash,
1334
+ "enabled_processors",
1335
+ enabled_array.into_value_with(ruby),
1336
+ )?;
1337
+ }
1338
+ if let Some(disabled_processors) = postprocessor.disabled_processors {
1339
+ let disabled_array = ruby.ary_from_vec(disabled_processors);
1340
+ set_hash_entry(
1341
+ ruby,
1342
+ &pp_hash,
1343
+ "disabled_processors",
1344
+ disabled_array.into_value_with(ruby),
1345
+ )?;
1346
+ }
1347
+ set_hash_entry(ruby, &hash, "postprocessor", pp_hash.into_value_with(ruby))?;
1348
+ }
1349
+
1350
+ if let Some(token_reduction) = config.token_reduction {
1351
+ let tr_hash = ruby.hash_new();
1352
+ set_hash_entry(
1353
+ ruby,
1354
+ &tr_hash,
1355
+ "mode",
1356
+ ruby.str_new(&token_reduction.mode).into_value_with(ruby),
1357
+ )?;
1358
+ set_hash_entry(
1359
+ ruby,
1360
+ &tr_hash,
1361
+ "preserve_important_words",
1362
+ if token_reduction.preserve_important_words {
1363
+ ruby.qtrue().as_value()
1364
+ } else {
1365
+ ruby.qfalse().as_value()
1366
+ },
1367
+ )?;
1368
+ set_hash_entry(ruby, &hash, "token_reduction", tr_hash.into_value_with(ruby))?;
1369
+ }
1370
+
1371
+ if let Some(keywords) = config.keywords {
1372
+ let keywords_hash = keyword_config_to_ruby_hash(ruby, &keywords)?;
1373
+ set_hash_entry(ruby, &hash, "keywords", keywords_hash.into_value_with(ruby))?;
1374
+ }
1375
+
1376
+ if let Some(html_options) = config.html_options {
1377
+ let html_hash = html_options_to_ruby_hash(ruby, &html_options)?;
1378
+ set_hash_entry(ruby, &hash, "html_options", html_hash.into_value_with(ruby))?;
1379
+ }
1380
+
1381
+ if let Some(max_concurrent) = config.max_concurrent_extractions {
1382
+ set_hash_entry(
1383
+ ruby,
1384
+ &hash,
1385
+ "max_concurrent_extractions",
1386
+ ruby.integer_from_u64(max_concurrent as u64).into_value_with(ruby),
1387
+ )?;
1388
+ }
1389
+
1390
+ Ok(hash)
1391
+ }
1392
+
1393
+ /// Load extraction configuration from a file.
1394
+ ///
1395
+ /// Detects the file format from the extension (.toml, .yaml, .json)
1396
+ /// and loads the configuration accordingly. Returns a hash to be used by Ruby.
1397
+ ///
1398
+ /// @param path [String] Path to the configuration file
1399
+ /// @return [Hash] Configuration hash
1400
+ ///
1401
+ /// @example Load from TOML
1402
+ /// hash = Kreuzberg._config_from_file_native("config.toml")
1403
+ ///
1404
+ /// @example Load from YAML
1405
+ /// hash = Kreuzberg._config_from_file_native("config.yaml")
1406
+ ///
1407
+ fn config_from_file(path: String) -> Result<RHash, Error> {
1408
+ let ruby = Ruby::get().expect("Ruby not initialized");
1409
+ let file_path = Path::new(&path);
1410
+
1411
+ let extension = file_path
1412
+ .extension()
1413
+ .and_then(|ext| ext.to_str())
1414
+ .ok_or_else(|| runtime_error("File path must have an extension (.toml, .yaml, or .json)"))?;
1415
+
1416
+ let config = match extension {
1417
+ "toml" => ExtractionConfig::from_toml_file(file_path).map_err(kreuzberg_error)?,
1418
+ "yaml" => ExtractionConfig::from_yaml_file(file_path).map_err(kreuzberg_error)?,
1419
+ "json" => ExtractionConfig::from_json_file(file_path).map_err(kreuzberg_error)?,
1420
+ _ => {
1421
+ return Err(runtime_error(format!(
1422
+ "Unsupported file extension '{}'. Supported: .toml, .yaml, .json",
1423
+ extension
1424
+ )));
1425
+ }
1426
+ };
1427
+
1428
+ extraction_config_to_ruby_hash(&ruby, config)
1429
+ }
1430
+
1431
+ /// Discover configuration file in current or parent directories.
1432
+ ///
1433
+ /// Searches for kreuzberg.toml, kreuzberg.yaml, or kreuzberg.json in the current
1434
+ /// directory and parent directories. Returns nil if no config file is found.
1435
+ ///
1436
+ /// @return [Hash, nil] Configuration hash or nil if not found
1437
+ ///
1438
+ /// @example
1439
+ /// hash = Kreuzberg._config_discover_native
1440
+ /// # => {...config hash...} or nil
1441
+ ///
1442
+ fn config_discover() -> Result<Value, Error> {
1443
+ let ruby = Ruby::get().expect("Ruby not initialized");
1444
+
1445
+ let maybe_config = ExtractionConfig::discover().map_err(kreuzberg_error)?;
1446
+
1447
+ match maybe_config {
1448
+ Some(config) => {
1449
+ let hash = extraction_config_to_ruby_hash(&ruby, config)?;
1450
+ Ok(hash.as_value())
1451
+ }
1452
+ None => Ok(ruby.qnil().as_value()),
1453
+ }
1454
+ }
1455
+
1456
+ /// Convert Rust ExtractionResult to Ruby Hash
1457
+ fn extraction_result_to_ruby(ruby: &Ruby, result: RustExtractionResult) -> Result<RHash, Error> {
1458
+ let hash = ruby.hash_new();
1459
+
1460
+ let content_value = ruby.str_new(result.content.as_str()).into_value_with(ruby);
1461
+ set_hash_entry(ruby, &hash, "content", content_value)?;
1462
+
1463
+ let mime_value = ruby.str_new(result.mime_type.as_str()).into_value_with(ruby);
1464
+ set_hash_entry(ruby, &hash, "mime_type", mime_value)?;
1465
+
1466
+ let metadata_json = serde_json::to_string(&result.metadata)
1467
+ .map_err(|e| runtime_error(format!("Failed to serialize metadata: {}", e)))?;
1468
+ let metadata_json_value = ruby.str_new(&metadata_json).into_value_with(ruby);
1469
+ set_hash_entry(ruby, &hash, "metadata_json", metadata_json_value)?;
1470
+ let metadata_value = serde_json::to_value(&result.metadata)
1471
+ .map_err(|e| runtime_error(format!("Failed to serialize metadata: {}", e)))?;
1472
+ let metadata_hash = json_value_to_ruby(ruby, &metadata_value)?;
1473
+ set_hash_entry(ruby, &hash, "metadata", metadata_hash)?;
1474
+
1475
+ let tables_array = ruby.ary_new();
1476
+ for table in result.tables {
1477
+ let table_hash = ruby.hash_new();
1478
+
1479
+ let cells_array = ruby.ary_new();
1480
+ for row in table.cells {
1481
+ let row_array = ruby.ary_from_vec(row);
1482
+ cells_array.push(row_array)?;
1483
+ }
1484
+ table_hash.aset("cells", cells_array)?;
1485
+
1486
+ table_hash.aset("markdown", table.markdown)?;
1487
+
1488
+ table_hash.aset("page_number", table.page_number)?;
1489
+
1490
+ tables_array.push(table_hash)?;
1491
+ }
1492
+ let tables_value = tables_array.into_value_with(ruby);
1493
+ set_hash_entry(ruby, &hash, "tables", tables_value)?;
1494
+
1495
+ if let Some(langs) = result.detected_languages {
1496
+ let langs_array = ruby.ary_from_vec(langs);
1497
+ let langs_value = langs_array.into_value_with(ruby);
1498
+ set_hash_entry(ruby, &hash, "detected_languages", langs_value)?;
1499
+ } else {
1500
+ set_hash_entry(ruby, &hash, "detected_languages", ruby.qnil().as_value())?;
1501
+ }
1502
+
1503
+ if let Some(chunks) = result.chunks {
1504
+ let chunks_array = ruby.ary_new();
1505
+ for chunk in chunks {
1506
+ let chunk_hash = ruby.hash_new();
1507
+ chunk_hash.aset("content", chunk.content)?;
1508
+ chunk_hash.aset("char_start", chunk.metadata.char_start)?;
1509
+ chunk_hash.aset("char_end", chunk.metadata.char_end)?;
1510
+ if let Some(token_count) = chunk.metadata.token_count {
1511
+ chunk_hash.aset("token_count", token_count)?;
1512
+ } else {
1513
+ chunk_hash.aset("token_count", ruby.qnil().as_value())?;
1514
+ }
1515
+ chunk_hash.aset("chunk_index", chunk.metadata.chunk_index)?;
1516
+ chunk_hash.aset("total_chunks", chunk.metadata.total_chunks)?;
1517
+ if let Some(embedding) = chunk.embedding {
1518
+ let embedding_array = ruby.ary_new();
1519
+ for value in embedding {
1520
+ embedding_array.push(ruby.float_from_f64(value as f64).into_value_with(ruby))?;
1521
+ }
1522
+ chunk_hash.aset("embedding", embedding_array)?;
1523
+ } else {
1524
+ chunk_hash.aset("embedding", ruby.qnil().as_value())?;
1525
+ }
1526
+ chunks_array.push(chunk_hash)?;
1527
+ }
1528
+ let chunks_value = chunks_array.into_value_with(ruby);
1529
+ set_hash_entry(ruby, &hash, "chunks", chunks_value)?;
1530
+ } else {
1531
+ set_hash_entry(ruby, &hash, "chunks", ruby.qnil().as_value())?;
1532
+ }
1533
+
1534
+ if let Some(images) = result.images {
1535
+ let images_array = ruby.ary_new();
1536
+ for image in images {
1537
+ let image_hash = ruby.hash_new();
1538
+ let data_value = ruby.str_from_slice(&image.data).into_value_with(ruby);
1539
+ image_hash.aset("data", data_value)?;
1540
+ image_hash.aset("format", image.format)?;
1541
+ image_hash.aset("image_index", image.image_index as i64)?;
1542
+ if let Some(page) = image.page_number {
1543
+ image_hash.aset("page_number", page as i64)?;
1544
+ } else {
1545
+ image_hash.aset("page_number", ruby.qnil().as_value())?;
1546
+ }
1547
+ if let Some(width) = image.width {
1548
+ image_hash.aset("width", width as i64)?;
1549
+ } else {
1550
+ image_hash.aset("width", ruby.qnil().as_value())?;
1551
+ }
1552
+ if let Some(height) = image.height {
1553
+ image_hash.aset("height", height as i64)?;
1554
+ } else {
1555
+ image_hash.aset("height", ruby.qnil().as_value())?;
1556
+ }
1557
+ if let Some(colorspace) = image.colorspace {
1558
+ image_hash.aset("colorspace", colorspace)?;
1559
+ } else {
1560
+ image_hash.aset("colorspace", ruby.qnil().as_value())?;
1561
+ }
1562
+ if let Some(bits) = image.bits_per_component {
1563
+ image_hash.aset("bits_per_component", bits as i64)?;
1564
+ } else {
1565
+ image_hash.aset("bits_per_component", ruby.qnil().as_value())?;
1566
+ }
1567
+ image_hash.aset(
1568
+ "is_mask",
1569
+ if image.is_mask {
1570
+ ruby.qtrue().as_value()
1571
+ } else {
1572
+ ruby.qfalse().as_value()
1573
+ },
1574
+ )?;
1575
+ if let Some(description) = image.description {
1576
+ image_hash.aset("description", description)?;
1577
+ } else {
1578
+ image_hash.aset("description", ruby.qnil().as_value())?;
1579
+ }
1580
+ if let Some(ocr_result) = image.ocr_result {
1581
+ let nested = extraction_result_to_ruby(ruby, *ocr_result)?;
1582
+ image_hash.aset("ocr_result", nested.into_value_with(ruby))?;
1583
+ } else {
1584
+ image_hash.aset("ocr_result", ruby.qnil().as_value())?;
1585
+ }
1586
+ images_array.push(image_hash)?;
1587
+ }
1588
+ set_hash_entry(ruby, &hash, "images", images_array.into_value_with(ruby))?;
1589
+ } else {
1590
+ set_hash_entry(ruby, &hash, "images", ruby.qnil().as_value())?;
1591
+ }
1592
+
1593
+ Ok(hash)
1594
+ }
1595
+
1596
+ /// Extract content from a file (synchronous).
1597
+ ///
1598
+ /// @param path [String] Path to the file
1599
+ /// @param mime_type [String, nil] Optional MIME type hint
1600
+ /// @param options [Hash] Extraction configuration
1601
+ /// @return [Hash] Extraction result with :content, :mime_type, :metadata, :tables, etc.
1602
+ ///
1603
+ /// @example Basic usage
1604
+ /// result = Kreuzberg.extract_file_sync("document.pdf")
1605
+ /// puts result[:content]
1606
+ ///
1607
+ /// @example With OCR
1608
+ /// result = Kreuzberg.extract_file_sync("scanned.pdf", nil, force_ocr: true)
1609
+ ///
1610
+ fn extract_file_sync(args: &[Value]) -> Result<RHash, Error> {
1611
+ let ruby = Ruby::get().expect("Ruby not initialized");
1612
+ let args = scan_args::<(String,), (Option<String>,), (), (), RHash, ()>(args)?;
1613
+ let (path,) = args.required;
1614
+ let (mime_type,) = args.optional;
1615
+ let opts = Some(args.keywords);
1616
+
1617
+ let config = parse_extraction_config(&ruby, opts)?;
1618
+
1619
+ let result = kreuzberg::extract_file_sync(&path, mime_type.as_deref(), &config).map_err(kreuzberg_error)?;
1620
+
1621
+ extraction_result_to_ruby(&ruby, result)
1622
+ }
1623
+
1624
+ /// Extract content from bytes (synchronous).
1625
+ ///
1626
+ /// @param data [String] Binary data to extract
1627
+ /// @param mime_type [String] MIME type of the data
1628
+ /// @param options [Hash] Extraction configuration
1629
+ /// @return [Hash] Extraction result
1630
+ ///
1631
+ /// @example
1632
+ /// data = File.binread("document.pdf")
1633
+ /// result = Kreuzberg.extract_bytes_sync(data, "application/pdf")
1634
+ ///
1635
+ fn extract_bytes_sync(args: &[Value]) -> Result<RHash, Error> {
1636
+ let ruby = Ruby::get().expect("Ruby not initialized");
1637
+ let args = scan_args::<(String, String), (), (), (), RHash, ()>(args)?;
1638
+ let (data, mime_type) = args.required;
1639
+ let opts = Some(args.keywords);
1640
+
1641
+ let config = parse_extraction_config(&ruby, opts)?;
1642
+
1643
+ let result = kreuzberg::extract_bytes_sync(data.as_bytes(), &mime_type, &config).map_err(kreuzberg_error)?;
1644
+
1645
+ extraction_result_to_ruby(&ruby, result)
1646
+ }
1647
+
1648
+ /// Batch extract content from multiple files (synchronous).
1649
+ ///
1650
+ /// @param paths [Array<String>] List of file paths
1651
+ /// @param options [Hash] Extraction configuration
1652
+ /// @return [Array<Hash>] Array of extraction results
1653
+ ///
1654
+ /// @example
1655
+ /// paths = ["doc1.pdf", "doc2.docx", "doc3.xlsx"]
1656
+ /// results = Kreuzberg.batch_extract_files_sync(paths)
1657
+ /// results.each { |r| puts r[:content] }
1658
+ ///
1659
+ fn batch_extract_files_sync(args: &[Value]) -> Result<RArray, Error> {
1660
+ let ruby = Ruby::get().expect("Ruby not initialized");
1661
+ let args = scan_args::<(RArray,), (), (), (), RHash, ()>(args)?;
1662
+ let (paths_array,) = args.required;
1663
+ let opts = Some(args.keywords);
1664
+
1665
+ let config = parse_extraction_config(&ruby, opts)?;
1666
+
1667
+ let paths: Vec<String> = paths_array.to_vec::<String>()?;
1668
+
1669
+ let results = kreuzberg::batch_extract_file_sync(paths, &config).map_err(kreuzberg_error)?;
1670
+
1671
+ let results_array = ruby.ary_new();
1672
+ for result in results {
1673
+ results_array.push(extraction_result_to_ruby(&ruby, result)?)?;
1674
+ }
1675
+
1676
+ Ok(results_array)
1677
+ }
1678
+
1679
+ /// Extract content from a file (asynchronous).
1680
+ ///
1681
+ /// Note: Ruby doesn't have native async/await, so this uses a blocking Tokio runtime.
1682
+ /// For true async behavior, use the synchronous version in a background thread.
1683
+ ///
1684
+ /// @param path [String] Path to the file
1685
+ /// @param mime_type [String, nil] Optional MIME type hint
1686
+ /// @param options [Hash] Extraction configuration
1687
+ /// @return [Hash] Extraction result
1688
+ ///
1689
+ fn extract_file(args: &[Value]) -> Result<RHash, Error> {
1690
+ let ruby = Ruby::get().expect("Ruby not initialized");
1691
+ let args = scan_args::<(String,), (Option<String>,), (), (), RHash, ()>(args)?;
1692
+ let (path,) = args.required;
1693
+ let (mime_type,) = args.optional;
1694
+ let opts = Some(args.keywords);
1695
+
1696
+ let config = parse_extraction_config(&ruby, opts)?;
1697
+
1698
+ let runtime =
1699
+ tokio::runtime::Runtime::new().map_err(|e| runtime_error(format!("Failed to create Tokio runtime: {}", e)))?;
1700
+
1701
+ let result = runtime
1702
+ .block_on(async { kreuzberg::extract_file(&path, mime_type.as_deref(), &config).await })
1703
+ .map_err(kreuzberg_error)?;
1704
+
1705
+ extraction_result_to_ruby(&ruby, result)
1706
+ }
1707
+
1708
+ /// Extract content from bytes (asynchronous).
1709
+ ///
1710
+ /// @param data [String] Binary data
1711
+ /// @param mime_type [String] MIME type
1712
+ /// @param options [Hash] Extraction configuration
1713
+ /// @return [Hash] Extraction result
1714
+ ///
1715
+ fn extract_bytes(args: &[Value]) -> Result<RHash, Error> {
1716
+ let ruby = Ruby::get().expect("Ruby not initialized");
1717
+ let args = scan_args::<(String, String), (), (), (), RHash, ()>(args)?;
1718
+ let (data, mime_type) = args.required;
1719
+ let opts = Some(args.keywords);
1720
+
1721
+ let config = parse_extraction_config(&ruby, opts)?;
1722
+
1723
+ let runtime =
1724
+ tokio::runtime::Runtime::new().map_err(|e| runtime_error(format!("Failed to create Tokio runtime: {}", e)))?;
1725
+
1726
+ let result = runtime
1727
+ .block_on(async { kreuzberg::extract_bytes(data.as_bytes(), &mime_type, &config).await })
1728
+ .map_err(kreuzberg_error)?;
1729
+
1730
+ extraction_result_to_ruby(&ruby, result)
1731
+ }
1732
+
1733
+ /// Batch extract content from multiple files (asynchronous).
1734
+ ///
1735
+ /// @param paths [Array<String>] List of file paths
1736
+ /// @param options [Hash] Extraction configuration
1737
+ /// @return [Array<Hash>] Array of extraction results
1738
+ ///
1739
+ fn batch_extract_files(args: &[Value]) -> Result<RArray, Error> {
1740
+ let ruby = Ruby::get().expect("Ruby not initialized");
1741
+ let args = scan_args::<(RArray,), (), (), (), RHash, ()>(args)?;
1742
+ let (paths_array,) = args.required;
1743
+ let opts = Some(args.keywords);
1744
+
1745
+ let config = parse_extraction_config(&ruby, opts)?;
1746
+
1747
+ let paths: Vec<String> = paths_array.to_vec::<String>()?;
1748
+
1749
+ let runtime =
1750
+ tokio::runtime::Runtime::new().map_err(|e| runtime_error(format!("Failed to create Tokio runtime: {}", e)))?;
1751
+
1752
+ let results = runtime
1753
+ .block_on(async { kreuzberg::batch_extract_file(paths, &config).await })
1754
+ .map_err(kreuzberg_error)?;
1755
+
1756
+ let results_array = ruby.ary_new();
1757
+ for result in results {
1758
+ results_array.push(extraction_result_to_ruby(&ruby, result)?)?;
1759
+ }
1760
+
1761
+ Ok(results_array)
1762
+ }
1763
+
1764
+ /// Batch extract content from multiple byte arrays (synchronous).
1765
+ ///
1766
+ /// @param bytes_array [Array<String>] List of binary data strings
1767
+ /// @param mime_types [Array<String>] List of MIME types corresponding to each byte array
1768
+ /// @param options [Hash] Extraction configuration
1769
+ /// @return [Array<Hash>] Array of extraction results
1770
+ ///
1771
+ /// @example
1772
+ /// data1 = File.binread("document.pdf")
1773
+ /// data2 = File.binread("invoice.docx")
1774
+ /// results = Kreuzberg.batch_extract_bytes_sync([data1, data2], ["application/pdf", "application/vnd.openxmlformats-officedocument.wordprocessingml.document"])
1775
+ ///
1776
+ fn batch_extract_bytes_sync(args: &[Value]) -> Result<RArray, Error> {
1777
+ let ruby = Ruby::get().expect("Ruby not initialized");
1778
+ let args = scan_args::<(RArray, RArray), (), (), (), RHash, ()>(args)?;
1779
+ let (bytes_array, mime_types_array) = args.required;
1780
+ let opts = Some(args.keywords);
1781
+
1782
+ let config = parse_extraction_config(&ruby, opts)?;
1783
+
1784
+ let bytes_vec: Vec<String> = bytes_array.to_vec::<String>()?;
1785
+ let mime_types: Vec<String> = mime_types_array.to_vec::<String>()?;
1786
+
1787
+ if bytes_vec.len() != mime_types.len() {
1788
+ return Err(runtime_error(format!(
1789
+ "bytes_array and mime_types must have the same length: {} vs {}",
1790
+ bytes_vec.len(),
1791
+ mime_types.len()
1792
+ )));
1793
+ }
1794
+
1795
+ let contents: Vec<(&[u8], &str)> = bytes_vec
1796
+ .iter()
1797
+ .zip(mime_types.iter())
1798
+ .map(|(bytes, mime)| (bytes.as_bytes(), mime.as_str()))
1799
+ .collect();
1800
+
1801
+ let results = kreuzberg::batch_extract_bytes_sync(contents, &config).map_err(kreuzberg_error)?;
1802
+
1803
+ let results_array = ruby.ary_new();
1804
+ for result in results {
1805
+ results_array.push(extraction_result_to_ruby(&ruby, result)?)?;
1806
+ }
1807
+
1808
+ Ok(results_array)
1809
+ }
1810
+
1811
+ /// Batch extract content from multiple byte arrays (asynchronous).
1812
+ ///
1813
+ /// @param bytes_array [Array<String>] List of binary data strings
1814
+ /// @param mime_types [Array<String>] List of MIME types corresponding to each byte array
1815
+ /// @param options [Hash] Extraction configuration
1816
+ /// @return [Array<Hash>] Array of extraction results
1817
+ ///
1818
+ fn batch_extract_bytes(args: &[Value]) -> Result<RArray, Error> {
1819
+ let ruby = Ruby::get().expect("Ruby not initialized");
1820
+ let args = scan_args::<(RArray, RArray), (), (), (), RHash, ()>(args)?;
1821
+ let (bytes_array, mime_types_array) = args.required;
1822
+ let opts = Some(args.keywords);
1823
+
1824
+ let config = parse_extraction_config(&ruby, opts)?;
1825
+
1826
+ let bytes_vec: Vec<String> = bytes_array.to_vec::<String>()?;
1827
+ let mime_types: Vec<String> = mime_types_array.to_vec::<String>()?;
1828
+
1829
+ if bytes_vec.len() != mime_types.len() {
1830
+ return Err(runtime_error(format!(
1831
+ "bytes_array and mime_types must have the same length: {} vs {}",
1832
+ bytes_vec.len(),
1833
+ mime_types.len()
1834
+ )));
1835
+ }
1836
+
1837
+ let contents: Vec<(&[u8], &str)> = bytes_vec
1838
+ .iter()
1839
+ .zip(mime_types.iter())
1840
+ .map(|(bytes, mime)| (bytes.as_bytes(), mime.as_str()))
1841
+ .collect();
1842
+
1843
+ let runtime =
1844
+ tokio::runtime::Runtime::new().map_err(|e| runtime_error(format!("Failed to create Tokio runtime: {}", e)))?;
1845
+
1846
+ let results = runtime
1847
+ .block_on(async { kreuzberg::batch_extract_bytes(contents, &config).await })
1848
+ .map_err(kreuzberg_error)?;
1849
+
1850
+ let results_array = ruby.ary_new();
1851
+ for result in results {
1852
+ results_array.push(extraction_result_to_ruby(&ruby, result)?)?;
1853
+ }
1854
+
1855
+ Ok(results_array)
1856
+ }
1857
+
1858
+ /// Clear all cache entries.
1859
+ ///
1860
+ /// @return [void]
1861
+ ///
1862
+ /// @example
1863
+ /// Kreuzberg.clear_cache
1864
+ ///
1865
+ fn ruby_clear_cache() -> Result<(), Error> {
1866
+ let cache_root = cache_root_dir()?;
1867
+ if !cache_root.exists() {
1868
+ return Ok(());
1869
+ }
1870
+
1871
+ for dir in cache_directories(&cache_root)? {
1872
+ let Some(dir_str) = dir.to_str() else {
1873
+ return Err(runtime_error("Cache directory path contains non-UTF8 characters"));
1874
+ };
1875
+
1876
+ // OSError/RuntimeError must bubble up - system errors need user reports ~keep
1877
+ kreuzberg::cache::clear_cache_directory(dir_str).map_err(kreuzberg_error)?;
1878
+ }
1879
+
1880
+ Ok(())
1881
+ }
1882
+
1883
+ /// Get cache statistics.
1884
+ ///
1885
+ /// @return [Hash] Cache statistics with :total_entries and :total_size_bytes
1886
+ ///
1887
+ /// @example
1888
+ /// stats = Kreuzberg.cache_stats
1889
+ /// puts "Cache entries: #{stats[:total_entries]}"
1890
+ /// puts "Cache size: #{stats[:total_size_bytes]} bytes"
1891
+ ///
1892
+ fn ruby_cache_stats() -> Result<RHash, Error> {
1893
+ let ruby = Ruby::get().expect("Ruby not initialized");
1894
+
1895
+ let hash = ruby.hash_new();
1896
+ let cache_root = cache_root_dir()?;
1897
+
1898
+ if !cache_root.exists() {
1899
+ hash.aset("total_entries", 0)?;
1900
+ hash.aset("total_size_bytes", 0)?;
1901
+ return Ok(hash);
1902
+ }
1903
+
1904
+ let mut total_entries: usize = 0;
1905
+ let mut total_bytes: f64 = 0.0;
1906
+
1907
+ for dir in cache_directories(&cache_root)? {
1908
+ let Some(dir_str) = dir.to_str() else {
1909
+ return Err(runtime_error("Cache directory path contains non-UTF8 characters"));
1910
+ };
1911
+
1912
+ // OSError/RuntimeError must bubble up - system errors need user reports ~keep
1913
+ let stats = kreuzberg::cache::get_cache_metadata(dir_str).map_err(kreuzberg_error)?;
1914
+ total_entries += stats.total_files;
1915
+ total_bytes += stats.total_size_mb * 1024.0 * 1024.0;
1916
+ }
1917
+
1918
+ set_hash_entry(
1919
+ &ruby,
1920
+ &hash,
1921
+ "total_entries",
1922
+ ruby.integer_from_u64(total_entries as u64).into_value_with(&ruby),
1923
+ )?;
1924
+ set_hash_entry(
1925
+ &ruby,
1926
+ &hash,
1927
+ "total_size_bytes",
1928
+ ruby.integer_from_u64(total_bytes.round() as u64).into_value_with(&ruby),
1929
+ )?;
1930
+
1931
+ Ok(hash)
1932
+ }
1933
+
1934
+ /// Register a post-processor plugin.
1935
+ ///
1936
+ /// @param name [String] Unique identifier for the post-processor
1937
+ /// @param processor [Proc] Ruby Proc/lambda that processes extraction results
1938
+ /// @param priority [Integer] Execution priority (default: 50, higher = runs first)
1939
+ /// @return [nil]
1940
+ ///
1941
+ /// # Example
1942
+ /// ```text
1943
+ /// Kreuzberg.register_post_processor("uppercase", ->(result) {
1944
+ /// result[:content] = result[:content].upcase
1945
+ /// result
1946
+ /// }, 100)
1947
+ /// ```
1948
+ fn register_post_processor(args: &[Value]) -> Result<(), Error> {
1949
+ let _ruby = Ruby::get().expect("Ruby not initialized");
1950
+ let args = scan_args::<(String, Value), (Option<i32>,), (), (), (), ()>(args)?;
1951
+ let (name, processor) = args.required;
1952
+ let (priority,) = args.optional;
1953
+ let priority = priority.unwrap_or(50);
1954
+
1955
+ if !processor.respond_to("call", true)? {
1956
+ return Err(runtime_error("Post-processor must be a Proc or respond to 'call'"));
1957
+ }
1958
+
1959
+ use async_trait::async_trait;
1960
+ use kreuzberg::plugins::{Plugin, PostProcessor, ProcessingStage};
1961
+ use std::sync::Arc;
1962
+
1963
+ struct RubyPostProcessor {
1964
+ name: String,
1965
+ processor: GcGuardedValue,
1966
+ }
1967
+
1968
+ unsafe impl Send for RubyPostProcessor {}
1969
+ unsafe impl Sync for RubyPostProcessor {}
1970
+
1971
+ impl Plugin for RubyPostProcessor {
1972
+ fn name(&self) -> &str {
1973
+ &self.name
1974
+ }
1975
+
1976
+ fn version(&self) -> String {
1977
+ "1.0.0".to_string()
1978
+ }
1979
+
1980
+ fn initialize(&self) -> kreuzberg::Result<()> {
1981
+ Ok(())
1982
+ }
1983
+
1984
+ fn shutdown(&self) -> kreuzberg::Result<()> {
1985
+ Ok(())
1986
+ }
1987
+ }
1988
+
1989
+ #[async_trait]
1990
+ impl PostProcessor for RubyPostProcessor {
1991
+ async fn process(
1992
+ &self,
1993
+ result: &mut kreuzberg::ExtractionResult,
1994
+ _config: &kreuzberg::ExtractionConfig,
1995
+ ) -> kreuzberg::Result<()> {
1996
+ let processor_name = self.name.clone();
1997
+ let processor = self.processor.value();
1998
+ let result_clone = result.clone();
1999
+
2000
+ // Use block_in_place to avoid GVL deadlocks (same pattern as Python PostProcessor)
2001
+ // See crates/kreuzberg-py/README.md:151-158 for explanation
2002
+ // CRITICAL: spawn_blocking causes GVL deadlocks, must use block_in_place
2003
+ let updated_result = tokio::task::block_in_place(|| {
2004
+ let ruby = Ruby::get().expect("Ruby not initialized");
2005
+ let result_hash = extraction_result_to_ruby(&ruby, result_clone.clone()).map_err(|e| {
2006
+ kreuzberg::KreuzbergError::Plugin {
2007
+ message: format!("Failed to convert result to Ruby: {}", e),
2008
+ plugin_name: processor_name.clone(),
2009
+ }
2010
+ })?;
2011
+
2012
+ let modified = processor
2013
+ .funcall::<_, _, magnus::Value>("call", (result_hash,))
2014
+ .map_err(|e| kreuzberg::KreuzbergError::Plugin {
2015
+ message: format!("Ruby post-processor failed: {}", e),
2016
+ plugin_name: processor_name.clone(),
2017
+ })?;
2018
+
2019
+ let modified_hash =
2020
+ magnus::RHash::try_convert(modified).map_err(|e| kreuzberg::KreuzbergError::Plugin {
2021
+ message: format!("Post-processor must return a Hash: {}", e),
2022
+ plugin_name: processor_name.clone(),
2023
+ })?;
2024
+
2025
+ let mut updated_result = result_clone;
2026
+
2027
+ if let Some(content_val) = get_kw(&ruby, modified_hash, "content") {
2028
+ let new_content =
2029
+ String::try_convert(content_val).map_err(|e| kreuzberg::KreuzbergError::Plugin {
2030
+ message: format!("Failed to convert content: {}", e),
2031
+ plugin_name: processor_name.clone(),
2032
+ })?;
2033
+ updated_result.content = new_content;
2034
+ }
2035
+
2036
+ if let Some(mime_val) = get_kw(&ruby, modified_hash, "mime_type") {
2037
+ let new_mime = String::try_convert(mime_val).map_err(|e| kreuzberg::KreuzbergError::Plugin {
2038
+ message: format!("Failed to convert mime_type: {}", e),
2039
+ plugin_name: processor_name.clone(),
2040
+ })?;
2041
+ updated_result.mime_type = new_mime;
2042
+ }
2043
+
2044
+ if let Some(metadata_val) = get_kw(&ruby, modified_hash, "metadata") {
2045
+ if metadata_val.is_nil() {
2046
+ updated_result.metadata = kreuzberg::types::Metadata::default();
2047
+ } else {
2048
+ let metadata_json =
2049
+ ruby_value_to_json(metadata_val).map_err(|e| kreuzberg::KreuzbergError::Plugin {
2050
+ message: format!("Metadata must be JSON-serializable: {}", e),
2051
+ plugin_name: processor_name.clone(),
2052
+ })?;
2053
+ let metadata: kreuzberg::types::Metadata =
2054
+ serde_json::from_value(metadata_json).map_err(|e| kreuzberg::KreuzbergError::Plugin {
2055
+ message: format!("Failed to deserialize metadata: {}", e),
2056
+ plugin_name: processor_name.clone(),
2057
+ })?;
2058
+ updated_result.metadata = metadata;
2059
+ }
2060
+ }
2061
+
2062
+ if let Some(tables_val) = get_kw(&ruby, modified_hash, "tables") {
2063
+ let tables_json =
2064
+ ruby_value_to_json(tables_val).map_err(|e| kreuzberg::KreuzbergError::Plugin {
2065
+ message: format!("Tables must be JSON-serializable: {}", e),
2066
+ plugin_name: processor_name.clone(),
2067
+ })?;
2068
+ if tables_json.is_null() {
2069
+ updated_result.tables.clear();
2070
+ } else {
2071
+ let tables: Vec<kreuzberg::types::Table> =
2072
+ serde_json::from_value(tables_json).map_err(|e| kreuzberg::KreuzbergError::Plugin {
2073
+ message: format!("Failed to deserialize tables: {}", e),
2074
+ plugin_name: processor_name.clone(),
2075
+ })?;
2076
+ updated_result.tables = tables;
2077
+ }
2078
+ }
2079
+
2080
+ if let Some(languages_val) = get_kw(&ruby, modified_hash, "detected_languages") {
2081
+ if languages_val.is_nil() {
2082
+ updated_result.detected_languages = None;
2083
+ } else {
2084
+ let langs_json =
2085
+ ruby_value_to_json(languages_val).map_err(|e| kreuzberg::KreuzbergError::Plugin {
2086
+ message: format!("detected_languages must be JSON-serializable: {}", e),
2087
+ plugin_name: processor_name.clone(),
2088
+ })?;
2089
+ let languages: Vec<String> =
2090
+ serde_json::from_value(langs_json).map_err(|e| kreuzberg::KreuzbergError::Plugin {
2091
+ message: format!("Failed to deserialize detected_languages: {}", e),
2092
+ plugin_name: processor_name.clone(),
2093
+ })?;
2094
+ updated_result.detected_languages = Some(languages);
2095
+ }
2096
+ }
2097
+
2098
+ if let Some(chunks_val) = get_kw(&ruby, modified_hash, "chunks") {
2099
+ if chunks_val.is_nil() {
2100
+ updated_result.chunks = None;
2101
+ } else {
2102
+ let chunks_json =
2103
+ ruby_value_to_json(chunks_val).map_err(|e| kreuzberg::KreuzbergError::Plugin {
2104
+ message: format!("Chunks must be JSON-serializable: {}", e),
2105
+ plugin_name: processor_name.clone(),
2106
+ })?;
2107
+ let chunks: Vec<kreuzberg::types::Chunk> =
2108
+ serde_json::from_value(chunks_json).map_err(|e| kreuzberg::KreuzbergError::Plugin {
2109
+ message: format!("Failed to deserialize chunks: {}", e),
2110
+ plugin_name: processor_name.clone(),
2111
+ })?;
2112
+ updated_result.chunks = Some(chunks);
2113
+ }
2114
+ }
2115
+
2116
+ Ok::<kreuzberg::ExtractionResult, kreuzberg::KreuzbergError>(updated_result)
2117
+ })?;
2118
+
2119
+ *result = updated_result;
2120
+ Ok(())
2121
+ }
2122
+
2123
+ fn processing_stage(&self) -> ProcessingStage {
2124
+ ProcessingStage::Late
2125
+ }
2126
+ }
2127
+
2128
+ let processor_impl = Arc::new(RubyPostProcessor {
2129
+ name: name.clone(),
2130
+ processor: GcGuardedValue::new(processor),
2131
+ });
2132
+
2133
+ let registry = kreuzberg::get_post_processor_registry();
2134
+ registry
2135
+ .write()
2136
+ .map_err(|e| runtime_error(format!("Failed to acquire registry lock: {}", e)))?
2137
+ .register(processor_impl, priority)
2138
+ .map_err(kreuzberg_error)?;
2139
+
2140
+ Ok(())
2141
+ }
2142
+
2143
+ /// Register a validator plugin.
2144
+ ///
2145
+ /// @param name [String] Unique identifier for the validator
2146
+ /// @param validator [Proc] Ruby Proc/lambda that validates extraction results
2147
+ /// @param priority [Integer] Execution priority (default: 50, higher = runs first)
2148
+ /// @return [nil]
2149
+ ///
2150
+ /// # Example
2151
+ /// ```text
2152
+ /// Kreuzberg.register_validator("min_length", ->(result) {
2153
+ /// raise "Content too short" if result[:content].length < 100
2154
+ /// }, 100)
2155
+ /// ```
2156
+ fn register_validator(args: &[Value]) -> Result<(), Error> {
2157
+ let _ruby = Ruby::get().expect("Ruby not initialized");
2158
+ let args = scan_args::<(String, Value), (Option<i32>,), (), (), (), ()>(args)?;
2159
+ let (name, validator) = args.required;
2160
+ let (priority,) = args.optional;
2161
+ let priority = priority.unwrap_or(50);
2162
+
2163
+ if !validator.respond_to("call", true)? {
2164
+ return Err(runtime_error("Validator must be a Proc or respond to 'call'"));
2165
+ }
2166
+
2167
+ use async_trait::async_trait;
2168
+ use kreuzberg::plugins::{Plugin, Validator};
2169
+ use std::sync::Arc;
2170
+
2171
+ struct RubyValidator {
2172
+ name: String,
2173
+ validator: GcGuardedValue,
2174
+ priority: i32,
2175
+ }
2176
+
2177
+ unsafe impl Send for RubyValidator {}
2178
+ unsafe impl Sync for RubyValidator {}
2179
+
2180
+ impl Plugin for RubyValidator {
2181
+ fn name(&self) -> &str {
2182
+ &self.name
2183
+ }
2184
+
2185
+ fn version(&self) -> String {
2186
+ "1.0.0".to_string()
2187
+ }
2188
+
2189
+ fn initialize(&self) -> kreuzberg::Result<()> {
2190
+ Ok(())
2191
+ }
2192
+
2193
+ fn shutdown(&self) -> kreuzberg::Result<()> {
2194
+ Ok(())
2195
+ }
2196
+ }
2197
+
2198
+ #[async_trait]
2199
+ impl Validator for RubyValidator {
2200
+ async fn validate(
2201
+ &self,
2202
+ result: &kreuzberg::ExtractionResult,
2203
+ _config: &kreuzberg::ExtractionConfig,
2204
+ ) -> kreuzberg::Result<()> {
2205
+ let validator_name = self.name.clone();
2206
+ let validator = self.validator.value();
2207
+ let result_clone = result.clone();
2208
+
2209
+ // Use block_in_place to avoid GVL deadlocks (same pattern as Python Validator)
2210
+ // See crates/kreuzberg-py/README.md:151-158 for explanation
2211
+ // CRITICAL: spawn_blocking causes GVL deadlocks, must use block_in_place
2212
+ tokio::task::block_in_place(|| {
2213
+ let ruby = Ruby::get().expect("Ruby not initialized");
2214
+ let result_hash =
2215
+ extraction_result_to_ruby(&ruby, result_clone).map_err(|e| kreuzberg::KreuzbergError::Plugin {
2216
+ message: format!("Failed to convert result to Ruby: {}", e),
2217
+ plugin_name: validator_name.clone(),
2218
+ })?;
2219
+
2220
+ validator
2221
+ .funcall::<_, _, magnus::Value>("call", (result_hash,))
2222
+ .map_err(|e| kreuzberg::KreuzbergError::Validation {
2223
+ message: format!("Validation failed: {}", e),
2224
+ source: None,
2225
+ })?;
2226
+
2227
+ Ok(())
2228
+ })
2229
+ }
2230
+
2231
+ fn priority(&self) -> i32 {
2232
+ self.priority
2233
+ }
2234
+ }
2235
+
2236
+ let validator_impl = Arc::new(RubyValidator {
2237
+ name: name.clone(),
2238
+ validator: GcGuardedValue::new(validator),
2239
+ priority,
2240
+ });
2241
+
2242
+ let registry = kreuzberg::get_validator_registry();
2243
+ registry
2244
+ .write()
2245
+ .map_err(|e| runtime_error(format!("Failed to acquire registry lock: {}", e)))?
2246
+ .register(validator_impl)
2247
+ .map_err(kreuzberg_error)?;
2248
+
2249
+ Ok(())
2250
+ }
2251
+
2252
+ /// Register an OCR backend plugin.
2253
+ ///
2254
+ /// @param name [String] Unique identifier for the OCR backend
2255
+ /// @param backend [Object] Ruby object implementing OCR backend interface
2256
+ /// @return [nil]
2257
+ ///
2258
+ /// # Example
2259
+ /// ```text
2260
+ /// class CustomOcr
2261
+ /// def process_image(image_bytes, language)
2262
+ /// # Return extracted text
2263
+ /// "Extracted text"
2264
+ /// end
2265
+ ///
2266
+ /// def supports_language?(lang)
2267
+ /// %w[eng deu fra].include?(lang)
2268
+ /// end
2269
+ /// end
2270
+ ///
2271
+ /// Kreuzberg.register_ocr_backend("custom", CustomOcr.new)
2272
+ /// ```
2273
+ fn register_ocr_backend(name: String, backend: Value) -> Result<(), Error> {
2274
+ if !backend.respond_to("name", true)? {
2275
+ return Err(runtime_error("OCR backend must respond to 'name'"));
2276
+ }
2277
+ if !backend.respond_to("process_image", true)? {
2278
+ return Err(runtime_error("OCR backend must respond to 'process_image'"));
2279
+ }
2280
+
2281
+ use async_trait::async_trait;
2282
+ use kreuzberg::plugins::{OcrBackend, OcrBackendType, Plugin};
2283
+ use std::sync::Arc;
2284
+
2285
+ struct RubyOcrBackend {
2286
+ name: String,
2287
+ backend: GcGuardedValue,
2288
+ }
2289
+
2290
+ unsafe impl Send for RubyOcrBackend {}
2291
+ unsafe impl Sync for RubyOcrBackend {}
2292
+
2293
+ impl Plugin for RubyOcrBackend {
2294
+ fn name(&self) -> &str {
2295
+ &self.name
2296
+ }
2297
+
2298
+ fn version(&self) -> String {
2299
+ "1.0.0".to_string()
2300
+ }
2301
+
2302
+ fn initialize(&self) -> kreuzberg::Result<()> {
2303
+ Ok(())
2304
+ }
2305
+
2306
+ fn shutdown(&self) -> kreuzberg::Result<()> {
2307
+ Ok(())
2308
+ }
2309
+ }
2310
+
2311
+ #[async_trait]
2312
+ impl OcrBackend for RubyOcrBackend {
2313
+ async fn process_image(
2314
+ &self,
2315
+ image_bytes: &[u8],
2316
+ config: &kreuzberg::OcrConfig,
2317
+ ) -> kreuzberg::Result<kreuzberg::ExtractionResult> {
2318
+ let ruby = Ruby::get().expect("Ruby not initialized");
2319
+ let image_str = ruby.str_from_slice(image_bytes);
2320
+
2321
+ let config_hash = ocr_config_to_ruby_hash(&ruby, config).map_err(|e| kreuzberg::KreuzbergError::Ocr {
2322
+ message: format!("Failed to convert OCR config: {}", e),
2323
+ source: None,
2324
+ })?;
2325
+
2326
+ let response = self
2327
+ .backend
2328
+ .value()
2329
+ .funcall::<_, _, Value>("process_image", (image_str, config_hash.into_value_with(&ruby)))
2330
+ .map_err(|e| kreuzberg::KreuzbergError::Ocr {
2331
+ message: format!("Ruby OCR backend failed: {}", e),
2332
+ source: None,
2333
+ })?;
2334
+
2335
+ let text = String::try_convert(response).map_err(|e| kreuzberg::KreuzbergError::Ocr {
2336
+ message: format!("OCR backend must return a String: {}", e),
2337
+ source: None,
2338
+ })?;
2339
+
2340
+ Ok(kreuzberg::ExtractionResult {
2341
+ content: text,
2342
+ mime_type: "text/plain".to_string(),
2343
+ metadata: kreuzberg::types::Metadata::default(),
2344
+ tables: vec![],
2345
+ detected_languages: None,
2346
+ chunks: None,
2347
+ images: None,
2348
+ })
2349
+ }
2350
+
2351
+ fn supports_language(&self, lang: &str) -> bool {
2352
+ match self.backend.value().respond_to("supports_language?", true) {
2353
+ Ok(true) => self
2354
+ .backend
2355
+ .value()
2356
+ .funcall::<_, _, bool>("supports_language?", (lang,))
2357
+ .unwrap_or(true),
2358
+ _ => true,
2359
+ }
2360
+ }
2361
+
2362
+ fn backend_type(&self) -> OcrBackendType {
2363
+ OcrBackendType::Custom
2364
+ }
2365
+ }
2366
+
2367
+ let backend_impl = Arc::new(RubyOcrBackend {
2368
+ name: name.clone(),
2369
+ backend: GcGuardedValue::new(backend),
2370
+ });
2371
+
2372
+ let registry = kreuzberg::get_ocr_backend_registry();
2373
+ registry
2374
+ .write()
2375
+ .map_err(|e| runtime_error(format!("Failed to acquire registry lock: {}", e)))?
2376
+ .register(backend_impl)
2377
+ .map_err(kreuzberg_error)?;
2378
+
2379
+ Ok(())
2380
+ }
2381
+
2382
+ /// Unregister a post-processor plugin.
2383
+ ///
2384
+ /// @param name [String] Name of the post-processor to remove
2385
+ /// @return [nil]
2386
+ ///
2387
+ fn unregister_post_processor(name: String) -> Result<(), Error> {
2388
+ let registry = kreuzberg::get_post_processor_registry();
2389
+ registry
2390
+ .write()
2391
+ .map_err(|e| runtime_error(format!("Failed to acquire registry lock: {}", e)))?
2392
+ .remove(&name)
2393
+ .map_err(kreuzberg_error)?;
2394
+ Ok(())
2395
+ }
2396
+
2397
+ /// Unregister a validator plugin.
2398
+ ///
2399
+ /// @param name [String] Name of the validator to remove
2400
+ /// @return [nil]
2401
+ ///
2402
+ fn unregister_validator(name: String) -> Result<(), Error> {
2403
+ let registry = kreuzberg::get_validator_registry();
2404
+ registry
2405
+ .write()
2406
+ .map_err(|e| runtime_error(format!("Failed to acquire registry lock: {}", e)))?
2407
+ .remove(&name)
2408
+ .map_err(kreuzberg_error)?;
2409
+ Ok(())
2410
+ }
2411
+
2412
+ /// Clear all registered post-processors.
2413
+ ///
2414
+ /// @return [nil]
2415
+ ///
2416
+ fn clear_post_processors() -> Result<(), Error> {
2417
+ let registry = kreuzberg::get_post_processor_registry();
2418
+ registry
2419
+ .write()
2420
+ .map_err(|e| runtime_error(format!("Failed to acquire registry lock: {}", e)))?
2421
+ .shutdown_all()
2422
+ .map_err(kreuzberg_error)?;
2423
+ Ok(())
2424
+ }
2425
+
2426
+ /// Clear all registered validators.
2427
+ ///
2428
+ /// @return [nil]
2429
+ ///
2430
+ fn clear_validators() -> Result<(), Error> {
2431
+ let registry = kreuzberg::get_validator_registry();
2432
+ registry
2433
+ .write()
2434
+ .map_err(|e| runtime_error(format!("Failed to acquire registry lock: {}", e)))?
2435
+ .shutdown_all()
2436
+ .map_err(kreuzberg_error)?;
2437
+ Ok(())
2438
+ }
2439
+
2440
+ /// List all registered validators.
2441
+ ///
2442
+ /// @return [Array<String>] Array of validator names
2443
+ ///
2444
+ fn list_validators() -> Result<Vec<String>, Error> {
2445
+ let registry = kreuzberg::get_validator_registry();
2446
+ let validators = registry
2447
+ .read()
2448
+ .map_err(|e| runtime_error(format!("Failed to acquire registry lock: {}", e)))?
2449
+ .list();
2450
+ Ok(validators)
2451
+ }
2452
+
2453
+ /// List all registered post-processors.
2454
+ ///
2455
+ /// @return [Array<String>] Array of post-processor names
2456
+ ///
2457
+ fn list_post_processors() -> Result<Vec<String>, Error> {
2458
+ let registry = kreuzberg::get_post_processor_registry();
2459
+ let processors = registry
2460
+ .read()
2461
+ .map_err(|e| runtime_error(format!("Failed to acquire registry lock: {}", e)))?
2462
+ .list();
2463
+ Ok(processors)
2464
+ }
2465
+
2466
+ /// Unregister an OCR backend by name.
2467
+ ///
2468
+ /// Removes a previously registered OCR backend from the global registry.
2469
+ ///
2470
+ /// @param name [String] Backend name to unregister
2471
+ /// @return [void]
2472
+ ///
2473
+ /// @example
2474
+ /// Kreuzberg.unregister_ocr_backend("my_ocr")
2475
+ ///
2476
+ fn unregister_ocr_backend(name: String) -> Result<(), Error> {
2477
+ kreuzberg::plugins::unregister_ocr_backend(&name).map_err(|e| runtime_error(e.to_string()))
2478
+ }
2479
+
2480
+ /// List all registered OCR backend names.
2481
+ ///
2482
+ /// Returns an array of all OCR backend names currently registered in the global registry.
2483
+ ///
2484
+ /// @return [Array<String>] Array of OCR backend names
2485
+ ///
2486
+ /// @example
2487
+ /// backends = Kreuzberg.list_ocr_backends
2488
+ /// #=> ["tesseract", "my_custom_ocr"]
2489
+ ///
2490
+ fn list_ocr_backends() -> Result<Vec<String>, Error> {
2491
+ kreuzberg::plugins::list_ocr_backends().map_err(|e| runtime_error(e.to_string()))
2492
+ }
2493
+
2494
+ /// Clear all registered OCR backends.
2495
+ ///
2496
+ /// Removes all OCR backends from the global registry and calls their shutdown methods.
2497
+ ///
2498
+ /// @return [void]
2499
+ ///
2500
+ /// @example
2501
+ /// Kreuzberg.clear_ocr_backends
2502
+ ///
2503
+ fn clear_ocr_backends() -> Result<(), Error> {
2504
+ kreuzberg::plugins::clear_ocr_backends().map_err(|e| runtime_error(e.to_string()))
2505
+ }
2506
+
2507
+ /// List all registered document extractor names.
2508
+ ///
2509
+ /// Returns an array of all document extractor names currently registered in the global registry.
2510
+ ///
2511
+ /// @return [Array<String>] Array of document extractor names
2512
+ ///
2513
+ /// @example
2514
+ /// extractors = Kreuzberg.list_document_extractors
2515
+ /// #=> ["pdf", "docx", "txt"]
2516
+ ///
2517
+ fn list_document_extractors() -> Result<Vec<String>, Error> {
2518
+ kreuzberg::plugins::list_extractors().map_err(|e| runtime_error(e.to_string()))
2519
+ }
2520
+
2521
+ /// Unregister a document extractor by name.
2522
+ ///
2523
+ /// Removes a previously registered document extractor from the global registry.
2524
+ ///
2525
+ /// @param name [String] Extractor name to unregister
2526
+ /// @return [void]
2527
+ ///
2528
+ /// @example
2529
+ /// Kreuzberg.unregister_document_extractor("my_extractor")
2530
+ ///
2531
+ fn unregister_document_extractor(name: String) -> Result<(), Error> {
2532
+ kreuzberg::plugins::unregister_extractor(&name).map_err(|e| runtime_error(e.to_string()))
2533
+ }
2534
+
2535
+ /// Clear all registered document extractors.
2536
+ ///
2537
+ /// Removes all document extractors from the global registry and calls their shutdown methods.
2538
+ ///
2539
+ /// @return [void]
2540
+ ///
2541
+ /// @example
2542
+ /// Kreuzberg.clear_document_extractors
2543
+ ///
2544
+ fn clear_document_extractors() -> Result<(), Error> {
2545
+ kreuzberg::plugins::clear_extractors().map_err(|e| runtime_error(e.to_string()))
2546
+ }
2547
+
2548
+ /// Validate that a MIME type is supported.
2549
+ ///
2550
+ /// @param mime_type [String] The MIME type to validate
2551
+ /// @return [String] The validated MIME type (may be normalized)
2552
+ ///
2553
+ /// @example
2554
+ /// validated = Kreuzberg.validate_mime_type("application/pdf")
2555
+ /// #=> "application/pdf"
2556
+ ///
2557
+ /// @example Validate image MIME type
2558
+ /// validated = Kreuzberg.validate_mime_type("image/jpeg")
2559
+ /// #=> "image/jpeg"
2560
+ ///
2561
+ fn validate_mime_type_native(mime_type: String) -> Result<String, Error> {
2562
+ kreuzberg::validate_mime_type(&mime_type).map_err(kreuzberg_error)
2563
+ }
2564
+
2565
+ /// Detect MIME type from byte content.
2566
+ ///
2567
+ /// Uses magic byte detection to determine the MIME type of content.
2568
+ ///
2569
+ /// @param bytes [String] The byte content to analyze
2570
+ /// @return [String] Detected MIME type
2571
+ ///
2572
+ /// @example
2573
+ /// pdf_bytes = "%PDF-1.4\n"
2574
+ /// mime = Kreuzberg.detect_mime_type(pdf_bytes)
2575
+ /// #=> "application/pdf"
2576
+ ///
2577
+ fn detect_mime_type_from_bytes(bytes: String) -> Result<String, Error> {
2578
+ let mime_type = kreuzberg::detect_mime_type_from_bytes(bytes.as_bytes()).map_err(kreuzberg_error)?;
2579
+ Ok(mime_type)
2580
+ }
2581
+
2582
+ /// Detect MIME type from a file path.
2583
+ ///
2584
+ /// Detects MIME type by reading the file's magic bytes.
2585
+ ///
2586
+ /// @param path [String] Path to the file
2587
+ /// @return [String] Detected MIME type
2588
+ ///
2589
+ /// @example
2590
+ /// mime = Kreuzberg.detect_mime_type_from_path("document.pdf")
2591
+ /// #=> "application/pdf"
2592
+ ///
2593
+ fn detect_mime_type_from_path_native(path: String) -> Result<String, Error> {
2594
+ let content = fs::read(&path).map_err(KreuzbergError::Io).map_err(kreuzberg_error)?;
2595
+ let mime_type = kreuzberg::detect_mime_type_from_bytes(&content).map_err(kreuzberg_error)?;
2596
+ Ok(mime_type)
2597
+ }
2598
+
2599
+ /// Get file extensions for a given MIME type.
2600
+ ///
2601
+ /// Returns an array of file extensions commonly associated with the MIME type.
2602
+ ///
2603
+ /// @param mime_type [String] The MIME type
2604
+ /// @return [Array<String>] Array of file extensions (without dots)
2605
+ ///
2606
+ /// @example
2607
+ /// exts = Kreuzberg.get_extensions_for_mime("application/pdf")
2608
+ /// #=> ["pdf"]
2609
+ ///
2610
+ /// @example
2611
+ /// exts = Kreuzberg.get_extensions_for_mime("image/jpeg")
2612
+ /// #=> ["jpg", "jpeg"]
2613
+ ///
2614
+ fn get_extensions_for_mime_native(mime_type: String) -> Result<Vec<String>, Error> {
2615
+ kreuzberg::get_extensions_for_mime(&mime_type).map_err(kreuzberg_error)
2616
+ }
2617
+
2618
+ /// List all available embedding preset names.
2619
+ ///
2620
+ /// Returns an array of preset names that can be used with get_embedding_preset.
2621
+ ///
2622
+ /// # Returns
2623
+ ///
2624
+ /// Array of 4 preset names: ["fast", "balanced", "quality", "multilingual"]
2625
+ ///
2626
+ /// # Example
2627
+ ///
2628
+ /// ```ruby
2629
+ /// require 'kreuzberg'
2630
+ ///
2631
+ /// presets = Kreuzberg.list_embedding_presets
2632
+ /// puts presets # => ["fast", "balanced", "quality", "multilingual"]
2633
+ /// ```
2634
+ fn list_embedding_presets(ruby: &Ruby) -> Result<RArray, Error> {
2635
+ let presets = kreuzberg::embeddings::list_presets();
2636
+ let array = ruby.ary_new();
2637
+ for name in presets {
2638
+ array.push(name)?;
2639
+ }
2640
+ Ok(array)
2641
+ }
2642
+
2643
+ /// Get a specific embedding preset by name.
2644
+ ///
2645
+ /// Returns a preset configuration hash, or nil if the preset name is not found.
2646
+ ///
2647
+ /// # Arguments
2648
+ ///
2649
+ /// * `name` - The preset name (case-sensitive)
2650
+ ///
2651
+ /// # Returns
2652
+ ///
2653
+ /// Hash with preset configuration or nil if not found
2654
+ ///
2655
+ /// Available presets:
2656
+ /// - "fast": AllMiniLML6V2Q (384 dimensions) - Quick prototyping, low-latency
2657
+ /// - "balanced": BGEBaseENV15 (768 dimensions) - General-purpose RAG
2658
+ /// - "quality": BGELargeENV15 (1024 dimensions) - High-quality embeddings
2659
+ /// - "multilingual": MultilingualE5Base (768 dimensions) - Multi-language support
2660
+ ///
2661
+ /// # Example
2662
+ ///
2663
+ /// ```ruby
2664
+ /// require 'kreuzberg'
2665
+ ///
2666
+ /// preset = Kreuzberg.get_embedding_preset("balanced")
2667
+ /// if preset
2668
+ /// puts "Model: #{preset[:model_name]}, Dims: #{preset[:dimensions]}"
2669
+ /// # => Model: BGEBaseENV15, Dims: 768
2670
+ /// end
2671
+ /// ```
2672
+ fn get_embedding_preset(ruby: &Ruby, name: String) -> Result<Value, Error> {
2673
+ let preset = kreuzberg::embeddings::get_preset(&name);
2674
+
2675
+ match preset {
2676
+ Some(preset) => {
2677
+ let hash = ruby.hash_new();
2678
+
2679
+ set_hash_entry(ruby, &hash, "name", ruby.str_new(preset.name).as_value())?;
2680
+ set_hash_entry(ruby, &hash, "chunk_size", preset.chunk_size.into_value_with(ruby))?;
2681
+ set_hash_entry(ruby, &hash, "overlap", preset.overlap.into_value_with(ruby))?;
2682
+
2683
+ // Note: When embeddings feature is enabled in kreuzberg, the model field is EmbeddingModel
2684
+ // Since Ruby bindings typically build with all features, we use the model field and format it.
2685
+ let model_name = format!("{:?}", preset.model);
2686
+
2687
+ set_hash_entry(ruby, &hash, "model_name", ruby.str_new(&model_name).as_value())?;
2688
+ set_hash_entry(ruby, &hash, "dimensions", preset.dimensions.into_value_with(ruby))?;
2689
+ set_hash_entry(ruby, &hash, "description", ruby.str_new(preset.description).as_value())?;
2690
+
2691
+ Ok(hash.as_value())
2692
+ }
2693
+ None => Ok(ruby.qnil().as_value()),
2694
+ }
2695
+ }
2696
+
2697
+ /// Initialize the Kreuzberg Ruby module
2698
+ #[magnus::init]
2699
+ fn init(ruby: &Ruby) -> Result<(), Error> {
2700
+ let module = ruby.define_module("Kreuzberg")?;
2701
+
2702
+ module.define_module_function("extract_file_sync", function!(extract_file_sync, -1))?;
2703
+ module.define_module_function("extract_bytes_sync", function!(extract_bytes_sync, -1))?;
2704
+ module.define_module_function("batch_extract_files_sync", function!(batch_extract_files_sync, -1))?;
2705
+ module.define_module_function("batch_extract_bytes_sync", function!(batch_extract_bytes_sync, -1))?;
2706
+
2707
+ module.define_module_function("extract_file", function!(extract_file, -1))?;
2708
+ module.define_module_function("extract_bytes", function!(extract_bytes, -1))?;
2709
+ module.define_module_function("batch_extract_files", function!(batch_extract_files, -1))?;
2710
+ module.define_module_function("batch_extract_bytes", function!(batch_extract_bytes, -1))?;
2711
+
2712
+ module.define_module_function("clear_cache", function!(ruby_clear_cache, 0))?;
2713
+ module.define_module_function("cache_stats", function!(ruby_cache_stats, 0))?;
2714
+
2715
+ module.define_module_function("register_post_processor", function!(register_post_processor, -1))?;
2716
+ module.define_module_function("register_validator", function!(register_validator, -1))?;
2717
+ module.define_module_function("register_ocr_backend", function!(register_ocr_backend, 2))?;
2718
+ module.define_module_function("unregister_post_processor", function!(unregister_post_processor, 1))?;
2719
+ module.define_module_function("unregister_validator", function!(unregister_validator, 1))?;
2720
+ module.define_module_function("clear_post_processors", function!(clear_post_processors, 0))?;
2721
+ module.define_module_function("clear_validators", function!(clear_validators, 0))?;
2722
+ module.define_module_function("list_post_processors", function!(list_post_processors, 0))?;
2723
+ module.define_module_function("list_validators", function!(list_validators, 0))?;
2724
+ module.define_module_function("unregister_ocr_backend", function!(unregister_ocr_backend, 1))?;
2725
+ module.define_module_function("list_ocr_backends", function!(list_ocr_backends, 0))?;
2726
+ module.define_module_function("clear_ocr_backends", function!(clear_ocr_backends, 0))?;
2727
+ module.define_module_function("list_document_extractors", function!(list_document_extractors, 0))?;
2728
+ module.define_module_function(
2729
+ "unregister_document_extractor",
2730
+ function!(unregister_document_extractor, 1),
2731
+ )?;
2732
+ module.define_module_function("clear_document_extractors", function!(clear_document_extractors, 0))?;
2733
+
2734
+ module.define_module_function("_config_from_file_native", function!(config_from_file, 1))?;
2735
+ module.define_module_function("_config_discover_native", function!(config_discover, 0))?;
2736
+
2737
+ module.define_module_function("detect_mime_type", function!(detect_mime_type_from_bytes, 1))?;
2738
+ module.define_module_function(
2739
+ "detect_mime_type_from_path",
2740
+ function!(detect_mime_type_from_path_native, 1),
2741
+ )?;
2742
+ module.define_module_function("get_extensions_for_mime", function!(get_extensions_for_mime_native, 1))?;
2743
+ module.define_module_function("validate_mime_type", function!(validate_mime_type_native, 1))?;
2744
+
2745
+ module.define_module_function("list_embedding_presets", function!(list_embedding_presets, 0))?;
2746
+ module.define_module_function("get_embedding_preset", function!(get_embedding_preset, 1))?;
2747
+
2748
+ Ok(())
2749
+ }
2750
+
2751
+ #[cfg(test)]
2752
+ mod tests {
2753
+ use super::*;
2754
+
2755
+ #[test]
2756
+ fn test_ruby_clear_cache_clears_directory() {
2757
+ use std::fs;
2758
+ use std::path::PathBuf;
2759
+
2760
+ let thread_id = std::thread::current().id();
2761
+ let cache_dir = PathBuf::from(format!("/tmp/kreuzberg_test_clear_{:?}", thread_id));
2762
+
2763
+ let _ = fs::remove_dir_all(&cache_dir);
2764
+
2765
+ fs::create_dir_all(&cache_dir).expect("Failed to create cache directory");
2766
+
2767
+ let test_file = cache_dir.join("test_cache.msgpack");
2768
+ fs::write(&test_file, b"test data").expect("Failed to write test file");
2769
+
2770
+ assert!(test_file.exists(), "Test file should exist before clear");
2771
+
2772
+ let cache_dir_str = cache_dir.to_str().expect("Cache dir must be valid UTF-8");
2773
+ let result = kreuzberg::cache::clear_cache_directory(cache_dir_str);
2774
+
2775
+ assert!(result.is_ok(), "Cache clear should succeed");
2776
+ let (removed, _) = result.unwrap();
2777
+ assert_eq!(removed, 1, "Should remove one file");
2778
+
2779
+ assert!(!test_file.exists(), "Test file should be removed after clear");
2780
+
2781
+ let _ = fs::remove_dir_all(&cache_dir);
2782
+ }
2783
+
2784
+ #[test]
2785
+ fn test_ruby_cache_stats_returns_correct_structure() {
2786
+ use std::fs;
2787
+ use std::path::PathBuf;
2788
+
2789
+ let thread_id = std::thread::current().id();
2790
+ let cache_dir = PathBuf::from(format!("/tmp/kreuzberg_test_stats_{:?}", thread_id));
2791
+
2792
+ let _ = fs::remove_dir_all(&cache_dir);
2793
+
2794
+ fs::create_dir_all(&cache_dir).expect("Failed to create cache directory");
2795
+
2796
+ let test_file1 = cache_dir.join("test1.msgpack");
2797
+ let test_file2 = cache_dir.join("test2.msgpack");
2798
+ fs::write(&test_file1, b"test data 1").expect("Failed to write test file 1");
2799
+ fs::write(&test_file2, b"test data 2").expect("Failed to write test file 2");
2800
+
2801
+ let cache_dir_str = cache_dir.to_str().expect("Cache dir must be valid UTF-8");
2802
+ let stats = kreuzberg::cache::get_cache_metadata(cache_dir_str);
2803
+
2804
+ assert!(stats.is_ok(), "Cache stats should succeed");
2805
+ let stats = stats.unwrap();
2806
+
2807
+ assert_eq!(stats.total_files, 2, "Should report 2 files");
2808
+ assert!(stats.total_size_mb > 0.0, "Total size should be greater than 0");
2809
+ assert!(
2810
+ stats.available_space_mb > 0.0,
2811
+ "Available space should be greater than 0"
2812
+ );
2813
+
2814
+ let _ = fs::remove_dir_all(&cache_dir);
2815
+ }
2816
+
2817
+ #[test]
2818
+ fn test_ruby_cache_stats_converts_mb_to_bytes() {
2819
+ let size_mb = 1.5;
2820
+ let size_bytes = (size_mb * 1024.0 * 1024.0) as u64;
2821
+ assert_eq!(size_bytes, 1_572_864, "Should convert MB to bytes correctly");
2822
+ }
2823
+
2824
+ #[test]
2825
+ fn test_ruby_clear_cache_handles_empty_directory() {
2826
+ use std::fs;
2827
+ use std::path::PathBuf;
2828
+
2829
+ let thread_id = std::thread::current().id();
2830
+ let cache_dir = PathBuf::from(format!("/tmp/kreuzberg_test_empty_{:?}", thread_id));
2831
+
2832
+ let _ = fs::remove_dir_all(&cache_dir);
2833
+
2834
+ fs::create_dir_all(&cache_dir).expect("Failed to create cache directory");
2835
+
2836
+ let cache_dir_str = cache_dir.to_str().expect("Cache dir must be valid UTF-8");
2837
+ let result = kreuzberg::cache::clear_cache_directory(cache_dir_str);
2838
+
2839
+ assert!(result.is_ok(), "Should handle empty directory");
2840
+ let (removed, freed) = result.unwrap();
2841
+ assert_eq!(removed, 0, "Should remove 0 files from empty directory");
2842
+ assert_eq!(freed, 0.0, "Should free 0 MB from empty directory");
2843
+
2844
+ let _ = fs::remove_dir_all(&cache_dir);
2845
+ }
2846
+
2847
+ #[test]
2848
+ fn test_image_extraction_config_conversion() {
2849
+ let config = ImageExtractionConfig {
2850
+ extract_images: true,
2851
+ target_dpi: 300,
2852
+ max_image_dimension: 4096,
2853
+ auto_adjust_dpi: true,
2854
+ min_dpi: 72,
2855
+ max_dpi: 600,
2856
+ };
2857
+
2858
+ assert!(config.extract_images);
2859
+ assert_eq!(config.target_dpi, 300);
2860
+ assert_eq!(config.max_image_dimension, 4096);
2861
+ assert!(config.auto_adjust_dpi);
2862
+ assert_eq!(config.min_dpi, 72);
2863
+ assert_eq!(config.max_dpi, 600);
2864
+ }
2865
+
2866
+ #[test]
2867
+ fn test_image_preprocessing_config_conversion() {
2868
+ let config = ImagePreprocessingConfig {
2869
+ target_dpi: 300,
2870
+ auto_rotate: true,
2871
+ deskew: true,
2872
+ denoise: false,
2873
+ contrast_enhance: false,
2874
+ binarization_method: "otsu".to_string(),
2875
+ invert_colors: false,
2876
+ };
2877
+
2878
+ assert_eq!(config.target_dpi, 300);
2879
+ assert!(config.auto_rotate);
2880
+ assert!(config.deskew);
2881
+ assert!(!config.denoise);
2882
+ assert!(!config.contrast_enhance);
2883
+ assert_eq!(config.binarization_method, "otsu");
2884
+ assert!(!config.invert_colors);
2885
+ }
2886
+
2887
+ #[test]
2888
+ fn test_postprocessor_config_conversion() {
2889
+ let config = PostProcessorConfig {
2890
+ enabled: true,
2891
+ enabled_processors: Some(vec!["processor1".to_string(), "processor2".to_string()]),
2892
+ disabled_processors: None,
2893
+ };
2894
+
2895
+ assert!(config.enabled);
2896
+ assert!(config.enabled_processors.is_some());
2897
+ assert_eq!(config.enabled_processors.unwrap().len(), 2);
2898
+ assert!(config.disabled_processors.is_none());
2899
+ }
2900
+
2901
+ #[test]
2902
+ fn test_token_reduction_config_conversion() {
2903
+ let config = TokenReductionConfig {
2904
+ mode: "moderate".to_string(),
2905
+ preserve_important_words: true,
2906
+ };
2907
+
2908
+ assert_eq!(config.mode, "moderate");
2909
+ assert!(config.preserve_important_words);
2910
+ }
2911
+
2912
+ #[test]
2913
+ fn test_extraction_config_with_new_fields() {
2914
+ let config = ExtractionConfig {
2915
+ images: Some(ImageExtractionConfig {
2916
+ extract_images: true,
2917
+ target_dpi: 300,
2918
+ max_image_dimension: 4096,
2919
+ auto_adjust_dpi: true,
2920
+ min_dpi: 72,
2921
+ max_dpi: 600,
2922
+ }),
2923
+ postprocessor: Some(PostProcessorConfig {
2924
+ enabled: true,
2925
+ enabled_processors: None,
2926
+ disabled_processors: None,
2927
+ }),
2928
+ token_reduction: Some(TokenReductionConfig {
2929
+ mode: "light".to_string(),
2930
+ preserve_important_words: true,
2931
+ }),
2932
+ ..Default::default()
2933
+ };
2934
+
2935
+ assert!(config.images.is_some());
2936
+ assert!(config.postprocessor.is_some());
2937
+ assert!(config.token_reduction.is_some());
2938
+ }
2939
+ }