RubyGems - kreuzberg - Versions diffs - 4.0.8 → 4.1.1 - Mend

kreuzberg 4.0.8 → 4.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (312) hide show

checksums.yaml +4 -4
data/Gemfile.lock +2 -2
data/README.md +1 -1
data/ext/kreuzberg_rb/native/Cargo.lock +94 -98
data/ext/kreuzberg_rb/native/Cargo.toml +4 -2
data/ext/kreuzberg_rb/native/src/batch.rs +139 -0
data/ext/kreuzberg_rb/native/src/config/mod.rs +10 -0
data/ext/kreuzberg_rb/native/src/config/types.rs +1058 -0
data/ext/kreuzberg_rb/native/src/error_handling.rs +125 -0
data/ext/kreuzberg_rb/native/src/extraction.rs +79 -0
data/ext/kreuzberg_rb/native/src/gc_guarded_value.rs +35 -0
data/ext/kreuzberg_rb/native/src/helpers.rs +176 -0
data/ext/kreuzberg_rb/native/src/lib.rs +342 -3622
data/ext/kreuzberg_rb/native/src/metadata.rs +34 -0
data/ext/kreuzberg_rb/native/src/plugins/mod.rs +92 -0
data/ext/kreuzberg_rb/native/src/plugins/ocr_backend.rs +159 -0
data/ext/kreuzberg_rb/native/src/plugins/post_processor.rs +126 -0
data/ext/kreuzberg_rb/native/src/plugins/validator.rs +99 -0
data/ext/kreuzberg_rb/native/src/result.rs +326 -0
data/ext/kreuzberg_rb/native/src/validation.rs +4 -0
data/lib/kreuzberg/config.rb +99 -2
data/lib/kreuzberg/result.rb +107 -2
data/lib/kreuzberg/types.rb +104 -0
data/lib/kreuzberg/version.rb +1 -1
data/lib/kreuzberg.rb +0 -4
data/sig/kreuzberg.rbs +105 -1
data/spec/fixtures/config.toml +1 -1
data/spec/fixtures/config.yaml +1 -1
data/vendor/Cargo.toml +3 -3
data/vendor/kreuzberg/Cargo.toml +5 -4
data/vendor/kreuzberg/README.md +1 -1
data/vendor/kreuzberg/src/api/config.rs +69 -0
data/vendor/kreuzberg/src/api/handlers.rs +99 -2
data/vendor/kreuzberg/src/api/mod.rs +14 -7
data/vendor/kreuzberg/src/api/router.rs +214 -0
data/vendor/kreuzberg/src/api/startup.rs +243 -0
data/vendor/kreuzberg/src/api/types.rs +78 -0
data/vendor/kreuzberg/src/cache/cleanup.rs +277 -0
data/vendor/kreuzberg/src/cache/core.rs +428 -0
data/vendor/kreuzberg/src/cache/mod.rs +21 -843
data/vendor/kreuzberg/src/cache/utilities.rs +156 -0
data/vendor/kreuzberg/src/chunking/boundaries.rs +301 -0
data/vendor/kreuzberg/src/chunking/builder.rs +294 -0
data/vendor/kreuzberg/src/chunking/config.rs +52 -0
data/vendor/kreuzberg/src/chunking/core.rs +1017 -0
data/vendor/kreuzberg/src/chunking/mod.rs +14 -2211
data/vendor/kreuzberg/src/chunking/processor.rs +10 -0
data/vendor/kreuzberg/src/chunking/validation.rs +686 -0
data/vendor/kreuzberg/src/core/config/extraction/core.rs +169 -0
data/vendor/kreuzberg/src/core/config/extraction/env.rs +179 -0
data/vendor/kreuzberg/src/core/config/extraction/loaders.rs +204 -0
data/vendor/kreuzberg/src/core/config/extraction/mod.rs +42 -0
data/vendor/kreuzberg/src/core/config/extraction/types.rs +93 -0
data/vendor/kreuzberg/src/core/config/formats.rs +135 -0
data/vendor/kreuzberg/src/core/config/mod.rs +20 -0
data/vendor/kreuzberg/src/core/config/ocr.rs +73 -0
data/vendor/kreuzberg/src/core/config/page.rs +57 -0
data/vendor/kreuzberg/src/core/config/pdf.rs +111 -0
data/vendor/kreuzberg/src/core/config/processing.rs +312 -0
data/vendor/kreuzberg/src/core/config_validation/dependencies.rs +187 -0
data/vendor/kreuzberg/src/core/config_validation/mod.rs +386 -0
data/vendor/kreuzberg/src/core/config_validation/sections.rs +401 -0
data/vendor/kreuzberg/src/core/extractor/batch.rs +246 -0
data/vendor/kreuzberg/src/core/extractor/bytes.rs +116 -0
data/vendor/kreuzberg/src/core/extractor/file.rs +240 -0
data/vendor/kreuzberg/src/core/extractor/helpers.rs +71 -0
data/vendor/kreuzberg/src/core/extractor/legacy.rs +62 -0
data/vendor/kreuzberg/src/core/extractor/mod.rs +490 -0
data/vendor/kreuzberg/src/core/extractor/sync.rs +208 -0
data/vendor/kreuzberg/src/core/mime.rs +15 -0
data/vendor/kreuzberg/src/core/mod.rs +4 -1
data/vendor/kreuzberg/src/core/pipeline/cache.rs +60 -0
data/vendor/kreuzberg/src/core/pipeline/execution.rs +89 -0
data/vendor/kreuzberg/src/core/pipeline/features.rs +108 -0
data/vendor/kreuzberg/src/core/pipeline/format.rs +392 -0
data/vendor/kreuzberg/src/core/pipeline/initialization.rs +67 -0
data/vendor/kreuzberg/src/core/pipeline/mod.rs +135 -0
data/vendor/kreuzberg/src/core/pipeline/tests.rs +975 -0
data/vendor/kreuzberg/src/core/server_config/env.rs +90 -0
data/vendor/kreuzberg/src/core/server_config/loader.rs +202 -0
data/vendor/kreuzberg/src/core/server_config/mod.rs +380 -0
data/vendor/kreuzberg/src/core/server_config/tests/basic_tests.rs +124 -0
data/vendor/kreuzberg/src/core/server_config/tests/env_tests.rs +216 -0
data/vendor/kreuzberg/src/core/server_config/tests/file_loading_tests.rs +341 -0
data/vendor/kreuzberg/src/core/server_config/tests/mod.rs +5 -0
data/vendor/kreuzberg/src/core/server_config/validation.rs +17 -0
data/vendor/kreuzberg/src/embeddings.rs +136 -13
data/vendor/kreuzberg/src/extraction/{archive.rs → archive/mod.rs} +45 -239
data/vendor/kreuzberg/src/extraction/archive/sevenz.rs +98 -0
data/vendor/kreuzberg/src/extraction/archive/tar.rs +118 -0
data/vendor/kreuzberg/src/extraction/archive/zip.rs +101 -0
data/vendor/kreuzberg/src/extraction/html/converter.rs +592 -0
data/vendor/kreuzberg/src/extraction/html/image_handling.rs +95 -0
data/vendor/kreuzberg/src/extraction/html/mod.rs +53 -0
data/vendor/kreuzberg/src/extraction/html/processor.rs +659 -0
data/vendor/kreuzberg/src/extraction/html/stack_management.rs +103 -0
data/vendor/kreuzberg/src/extraction/html/types.rs +28 -0
data/vendor/kreuzberg/src/extraction/mod.rs +6 -2
data/vendor/kreuzberg/src/extraction/pptx/container.rs +159 -0
data/vendor/kreuzberg/src/extraction/pptx/content_builder.rs +168 -0
data/vendor/kreuzberg/src/extraction/pptx/elements.rs +132 -0
data/vendor/kreuzberg/src/extraction/pptx/image_handling.rs +57 -0
data/vendor/kreuzberg/src/extraction/pptx/metadata.rs +160 -0
data/vendor/kreuzberg/src/extraction/pptx/mod.rs +558 -0
data/vendor/kreuzberg/src/extraction/pptx/parser.rs +388 -0
data/vendor/kreuzberg/src/extraction/transform/content.rs +205 -0
data/vendor/kreuzberg/src/extraction/transform/elements.rs +211 -0
data/vendor/kreuzberg/src/extraction/transform/mod.rs +480 -0
data/vendor/kreuzberg/src/extraction/transform/types.rs +27 -0
data/vendor/kreuzberg/src/extractors/archive.rs +2 -0
data/vendor/kreuzberg/src/extractors/bibtex.rs +2 -0
data/vendor/kreuzberg/src/extractors/djot_format/attributes.rs +134 -0
data/vendor/kreuzberg/src/extractors/djot_format/conversion.rs +223 -0
data/vendor/kreuzberg/src/extractors/djot_format/extractor.rs +172 -0
data/vendor/kreuzberg/src/extractors/djot_format/mod.rs +24 -0
data/vendor/kreuzberg/src/extractors/djot_format/parsing/block_handlers.rs +271 -0
data/vendor/kreuzberg/src/extractors/djot_format/parsing/content_extraction.rs +257 -0
data/vendor/kreuzberg/src/extractors/djot_format/parsing/event_handlers.rs +101 -0
data/vendor/kreuzberg/src/extractors/djot_format/parsing/inline_handlers.rs +201 -0
data/vendor/kreuzberg/src/extractors/djot_format/parsing/mod.rs +16 -0
data/vendor/kreuzberg/src/extractors/djot_format/parsing/state.rs +78 -0
data/vendor/kreuzberg/src/extractors/djot_format/parsing/table_extraction.rs +68 -0
data/vendor/kreuzberg/src/extractors/djot_format/parsing/text_extraction.rs +61 -0
data/vendor/kreuzberg/src/extractors/djot_format/rendering.rs +452 -0
data/vendor/kreuzberg/src/extractors/docbook.rs +2 -0
data/vendor/kreuzberg/src/extractors/docx.rs +12 -1
data/vendor/kreuzberg/src/extractors/email.rs +2 -0
data/vendor/kreuzberg/src/extractors/epub/content.rs +333 -0
data/vendor/kreuzberg/src/extractors/epub/metadata.rs +137 -0
data/vendor/kreuzberg/src/extractors/epub/mod.rs +186 -0
data/vendor/kreuzberg/src/extractors/epub/parsing.rs +86 -0
data/vendor/kreuzberg/src/extractors/excel.rs +4 -0
data/vendor/kreuzberg/src/extractors/fictionbook.rs +2 -0
data/vendor/kreuzberg/src/extractors/frontmatter_utils.rs +466 -0
data/vendor/kreuzberg/src/extractors/html.rs +80 -8
data/vendor/kreuzberg/src/extractors/image.rs +8 -1
data/vendor/kreuzberg/src/extractors/jats/elements.rs +350 -0
data/vendor/kreuzberg/src/extractors/jats/metadata.rs +21 -0
data/vendor/kreuzberg/src/extractors/{jats.rs → jats/mod.rs} +10 -412
data/vendor/kreuzberg/src/extractors/jats/parser.rs +52 -0
data/vendor/kreuzberg/src/extractors/jupyter.rs +2 -0
data/vendor/kreuzberg/src/extractors/latex/commands.rs +93 -0
data/vendor/kreuzberg/src/extractors/latex/environments.rs +157 -0
data/vendor/kreuzberg/src/extractors/latex/metadata.rs +27 -0
data/vendor/kreuzberg/src/extractors/latex/mod.rs +146 -0
data/vendor/kreuzberg/src/extractors/latex/parser.rs +231 -0
data/vendor/kreuzberg/src/extractors/latex/utilities.rs +126 -0
data/vendor/kreuzberg/src/extractors/markdown.rs +39 -162
data/vendor/kreuzberg/src/extractors/mod.rs +9 -1
data/vendor/kreuzberg/src/extractors/odt.rs +2 -0
data/vendor/kreuzberg/src/extractors/opml/core.rs +165 -0
data/vendor/kreuzberg/src/extractors/opml/mod.rs +31 -0
data/vendor/kreuzberg/src/extractors/opml/parser.rs +479 -0
data/vendor/kreuzberg/src/extractors/orgmode.rs +2 -0
data/vendor/kreuzberg/src/extractors/pdf/extraction.rs +106 -0
data/vendor/kreuzberg/src/extractors/{pdf.rs → pdf/mod.rs} +25 -324
data/vendor/kreuzberg/src/extractors/pdf/ocr.rs +214 -0
data/vendor/kreuzberg/src/extractors/pdf/pages.rs +51 -0
data/vendor/kreuzberg/src/extractors/pptx.rs +9 -2
data/vendor/kreuzberg/src/extractors/rst.rs +2 -0
data/vendor/kreuzberg/src/extractors/rtf/encoding.rs +116 -0
data/vendor/kreuzberg/src/extractors/rtf/formatting.rs +24 -0
data/vendor/kreuzberg/src/extractors/rtf/images.rs +72 -0
data/vendor/kreuzberg/src/extractors/rtf/metadata.rs +216 -0
data/vendor/kreuzberg/src/extractors/rtf/mod.rs +142 -0
data/vendor/kreuzberg/src/extractors/rtf/parser.rs +259 -0
data/vendor/kreuzberg/src/extractors/rtf/tables.rs +83 -0
data/vendor/kreuzberg/src/extractors/structured.rs +2 -0
data/vendor/kreuzberg/src/extractors/text.rs +4 -0
data/vendor/kreuzberg/src/extractors/typst.rs +2 -0
data/vendor/kreuzberg/src/extractors/xml.rs +2 -0
data/vendor/kreuzberg/src/keywords/processor.rs +14 -0
data/vendor/kreuzberg/src/language_detection/processor.rs +10 -0
data/vendor/kreuzberg/src/lib.rs +2 -2
data/vendor/kreuzberg/src/mcp/errors.rs +312 -0
data/vendor/kreuzberg/src/mcp/format.rs +211 -0
data/vendor/kreuzberg/src/mcp/mod.rs +9 -3
data/vendor/kreuzberg/src/mcp/params.rs +196 -0
data/vendor/kreuzberg/src/mcp/server.rs +39 -1438
data/vendor/kreuzberg/src/mcp/tools/cache.rs +179 -0
data/vendor/kreuzberg/src/mcp/tools/extraction.rs +403 -0
data/vendor/kreuzberg/src/mcp/tools/mime.rs +150 -0
data/vendor/kreuzberg/src/mcp/tools/mod.rs +11 -0
data/vendor/kreuzberg/src/ocr/backends/easyocr.rs +96 -0
data/vendor/kreuzberg/src/ocr/backends/mod.rs +7 -0
data/vendor/kreuzberg/src/ocr/backends/paddleocr.rs +27 -0
data/vendor/kreuzberg/src/ocr/backends/tesseract.rs +134 -0
data/vendor/kreuzberg/src/ocr/hocr.rs +60 -16
data/vendor/kreuzberg/src/ocr/language_registry.rs +11 -235
data/vendor/kreuzberg/src/ocr/mod.rs +1 -0
data/vendor/kreuzberg/src/ocr/processor/config.rs +203 -0
data/vendor/kreuzberg/src/ocr/processor/execution.rs +494 -0
data/vendor/kreuzberg/src/ocr/processor/mod.rs +265 -0
data/vendor/kreuzberg/src/ocr/processor/validation.rs +145 -0
data/vendor/kreuzberg/src/ocr/tesseract_backend.rs +41 -24
data/vendor/kreuzberg/src/pdf/bindings.rs +21 -8
data/vendor/kreuzberg/src/pdf/hierarchy/bounding_box.rs +289 -0
data/vendor/kreuzberg/src/pdf/hierarchy/clustering.rs +199 -0
data/vendor/kreuzberg/src/pdf/{hierarchy.rs → hierarchy/extraction.rs} +6 -346
data/vendor/kreuzberg/src/pdf/hierarchy/mod.rs +18 -0
data/vendor/kreuzberg/src/plugins/extractor/mod.rs +319 -0
data/vendor/kreuzberg/src/plugins/extractor/registry.rs +434 -0
data/vendor/kreuzberg/src/plugins/extractor/trait.rs +391 -0
data/vendor/kreuzberg/src/plugins/mod.rs +13 -0
data/vendor/kreuzberg/src/plugins/ocr.rs +11 -0
data/vendor/kreuzberg/src/plugins/processor/mod.rs +365 -0
data/vendor/kreuzberg/src/plugins/processor/registry.rs +37 -0
data/vendor/kreuzberg/src/plugins/processor/trait.rs +284 -0
data/vendor/kreuzberg/src/plugins/registry/extractor.rs +416 -0
data/vendor/kreuzberg/src/plugins/registry/mod.rs +116 -0
data/vendor/kreuzberg/src/plugins/registry/ocr.rs +293 -0
data/vendor/kreuzberg/src/plugins/registry/processor.rs +304 -0
data/vendor/kreuzberg/src/plugins/registry/validator.rs +238 -0
data/vendor/kreuzberg/src/plugins/validator/mod.rs +424 -0
data/vendor/kreuzberg/src/plugins/validator/registry.rs +355 -0
data/vendor/kreuzberg/src/plugins/validator/trait.rs +276 -0
data/vendor/kreuzberg/src/stopwords/languages/asian.rs +40 -0
data/vendor/kreuzberg/src/stopwords/languages/germanic.rs +36 -0
data/vendor/kreuzberg/src/stopwords/languages/mod.rs +10 -0
data/vendor/kreuzberg/src/stopwords/languages/other.rs +44 -0
data/vendor/kreuzberg/src/stopwords/languages/romance.rs +36 -0
data/vendor/kreuzberg/src/stopwords/languages/slavic.rs +36 -0
data/vendor/kreuzberg/src/stopwords/mod.rs +7 -33
data/vendor/kreuzberg/src/text/quality.rs +1 -1
data/vendor/kreuzberg/src/text/quality_processor.rs +10 -0
data/vendor/kreuzberg/src/text/token_reduction/core/analysis.rs +238 -0
data/vendor/kreuzberg/src/text/token_reduction/core/mod.rs +8 -0
data/vendor/kreuzberg/src/text/token_reduction/core/punctuation.rs +54 -0
data/vendor/kreuzberg/src/text/token_reduction/core/reducer.rs +384 -0
data/vendor/kreuzberg/src/text/token_reduction/core/sentence_selection.rs +68 -0
data/vendor/kreuzberg/src/text/token_reduction/core/word_filtering.rs +156 -0
data/vendor/kreuzberg/src/text/token_reduction/filters/general.rs +377 -0
data/vendor/kreuzberg/src/text/token_reduction/filters/html.rs +51 -0
data/vendor/kreuzberg/src/text/token_reduction/filters/markdown.rs +285 -0
data/vendor/kreuzberg/src/text/token_reduction/filters.rs +131 -246
data/vendor/kreuzberg/src/types/djot.rs +209 -0
data/vendor/kreuzberg/src/types/extraction.rs +301 -0
data/vendor/kreuzberg/src/types/formats.rs +443 -0
data/vendor/kreuzberg/src/types/metadata.rs +560 -0
data/vendor/kreuzberg/src/types/mod.rs +281 -0
data/vendor/kreuzberg/src/types/page.rs +182 -0
data/vendor/kreuzberg/src/types/serde_helpers.rs +132 -0
data/vendor/kreuzberg/src/types/tables.rs +39 -0
data/vendor/kreuzberg/src/utils/quality/heuristics.rs +58 -0
data/vendor/kreuzberg/src/utils/{quality.rs → quality/mod.rs} +168 -489
data/vendor/kreuzberg/src/utils/quality/patterns.rs +117 -0
data/vendor/kreuzberg/src/utils/quality/scoring.rs +178 -0
data/vendor/kreuzberg/src/utils/string_pool/buffer_pool.rs +325 -0
data/vendor/kreuzberg/src/utils/string_pool/interned.rs +102 -0
data/vendor/kreuzberg/src/utils/string_pool/language_pool.rs +119 -0
data/vendor/kreuzberg/src/utils/string_pool/mime_pool.rs +235 -0
data/vendor/kreuzberg/src/utils/string_pool/mod.rs +41 -0
data/vendor/kreuzberg/tests/api_chunk.rs +313 -0
data/vendor/kreuzberg/tests/api_embed.rs +6 -9
data/vendor/kreuzberg/tests/batch_orchestration.rs +1 -0
data/vendor/kreuzberg/tests/concurrency_stress.rs +7 -0
data/vendor/kreuzberg/tests/core_integration.rs +1 -0
data/vendor/kreuzberg/tests/docx_metadata_extraction_test.rs +130 -0
data/vendor/kreuzberg/tests/epub_native_extractor_tests.rs +5 -14
data/vendor/kreuzberg/tests/format_integration.rs +2 -0
data/vendor/kreuzberg/tests/helpers/mod.rs +1 -0
data/vendor/kreuzberg/tests/html_table_test.rs +11 -11
data/vendor/kreuzberg/tests/ocr_configuration.rs +16 -0
data/vendor/kreuzberg/tests/ocr_errors.rs +18 -0
data/vendor/kreuzberg/tests/ocr_quality.rs +9 -0
data/vendor/kreuzberg/tests/ocr_stress.rs +1 -0
data/vendor/kreuzberg/tests/pipeline_integration.rs +50 -0
data/vendor/kreuzberg/tests/plugin_ocr_backend_test.rs +13 -0
data/vendor/kreuzberg/tests/plugin_system.rs +12 -0
data/vendor/kreuzberg/tests/pptx_regression_tests.rs +504 -0
data/vendor/kreuzberg/tests/registry_integration_tests.rs +2 -0
data/vendor/kreuzberg-ffi/Cargo.toml +2 -1
data/vendor/kreuzberg-ffi/benches/result_view_benchmark.rs +2 -0
data/vendor/kreuzberg-ffi/kreuzberg.h +347 -178
data/vendor/kreuzberg-ffi/src/config/html.rs +318 -0
data/vendor/kreuzberg-ffi/src/config/loader.rs +154 -0
data/vendor/kreuzberg-ffi/src/config/merge.rs +104 -0
data/vendor/kreuzberg-ffi/src/config/mod.rs +385 -0
data/vendor/kreuzberg-ffi/src/config/parse.rs +91 -0
data/vendor/kreuzberg-ffi/src/config/serialize.rs +118 -0
data/vendor/kreuzberg-ffi/src/config_builder.rs +598 -0
data/vendor/kreuzberg-ffi/src/error.rs +46 -14
data/vendor/kreuzberg-ffi/src/helpers.rs +10 -0
data/vendor/kreuzberg-ffi/src/html_options.rs +421 -0
data/vendor/kreuzberg-ffi/src/lib.rs +16 -0
data/vendor/kreuzberg-ffi/src/panic_shield.rs +11 -0
data/vendor/kreuzberg-ffi/src/plugins/ocr_backend.rs +2 -0
data/vendor/kreuzberg-ffi/src/result.rs +148 -122
data/vendor/kreuzberg-ffi/src/result_view.rs +4 -0
data/vendor/kreuzberg-tesseract/Cargo.toml +2 -2
metadata +201 -28
data/vendor/kreuzberg/src/api/server.rs +0 -518
data/vendor/kreuzberg/src/core/config.rs +0 -1914
data/vendor/kreuzberg/src/core/config_validation.rs +0 -949
data/vendor/kreuzberg/src/core/extractor.rs +0 -1200
data/vendor/kreuzberg/src/core/pipeline.rs +0 -1223
data/vendor/kreuzberg/src/core/server_config.rs +0 -1220
data/vendor/kreuzberg/src/extraction/html.rs +0 -1830
data/vendor/kreuzberg/src/extraction/pptx.rs +0 -3102
data/vendor/kreuzberg/src/extractors/epub.rs +0 -696
data/vendor/kreuzberg/src/extractors/latex.rs +0 -653
data/vendor/kreuzberg/src/extractors/opml.rs +0 -635
data/vendor/kreuzberg/src/extractors/rtf.rs +0 -809
data/vendor/kreuzberg/src/ocr/processor.rs +0 -858
data/vendor/kreuzberg/src/plugins/extractor.rs +0 -1042
data/vendor/kreuzberg/src/plugins/processor.rs +0 -650
data/vendor/kreuzberg/src/plugins/registry.rs +0 -1339
data/vendor/kreuzberg/src/plugins/validator.rs +0 -967
data/vendor/kreuzberg/src/text/token_reduction/core.rs +0 -832
data/vendor/kreuzberg/src/types.rs +0 -1713
data/vendor/kreuzberg/src/utils/string_pool.rs +0 -762
data/vendor/kreuzberg-ffi/src/config.rs +0 -1341

checksums.yaml CHANGED Viewed

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f5742beaf511e059ff6d34f536e90a5636097b48cd1262a34eaa7692261de7aa
-  data.tar.gz: b913234c3ecda47e6e4944181309c2e60c09a8e708eb651cd705258048d3f7bb
+  metadata.gz: 0c1c0519fb3a58c45ec553994bd982b4f284835bd35ea0758461f6f381accfd6
+  data.tar.gz: 161c18cfabdd20bdaa520abda521cb16072dcc00f5fd2e41152d9da4acdb9d08
 SHA512:
-  metadata.gz: 1a459be58b389e806f6c46ffadaf0f01ea095154c3f9265b54bcd1c722493d8a094df41cc5a008bea6ec3cadc2622ecd0379f36a0ae7404f93e2a6c39f44ebfd
-  data.tar.gz: 31c2d8cb14a9bd72e7fc4fff84f598ec6ecb24c5d982394f642fcc04e09cc74d4280550e10ca1b5d703a0a824fc3c63330d5608c42785a825ea4f46ee3ad8c0a
+  metadata.gz: 7c6e1768022dcfdef5eaaaa3557a8388e8ad45158a69ed022d852b07f7658cbb885ca7860fa32eda8f29b5f7ea4216f93033aad77614afb82578b9157ed92710
+  data.tar.gz: 96ca3f1f3c6d6f9ea6dc826f7704c1d741ffaab24f524f099c4eb294652211ac6b9f324874974bb24d417193e76b1bce951315330d6f1ccd8eb5b014fa7fc71f

data/Gemfile.lock CHANGED Viewed

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    kreuzberg (4.0.8)
+    kreuzberg (4.1.1)
 GEM
   remote: https://rubygems.org/
@@ -207,7 +207,7 @@ CHECKSUMS
   i18n (1.14.8) sha256=285778639134865c5e0f6269e0b818256017e8cde89993fdfcbfb64d088824a5
   io-console (0.8.2) sha256=d6e3ae7a7cc7574f4b8893b4fca2162e57a825b223a177b7afa236c5ef9814cc
   json (2.18.0) sha256=b10506aee4183f5cf49e0efc48073d7b75843ce3782c68dbeb763351c08fd505
-  kreuzberg (4.0.8)
+  kreuzberg (4.1.1)
   language_server-protocol (3.17.0.5) sha256=fd1e39a51a28bf3eec959379985a72e296e9f9acfce46f6a79d31ca8760803cc
   lint_roller (1.1.0) sha256=2c0c845b632a7d172cb849cc90c1bce937a28c5c8ccccb50dfd46a485003cc87
   listen (3.10.0) sha256=c6e182db62143aeccc2e1960033bebe7445309c7272061979bb098d03760c9d2

data/README.md CHANGED Viewed

@@ -22,7 +22,7 @@
     <img src="https://img.shields.io/maven-central/v/dev.kreuzberg/kreuzberg?label=Java&color=007ec6" alt="Java">
   </a>
   <a href="https://github.com/kreuzberg-dev/kreuzberg/releases">
-    <img src="https://img.shields.io/github/v/tag/kreuzberg-dev/kreuzberg?label=Go&color=007ec6&filter=v4.0.0" alt="Go">
+    <img src="https://img.shields.io/github/v/tag/kreuzberg-dev/kreuzberg?label=Go&color=007ec6&filter=v4.1.1" alt="Go">
   </a>
   <a href="https://www.nuget.org/packages/Kreuzberg/">
     <img src="https://img.shields.io/nuget/v/Kreuzberg?label=C%23&color=007ec6" alt="C#">

data/ext/kreuzberg_rb/native/Cargo.lock CHANGED Viewed

@@ -307,7 +307,7 @@ dependencies = [
  "num-traits",
  "pastey 0.1.1",
  "rayon",
- "thiserror 2.0.17",
+ "thiserror 2.0.18",
  "v_frame",
  "y4m",
 ]
@@ -711,14 +711,14 @@ dependencies = [
  "serde_json",
  "syn",
  "tempfile",
- "toml 0.9.10+spec-1.1.0",
+ "toml 0.9.11+spec-1.1.0",
 ]
 [[package]]
 name = "cc"
-version = "1.2.51"
+version = "1.2.53"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7a0aeaff4ff1a90589618835a598e545176939b97874f7abc7851caa0618f203"
+checksum = "755d2fce177175ffca841e9a06afdb2c4ab0f593d53b4dee48147dfaade85932"
 dependencies = [
  "find-msvc-tools",
  "jobserver",
@@ -1515,7 +1515,7 @@ dependencies = [
  "lebe",
  "miniz_oxide",
  "rayon-core",
- "smallvec 1.15.1",
+ "smallvec",
  "zune-inflate",
 ]
@@ -1544,27 +1544,28 @@ checksum = "f8eb564c5c7423d25c886fb561d1e4ee69f72354d16918afa32c08811f6b6a55"
 [[package]]
 name = "fast_image_resize"
-version = "5.6.0"
+version = "6.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6b6e793dfd0ee192d1999c655797ecc956c82d1f6d367be20bf6b81d6a1c87ac"
+checksum = "12dd43e5011e8d8411a3215a0d57a2ec5c68282fb90eb5d7221fab0113442174"
 dependencies = [
  "cfg-if",
  "document-features",
  "num-traits",
- "thiserror 2.0.17",
+ "thiserror 2.0.18",
 ]
 [[package]]
 name = "fastembed"
-version = "5.7.0"
+version = "5.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "158bd4a909fb7edd96013796d6b6c57660615a90da81811f2a97bf4ea832d6e4"
+checksum = "59a3f841f27a44bcc32214f8df75cc9b6cea55dbbebbfe546735690eab5bb2d2"
 dependencies = [
  "anyhow",
  "hf-hub",
- "ndarray 0.16.1",
+ "ndarray",
  "ort",
  "safetensors",
+ "serde",
  "serde_json",
  "tokenizers",
 ]
@@ -1630,9 +1631,9 @@ dependencies = [
 [[package]]
 name = "find-msvc-tools"
-version = "0.1.6"
+version = "0.1.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "645cbb3a84e60b7531617d5ae4e57f7e27308f6445f5abf653209ea76dec8dff"
+checksum = "8591b0bcc8a98a64310a2fae1bb3e9b8564dd10e381e6e28010fde8e8e8568db"
 [[package]]
 name = "flate2"
@@ -1992,7 +1993,7 @@ dependencies = [
  "reqwest 0.12.28",
  "serde",
  "serde_json",
- "thiserror 2.0.17",
+ "thiserror 2.0.18",
  "ureq 2.12.1",
  "windows-sys 0.60.2",
 ]
@@ -2026,9 +2027,9 @@ dependencies = [
 [[package]]
 name = "html-to-markdown-rs"
-version = "2.22.2"
+version = "2.23.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bc035a13874f15114115e664e096596b9ac54c9938befb5bfc2f2e35a5492e7d"
+checksum = "e1f4d6781ac8dd203853803d27054ca4153c7fd0f3956cb7fc95dc06f42a1c46"
 dependencies = [
  "astral-tl",
  "base64 0.22.1",
@@ -2041,7 +2042,7 @@ dependencies = [
  "regex",
  "serde",
  "serde_json",
- "thiserror 2.0.17",
+ "thiserror 2.0.18",
 ]
 [[package]]
@@ -2135,7 +2136,7 @@ dependencies = [
  "itoa",
  "pin-project-lite",
  "pin-utils",
- "smallvec 1.15.1",
+ "smallvec",
  "tokio",
  "want",
 ]
@@ -2282,7 +2283,7 @@ dependencies = [
  "icu_normalizer_data",
  "icu_properties",
  "icu_provider",
- "smallvec 1.15.1",
+ "smallvec",
  "zerovec",
 ]
@@ -2364,7 +2365,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de"
 dependencies = [
  "idna_adapter",
- "smallvec 1.15.1",
+ "smallvec",
  "utf8_iter",
 ]
@@ -2599,6 +2600,12 @@ dependencies = [
  "libc",
 ]
+[[package]]
+name = "jotdown"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "086b08ec7a274cd60cd575ed3651ba081ee72dec0d39a6210e8adcff9efe3880"
 [[package]]
 name = "js-sys"
 version = "0.3.83"
@@ -2620,7 +2627,7 @@ dependencies = [
 [[package]]
 name = "kreuzberg"
-version = "4.0.0-rc.28"
+version = "4.1.0"
 dependencies = [
  "ahash",
  "async-trait",
@@ -2644,6 +2651,7 @@ dependencies = [
  "image",
  "indexmap",
  "infer",
+ "jotdown",
  "kamadak-exif",
  "kreuzberg-pdfium-render",
  "kreuzberg-tesseract",
@@ -2655,7 +2663,7 @@ dependencies = [
  "memchr",
  "mime_guess",
  "msg_parser",
- "ndarray 0.17.1",
+ "ndarray",
  "num_cpus",
  "once_cell",
  "opentelemetry",
@@ -2666,7 +2674,7 @@ dependencies = [
  "pkg-config",
  "polars",
  "pulldown-cmark",
- "quick-xml 0.38.4",
+ "quick-xml 0.39.0",
  "rake",
  "rayon",
  "regex",
@@ -2684,10 +2692,10 @@ dependencies = [
  "smartcore",
  "tar",
  "text-splitter",
- "thiserror 2.0.17",
+ "thiserror 2.0.18",
  "tiff 0.11.0",
  "tokio",
- "toml 0.9.10+spec-1.1.0",
+ "toml 0.9.11+spec-1.1.0",
  "tower",
  "tower-http",
  "tracing",
@@ -2698,12 +2706,12 @@ dependencies = [
  "uuid",
  "whatlang",
  "yake-rust",
- "zip 7.0.0",
+ "zip 7.1.0",
 ]
 [[package]]
 name = "kreuzberg-ffi"
-version = "4.0.0-rc.28"
+version = "4.1.0"
 dependencies = [
  "async-trait",
  "cbindgen",
@@ -2752,19 +2760,21 @@ dependencies = [
  "pretty_assertions",
  "rb-sys",
  "serde_json",
+ "serde_yaml_ng",
  "tokio",
+ "toml 0.8.23",
 ]
 [[package]]
 name = "kreuzberg-tesseract"
-version = "4.0.0-rc.28"
+version = "4.1.0"
 dependencies = [
  "cc",
  "cmake",
  "libc",
  "reqwest 0.13.1",
- "thiserror 2.0.17",
- "zip 7.0.0",
+ "thiserror 2.0.18",
+ "zip 7.1.0",
 ]
 [[package]]
@@ -2912,9 +2922,9 @@ dependencies = [
 [[package]]
 name = "lopdf"
-version = "0.38.0"
+version = "0.39.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c7184fdea2bc3cd272a1acec4030c321a8f9875e877b3f92a53f2f6033fdc289"
+checksum = "f560f57dfb9142a02d673e137622fd515d4231e51feb8b4af28d92647d83f35b"
 dependencies = [
  "aes",
  "bitflags",
@@ -2936,7 +2946,7 @@ dependencies = [
  "rayon",
  "sha2",
  "stringprep",
- "thiserror 2.0.17",
+ "thiserror 2.0.18",
  "time",
  "ttf-parser",
  "weezl",
@@ -2978,9 +2988,9 @@ dependencies = [
 [[package]]
 name = "lzma-rust2"
-version = "0.15.6"
+version = "0.15.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "17f7337d278fec032975dc884152491580dd23750ee957047856735fe0e61ede"
+checksum = "1670343e58806300d87950e3401e820b519b9384281bbabfb15e3636689ffd69"
 dependencies = [
  "crc",
  "sha2",
@@ -3255,24 +3265,9 @@ dependencies = [
 [[package]]
 name = "ndarray"
-version = "0.16.1"
+version = "0.17.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "882ed72dce9365842bf196bdeedf5055305f11fc8c03dee7bb0194a6cad34841"
-dependencies = [
- "matrixmultiply",
- "num-complex",
- "num-integer",
- "num-traits",
- "portable-atomic",
- "portable-atomic-util",
- "rawpointer",
-]
-[[package]]
-name = "ndarray"
-version = "0.17.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c7c9125e8f6f10c9da3aad044cc918cf8784fa34de857b1aa68038eb05a50a9"
+checksum = "520080814a7a6b4a6e9070823bb24b4531daac8c4627e08ba5de8c5ef2f2752d"
 dependencies = [
  "matrixmultiply",
  "num-complex",
@@ -3476,7 +3471,7 @@ dependencies = [
  "serde",
  "serde_json",
  "serde_urlencoded",
- "thiserror 2.0.17",
+ "thiserror 2.0.18",
  "tokio",
  "tracing",
  "url",
@@ -3579,7 +3574,7 @@ dependencies = [
  "futures-sink",
  "js-sys",
  "pin-project-lite",
- "thiserror 2.0.17",
+ "thiserror 2.0.18",
  "tracing",
 ]
@@ -3595,7 +3590,7 @@ dependencies = [
  "opentelemetry",
  "percent-encoding",
  "rand 0.9.2",
- "thiserror 2.0.17",
+ "thiserror 2.0.18",
  "tokio",
  "tokio-stream",
 ]
@@ -3623,25 +3618,22 @@ checksum = "cfdf547b633735ad9d67353aba48b3e685ab5ffb3195aaa9a1b1d8613e11b98c"
 [[package]]
 name = "ort"
-version = "2.0.0-rc.10"
+version = "2.0.0-rc.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1fa7e49bd669d32d7bc2a15ec540a527e7764aec722a45467814005725bcd721"
+checksum = "4a5df903c0d2c07b56950f1058104ab0c8557159f2741782223704de9be73c3c"
 dependencies = [
- "libloading 0.8.9",
- "ndarray 0.16.1",
+ "libloading 0.9.0",
+ "ndarray",
  "ort-sys",
- "smallvec 2.0.0-alpha.10",
+ "smallvec",
  "tracing",
 ]
 [[package]]
 name = "ort-sys"
-version = "2.0.0-rc.10"
+version = "2.0.0-rc.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e2aba9f5c7c479925205799216e7e5d07cc1d4fa76ea8058c60a9a30f6a4e890"
-dependencies = [
- "pkg-config",
-]
+checksum = "06503bb33f294c5f1ba484011e053bfa6ae227074bdb841e9863492dc5960d4b"
 [[package]]
 name = "outref"
@@ -3674,7 +3666,7 @@ dependencies = [
  "cfg-if",
  "libc",
  "redox_syscall 0.5.18",
- "smallvec 1.15.1",
+ "smallvec",
  "windows-link",
 ]
@@ -4536,6 +4528,16 @@ dependencies = [
  "serde",
 ]
+[[package]]
+name = "quick-xml"
+version = "0.39.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2e3bf4aa9d243beeb01a7b3bc30b77cfe2c44e24ec02d751a7104a53c2c49a1"
+dependencies = [
+ "memchr",
+ "serde",
+]
 [[package]]
 name = "quinn"
 version = "0.11.9"
@@ -4550,7 +4552,7 @@ dependencies = [
  "rustc-hash 2.1.1",
  "rustls",
  "socket2",
- "thiserror 2.0.17",
+ "thiserror 2.0.18",
  "tokio",
  "tracing",
  "web-time",
@@ -4572,7 +4574,7 @@ dependencies = [
  "rustls",
  "rustls-pki-types",
  "slab",
- "thiserror 2.0.17",
+ "thiserror 2.0.18",
  "tinyvec",
  "tracing",
  "web-time",
@@ -4714,7 +4716,7 @@ dependencies = [
  "rand 0.9.2",
  "rand_chacha",
  "simd_helpers",
- "thiserror 2.0.17",
+ "thiserror 2.0.18",
  "v_frame",
  "wasm-bindgen",
 ]
@@ -4856,7 +4858,7 @@ checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac"
 dependencies = [
  "getrandom 0.2.16",
  "libredox",
- "thiserror 2.0.17",
+ "thiserror 2.0.18",
 ]
 [[package]]
@@ -5020,9 +5022,9 @@ dependencies = [
 [[package]]
 name = "rmcp"
-version = "0.12.0"
+version = "0.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "528d42f8176e6e5e71ea69182b17d1d0a19a6b3b894b564678b74cd7cab13cfa"
+checksum = "d1815dbc06c414d720f8bc1951eccd66bc99efc6376331f1e7093a119b3eb508"
 dependencies = [
  "async-trait",
  "axum",
@@ -5041,7 +5043,7 @@ dependencies = [
  "serde",
  "serde_json",
  "sse-stream",
- "thiserror 2.0.17",
+ "thiserror 2.0.18",
  "tokio",
  "tokio-stream",
  "tokio-util",
@@ -5052,9 +5054,9 @@ dependencies = [
 [[package]]
 name = "rmcp-macros"
-version = "0.12.0"
+version = "0.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e3f81daaa494eb8e985c9462f7d6ce1ab05e5299f48aafd76cdd3d8b060e6f59"
+checksum = "11f0bc7008fa102e771a76c6d2c9b253be3f2baa5964e060464d038ae1cbc573"
 dependencies = [
  "darling 0.23.0",
  "proc-macro2",
@@ -5236,10 +5238,11 @@ checksum = "a50f4cf475b65d88e057964e0e9bb1f0aa9bbb2036dc65c64596b42932536984"
 [[package]]
 name = "safetensors"
-version = "0.4.5"
+version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "44560c11236a6130a46ce36c836a62936dc81ebf8c36a37947423571be0e55b6"
+checksum = "675656c1eabb620b921efea4f9199f97fc86e36dd6ffd1fbbe48d0f59a4987f5"
 dependencies = [
+ "hashbrown 0.16.1",
  "serde",
  "serde_json",
 ]
@@ -5607,12 +5610,6 @@ version = "1.15.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03"
-[[package]]
-name = "smallvec"
-version = "2.0.0-alpha.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "51d44cfb396c3caf6fbfd0ab422af02631b69ddd96d2eff0b0f0724f9024051b"
 [[package]]
 name = "smartcore"
 version = "0.4.8"
@@ -5922,7 +5919,7 @@ dependencies = [
  "memchr",
  "pulldown-cmark",
  "strum",
- "thiserror 2.0.17",
+ "thiserror 2.0.18",
 ]
 [[package]]
@@ -5942,11 +5939,11 @@ dependencies = [
 [[package]]
 name = "thiserror"
-version = "2.0.17"
+version = "2.0.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8"
+checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4"
 dependencies = [
- "thiserror-impl 2.0.17",
+ "thiserror-impl 2.0.18",
 ]
 [[package]]
@@ -5962,9 +5959,9 @@ dependencies = [
 [[package]]
 name = "thiserror-impl"
-version = "2.0.17"
+version = "2.0.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913"
+checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -6092,7 +6089,7 @@ dependencies = [
  "serde",
  "serde_json",
  "spm_precompiled",
- "thiserror 2.0.17",
+ "thiserror 2.0.18",
  "unicode-normalization-alignments",
  "unicode-segmentation",
  "unicode_categories",
@@ -6183,9 +6180,9 @@ dependencies = [
 [[package]]
 name = "toml"
-version = "0.9.10+spec-1.1.0"
+version = "0.9.11+spec-1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0825052159284a1a8b4d6c0c86cbc801f2da5afd2b225fa548c72f2e74002f48"
+checksum = "f3afc9a848309fe1aaffaed6e1546a7a14de1f935dc9d89d32afd9a44bab7c46"
 dependencies = [
  "indexmap",
  "serde_core",
@@ -6351,8 +6348,8 @@ dependencies = [
  "opentelemetry",
  "opentelemetry_sdk",
  "rustversion",
- "smallvec 1.15.1",
- "thiserror 2.0.17",
+ "smallvec",
+ "thiserror 2.0.18",
  "tracing",
  "tracing-core",
  "tracing-log",
@@ -6534,7 +6531,7 @@ version = "0.1.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de"
 dependencies = [
- "smallvec 1.15.1",
+ "smallvec",
 ]
 [[package]]
@@ -7501,12 +7498,11 @@ dependencies = [
 [[package]]
 name = "zip"
-version = "7.0.0"
+version = "7.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bdd8a47718a4ee5fe78e07667cd36f3de80e7c2bfe727c7074245ffc7303c037"
+checksum = "9013f1222db8a6d680f13a7ccdc60a781199cd09c2fa4eff58e728bb181757fc"
 dependencies = [
  "aes",
- "arbitrary",
  "bzip2",
  "constant_time_eq",
  "crc32fast",

data/ext/kreuzberg_rb/native/Cargo.toml CHANGED Viewed

@@ -31,7 +31,7 @@ embeddings = ["kreuzberg/embeddings"]
 [dependencies]
 async-trait = "0.1.89"
-kreuzberg = { path = "../../../vendor/kreuzberg", default-features = false, features = [
+kreuzberg = { path = "../../../../../crates/kreuzberg", default-features = false, features = [
     "pdf",
     "excel",
     "office",
@@ -51,7 +51,7 @@ kreuzberg = { path = "../../../vendor/kreuzberg", default-features = false, feat
     "bundled-pdfium",
     "tokio-runtime",
 ] }
-kreuzberg-ffi = { path = "../../../vendor/kreuzberg-ffi" }
+kreuzberg-ffi = { path = "../../../../../crates/kreuzberg-ffi" }
 magnus = { git = "https://github.com/matsadler/magnus", rev = "f6db11769efb517427bf7f121f9c32e18b059b38", features = [
     "rb-sys",
 ] }
@@ -59,6 +59,8 @@ rb-sys = { version = "0.9.119", default-features = false, features = [
     "stable-api-compiled-fallback",
 ] }
 serde_json = "1.0.145"
+toml = "0.8"
+serde_yaml_ng = "0.10"
 tokio = { version = "1.48.0", features = [
     "rt",
     "rt-multi-thread",