kreuzberg 4.0.0.pre.rc.29 → 4.0.0.rc1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +0 -6
- data/.rubocop.yaml +534 -1
- data/Gemfile +2 -1
- data/Gemfile.lock +28 -116
- data/README.md +269 -629
- data/Rakefile +0 -9
- data/Steepfile +4 -8
- data/examples/async_patterns.rb +58 -1
- data/ext/kreuzberg_rb/extconf.rb +5 -35
- data/ext/kreuzberg_rb/native/Cargo.toml +16 -55
- data/ext/kreuzberg_rb/native/build.rs +14 -12
- data/ext/kreuzberg_rb/native/include/ieeefp.h +1 -1
- data/ext/kreuzberg_rb/native/include/msvc_compat/strings.h +1 -1
- data/ext/kreuzberg_rb/native/include/strings.h +2 -2
- data/ext/kreuzberg_rb/native/include/unistd.h +1 -1
- data/ext/kreuzberg_rb/native/src/lib.rs +34 -897
- data/extconf.rb +6 -38
- data/kreuzberg.gemspec +20 -114
- data/lib/kreuzberg/api_proxy.rb +18 -2
- data/lib/kreuzberg/cache_api.rb +0 -22
- data/lib/kreuzberg/cli.rb +10 -2
- data/lib/kreuzberg/cli_proxy.rb +10 -0
- data/lib/kreuzberg/config.rb +22 -274
- data/lib/kreuzberg/errors.rb +7 -73
- data/lib/kreuzberg/extraction_api.rb +8 -237
- data/lib/kreuzberg/mcp_proxy.rb +11 -2
- data/lib/kreuzberg/ocr_backend_protocol.rb +73 -0
- data/lib/kreuzberg/post_processor_protocol.rb +71 -0
- data/lib/kreuzberg/result.rb +33 -151
- data/lib/kreuzberg/setup_lib_path.rb +2 -22
- data/lib/kreuzberg/validator_protocol.rb +73 -0
- data/lib/kreuzberg/version.rb +1 -1
- data/lib/kreuzberg.rb +13 -27
- data/pkg/kreuzberg-4.0.0.rc1.gem +0 -0
- data/sig/kreuzberg.rbs +12 -105
- data/spec/binding/cache_spec.rb +22 -22
- data/spec/binding/cli_proxy_spec.rb +4 -2
- data/spec/binding/cli_spec.rb +11 -12
- data/spec/binding/config_spec.rb +0 -74
- data/spec/binding/config_validation_spec.rb +6 -100
- data/spec/binding/error_handling_spec.rb +97 -283
- data/spec/binding/plugins/ocr_backend_spec.rb +8 -8
- data/spec/binding/plugins/postprocessor_spec.rb +11 -11
- data/spec/binding/plugins/validator_spec.rb +13 -12
- data/spec/examples.txt +104 -0
- data/spec/fixtures/config.toml +1 -0
- data/spec/fixtures/config.yaml +1 -0
- data/spec/fixtures/invalid_config.toml +1 -0
- data/spec/smoke/package_spec.rb +3 -2
- data/spec/spec_helper.rb +3 -1
- data/vendor/kreuzberg/Cargo.toml +67 -192
- data/vendor/kreuzberg/README.md +9 -97
- data/vendor/kreuzberg/build.rs +194 -516
- data/vendor/kreuzberg/src/api/handlers.rs +9 -130
- data/vendor/kreuzberg/src/api/mod.rs +3 -18
- data/vendor/kreuzberg/src/api/server.rs +71 -236
- data/vendor/kreuzberg/src/api/types.rs +7 -43
- data/vendor/kreuzberg/src/bin/profile_extract.rs +455 -0
- data/vendor/kreuzberg/src/cache/mod.rs +3 -27
- data/vendor/kreuzberg/src/chunking/mod.rs +79 -1705
- data/vendor/kreuzberg/src/core/batch_mode.rs +0 -60
- data/vendor/kreuzberg/src/core/config.rs +23 -905
- data/vendor/kreuzberg/src/core/extractor.rs +106 -403
- data/vendor/kreuzberg/src/core/io.rs +2 -4
- data/vendor/kreuzberg/src/core/mime.rs +12 -2
- data/vendor/kreuzberg/src/core/mod.rs +3 -22
- data/vendor/kreuzberg/src/core/pipeline.rs +78 -395
- data/vendor/kreuzberg/src/embeddings.rs +21 -169
- data/vendor/kreuzberg/src/error.rs +2 -2
- data/vendor/kreuzberg/src/extraction/archive.rs +31 -36
- data/vendor/kreuzberg/src/extraction/docx.rs +1 -365
- data/vendor/kreuzberg/src/extraction/email.rs +11 -12
- data/vendor/kreuzberg/src/extraction/excel.rs +129 -138
- data/vendor/kreuzberg/src/extraction/html.rs +170 -1447
- data/vendor/kreuzberg/src/extraction/image.rs +14 -138
- data/vendor/kreuzberg/src/extraction/libreoffice.rs +3 -13
- data/vendor/kreuzberg/src/extraction/mod.rs +5 -21
- data/vendor/kreuzberg/src/extraction/office_metadata/mod.rs +0 -2
- data/vendor/kreuzberg/src/extraction/pandoc/batch.rs +275 -0
- data/vendor/kreuzberg/src/extraction/pandoc/mime_types.rs +178 -0
- data/vendor/kreuzberg/src/extraction/pandoc/mod.rs +491 -0
- data/vendor/kreuzberg/src/extraction/pandoc/server.rs +496 -0
- data/vendor/kreuzberg/src/extraction/pandoc/subprocess.rs +1188 -0
- data/vendor/kreuzberg/src/extraction/pandoc/version.rs +162 -0
- data/vendor/kreuzberg/src/extraction/pptx.rs +94 -196
- data/vendor/kreuzberg/src/extraction/structured.rs +4 -5
- data/vendor/kreuzberg/src/extraction/table.rs +1 -2
- data/vendor/kreuzberg/src/extraction/text.rs +10 -18
- data/vendor/kreuzberg/src/extractors/archive.rs +0 -22
- data/vendor/kreuzberg/src/extractors/docx.rs +148 -69
- data/vendor/kreuzberg/src/extractors/email.rs +9 -37
- data/vendor/kreuzberg/src/extractors/excel.rs +40 -81
- data/vendor/kreuzberg/src/extractors/html.rs +173 -182
- data/vendor/kreuzberg/src/extractors/image.rs +8 -32
- data/vendor/kreuzberg/src/extractors/mod.rs +10 -171
- data/vendor/kreuzberg/src/extractors/pandoc.rs +201 -0
- data/vendor/kreuzberg/src/extractors/pdf.rs +64 -329
- data/vendor/kreuzberg/src/extractors/pptx.rs +34 -79
- data/vendor/kreuzberg/src/extractors/structured.rs +0 -16
- data/vendor/kreuzberg/src/extractors/text.rs +7 -30
- data/vendor/kreuzberg/src/extractors/xml.rs +8 -27
- data/vendor/kreuzberg/src/keywords/processor.rs +1 -9
- data/vendor/kreuzberg/src/keywords/rake.rs +1 -0
- data/vendor/kreuzberg/src/language_detection/mod.rs +51 -94
- data/vendor/kreuzberg/src/lib.rs +5 -17
- data/vendor/kreuzberg/src/mcp/mod.rs +1 -4
- data/vendor/kreuzberg/src/mcp/server.rs +21 -145
- data/vendor/kreuzberg/src/ocr/mod.rs +0 -2
- data/vendor/kreuzberg/src/ocr/processor.rs +8 -19
- data/vendor/kreuzberg/src/ocr/tesseract_backend.rs +0 -2
- data/vendor/kreuzberg/src/pdf/error.rs +1 -93
- data/vendor/kreuzberg/src/pdf/metadata.rs +100 -263
- data/vendor/kreuzberg/src/pdf/mod.rs +2 -33
- data/vendor/kreuzberg/src/pdf/rendering.rs +12 -12
- data/vendor/kreuzberg/src/pdf/table.rs +64 -61
- data/vendor/kreuzberg/src/pdf/text.rs +24 -416
- data/vendor/kreuzberg/src/plugins/extractor.rs +8 -40
- data/vendor/kreuzberg/src/plugins/mod.rs +0 -3
- data/vendor/kreuzberg/src/plugins/ocr.rs +14 -22
- data/vendor/kreuzberg/src/plugins/processor.rs +1 -10
- data/vendor/kreuzberg/src/plugins/registry.rs +0 -15
- data/vendor/kreuzberg/src/plugins/validator.rs +8 -20
- data/vendor/kreuzberg/src/stopwords/mod.rs +2 -2
- data/vendor/kreuzberg/src/text/mod.rs +0 -8
- data/vendor/kreuzberg/src/text/quality.rs +15 -28
- data/vendor/kreuzberg/src/text/string_utils.rs +10 -22
- data/vendor/kreuzberg/src/text/token_reduction/core.rs +50 -86
- data/vendor/kreuzberg/src/text/token_reduction/filters.rs +16 -37
- data/vendor/kreuzberg/src/text/token_reduction/simd_text.rs +1 -2
- data/vendor/kreuzberg/src/types.rs +67 -907
- data/vendor/kreuzberg/src/utils/mod.rs +0 -14
- data/vendor/kreuzberg/src/utils/quality.rs +3 -12
- data/vendor/kreuzberg/tests/api_tests.rs +0 -506
- data/vendor/kreuzberg/tests/archive_integration.rs +0 -2
- data/vendor/kreuzberg/tests/batch_orchestration.rs +12 -57
- data/vendor/kreuzberg/tests/batch_processing.rs +8 -32
- data/vendor/kreuzberg/tests/chunking_offset_demo.rs +92 -0
- data/vendor/kreuzberg/tests/concurrency_stress.rs +8 -40
- data/vendor/kreuzberg/tests/config_features.rs +1 -33
- data/vendor/kreuzberg/tests/config_loading_tests.rs +39 -16
- data/vendor/kreuzberg/tests/core_integration.rs +9 -35
- data/vendor/kreuzberg/tests/csv_integration.rs +81 -71
- data/vendor/kreuzberg/tests/docx_metadata_extraction_test.rs +25 -23
- data/vendor/kreuzberg/tests/email_integration.rs +1 -3
- data/vendor/kreuzberg/tests/error_handling.rs +34 -43
- data/vendor/kreuzberg/tests/format_integration.rs +1 -7
- data/vendor/kreuzberg/tests/helpers/mod.rs +0 -60
- data/vendor/kreuzberg/tests/image_integration.rs +0 -2
- data/vendor/kreuzberg/tests/mime_detection.rs +16 -17
- data/vendor/kreuzberg/tests/ocr_configuration.rs +0 -4
- data/vendor/kreuzberg/tests/ocr_errors.rs +0 -22
- data/vendor/kreuzberg/tests/ocr_quality.rs +0 -2
- data/vendor/kreuzberg/tests/pandoc_integration.rs +503 -0
- data/vendor/kreuzberg/tests/pdf_integration.rs +0 -2
- data/vendor/kreuzberg/tests/pipeline_integration.rs +2 -36
- data/vendor/kreuzberg/tests/plugin_ocr_backend_test.rs +0 -5
- data/vendor/kreuzberg/tests/plugin_postprocessor_test.rs +1 -17
- data/vendor/kreuzberg/tests/plugin_system.rs +0 -6
- data/vendor/kreuzberg/tests/registry_integration_tests.rs +22 -2
- data/vendor/kreuzberg/tests/security_validation.rs +1 -13
- data/vendor/kreuzberg/tests/test_fastembed.rs +23 -45
- metadata +25 -171
- data/.rubocop.yml +0 -543
- data/ext/kreuzberg_rb/native/.cargo/config.toml +0 -23
- data/ext/kreuzberg_rb/native/Cargo.lock +0 -7619
- data/lib/kreuzberg/error_context.rb +0 -136
- data/lib/kreuzberg/types.rb +0 -170
- data/lib/libpdfium.so +0 -0
- data/spec/binding/async_operations_spec.rb +0 -473
- data/spec/binding/batch_operations_spec.rb +0 -595
- data/spec/binding/batch_spec.rb +0 -359
- data/spec/binding/config_result_spec.rb +0 -377
- data/spec/binding/embeddings_spec.rb +0 -816
- data/spec/binding/error_recovery_spec.rb +0 -488
- data/spec/binding/font_config_spec.rb +0 -220
- data/spec/binding/images_spec.rb +0 -738
- data/spec/binding/keywords_extraction_spec.rb +0 -600
- data/spec/binding/metadata_types_spec.rb +0 -1228
- data/spec/binding/pages_extraction_spec.rb +0 -471
- data/spec/binding/tables_spec.rb +0 -641
- data/spec/unit/config/chunking_config_spec.rb +0 -213
- data/spec/unit/config/embedding_config_spec.rb +0 -343
- data/spec/unit/config/extraction_config_spec.rb +0 -438
- data/spec/unit/config/font_config_spec.rb +0 -285
- data/spec/unit/config/hierarchy_config_spec.rb +0 -314
- data/spec/unit/config/image_extraction_config_spec.rb +0 -209
- data/spec/unit/config/image_preprocessing_config_spec.rb +0 -249
- data/spec/unit/config/keyword_config_spec.rb +0 -229
- data/spec/unit/config/language_detection_config_spec.rb +0 -258
- data/spec/unit/config/ocr_config_spec.rb +0 -171
- data/spec/unit/config/page_config_spec.rb +0 -221
- data/spec/unit/config/pdf_config_spec.rb +0 -267
- data/spec/unit/config/postprocessor_config_spec.rb +0 -290
- data/spec/unit/config/tesseract_config_spec.rb +0 -181
- data/spec/unit/config/token_reduction_config_spec.rb +0 -251
- data/test/metadata_types_test.rb +0 -959
- data/vendor/Cargo.toml +0 -61
- data/vendor/kreuzberg/examples/bench_fixes.rs +0 -71
- data/vendor/kreuzberg/examples/test_pdfium_fork.rs +0 -62
- data/vendor/kreuzberg/src/chunking/processor.rs +0 -219
- data/vendor/kreuzberg/src/core/batch_optimizations.rs +0 -385
- data/vendor/kreuzberg/src/core/config_validation.rs +0 -949
- data/vendor/kreuzberg/src/core/formats.rs +0 -235
- data/vendor/kreuzberg/src/core/server_config.rs +0 -1220
- data/vendor/kreuzberg/src/extraction/capacity.rs +0 -263
- data/vendor/kreuzberg/src/extraction/markdown.rs +0 -216
- data/vendor/kreuzberg/src/extraction/office_metadata/odt_properties.rs +0 -284
- data/vendor/kreuzberg/src/extractors/bibtex.rs +0 -470
- data/vendor/kreuzberg/src/extractors/docbook.rs +0 -504
- data/vendor/kreuzberg/src/extractors/epub.rs +0 -696
- data/vendor/kreuzberg/src/extractors/fictionbook.rs +0 -492
- data/vendor/kreuzberg/src/extractors/jats.rs +0 -1054
- data/vendor/kreuzberg/src/extractors/jupyter.rs +0 -368
- data/vendor/kreuzberg/src/extractors/latex.rs +0 -653
- data/vendor/kreuzberg/src/extractors/markdown.rs +0 -701
- data/vendor/kreuzberg/src/extractors/odt.rs +0 -628
- data/vendor/kreuzberg/src/extractors/opml.rs +0 -635
- data/vendor/kreuzberg/src/extractors/orgmode.rs +0 -529
- data/vendor/kreuzberg/src/extractors/rst.rs +0 -577
- data/vendor/kreuzberg/src/extractors/rtf.rs +0 -809
- data/vendor/kreuzberg/src/extractors/security.rs +0 -484
- data/vendor/kreuzberg/src/extractors/security_tests.rs +0 -367
- data/vendor/kreuzberg/src/extractors/typst.rs +0 -651
- data/vendor/kreuzberg/src/language_detection/processor.rs +0 -218
- data/vendor/kreuzberg/src/ocr/language_registry.rs +0 -520
- data/vendor/kreuzberg/src/panic_context.rs +0 -154
- data/vendor/kreuzberg/src/pdf/bindings.rs +0 -306
- data/vendor/kreuzberg/src/pdf/bundled.rs +0 -408
- data/vendor/kreuzberg/src/pdf/fonts.rs +0 -358
- data/vendor/kreuzberg/src/pdf/hierarchy.rs +0 -903
- data/vendor/kreuzberg/src/text/quality_processor.rs +0 -231
- data/vendor/kreuzberg/src/text/utf8_validation.rs +0 -193
- data/vendor/kreuzberg/src/utils/pool.rs +0 -503
- data/vendor/kreuzberg/src/utils/pool_sizing.rs +0 -364
- data/vendor/kreuzberg/src/utils/string_pool.rs +0 -761
- data/vendor/kreuzberg/tests/api_embed.rs +0 -360
- data/vendor/kreuzberg/tests/api_extract_multipart.rs +0 -52
- data/vendor/kreuzberg/tests/api_large_pdf_extraction.rs +0 -471
- data/vendor/kreuzberg/tests/api_large_pdf_extraction_diagnostics.rs +0 -289
- data/vendor/kreuzberg/tests/batch_pooling_benchmark.rs +0 -154
- data/vendor/kreuzberg/tests/bibtex_parity_test.rs +0 -421
- data/vendor/kreuzberg/tests/config_integration_test.rs +0 -753
- data/vendor/kreuzberg/tests/data/hierarchy_ground_truth.json +0 -294
- data/vendor/kreuzberg/tests/docbook_extractor_tests.rs +0 -500
- data/vendor/kreuzberg/tests/docx_vs_pandoc_comparison.rs +0 -370
- data/vendor/kreuzberg/tests/epub_native_extractor_tests.rs +0 -275
- data/vendor/kreuzberg/tests/fictionbook_extractor_tests.rs +0 -228
- data/vendor/kreuzberg/tests/html_table_test.rs +0 -551
- data/vendor/kreuzberg/tests/instrumentation_test.rs +0 -139
- data/vendor/kreuzberg/tests/jats_extractor_tests.rs +0 -639
- data/vendor/kreuzberg/tests/jupyter_extractor_tests.rs +0 -704
- data/vendor/kreuzberg/tests/latex_extractor_tests.rs +0 -496
- data/vendor/kreuzberg/tests/markdown_extractor_tests.rs +0 -490
- data/vendor/kreuzberg/tests/ocr_language_registry.rs +0 -191
- data/vendor/kreuzberg/tests/odt_extractor_tests.rs +0 -674
- data/vendor/kreuzberg/tests/opml_extractor_tests.rs +0 -616
- data/vendor/kreuzberg/tests/orgmode_extractor_tests.rs +0 -822
- data/vendor/kreuzberg/tests/page_markers.rs +0 -297
- data/vendor/kreuzberg/tests/pdf_hierarchy_detection.rs +0 -301
- data/vendor/kreuzberg/tests/pdf_hierarchy_quality.rs +0 -589
- data/vendor/kreuzberg/tests/pdf_ocr_triggering.rs +0 -301
- data/vendor/kreuzberg/tests/pdf_text_merging.rs +0 -475
- data/vendor/kreuzberg/tests/pdfium_linking.rs +0 -340
- data/vendor/kreuzberg/tests/rst_extractor_tests.rs +0 -694
- data/vendor/kreuzberg/tests/rtf_extractor_tests.rs +0 -775
- data/vendor/kreuzberg/tests/typst_behavioral_tests.rs +0 -1260
- data/vendor/kreuzberg/tests/typst_extractor_tests.rs +0 -648
- data/vendor/kreuzberg-ffi/Cargo.toml +0 -67
- data/vendor/kreuzberg-ffi/README.md +0 -851
- data/vendor/kreuzberg-ffi/benches/result_view_benchmark.rs +0 -227
- data/vendor/kreuzberg-ffi/build.rs +0 -168
- data/vendor/kreuzberg-ffi/cbindgen.toml +0 -37
- data/vendor/kreuzberg-ffi/kreuzberg-ffi.pc.in +0 -12
- data/vendor/kreuzberg-ffi/kreuzberg.h +0 -3012
- data/vendor/kreuzberg-ffi/src/batch_streaming.rs +0 -588
- data/vendor/kreuzberg-ffi/src/config.rs +0 -1341
- data/vendor/kreuzberg-ffi/src/error.rs +0 -901
- data/vendor/kreuzberg-ffi/src/extraction.rs +0 -555
- data/vendor/kreuzberg-ffi/src/helpers.rs +0 -879
- data/vendor/kreuzberg-ffi/src/lib.rs +0 -977
- data/vendor/kreuzberg-ffi/src/memory.rs +0 -493
- data/vendor/kreuzberg-ffi/src/mime.rs +0 -329
- data/vendor/kreuzberg-ffi/src/panic_shield.rs +0 -265
- data/vendor/kreuzberg-ffi/src/plugins/document_extractor.rs +0 -442
- data/vendor/kreuzberg-ffi/src/plugins/mod.rs +0 -14
- data/vendor/kreuzberg-ffi/src/plugins/ocr_backend.rs +0 -628
- data/vendor/kreuzberg-ffi/src/plugins/post_processor.rs +0 -438
- data/vendor/kreuzberg-ffi/src/plugins/validator.rs +0 -329
- data/vendor/kreuzberg-ffi/src/result.rs +0 -510
- data/vendor/kreuzberg-ffi/src/result_pool.rs +0 -639
- data/vendor/kreuzberg-ffi/src/result_view.rs +0 -773
- data/vendor/kreuzberg-ffi/src/string_intern.rs +0 -568
- data/vendor/kreuzberg-ffi/src/types.rs +0 -363
- data/vendor/kreuzberg-ffi/src/util.rs +0 -210
- data/vendor/kreuzberg-ffi/src/validation.rs +0 -848
- data/vendor/kreuzberg-ffi/tests.disabled/README.md +0 -48
- data/vendor/kreuzberg-ffi/tests.disabled/config_loading_tests.rs +0 -299
- data/vendor/kreuzberg-ffi/tests.disabled/config_tests.rs +0 -346
- data/vendor/kreuzberg-ffi/tests.disabled/extractor_tests.rs +0 -232
- data/vendor/kreuzberg-ffi/tests.disabled/plugin_registration_tests.rs +0 -470
- data/vendor/kreuzberg-tesseract/.commitlintrc.json +0 -13
- data/vendor/kreuzberg-tesseract/.crate-ignore +0 -2
- data/vendor/kreuzberg-tesseract/Cargo.lock +0 -2933
- data/vendor/kreuzberg-tesseract/Cargo.toml +0 -57
- data/vendor/kreuzberg-tesseract/LICENSE +0 -22
- data/vendor/kreuzberg-tesseract/README.md +0 -399
- data/vendor/kreuzberg-tesseract/build.rs +0 -1127
- data/vendor/kreuzberg-tesseract/patches/README.md +0 -71
- data/vendor/kreuzberg-tesseract/patches/tesseract.diff +0 -199
- data/vendor/kreuzberg-tesseract/src/api.rs +0 -1371
- data/vendor/kreuzberg-tesseract/src/choice_iterator.rs +0 -77
- data/vendor/kreuzberg-tesseract/src/enums.rs +0 -297
- data/vendor/kreuzberg-tesseract/src/error.rs +0 -81
- data/vendor/kreuzberg-tesseract/src/lib.rs +0 -145
- data/vendor/kreuzberg-tesseract/src/monitor.rs +0 -57
- data/vendor/kreuzberg-tesseract/src/mutable_iterator.rs +0 -197
- data/vendor/kreuzberg-tesseract/src/page_iterator.rs +0 -253
- data/vendor/kreuzberg-tesseract/src/result_iterator.rs +0 -286
- data/vendor/kreuzberg-tesseract/src/result_renderer.rs +0 -183
- data/vendor/kreuzberg-tesseract/tests/integration_test.rs +0 -211
|
@@ -144,39 +144,33 @@ async fn test_generate_embeddings_for_chunks_basic() {
|
|
|
144
144
|
content: "Hello world, this is the first chunk.".to_string(),
|
|
145
145
|
embedding: None,
|
|
146
146
|
metadata: ChunkMetadata {
|
|
147
|
-
|
|
148
|
-
|
|
147
|
+
char_start: 0,
|
|
148
|
+
char_end: 38,
|
|
149
149
|
chunk_index: 0,
|
|
150
150
|
total_chunks: 1,
|
|
151
151
|
token_count: None,
|
|
152
|
-
first_page: None,
|
|
153
|
-
last_page: None,
|
|
154
152
|
},
|
|
155
153
|
},
|
|
156
154
|
Chunk {
|
|
157
155
|
content: "This is the second chunk with different content.".to_string(),
|
|
158
156
|
embedding: None,
|
|
159
157
|
metadata: ChunkMetadata {
|
|
160
|
-
|
|
161
|
-
|
|
158
|
+
char_start: 39,
|
|
159
|
+
char_end: 87,
|
|
162
160
|
chunk_index: 1,
|
|
163
161
|
total_chunks: 1,
|
|
164
162
|
token_count: None,
|
|
165
|
-
first_page: None,
|
|
166
|
-
last_page: None,
|
|
167
163
|
},
|
|
168
164
|
},
|
|
169
165
|
Chunk {
|
|
170
166
|
content: "And this is the third and final chunk.".to_string(),
|
|
171
167
|
embedding: None,
|
|
172
168
|
metadata: ChunkMetadata {
|
|
173
|
-
|
|
174
|
-
|
|
169
|
+
char_start: 88,
|
|
170
|
+
char_end: 126,
|
|
175
171
|
chunk_index: 2,
|
|
176
172
|
total_chunks: 1,
|
|
177
173
|
token_count: None,
|
|
178
|
-
first_page: None,
|
|
179
|
-
last_page: None,
|
|
180
174
|
},
|
|
181
175
|
},
|
|
182
176
|
];
|
|
@@ -220,13 +214,11 @@ async fn test_generate_embeddings_for_chunks_normalization() {
|
|
|
220
214
|
content: test_text.to_string(),
|
|
221
215
|
embedding: None,
|
|
222
216
|
metadata: ChunkMetadata {
|
|
223
|
-
|
|
224
|
-
|
|
217
|
+
char_start: 0,
|
|
218
|
+
char_end: test_text.len(),
|
|
225
219
|
chunk_index: 0,
|
|
226
220
|
total_chunks: 1,
|
|
227
221
|
token_count: None,
|
|
228
|
-
first_page: None,
|
|
229
|
-
last_page: None,
|
|
230
222
|
},
|
|
231
223
|
}];
|
|
232
224
|
|
|
@@ -247,13 +239,11 @@ async fn test_generate_embeddings_for_chunks_normalization() {
|
|
|
247
239
|
content: test_text.to_string(),
|
|
248
240
|
embedding: None,
|
|
249
241
|
metadata: ChunkMetadata {
|
|
250
|
-
|
|
251
|
-
|
|
242
|
+
char_start: 0,
|
|
243
|
+
char_end: test_text.len(),
|
|
252
244
|
chunk_index: 0,
|
|
253
245
|
total_chunks: 1,
|
|
254
246
|
token_count: None,
|
|
255
|
-
first_page: None,
|
|
256
|
-
last_page: None,
|
|
257
247
|
},
|
|
258
248
|
}];
|
|
259
249
|
|
|
@@ -329,13 +319,11 @@ async fn test_generate_embeddings_for_chunks_model_caching() {
|
|
|
329
319
|
content: "First batch of text.".to_string(),
|
|
330
320
|
embedding: None,
|
|
331
321
|
metadata: ChunkMetadata {
|
|
332
|
-
|
|
333
|
-
|
|
322
|
+
char_start: 0,
|
|
323
|
+
char_end: 20,
|
|
334
324
|
chunk_index: 0,
|
|
335
325
|
total_chunks: 1,
|
|
336
326
|
token_count: None,
|
|
337
|
-
first_page: None,
|
|
338
|
-
last_page: None,
|
|
339
327
|
},
|
|
340
328
|
}];
|
|
341
329
|
|
|
@@ -357,13 +345,11 @@ async fn test_generate_embeddings_for_chunks_model_caching() {
|
|
|
357
345
|
content: "Second batch of text.".to_string(),
|
|
358
346
|
embedding: None,
|
|
359
347
|
metadata: ChunkMetadata {
|
|
360
|
-
|
|
361
|
-
|
|
348
|
+
char_start: 0,
|
|
349
|
+
char_end: 21,
|
|
362
350
|
chunk_index: 0,
|
|
363
351
|
total_chunks: 1,
|
|
364
352
|
token_count: None,
|
|
365
|
-
first_page: None,
|
|
366
|
-
last_page: None,
|
|
367
353
|
},
|
|
368
354
|
}];
|
|
369
355
|
|
|
@@ -391,13 +377,11 @@ async fn test_generate_embeddings_for_chunks_invalid_preset() {
|
|
|
391
377
|
content: "Test content".to_string(),
|
|
392
378
|
embedding: None,
|
|
393
379
|
metadata: ChunkMetadata {
|
|
394
|
-
|
|
395
|
-
|
|
380
|
+
char_start: 0,
|
|
381
|
+
char_end: 12,
|
|
396
382
|
chunk_index: 0,
|
|
397
383
|
total_chunks: 1,
|
|
398
384
|
token_count: None,
|
|
399
|
-
first_page: None,
|
|
400
|
-
last_page: None,
|
|
401
385
|
},
|
|
402
386
|
}];
|
|
403
387
|
|
|
@@ -436,13 +420,11 @@ async fn test_generate_embeddings_for_chunks_unknown_model() {
|
|
|
436
420
|
content: "Test content".to_string(),
|
|
437
421
|
embedding: None,
|
|
438
422
|
metadata: ChunkMetadata {
|
|
439
|
-
|
|
440
|
-
|
|
423
|
+
char_start: 0,
|
|
424
|
+
char_end: 12,
|
|
441
425
|
chunk_index: 0,
|
|
442
426
|
total_chunks: 1,
|
|
443
427
|
token_count: None,
|
|
444
|
-
first_page: None,
|
|
445
|
-
last_page: None,
|
|
446
428
|
},
|
|
447
429
|
}];
|
|
448
430
|
|
|
@@ -482,13 +464,11 @@ async fn test_generate_embeddings_for_chunks_custom_model_not_supported() {
|
|
|
482
464
|
content: "Test content".to_string(),
|
|
483
465
|
embedding: None,
|
|
484
466
|
metadata: ChunkMetadata {
|
|
485
|
-
|
|
486
|
-
|
|
467
|
+
char_start: 0,
|
|
468
|
+
char_end: 12,
|
|
487
469
|
chunk_index: 0,
|
|
488
470
|
total_chunks: 1,
|
|
489
471
|
token_count: None,
|
|
490
|
-
first_page: None,
|
|
491
|
-
last_page: None,
|
|
492
472
|
},
|
|
493
473
|
}];
|
|
494
474
|
|
|
@@ -529,13 +509,11 @@ async fn test_generate_embeddings_for_chunks_batch_size() {
|
|
|
529
509
|
content: format!("This is test chunk number {}.", i),
|
|
530
510
|
embedding: None,
|
|
531
511
|
metadata: ChunkMetadata {
|
|
532
|
-
|
|
533
|
-
|
|
512
|
+
char_start: i * 30,
|
|
513
|
+
char_end: (i + 1) * 30,
|
|
534
514
|
chunk_index: i,
|
|
535
515
|
total_chunks: 10,
|
|
536
516
|
token_count: None,
|
|
537
|
-
first_page: None,
|
|
538
|
-
last_page: None,
|
|
539
517
|
},
|
|
540
518
|
})
|
|
541
519
|
.collect();
|
|
@@ -588,7 +566,7 @@ async fn test_generate_embeddings_chunking_integration() {
|
|
|
588
566
|
..Default::default()
|
|
589
567
|
};
|
|
590
568
|
|
|
591
|
-
let mut chunking_result = chunk_text(text, &chunking_config
|
|
569
|
+
let mut chunking_result = chunk_text(text, &chunking_config).expect("Chunking failed");
|
|
592
570
|
|
|
593
571
|
assert!(
|
|
594
572
|
chunking_result.chunks.len() > 1,
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: kreuzberg
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 4.0.0.
|
|
4
|
+
version: 4.0.0.rc1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Na'aman Hirschfeld
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2025-11-23 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: bundler
|
|
@@ -16,14 +16,14 @@ dependencies:
|
|
|
16
16
|
requirements:
|
|
17
17
|
- - "~>"
|
|
18
18
|
- !ruby/object:Gem::Version
|
|
19
|
-
version: '
|
|
19
|
+
version: '2.0'
|
|
20
20
|
type: :development
|
|
21
21
|
prerelease: false
|
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
|
23
23
|
requirements:
|
|
24
24
|
- - "~>"
|
|
25
25
|
- !ruby/object:Gem::Version
|
|
26
|
-
version: '
|
|
26
|
+
version: '2.0'
|
|
27
27
|
- !ruby/object:Gem::Dependency
|
|
28
28
|
name: rake
|
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -56,16 +56,16 @@ dependencies:
|
|
|
56
56
|
name: rb_sys
|
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
|
58
58
|
requirements:
|
|
59
|
-
- -
|
|
59
|
+
- - "~>"
|
|
60
60
|
- !ruby/object:Gem::Version
|
|
61
|
-
version: 0.9
|
|
61
|
+
version: '0.9'
|
|
62
62
|
type: :development
|
|
63
63
|
prerelease: false
|
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
|
65
65
|
requirements:
|
|
66
|
-
- -
|
|
66
|
+
- - "~>"
|
|
67
67
|
- !ruby/object:Gem::Version
|
|
68
|
-
version: 0.9
|
|
68
|
+
version: '0.9'
|
|
69
69
|
- !ruby/object:Gem::Dependency
|
|
70
70
|
name: rspec
|
|
71
71
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -178,7 +178,6 @@ files:
|
|
|
178
178
|
- ".gitignore"
|
|
179
179
|
- ".rspec"
|
|
180
180
|
- ".rubocop.yaml"
|
|
181
|
-
- ".rubocop.yml"
|
|
182
181
|
- Gemfile
|
|
183
182
|
- Gemfile.lock
|
|
184
183
|
- README.md
|
|
@@ -186,8 +185,6 @@ files:
|
|
|
186
185
|
- Steepfile
|
|
187
186
|
- examples/async_patterns.rb
|
|
188
187
|
- ext/kreuzberg_rb/extconf.rb
|
|
189
|
-
- ext/kreuzberg_rb/native/.cargo/config.toml
|
|
190
|
-
- ext/kreuzberg_rb/native/Cargo.lock
|
|
191
188
|
- ext/kreuzberg_rb/native/Cargo.toml
|
|
192
189
|
- ext/kreuzberg_rb/native/README.md
|
|
193
190
|
- ext/kreuzberg_rb/native/build.rs
|
|
@@ -204,7 +201,6 @@ files:
|
|
|
204
201
|
- lib/kreuzberg/cli.rb
|
|
205
202
|
- lib/kreuzberg/cli_proxy.rb
|
|
206
203
|
- lib/kreuzberg/config.rb
|
|
207
|
-
- lib/kreuzberg/error_context.rb
|
|
208
204
|
- lib/kreuzberg/errors.rb
|
|
209
205
|
- lib/kreuzberg/extraction_api.rb
|
|
210
206
|
- lib/kreuzberg/mcp_proxy.rb
|
|
@@ -212,182 +208,82 @@ files:
|
|
|
212
208
|
- lib/kreuzberg/post_processor_protocol.rb
|
|
213
209
|
- lib/kreuzberg/result.rb
|
|
214
210
|
- lib/kreuzberg/setup_lib_path.rb
|
|
215
|
-
- lib/kreuzberg/types.rb
|
|
216
211
|
- lib/kreuzberg/validator_protocol.rb
|
|
217
212
|
- lib/kreuzberg/version.rb
|
|
218
|
-
-
|
|
213
|
+
- pkg/kreuzberg-4.0.0.rc1.gem
|
|
219
214
|
- sig/kreuzberg.rbs
|
|
220
215
|
- sig/kreuzberg/internal.rbs
|
|
221
|
-
- spec/binding/async_operations_spec.rb
|
|
222
|
-
- spec/binding/batch_operations_spec.rb
|
|
223
|
-
- spec/binding/batch_spec.rb
|
|
224
216
|
- spec/binding/cache_spec.rb
|
|
225
217
|
- spec/binding/cli_proxy_spec.rb
|
|
226
218
|
- spec/binding/cli_spec.rb
|
|
227
|
-
- spec/binding/config_result_spec.rb
|
|
228
219
|
- spec/binding/config_spec.rb
|
|
229
220
|
- spec/binding/config_validation_spec.rb
|
|
230
|
-
- spec/binding/embeddings_spec.rb
|
|
231
221
|
- spec/binding/error_handling_spec.rb
|
|
232
|
-
- spec/binding/error_recovery_spec.rb
|
|
233
222
|
- spec/binding/errors_spec.rb
|
|
234
|
-
- spec/binding/font_config_spec.rb
|
|
235
|
-
- spec/binding/images_spec.rb
|
|
236
|
-
- spec/binding/keywords_extraction_spec.rb
|
|
237
|
-
- spec/binding/metadata_types_spec.rb
|
|
238
|
-
- spec/binding/pages_extraction_spec.rb
|
|
239
223
|
- spec/binding/plugins/ocr_backend_spec.rb
|
|
240
224
|
- spec/binding/plugins/postprocessor_spec.rb
|
|
241
225
|
- spec/binding/plugins/validator_spec.rb
|
|
242
|
-
- spec/
|
|
226
|
+
- spec/examples.txt
|
|
243
227
|
- spec/fixtures/config.toml
|
|
244
228
|
- spec/fixtures/config.yaml
|
|
245
229
|
- spec/fixtures/invalid_config.toml
|
|
246
230
|
- spec/smoke/package_spec.rb
|
|
247
231
|
- spec/spec_helper.rb
|
|
248
|
-
- spec/unit/config/chunking_config_spec.rb
|
|
249
|
-
- spec/unit/config/embedding_config_spec.rb
|
|
250
|
-
- spec/unit/config/extraction_config_spec.rb
|
|
251
|
-
- spec/unit/config/font_config_spec.rb
|
|
252
|
-
- spec/unit/config/hierarchy_config_spec.rb
|
|
253
|
-
- spec/unit/config/image_extraction_config_spec.rb
|
|
254
|
-
- spec/unit/config/image_preprocessing_config_spec.rb
|
|
255
|
-
- spec/unit/config/keyword_config_spec.rb
|
|
256
|
-
- spec/unit/config/language_detection_config_spec.rb
|
|
257
|
-
- spec/unit/config/ocr_config_spec.rb
|
|
258
|
-
- spec/unit/config/page_config_spec.rb
|
|
259
|
-
- spec/unit/config/pdf_config_spec.rb
|
|
260
|
-
- spec/unit/config/postprocessor_config_spec.rb
|
|
261
|
-
- spec/unit/config/tesseract_config_spec.rb
|
|
262
|
-
- spec/unit/config/token_reduction_config_spec.rb
|
|
263
|
-
- test/metadata_types_test.rb
|
|
264
|
-
- vendor/Cargo.toml
|
|
265
|
-
- vendor/kreuzberg-ffi/Cargo.toml
|
|
266
|
-
- vendor/kreuzberg-ffi/README.md
|
|
267
|
-
- vendor/kreuzberg-ffi/benches/result_view_benchmark.rs
|
|
268
|
-
- vendor/kreuzberg-ffi/build.rs
|
|
269
|
-
- vendor/kreuzberg-ffi/cbindgen.toml
|
|
270
|
-
- vendor/kreuzberg-ffi/kreuzberg-ffi.pc.in
|
|
271
|
-
- vendor/kreuzberg-ffi/kreuzberg.h
|
|
272
|
-
- vendor/kreuzberg-ffi/src/batch_streaming.rs
|
|
273
|
-
- vendor/kreuzberg-ffi/src/config.rs
|
|
274
|
-
- vendor/kreuzberg-ffi/src/error.rs
|
|
275
|
-
- vendor/kreuzberg-ffi/src/extraction.rs
|
|
276
|
-
- vendor/kreuzberg-ffi/src/helpers.rs
|
|
277
|
-
- vendor/kreuzberg-ffi/src/lib.rs
|
|
278
|
-
- vendor/kreuzberg-ffi/src/memory.rs
|
|
279
|
-
- vendor/kreuzberg-ffi/src/mime.rs
|
|
280
|
-
- vendor/kreuzberg-ffi/src/panic_shield.rs
|
|
281
|
-
- vendor/kreuzberg-ffi/src/plugins/document_extractor.rs
|
|
282
|
-
- vendor/kreuzberg-ffi/src/plugins/mod.rs
|
|
283
|
-
- vendor/kreuzberg-ffi/src/plugins/ocr_backend.rs
|
|
284
|
-
- vendor/kreuzberg-ffi/src/plugins/post_processor.rs
|
|
285
|
-
- vendor/kreuzberg-ffi/src/plugins/validator.rs
|
|
286
|
-
- vendor/kreuzberg-ffi/src/result.rs
|
|
287
|
-
- vendor/kreuzberg-ffi/src/result_pool.rs
|
|
288
|
-
- vendor/kreuzberg-ffi/src/result_view.rs
|
|
289
|
-
- vendor/kreuzberg-ffi/src/string_intern.rs
|
|
290
|
-
- vendor/kreuzberg-ffi/src/types.rs
|
|
291
|
-
- vendor/kreuzberg-ffi/src/util.rs
|
|
292
|
-
- vendor/kreuzberg-ffi/src/validation.rs
|
|
293
|
-
- vendor/kreuzberg-ffi/tests.disabled/README.md
|
|
294
|
-
- vendor/kreuzberg-ffi/tests.disabled/config_loading_tests.rs
|
|
295
|
-
- vendor/kreuzberg-ffi/tests.disabled/config_tests.rs
|
|
296
|
-
- vendor/kreuzberg-ffi/tests.disabled/extractor_tests.rs
|
|
297
|
-
- vendor/kreuzberg-ffi/tests.disabled/plugin_registration_tests.rs
|
|
298
|
-
- vendor/kreuzberg-tesseract/.commitlintrc.json
|
|
299
|
-
- vendor/kreuzberg-tesseract/.crate-ignore
|
|
300
|
-
- vendor/kreuzberg-tesseract/Cargo.lock
|
|
301
|
-
- vendor/kreuzberg-tesseract/Cargo.toml
|
|
302
|
-
- vendor/kreuzberg-tesseract/LICENSE
|
|
303
|
-
- vendor/kreuzberg-tesseract/README.md
|
|
304
|
-
- vendor/kreuzberg-tesseract/build.rs
|
|
305
|
-
- vendor/kreuzberg-tesseract/patches/README.md
|
|
306
|
-
- vendor/kreuzberg-tesseract/patches/tesseract.diff
|
|
307
|
-
- vendor/kreuzberg-tesseract/src/api.rs
|
|
308
|
-
- vendor/kreuzberg-tesseract/src/choice_iterator.rs
|
|
309
|
-
- vendor/kreuzberg-tesseract/src/enums.rs
|
|
310
|
-
- vendor/kreuzberg-tesseract/src/error.rs
|
|
311
|
-
- vendor/kreuzberg-tesseract/src/lib.rs
|
|
312
|
-
- vendor/kreuzberg-tesseract/src/monitor.rs
|
|
313
|
-
- vendor/kreuzberg-tesseract/src/mutable_iterator.rs
|
|
314
|
-
- vendor/kreuzberg-tesseract/src/page_iterator.rs
|
|
315
|
-
- vendor/kreuzberg-tesseract/src/result_iterator.rs
|
|
316
|
-
- vendor/kreuzberg-tesseract/src/result_renderer.rs
|
|
317
|
-
- vendor/kreuzberg-tesseract/tests/integration_test.rs
|
|
318
232
|
- vendor/kreuzberg/Cargo.toml
|
|
319
233
|
- vendor/kreuzberg/README.md
|
|
320
234
|
- vendor/kreuzberg/build.rs
|
|
321
|
-
- vendor/kreuzberg/examples/bench_fixes.rs
|
|
322
|
-
- vendor/kreuzberg/examples/test_pdfium_fork.rs
|
|
323
235
|
- vendor/kreuzberg/src/api/error.rs
|
|
324
236
|
- vendor/kreuzberg/src/api/handlers.rs
|
|
325
237
|
- vendor/kreuzberg/src/api/mod.rs
|
|
326
238
|
- vendor/kreuzberg/src/api/server.rs
|
|
327
239
|
- vendor/kreuzberg/src/api/types.rs
|
|
240
|
+
- vendor/kreuzberg/src/bin/profile_extract.rs
|
|
328
241
|
- vendor/kreuzberg/src/cache/mod.rs
|
|
329
242
|
- vendor/kreuzberg/src/chunking/mod.rs
|
|
330
|
-
- vendor/kreuzberg/src/chunking/processor.rs
|
|
331
243
|
- vendor/kreuzberg/src/core/batch_mode.rs
|
|
332
|
-
- vendor/kreuzberg/src/core/batch_optimizations.rs
|
|
333
244
|
- vendor/kreuzberg/src/core/config.rs
|
|
334
|
-
- vendor/kreuzberg/src/core/config_validation.rs
|
|
335
245
|
- vendor/kreuzberg/src/core/extractor.rs
|
|
336
|
-
- vendor/kreuzberg/src/core/formats.rs
|
|
337
246
|
- vendor/kreuzberg/src/core/io.rs
|
|
338
247
|
- vendor/kreuzberg/src/core/mime.rs
|
|
339
248
|
- vendor/kreuzberg/src/core/mod.rs
|
|
340
249
|
- vendor/kreuzberg/src/core/pipeline.rs
|
|
341
|
-
- vendor/kreuzberg/src/core/server_config.rs
|
|
342
250
|
- vendor/kreuzberg/src/embeddings.rs
|
|
343
251
|
- vendor/kreuzberg/src/error.rs
|
|
344
252
|
- vendor/kreuzberg/src/extraction/archive.rs
|
|
345
|
-
- vendor/kreuzberg/src/extraction/capacity.rs
|
|
346
253
|
- vendor/kreuzberg/src/extraction/docx.rs
|
|
347
254
|
- vendor/kreuzberg/src/extraction/email.rs
|
|
348
255
|
- vendor/kreuzberg/src/extraction/excel.rs
|
|
349
256
|
- vendor/kreuzberg/src/extraction/html.rs
|
|
350
257
|
- vendor/kreuzberg/src/extraction/image.rs
|
|
351
258
|
- vendor/kreuzberg/src/extraction/libreoffice.rs
|
|
352
|
-
- vendor/kreuzberg/src/extraction/markdown.rs
|
|
353
259
|
- vendor/kreuzberg/src/extraction/mod.rs
|
|
354
260
|
- vendor/kreuzberg/src/extraction/office_metadata/app_properties.rs
|
|
355
261
|
- vendor/kreuzberg/src/extraction/office_metadata/core_properties.rs
|
|
356
262
|
- vendor/kreuzberg/src/extraction/office_metadata/custom_properties.rs
|
|
357
263
|
- vendor/kreuzberg/src/extraction/office_metadata/mod.rs
|
|
358
|
-
- vendor/kreuzberg/src/extraction/
|
|
264
|
+
- vendor/kreuzberg/src/extraction/pandoc/batch.rs
|
|
265
|
+
- vendor/kreuzberg/src/extraction/pandoc/mime_types.rs
|
|
266
|
+
- vendor/kreuzberg/src/extraction/pandoc/mod.rs
|
|
267
|
+
- vendor/kreuzberg/src/extraction/pandoc/server.rs
|
|
268
|
+
- vendor/kreuzberg/src/extraction/pandoc/subprocess.rs
|
|
269
|
+
- vendor/kreuzberg/src/extraction/pandoc/version.rs
|
|
359
270
|
- vendor/kreuzberg/src/extraction/pptx.rs
|
|
360
271
|
- vendor/kreuzberg/src/extraction/structured.rs
|
|
361
272
|
- vendor/kreuzberg/src/extraction/table.rs
|
|
362
273
|
- vendor/kreuzberg/src/extraction/text.rs
|
|
363
274
|
- vendor/kreuzberg/src/extraction/xml.rs
|
|
364
275
|
- vendor/kreuzberg/src/extractors/archive.rs
|
|
365
|
-
- vendor/kreuzberg/src/extractors/bibtex.rs
|
|
366
|
-
- vendor/kreuzberg/src/extractors/docbook.rs
|
|
367
276
|
- vendor/kreuzberg/src/extractors/docx.rs
|
|
368
277
|
- vendor/kreuzberg/src/extractors/email.rs
|
|
369
|
-
- vendor/kreuzberg/src/extractors/epub.rs
|
|
370
278
|
- vendor/kreuzberg/src/extractors/excel.rs
|
|
371
|
-
- vendor/kreuzberg/src/extractors/fictionbook.rs
|
|
372
279
|
- vendor/kreuzberg/src/extractors/html.rs
|
|
373
280
|
- vendor/kreuzberg/src/extractors/image.rs
|
|
374
|
-
- vendor/kreuzberg/src/extractors/jats.rs
|
|
375
|
-
- vendor/kreuzberg/src/extractors/jupyter.rs
|
|
376
|
-
- vendor/kreuzberg/src/extractors/latex.rs
|
|
377
|
-
- vendor/kreuzberg/src/extractors/markdown.rs
|
|
378
281
|
- vendor/kreuzberg/src/extractors/mod.rs
|
|
379
|
-
- vendor/kreuzberg/src/extractors/
|
|
380
|
-
- vendor/kreuzberg/src/extractors/opml.rs
|
|
381
|
-
- vendor/kreuzberg/src/extractors/orgmode.rs
|
|
282
|
+
- vendor/kreuzberg/src/extractors/pandoc.rs
|
|
382
283
|
- vendor/kreuzberg/src/extractors/pdf.rs
|
|
383
284
|
- vendor/kreuzberg/src/extractors/pptx.rs
|
|
384
|
-
- vendor/kreuzberg/src/extractors/rst.rs
|
|
385
|
-
- vendor/kreuzberg/src/extractors/rtf.rs
|
|
386
|
-
- vendor/kreuzberg/src/extractors/security.rs
|
|
387
|
-
- vendor/kreuzberg/src/extractors/security_tests.rs
|
|
388
285
|
- vendor/kreuzberg/src/extractors/structured.rs
|
|
389
286
|
- vendor/kreuzberg/src/extractors/text.rs
|
|
390
|
-
- vendor/kreuzberg/src/extractors/typst.rs
|
|
391
287
|
- vendor/kreuzberg/src/extractors/xml.rs
|
|
392
288
|
- vendor/kreuzberg/src/image/dpi.rs
|
|
393
289
|
- vendor/kreuzberg/src/image/mod.rs
|
|
@@ -400,14 +296,12 @@ files:
|
|
|
400
296
|
- vendor/kreuzberg/src/keywords/types.rs
|
|
401
297
|
- vendor/kreuzberg/src/keywords/yake.rs
|
|
402
298
|
- vendor/kreuzberg/src/language_detection/mod.rs
|
|
403
|
-
- vendor/kreuzberg/src/language_detection/processor.rs
|
|
404
299
|
- vendor/kreuzberg/src/lib.rs
|
|
405
300
|
- vendor/kreuzberg/src/mcp/mod.rs
|
|
406
301
|
- vendor/kreuzberg/src/mcp/server.rs
|
|
407
302
|
- vendor/kreuzberg/src/ocr/cache.rs
|
|
408
303
|
- vendor/kreuzberg/src/ocr/error.rs
|
|
409
304
|
- vendor/kreuzberg/src/ocr/hocr.rs
|
|
410
|
-
- vendor/kreuzberg/src/ocr/language_registry.rs
|
|
411
305
|
- vendor/kreuzberg/src/ocr/mod.rs
|
|
412
306
|
- vendor/kreuzberg/src/ocr/processor.rs
|
|
413
307
|
- vendor/kreuzberg/src/ocr/table/mod.rs
|
|
@@ -416,12 +310,7 @@ files:
|
|
|
416
310
|
- vendor/kreuzberg/src/ocr/types.rs
|
|
417
311
|
- vendor/kreuzberg/src/ocr/utils.rs
|
|
418
312
|
- vendor/kreuzberg/src/ocr/validation.rs
|
|
419
|
-
- vendor/kreuzberg/src/panic_context.rs
|
|
420
|
-
- vendor/kreuzberg/src/pdf/bindings.rs
|
|
421
|
-
- vendor/kreuzberg/src/pdf/bundled.rs
|
|
422
313
|
- vendor/kreuzberg/src/pdf/error.rs
|
|
423
|
-
- vendor/kreuzberg/src/pdf/fonts.rs
|
|
424
|
-
- vendor/kreuzberg/src/pdf/hierarchy.rs
|
|
425
314
|
- vendor/kreuzberg/src/pdf/images.rs
|
|
426
315
|
- vendor/kreuzberg/src/pdf/metadata.rs
|
|
427
316
|
- vendor/kreuzberg/src/pdf/mod.rs
|
|
@@ -438,7 +327,6 @@ files:
|
|
|
438
327
|
- vendor/kreuzberg/src/stopwords/mod.rs
|
|
439
328
|
- vendor/kreuzberg/src/text/mod.rs
|
|
440
329
|
- vendor/kreuzberg/src/text/quality.rs
|
|
441
|
-
- vendor/kreuzberg/src/text/quality_processor.rs
|
|
442
330
|
- vendor/kreuzberg/src/text/string_utils.rs
|
|
443
331
|
- vendor/kreuzberg/src/text/token_reduction/cjk_utils.rs
|
|
444
332
|
- vendor/kreuzberg/src/text/token_reduction/config.rs
|
|
@@ -447,13 +335,9 @@ files:
|
|
|
447
335
|
- vendor/kreuzberg/src/text/token_reduction/mod.rs
|
|
448
336
|
- vendor/kreuzberg/src/text/token_reduction/semantic.rs
|
|
449
337
|
- vendor/kreuzberg/src/text/token_reduction/simd_text.rs
|
|
450
|
-
- vendor/kreuzberg/src/text/utf8_validation.rs
|
|
451
338
|
- vendor/kreuzberg/src/types.rs
|
|
452
339
|
- vendor/kreuzberg/src/utils/mod.rs
|
|
453
|
-
- vendor/kreuzberg/src/utils/pool.rs
|
|
454
|
-
- vendor/kreuzberg/src/utils/pool_sizing.rs
|
|
455
340
|
- vendor/kreuzberg/src/utils/quality.rs
|
|
456
|
-
- vendor/kreuzberg/src/utils/string_pool.rs
|
|
457
341
|
- vendor/kreuzberg/src/utils/string_utils.rs
|
|
458
342
|
- vendor/kreuzberg/stopwords/af_stopwords.json
|
|
459
343
|
- vendor/kreuzberg/stopwords/ar_stopwords.json
|
|
@@ -519,80 +403,50 @@ files:
|
|
|
519
403
|
- vendor/kreuzberg/stopwords/yo_stopwords.json
|
|
520
404
|
- vendor/kreuzberg/stopwords/zh_stopwords.json
|
|
521
405
|
- vendor/kreuzberg/stopwords/zu_stopwords.json
|
|
522
|
-
- vendor/kreuzberg/tests/api_embed.rs
|
|
523
|
-
- vendor/kreuzberg/tests/api_extract_multipart.rs
|
|
524
|
-
- vendor/kreuzberg/tests/api_large_pdf_extraction.rs
|
|
525
|
-
- vendor/kreuzberg/tests/api_large_pdf_extraction_diagnostics.rs
|
|
526
406
|
- vendor/kreuzberg/tests/api_tests.rs
|
|
527
407
|
- vendor/kreuzberg/tests/archive_integration.rs
|
|
528
408
|
- vendor/kreuzberg/tests/batch_orchestration.rs
|
|
529
|
-
- vendor/kreuzberg/tests/batch_pooling_benchmark.rs
|
|
530
409
|
- vendor/kreuzberg/tests/batch_processing.rs
|
|
531
|
-
- vendor/kreuzberg/tests/
|
|
410
|
+
- vendor/kreuzberg/tests/chunking_offset_demo.rs
|
|
532
411
|
- vendor/kreuzberg/tests/concurrency_stress.rs
|
|
533
412
|
- vendor/kreuzberg/tests/config_features.rs
|
|
534
|
-
- vendor/kreuzberg/tests/config_integration_test.rs
|
|
535
413
|
- vendor/kreuzberg/tests/config_loading_tests.rs
|
|
536
414
|
- vendor/kreuzberg/tests/core_integration.rs
|
|
537
415
|
- vendor/kreuzberg/tests/csv_integration.rs
|
|
538
|
-
- vendor/kreuzberg/tests/data/hierarchy_ground_truth.json
|
|
539
|
-
- vendor/kreuzberg/tests/docbook_extractor_tests.rs
|
|
540
416
|
- vendor/kreuzberg/tests/docx_metadata_extraction_test.rs
|
|
541
|
-
- vendor/kreuzberg/tests/docx_vs_pandoc_comparison.rs
|
|
542
417
|
- vendor/kreuzberg/tests/email_integration.rs
|
|
543
|
-
- vendor/kreuzberg/tests/epub_native_extractor_tests.rs
|
|
544
418
|
- vendor/kreuzberg/tests/error_handling.rs
|
|
545
|
-
- vendor/kreuzberg/tests/fictionbook_extractor_tests.rs
|
|
546
419
|
- vendor/kreuzberg/tests/format_integration.rs
|
|
547
420
|
- vendor/kreuzberg/tests/helpers/mod.rs
|
|
548
|
-
- vendor/kreuzberg/tests/html_table_test.rs
|
|
549
421
|
- vendor/kreuzberg/tests/image_integration.rs
|
|
550
|
-
- vendor/kreuzberg/tests/instrumentation_test.rs
|
|
551
|
-
- vendor/kreuzberg/tests/jats_extractor_tests.rs
|
|
552
|
-
- vendor/kreuzberg/tests/jupyter_extractor_tests.rs
|
|
553
422
|
- vendor/kreuzberg/tests/keywords_integration.rs
|
|
554
423
|
- vendor/kreuzberg/tests/keywords_quality.rs
|
|
555
|
-
- vendor/kreuzberg/tests/latex_extractor_tests.rs
|
|
556
|
-
- vendor/kreuzberg/tests/markdown_extractor_tests.rs
|
|
557
424
|
- vendor/kreuzberg/tests/mime_detection.rs
|
|
558
425
|
- vendor/kreuzberg/tests/ocr_configuration.rs
|
|
559
426
|
- vendor/kreuzberg/tests/ocr_errors.rs
|
|
560
|
-
- vendor/kreuzberg/tests/ocr_language_registry.rs
|
|
561
427
|
- vendor/kreuzberg/tests/ocr_quality.rs
|
|
562
428
|
- vendor/kreuzberg/tests/ocr_stress.rs
|
|
563
|
-
- vendor/kreuzberg/tests/
|
|
564
|
-
- vendor/kreuzberg/tests/opml_extractor_tests.rs
|
|
565
|
-
- vendor/kreuzberg/tests/orgmode_extractor_tests.rs
|
|
566
|
-
- vendor/kreuzberg/tests/page_markers.rs
|
|
567
|
-
- vendor/kreuzberg/tests/pdf_hierarchy_detection.rs
|
|
568
|
-
- vendor/kreuzberg/tests/pdf_hierarchy_quality.rs
|
|
429
|
+
- vendor/kreuzberg/tests/pandoc_integration.rs
|
|
569
430
|
- vendor/kreuzberg/tests/pdf_integration.rs
|
|
570
|
-
- vendor/kreuzberg/tests/pdf_ocr_triggering.rs
|
|
571
|
-
- vendor/kreuzberg/tests/pdf_text_merging.rs
|
|
572
|
-
- vendor/kreuzberg/tests/pdfium_linking.rs
|
|
573
431
|
- vendor/kreuzberg/tests/pipeline_integration.rs
|
|
574
432
|
- vendor/kreuzberg/tests/plugin_ocr_backend_test.rs
|
|
575
433
|
- vendor/kreuzberg/tests/plugin_postprocessor_test.rs
|
|
576
434
|
- vendor/kreuzberg/tests/plugin_system.rs
|
|
577
435
|
- vendor/kreuzberg/tests/plugin_validator_test.rs
|
|
578
436
|
- vendor/kreuzberg/tests/registry_integration_tests.rs
|
|
579
|
-
- vendor/kreuzberg/tests/rst_extractor_tests.rs
|
|
580
|
-
- vendor/kreuzberg/tests/rtf_extractor_tests.rs
|
|
581
437
|
- vendor/kreuzberg/tests/security_validation.rs
|
|
582
438
|
- vendor/kreuzberg/tests/stopwords_integration_test.rs
|
|
583
439
|
- vendor/kreuzberg/tests/test_fastembed.rs
|
|
584
|
-
- vendor/kreuzberg/tests/typst_behavioral_tests.rs
|
|
585
|
-
- vendor/kreuzberg/tests/typst_extractor_tests.rs
|
|
586
440
|
- vendor/kreuzberg/tests/xlsx_metadata_extraction_test.rs
|
|
587
|
-
homepage: https://github.com/
|
|
441
|
+
homepage: https://github.com/Goldziher/kreuzberg
|
|
588
442
|
licenses:
|
|
589
443
|
- MIT
|
|
590
444
|
metadata:
|
|
591
|
-
homepage_uri: https://github.com/
|
|
592
|
-
source_code_uri: https://github.com/
|
|
593
|
-
changelog_uri: https://github.com/
|
|
445
|
+
homepage_uri: https://github.com/Goldziher/kreuzberg
|
|
446
|
+
source_code_uri: https://github.com/Goldziher/kreuzberg
|
|
447
|
+
changelog_uri: https://github.com/Goldziher/kreuzberg/blob/main/CHANGELOG.md
|
|
594
448
|
documentation_uri: https://docs.kreuzberg.dev
|
|
595
|
-
bug_tracker_uri: https://github.com/
|
|
449
|
+
bug_tracker_uri: https://github.com/Goldziher/kreuzberg/issues
|
|
596
450
|
rubygems_mfa_required: 'true'
|
|
597
451
|
keywords: document-intelligence,document-extraction,ocr,rust,bindings
|
|
598
452
|
post_install_message:
|