kreuzberg 4.0.0.rc1 → 4.0.0.rc2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +14 -8
- data/.rspec +3 -3
- data/.rubocop.yaml +1 -534
- data/.rubocop.yml +538 -0
- data/Gemfile +8 -9
- data/Gemfile.lock +9 -109
- data/README.md +426 -421
- data/Rakefile +25 -25
- data/Steepfile +47 -47
- data/examples/async_patterns.rb +341 -340
- data/ext/kreuzberg_rb/extconf.rb +45 -35
- data/ext/kreuzberg_rb/native/Cargo.lock +6535 -0
- data/ext/kreuzberg_rb/native/Cargo.toml +44 -36
- data/ext/kreuzberg_rb/native/README.md +425 -425
- data/ext/kreuzberg_rb/native/build.rs +15 -17
- data/ext/kreuzberg_rb/native/include/ieeefp.h +11 -11
- data/ext/kreuzberg_rb/native/include/msvc_compat/strings.h +14 -14
- data/ext/kreuzberg_rb/native/include/strings.h +20 -20
- data/ext/kreuzberg_rb/native/include/unistd.h +47 -47
- data/ext/kreuzberg_rb/native/src/lib.rs +2998 -2939
- data/extconf.rb +28 -28
- data/kreuzberg.gemspec +148 -105
- data/lib/kreuzberg/api_proxy.rb +142 -142
- data/lib/kreuzberg/cache_api.rb +46 -45
- data/lib/kreuzberg/cli.rb +55 -55
- data/lib/kreuzberg/cli_proxy.rb +127 -127
- data/lib/kreuzberg/config.rb +691 -684
- data/lib/kreuzberg/error_context.rb +32 -0
- data/lib/kreuzberg/errors.rb +118 -50
- data/lib/kreuzberg/extraction_api.rb +85 -84
- data/lib/kreuzberg/mcp_proxy.rb +186 -186
- data/lib/kreuzberg/ocr_backend_protocol.rb +113 -113
- data/lib/kreuzberg/post_processor_protocol.rb +86 -86
- data/lib/kreuzberg/result.rb +216 -216
- data/lib/kreuzberg/setup_lib_path.rb +80 -79
- data/lib/kreuzberg/validator_protocol.rb +89 -89
- data/lib/kreuzberg/version.rb +5 -5
- data/lib/kreuzberg.rb +103 -82
- data/sig/kreuzberg/internal.rbs +184 -184
- data/sig/kreuzberg.rbs +520 -468
- data/spec/binding/cache_spec.rb +227 -227
- data/spec/binding/cli_proxy_spec.rb +85 -87
- data/spec/binding/cli_spec.rb +55 -54
- data/spec/binding/config_spec.rb +345 -345
- data/spec/binding/config_validation_spec.rb +283 -283
- data/spec/binding/error_handling_spec.rb +213 -213
- data/spec/binding/errors_spec.rb +66 -66
- data/spec/binding/plugins/ocr_backend_spec.rb +307 -307
- data/spec/binding/plugins/postprocessor_spec.rb +269 -269
- data/spec/binding/plugins/validator_spec.rb +274 -274
- data/spec/fixtures/config.toml +39 -39
- data/spec/fixtures/config.yaml +41 -42
- data/spec/fixtures/invalid_config.toml +4 -4
- data/spec/smoke/package_spec.rb +178 -178
- data/spec/spec_helper.rb +42 -42
- data/vendor/kreuzberg/Cargo.toml +204 -134
- data/vendor/kreuzberg/README.md +175 -175
- data/vendor/kreuzberg/benches/otel_overhead.rs +48 -0
- data/vendor/kreuzberg/build.rs +474 -460
- data/vendor/kreuzberg/src/api/error.rs +81 -81
- data/vendor/kreuzberg/src/api/handlers.rs +199 -199
- data/vendor/kreuzberg/src/api/mod.rs +79 -79
- data/vendor/kreuzberg/src/api/server.rs +353 -353
- data/vendor/kreuzberg/src/api/types.rs +170 -170
- data/vendor/kreuzberg/src/cache/mod.rs +1167 -1143
- data/vendor/kreuzberg/src/chunking/mod.rs +677 -677
- data/vendor/kreuzberg/src/core/batch_mode.rs +95 -35
- data/vendor/kreuzberg/src/core/config.rs +1032 -1032
- data/vendor/kreuzberg/src/core/extractor.rs +1024 -903
- data/vendor/kreuzberg/src/core/io.rs +329 -327
- data/vendor/kreuzberg/src/core/mime.rs +605 -615
- data/vendor/kreuzberg/src/core/mod.rs +45 -42
- data/vendor/kreuzberg/src/core/pipeline.rs +984 -906
- data/vendor/kreuzberg/src/embeddings.rs +432 -323
- data/vendor/kreuzberg/src/error.rs +431 -431
- data/vendor/kreuzberg/src/extraction/archive.rs +954 -954
- data/vendor/kreuzberg/src/extraction/docx.rs +40 -40
- data/vendor/kreuzberg/src/extraction/email.rs +854 -854
- data/vendor/kreuzberg/src/extraction/excel.rs +688 -688
- data/vendor/kreuzberg/src/extraction/html.rs +553 -553
- data/vendor/kreuzberg/src/extraction/image.rs +368 -368
- data/vendor/kreuzberg/src/extraction/libreoffice.rs +563 -564
- data/vendor/kreuzberg/src/extraction/markdown.rs +213 -0
- data/vendor/kreuzberg/src/extraction/mod.rs +81 -77
- data/vendor/kreuzberg/src/extraction/office_metadata/app_properties.rs +398 -398
- data/vendor/kreuzberg/src/extraction/office_metadata/core_properties.rs +247 -247
- data/vendor/kreuzberg/src/extraction/office_metadata/custom_properties.rs +240 -240
- data/vendor/kreuzberg/src/extraction/office_metadata/mod.rs +130 -128
- data/vendor/kreuzberg/src/extraction/office_metadata/odt_properties.rs +287 -0
- data/vendor/kreuzberg/src/extraction/pptx.rs +3000 -3000
- data/vendor/kreuzberg/src/extraction/structured.rs +490 -490
- data/vendor/kreuzberg/src/extraction/table.rs +328 -328
- data/vendor/kreuzberg/src/extraction/text.rs +269 -269
- data/vendor/kreuzberg/src/extraction/xml.rs +333 -333
- data/vendor/kreuzberg/src/extractors/archive.rs +446 -425
- data/vendor/kreuzberg/src/extractors/bibtex.rs +469 -0
- data/vendor/kreuzberg/src/extractors/docbook.rs +502 -0
- data/vendor/kreuzberg/src/extractors/docx.rs +367 -479
- data/vendor/kreuzberg/src/extractors/email.rs +143 -129
- data/vendor/kreuzberg/src/extractors/epub.rs +707 -0
- data/vendor/kreuzberg/src/extractors/excel.rs +343 -344
- data/vendor/kreuzberg/src/extractors/fictionbook.rs +491 -0
- data/vendor/kreuzberg/src/extractors/fictionbook.rs.backup2 +738 -0
- data/vendor/kreuzberg/src/extractors/html.rs +393 -410
- data/vendor/kreuzberg/src/extractors/image.rs +198 -195
- data/vendor/kreuzberg/src/extractors/jats.rs +1051 -0
- data/vendor/kreuzberg/src/extractors/jupyter.rs +367 -0
- data/vendor/kreuzberg/src/extractors/latex.rs +652 -0
- data/vendor/kreuzberg/src/extractors/markdown.rs +700 -0
- data/vendor/kreuzberg/src/extractors/mod.rs +365 -268
- data/vendor/kreuzberg/src/extractors/odt.rs +628 -0
- data/vendor/kreuzberg/src/extractors/opml.rs +634 -0
- data/vendor/kreuzberg/src/extractors/orgmode.rs +528 -0
- data/vendor/kreuzberg/src/extractors/pdf.rs +493 -496
- data/vendor/kreuzberg/src/extractors/pptx.rs +248 -234
- data/vendor/kreuzberg/src/extractors/rst.rs +576 -0
- data/vendor/kreuzberg/src/extractors/rtf.rs +810 -0
- data/vendor/kreuzberg/src/extractors/security.rs +484 -0
- data/vendor/kreuzberg/src/extractors/security_tests.rs +367 -0
- data/vendor/kreuzberg/src/extractors/structured.rs +140 -126
- data/vendor/kreuzberg/src/extractors/text.rs +260 -242
- data/vendor/kreuzberg/src/extractors/typst.rs +650 -0
- data/vendor/kreuzberg/src/extractors/xml.rs +135 -128
- data/vendor/kreuzberg/src/image/dpi.rs +164 -164
- data/vendor/kreuzberg/src/image/mod.rs +6 -6
- data/vendor/kreuzberg/src/image/preprocessing.rs +417 -417
- data/vendor/kreuzberg/src/image/resize.rs +89 -89
- data/vendor/kreuzberg/src/keywords/config.rs +154 -154
- data/vendor/kreuzberg/src/keywords/mod.rs +237 -237
- data/vendor/kreuzberg/src/keywords/processor.rs +267 -267
- data/vendor/kreuzberg/src/keywords/rake.rs +293 -294
- data/vendor/kreuzberg/src/keywords/types.rs +68 -68
- data/vendor/kreuzberg/src/keywords/yake.rs +163 -163
- data/vendor/kreuzberg/src/language_detection/mod.rs +942 -942
- data/vendor/kreuzberg/src/lib.rs +105 -102
- data/vendor/kreuzberg/src/mcp/mod.rs +32 -32
- data/vendor/kreuzberg/src/mcp/server.rs +1968 -1966
- data/vendor/kreuzberg/src/ocr/cache.rs +469 -469
- data/vendor/kreuzberg/src/ocr/error.rs +37 -37
- data/vendor/kreuzberg/src/ocr/hocr.rs +216 -216
- data/vendor/kreuzberg/src/ocr/mod.rs +58 -58
- data/vendor/kreuzberg/src/ocr/processor.rs +863 -847
- data/vendor/kreuzberg/src/ocr/table/mod.rs +4 -4
- data/vendor/kreuzberg/src/ocr/table/tsv_parser.rs +144 -144
- data/vendor/kreuzberg/src/ocr/tesseract_backend.rs +450 -450
- data/vendor/kreuzberg/src/ocr/types.rs +393 -393
- data/vendor/kreuzberg/src/ocr/utils.rs +47 -47
- data/vendor/kreuzberg/src/ocr/validation.rs +206 -206
- data/vendor/kreuzberg/src/panic_context.rs +154 -0
- data/vendor/kreuzberg/src/pdf/error.rs +122 -122
- data/vendor/kreuzberg/src/pdf/images.rs +139 -139
- data/vendor/kreuzberg/src/pdf/metadata.rs +346 -346
- data/vendor/kreuzberg/src/pdf/mod.rs +50 -50
- data/vendor/kreuzberg/src/pdf/rendering.rs +369 -369
- data/vendor/kreuzberg/src/pdf/table.rs +393 -420
- data/vendor/kreuzberg/src/pdf/text.rs +158 -161
- data/vendor/kreuzberg/src/plugins/extractor.rs +1013 -1010
- data/vendor/kreuzberg/src/plugins/mod.rs +209 -209
- data/vendor/kreuzberg/src/plugins/ocr.rs +620 -629
- data/vendor/kreuzberg/src/plugins/processor.rs +642 -641
- data/vendor/kreuzberg/src/plugins/registry.rs +1337 -1324
- data/vendor/kreuzberg/src/plugins/traits.rs +258 -258
- data/vendor/kreuzberg/src/plugins/validator.rs +956 -955
- data/vendor/kreuzberg/src/stopwords/mod.rs +1470 -1470
- data/vendor/kreuzberg/src/text/mod.rs +19 -19
- data/vendor/kreuzberg/src/text/quality.rs +697 -697
- data/vendor/kreuzberg/src/text/string_utils.rs +217 -217
- data/vendor/kreuzberg/src/text/token_reduction/cjk_utils.rs +164 -164
- data/vendor/kreuzberg/src/text/token_reduction/config.rs +100 -100
- data/vendor/kreuzberg/src/text/token_reduction/core.rs +796 -796
- data/vendor/kreuzberg/src/text/token_reduction/filters.rs +902 -902
- data/vendor/kreuzberg/src/text/token_reduction/mod.rs +160 -160
- data/vendor/kreuzberg/src/text/token_reduction/semantic.rs +619 -619
- data/vendor/kreuzberg/src/text/token_reduction/simd_text.rs +147 -147
- data/vendor/kreuzberg/src/types.rs +903 -873
- data/vendor/kreuzberg/src/utils/mod.rs +17 -17
- data/vendor/kreuzberg/src/utils/quality.rs +959 -959
- data/vendor/kreuzberg/src/utils/string_utils.rs +381 -381
- data/vendor/kreuzberg/stopwords/af_stopwords.json +53 -53
- data/vendor/kreuzberg/stopwords/ar_stopwords.json +482 -482
- data/vendor/kreuzberg/stopwords/bg_stopwords.json +261 -261
- data/vendor/kreuzberg/stopwords/bn_stopwords.json +400 -400
- data/vendor/kreuzberg/stopwords/br_stopwords.json +1205 -1205
- data/vendor/kreuzberg/stopwords/ca_stopwords.json +280 -280
- data/vendor/kreuzberg/stopwords/cs_stopwords.json +425 -425
- data/vendor/kreuzberg/stopwords/da_stopwords.json +172 -172
- data/vendor/kreuzberg/stopwords/de_stopwords.json +622 -622
- data/vendor/kreuzberg/stopwords/el_stopwords.json +849 -849
- data/vendor/kreuzberg/stopwords/en_stopwords.json +1300 -1300
- data/vendor/kreuzberg/stopwords/eo_stopwords.json +175 -175
- data/vendor/kreuzberg/stopwords/es_stopwords.json +734 -734
- data/vendor/kreuzberg/stopwords/et_stopwords.json +37 -37
- data/vendor/kreuzberg/stopwords/eu_stopwords.json +100 -100
- data/vendor/kreuzberg/stopwords/fa_stopwords.json +801 -801
- data/vendor/kreuzberg/stopwords/fi_stopwords.json +849 -849
- data/vendor/kreuzberg/stopwords/fr_stopwords.json +693 -693
- data/vendor/kreuzberg/stopwords/ga_stopwords.json +111 -111
- data/vendor/kreuzberg/stopwords/gl_stopwords.json +162 -162
- data/vendor/kreuzberg/stopwords/gu_stopwords.json +226 -226
- data/vendor/kreuzberg/stopwords/ha_stopwords.json +41 -41
- data/vendor/kreuzberg/stopwords/he_stopwords.json +196 -196
- data/vendor/kreuzberg/stopwords/hi_stopwords.json +227 -227
- data/vendor/kreuzberg/stopwords/hr_stopwords.json +181 -181
- data/vendor/kreuzberg/stopwords/hu_stopwords.json +791 -791
- data/vendor/kreuzberg/stopwords/hy_stopwords.json +47 -47
- data/vendor/kreuzberg/stopwords/id_stopwords.json +760 -760
- data/vendor/kreuzberg/stopwords/it_stopwords.json +634 -634
- data/vendor/kreuzberg/stopwords/ja_stopwords.json +136 -136
- data/vendor/kreuzberg/stopwords/kn_stopwords.json +84 -84
- data/vendor/kreuzberg/stopwords/ko_stopwords.json +681 -681
- data/vendor/kreuzberg/stopwords/ku_stopwords.json +64 -64
- data/vendor/kreuzberg/stopwords/la_stopwords.json +51 -51
- data/vendor/kreuzberg/stopwords/lt_stopwords.json +476 -476
- data/vendor/kreuzberg/stopwords/lv_stopwords.json +163 -163
- data/vendor/kreuzberg/stopwords/ml_stopwords.json +1 -1
- data/vendor/kreuzberg/stopwords/mr_stopwords.json +101 -101
- data/vendor/kreuzberg/stopwords/ms_stopwords.json +477 -477
- data/vendor/kreuzberg/stopwords/ne_stopwords.json +490 -490
- data/vendor/kreuzberg/stopwords/nl_stopwords.json +415 -415
- data/vendor/kreuzberg/stopwords/no_stopwords.json +223 -223
- data/vendor/kreuzberg/stopwords/pl_stopwords.json +331 -331
- data/vendor/kreuzberg/stopwords/pt_stopwords.json +562 -562
- data/vendor/kreuzberg/stopwords/ro_stopwords.json +436 -436
- data/vendor/kreuzberg/stopwords/ru_stopwords.json +561 -561
- data/vendor/kreuzberg/stopwords/si_stopwords.json +193 -193
- data/vendor/kreuzberg/stopwords/sk_stopwords.json +420 -420
- data/vendor/kreuzberg/stopwords/sl_stopwords.json +448 -448
- data/vendor/kreuzberg/stopwords/so_stopwords.json +32 -32
- data/vendor/kreuzberg/stopwords/st_stopwords.json +33 -33
- data/vendor/kreuzberg/stopwords/sv_stopwords.json +420 -420
- data/vendor/kreuzberg/stopwords/sw_stopwords.json +76 -76
- data/vendor/kreuzberg/stopwords/ta_stopwords.json +129 -129
- data/vendor/kreuzberg/stopwords/te_stopwords.json +54 -54
- data/vendor/kreuzberg/stopwords/th_stopwords.json +118 -118
- data/vendor/kreuzberg/stopwords/tl_stopwords.json +149 -149
- data/vendor/kreuzberg/stopwords/tr_stopwords.json +506 -506
- data/vendor/kreuzberg/stopwords/uk_stopwords.json +75 -75
- data/vendor/kreuzberg/stopwords/ur_stopwords.json +519 -519
- data/vendor/kreuzberg/stopwords/vi_stopwords.json +647 -647
- data/vendor/kreuzberg/stopwords/yo_stopwords.json +62 -62
- data/vendor/kreuzberg/stopwords/zh_stopwords.json +796 -796
- data/vendor/kreuzberg/stopwords/zu_stopwords.json +31 -31
- data/vendor/kreuzberg/tests/api_extract_multipart.rs +52 -0
- data/vendor/kreuzberg/tests/api_tests.rs +966 -966
- data/vendor/kreuzberg/tests/archive_integration.rs +543 -543
- data/vendor/kreuzberg/tests/batch_orchestration.rs +556 -542
- data/vendor/kreuzberg/tests/batch_processing.rs +316 -304
- data/vendor/kreuzberg/tests/bibtex_parity_test.rs +421 -0
- data/vendor/kreuzberg/tests/concurrency_stress.rs +525 -509
- data/vendor/kreuzberg/tests/config_features.rs +598 -580
- data/vendor/kreuzberg/tests/config_loading_tests.rs +415 -439
- data/vendor/kreuzberg/tests/core_integration.rs +510 -493
- data/vendor/kreuzberg/tests/csv_integration.rs +414 -424
- data/vendor/kreuzberg/tests/docbook_extractor_tests.rs +498 -0
- data/vendor/kreuzberg/tests/docx_metadata_extraction_test.rs +122 -124
- data/vendor/kreuzberg/tests/docx_vs_pandoc_comparison.rs +370 -0
- data/vendor/kreuzberg/tests/email_integration.rs +325 -325
- data/vendor/kreuzberg/tests/epub_native_extractor_tests.rs +275 -0
- data/vendor/kreuzberg/tests/error_handling.rs +393 -393
- data/vendor/kreuzberg/tests/fictionbook_extractor_tests.rs +228 -0
- data/vendor/kreuzberg/tests/format_integration.rs +159 -159
- data/vendor/kreuzberg/tests/helpers/mod.rs +142 -142
- data/vendor/kreuzberg/tests/html_table_test.rs +551 -0
- data/vendor/kreuzberg/tests/image_integration.rs +253 -253
- data/vendor/kreuzberg/tests/instrumentation_test.rs +139 -0
- data/vendor/kreuzberg/tests/jats_extractor_tests.rs +639 -0
- data/vendor/kreuzberg/tests/jupyter_extractor_tests.rs +704 -0
- data/vendor/kreuzberg/tests/keywords_integration.rs +479 -479
- data/vendor/kreuzberg/tests/keywords_quality.rs +509 -509
- data/vendor/kreuzberg/tests/latex_extractor_tests.rs +496 -0
- data/vendor/kreuzberg/tests/markdown_extractor_tests.rs +490 -0
- data/vendor/kreuzberg/tests/mime_detection.rs +428 -428
- data/vendor/kreuzberg/tests/ocr_configuration.rs +510 -510
- data/vendor/kreuzberg/tests/ocr_errors.rs +676 -676
- data/vendor/kreuzberg/tests/ocr_quality.rs +627 -627
- data/vendor/kreuzberg/tests/ocr_stress.rs +469 -469
- data/vendor/kreuzberg/tests/odt_extractor_tests.rs +695 -0
- data/vendor/kreuzberg/tests/opml_extractor_tests.rs +616 -0
- data/vendor/kreuzberg/tests/orgmode_extractor_tests.rs +822 -0
- data/vendor/kreuzberg/tests/pdf_integration.rs +43 -43
- data/vendor/kreuzberg/tests/pipeline_integration.rs +1411 -1412
- data/vendor/kreuzberg/tests/plugin_ocr_backend_test.rs +771 -771
- data/vendor/kreuzberg/tests/plugin_postprocessor_test.rs +560 -561
- data/vendor/kreuzberg/tests/plugin_system.rs +921 -921
- data/vendor/kreuzberg/tests/plugin_validator_test.rs +783 -783
- data/vendor/kreuzberg/tests/registry_integration_tests.rs +586 -607
- data/vendor/kreuzberg/tests/rst_extractor_tests.rs +692 -0
- data/vendor/kreuzberg/tests/rtf_extractor_tests.rs +776 -0
- data/vendor/kreuzberg/tests/security_validation.rs +415 -404
- data/vendor/kreuzberg/tests/stopwords_integration_test.rs +888 -888
- data/vendor/kreuzberg/tests/test_fastembed.rs +609 -609
- data/vendor/kreuzberg/tests/typst_behavioral_tests.rs +1259 -0
- data/vendor/kreuzberg/tests/typst_extractor_tests.rs +647 -0
- data/vendor/kreuzberg/tests/xlsx_metadata_extraction_test.rs +87 -87
- data/vendor/rb-sys/.cargo-ok +1 -0
- data/vendor/rb-sys/.cargo_vcs_info.json +6 -0
- data/vendor/rb-sys/Cargo.lock +393 -0
- data/vendor/rb-sys/Cargo.toml +70 -0
- data/vendor/rb-sys/Cargo.toml.orig +57 -0
- data/vendor/rb-sys/LICENSE-APACHE +190 -0
- data/vendor/rb-sys/LICENSE-MIT +21 -0
- data/vendor/rb-sys/bin/release.sh +21 -0
- data/vendor/rb-sys/build/features.rs +108 -0
- data/vendor/rb-sys/build/main.rs +246 -0
- data/vendor/rb-sys/build/stable_api_config.rs +153 -0
- data/vendor/rb-sys/build/version.rs +48 -0
- data/vendor/rb-sys/readme.md +36 -0
- data/vendor/rb-sys/src/bindings.rs +21 -0
- data/vendor/rb-sys/src/hidden.rs +11 -0
- data/vendor/rb-sys/src/lib.rs +34 -0
- data/vendor/rb-sys/src/macros.rs +371 -0
- data/vendor/rb-sys/src/memory.rs +53 -0
- data/vendor/rb-sys/src/ruby_abi_version.rs +38 -0
- data/vendor/rb-sys/src/special_consts.rs +31 -0
- data/vendor/rb-sys/src/stable_api/compiled.c +179 -0
- data/vendor/rb-sys/src/stable_api/compiled.rs +257 -0
- data/vendor/rb-sys/src/stable_api/ruby_2_6.rs +316 -0
- data/vendor/rb-sys/src/stable_api/ruby_2_7.rs +316 -0
- data/vendor/rb-sys/src/stable_api/ruby_3_0.rs +324 -0
- data/vendor/rb-sys/src/stable_api/ruby_3_1.rs +317 -0
- data/vendor/rb-sys/src/stable_api/ruby_3_2.rs +315 -0
- data/vendor/rb-sys/src/stable_api/ruby_3_3.rs +326 -0
- data/vendor/rb-sys/src/stable_api/ruby_3_4.rs +327 -0
- data/vendor/rb-sys/src/stable_api.rs +261 -0
- data/vendor/rb-sys/src/symbol.rs +31 -0
- data/vendor/rb-sys/src/tracking_allocator.rs +332 -0
- data/vendor/rb-sys/src/utils.rs +89 -0
- data/vendor/rb-sys/src/value_type.rs +7 -0
- metadata +90 -95
- data/pkg/kreuzberg-4.0.0.rc1.gem +0 -0
- data/spec/examples.txt +0 -104
- data/vendor/kreuzberg/src/bin/profile_extract.rs +0 -455
- data/vendor/kreuzberg/src/extraction/pandoc/batch.rs +0 -275
- data/vendor/kreuzberg/src/extraction/pandoc/mime_types.rs +0 -178
- data/vendor/kreuzberg/src/extraction/pandoc/mod.rs +0 -491
- data/vendor/kreuzberg/src/extraction/pandoc/server.rs +0 -496
- data/vendor/kreuzberg/src/extraction/pandoc/subprocess.rs +0 -1188
- data/vendor/kreuzberg/src/extraction/pandoc/version.rs +0 -162
- data/vendor/kreuzberg/src/extractors/pandoc.rs +0 -201
- data/vendor/kreuzberg/tests/chunking_offset_demo.rs +0 -92
- data/vendor/kreuzberg/tests/pandoc_integration.rs +0 -503
|
@@ -1,954 +1,954 @@
|
|
|
1
|
-
//! Archive extraction functionality.
|
|
2
|
-
//!
|
|
3
|
-
//! This module provides functions for extracting file lists and contents from archives.
|
|
4
|
-
|
|
5
|
-
use crate::error::{KreuzbergError, Result};
|
|
6
|
-
use sevenz_rust::SevenZReader;
|
|
7
|
-
use std::collections::HashMap;
|
|
8
|
-
use std::io::{Cursor, Read};
|
|
9
|
-
use tar::Archive as TarArchive;
|
|
10
|
-
use zip::ZipArchive;
|
|
11
|
-
|
|
12
|
-
/// Archive metadata extracted from an archive file.
|
|
13
|
-
#[derive(Debug, Clone)]
|
|
14
|
-
pub struct ArchiveMetadata {
|
|
15
|
-
/// Archive format (e.g., "ZIP", "TAR")
|
|
16
|
-
pub format: String,
|
|
17
|
-
/// List of files in the archive
|
|
18
|
-
pub file_list: Vec<ArchiveEntry>,
|
|
19
|
-
/// Total number of files
|
|
20
|
-
pub file_count: usize,
|
|
21
|
-
/// Total uncompressed size in bytes
|
|
22
|
-
pub total_size: u64,
|
|
23
|
-
}
|
|
24
|
-
|
|
25
|
-
/// Information about a single file in an archive.
|
|
26
|
-
#[derive(Debug, Clone)]
|
|
27
|
-
pub struct ArchiveEntry {
|
|
28
|
-
/// File path within the archive
|
|
29
|
-
pub path: String,
|
|
30
|
-
/// File size in bytes
|
|
31
|
-
pub size: u64,
|
|
32
|
-
/// Whether this is a directory
|
|
33
|
-
pub is_dir: bool,
|
|
34
|
-
}
|
|
35
|
-
|
|
36
|
-
/// Extract metadata from a ZIP archive.
|
|
37
|
-
pub fn extract_zip_metadata(bytes: &[u8]) -> Result<ArchiveMetadata> {
|
|
38
|
-
let cursor = Cursor::new(bytes);
|
|
39
|
-
let mut archive =
|
|
40
|
-
ZipArchive::new(cursor).map_err(|e| KreuzbergError::parsing(format!("Failed to read ZIP archive: {}", e)))?;
|
|
41
|
-
|
|
42
|
-
let mut file_list = Vec::new();
|
|
43
|
-
let mut total_size = 0u64;
|
|
44
|
-
|
|
45
|
-
for i in 0..archive.len() {
|
|
46
|
-
let file = archive
|
|
47
|
-
.by_index(i)
|
|
48
|
-
.map_err(|e| KreuzbergError::parsing(format!("Failed to read ZIP entry: {}", e)))?;
|
|
49
|
-
|
|
50
|
-
let path = file.name().to_string();
|
|
51
|
-
let size = file.size();
|
|
52
|
-
let is_dir = file.is_dir();
|
|
53
|
-
|
|
54
|
-
if !is_dir {
|
|
55
|
-
total_size += size;
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
file_list.push(ArchiveEntry { path, size, is_dir });
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
Ok(ArchiveMetadata {
|
|
62
|
-
format: "ZIP".to_string(),
|
|
63
|
-
file_list,
|
|
64
|
-
file_count: archive.len(),
|
|
65
|
-
total_size,
|
|
66
|
-
})
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
/// Extract metadata from a TAR archive.
|
|
70
|
-
pub fn extract_tar_metadata(bytes: &[u8]) -> Result<ArchiveMetadata> {
|
|
71
|
-
let cursor = Cursor::new(bytes);
|
|
72
|
-
let mut archive = TarArchive::new(cursor);
|
|
73
|
-
|
|
74
|
-
let mut file_list = Vec::new();
|
|
75
|
-
let mut total_size = 0u64;
|
|
76
|
-
let mut file_count = 0;
|
|
77
|
-
|
|
78
|
-
let entries = archive
|
|
79
|
-
.entries()
|
|
80
|
-
.map_err(|e| KreuzbergError::parsing(format!("Failed to read TAR archive: {}", e)))?;
|
|
81
|
-
|
|
82
|
-
for entry_result in entries {
|
|
83
|
-
let entry = entry_result.map_err(|e| KreuzbergError::parsing(format!("Failed to read TAR entry: {}", e)))?;
|
|
84
|
-
|
|
85
|
-
let path = entry
|
|
86
|
-
.path()
|
|
87
|
-
.map_err(|e| KreuzbergError::parsing(format!("Failed to read TAR entry path: {}", e)))?
|
|
88
|
-
.to_string_lossy()
|
|
89
|
-
.to_string();
|
|
90
|
-
|
|
91
|
-
let size = entry.size();
|
|
92
|
-
let is_dir = entry.header().entry_type().is_dir();
|
|
93
|
-
|
|
94
|
-
if !is_dir {
|
|
95
|
-
total_size += size;
|
|
96
|
-
}
|
|
97
|
-
|
|
98
|
-
file_count += 1;
|
|
99
|
-
file_list.push(ArchiveEntry { path, size, is_dir });
|
|
100
|
-
}
|
|
101
|
-
|
|
102
|
-
Ok(ArchiveMetadata {
|
|
103
|
-
format: "TAR".to_string(),
|
|
104
|
-
file_list,
|
|
105
|
-
file_count,
|
|
106
|
-
total_size,
|
|
107
|
-
})
|
|
108
|
-
}
|
|
109
|
-
|
|
110
|
-
/// Extract text content from files within a ZIP archive.
|
|
111
|
-
///
|
|
112
|
-
/// Only extracts files with common text extensions: .txt, .md, .json, .xml, .html, .csv, .log
|
|
113
|
-
pub fn extract_zip_text_content(bytes: &[u8]) -> Result<HashMap<String, String>> {
|
|
114
|
-
let cursor = Cursor::new(bytes);
|
|
115
|
-
let mut archive =
|
|
116
|
-
ZipArchive::new(cursor).map_err(|e| KreuzbergError::parsing(format!("Failed to read ZIP archive: {}", e)))?;
|
|
117
|
-
|
|
118
|
-
let mut contents = HashMap::new();
|
|
119
|
-
let text_extensions = [
|
|
120
|
-
".txt", ".md", ".json", ".xml", ".html", ".csv", ".log", ".yaml", ".toml",
|
|
121
|
-
];
|
|
122
|
-
|
|
123
|
-
for i in 0..archive.len() {
|
|
124
|
-
let mut file = archive
|
|
125
|
-
.by_index(i)
|
|
126
|
-
.map_err(|e| KreuzbergError::parsing(format!("Failed to read ZIP entry: {}", e)))?;
|
|
127
|
-
|
|
128
|
-
let path = file.name().to_string();
|
|
129
|
-
|
|
130
|
-
if !file.is_dir() && text_extensions.iter().any(|ext| path.to_lowercase().ends_with(ext)) {
|
|
131
|
-
let mut content = String::new();
|
|
132
|
-
if file.read_to_string(&mut content).is_ok() {
|
|
133
|
-
contents.insert(path, content);
|
|
134
|
-
}
|
|
135
|
-
}
|
|
136
|
-
}
|
|
137
|
-
|
|
138
|
-
Ok(contents)
|
|
139
|
-
}
|
|
140
|
-
|
|
141
|
-
/// Extract text content from files within a TAR archive.
|
|
142
|
-
///
|
|
143
|
-
/// Only extracts files with common text extensions: .txt, .md, .json, .xml, .html, .csv, .log
|
|
144
|
-
pub fn extract_tar_text_content(bytes: &[u8]) -> Result<HashMap<String, String>> {
|
|
145
|
-
let cursor = Cursor::new(bytes);
|
|
146
|
-
let mut archive = TarArchive::new(cursor);
|
|
147
|
-
|
|
148
|
-
let mut contents = HashMap::new();
|
|
149
|
-
let text_extensions = [
|
|
150
|
-
".txt", ".md", ".json", ".xml", ".html", ".csv", ".log", ".yaml", ".toml",
|
|
151
|
-
];
|
|
152
|
-
|
|
153
|
-
let entries = archive
|
|
154
|
-
.entries()
|
|
155
|
-
.map_err(|e| KreuzbergError::parsing(format!("Failed to read TAR archive: {}", e)))?;
|
|
156
|
-
|
|
157
|
-
for entry_result in entries {
|
|
158
|
-
let mut entry =
|
|
159
|
-
entry_result.map_err(|e| KreuzbergError::parsing(format!("Failed to read TAR entry: {}", e)))?;
|
|
160
|
-
|
|
161
|
-
let path = entry
|
|
162
|
-
.path()
|
|
163
|
-
.map_err(|e| KreuzbergError::parsing(format!("Failed to read TAR entry path: {}", e)))?
|
|
164
|
-
.to_string_lossy()
|
|
165
|
-
.to_string();
|
|
166
|
-
|
|
167
|
-
if !entry.header().entry_type().is_dir() && text_extensions.iter().any(|ext| path.to_lowercase().ends_with(ext))
|
|
168
|
-
{
|
|
169
|
-
let mut content = String::new();
|
|
170
|
-
if entry.read_to_string(&mut content).is_ok() {
|
|
171
|
-
contents.insert(path, content);
|
|
172
|
-
}
|
|
173
|
-
}
|
|
174
|
-
}
|
|
175
|
-
|
|
176
|
-
Ok(contents)
|
|
177
|
-
}
|
|
178
|
-
|
|
179
|
-
/// Extract metadata from a 7z archive.
|
|
180
|
-
pub fn extract_7z_metadata(bytes: &[u8]) -> Result<ArchiveMetadata> {
|
|
181
|
-
let cursor = Cursor::new(bytes);
|
|
182
|
-
let archive = SevenZReader::new(cursor, bytes.len() as u64, "".into())
|
|
183
|
-
.map_err(|e| KreuzbergError::parsing(format!("Failed to read 7z archive: {}", e)))?;
|
|
184
|
-
|
|
185
|
-
let mut file_list = Vec::new();
|
|
186
|
-
let mut total_size = 0u64;
|
|
187
|
-
|
|
188
|
-
for entry in &archive.archive().files {
|
|
189
|
-
let path = entry.name().to_string();
|
|
190
|
-
let size = entry.size();
|
|
191
|
-
let is_dir = entry.is_directory();
|
|
192
|
-
|
|
193
|
-
if !is_dir {
|
|
194
|
-
total_size += size;
|
|
195
|
-
}
|
|
196
|
-
|
|
197
|
-
file_list.push(ArchiveEntry { path, size, is_dir });
|
|
198
|
-
}
|
|
199
|
-
|
|
200
|
-
let file_count = file_list.len();
|
|
201
|
-
|
|
202
|
-
Ok(ArchiveMetadata {
|
|
203
|
-
format: "7Z".to_string(),
|
|
204
|
-
file_list,
|
|
205
|
-
file_count,
|
|
206
|
-
total_size,
|
|
207
|
-
})
|
|
208
|
-
}
|
|
209
|
-
|
|
210
|
-
/// Extract text content from files within a 7z archive.
|
|
211
|
-
///
|
|
212
|
-
/// Only extracts files with common text extensions: .txt, .md, .json, .xml, .html, .csv, .log
|
|
213
|
-
pub fn extract_7z_text_content(bytes: &[u8]) -> Result<HashMap<String, String>> {
|
|
214
|
-
let cursor = Cursor::new(bytes);
|
|
215
|
-
let mut archive = SevenZReader::new(cursor, bytes.len() as u64, "".into())
|
|
216
|
-
.map_err(|e| KreuzbergError::parsing(format!("Failed to read 7z archive: {}", e)))?;
|
|
217
|
-
|
|
218
|
-
let mut contents = HashMap::new();
|
|
219
|
-
let text_extensions = [
|
|
220
|
-
".txt", ".md", ".json", ".xml", ".html", ".csv", ".log", ".yaml", ".toml",
|
|
221
|
-
];
|
|
222
|
-
|
|
223
|
-
archive
|
|
224
|
-
.for_each_entries(|entry, reader| {
|
|
225
|
-
let path = entry.name().to_string();
|
|
226
|
-
|
|
227
|
-
if !entry.is_directory() && text_extensions.iter().any(|ext| path.to_lowercase().ends_with(ext)) {
|
|
228
|
-
let mut content = Vec::new();
|
|
229
|
-
if let Ok(_) = reader.read_to_end(&mut content)
|
|
230
|
-
&& let Ok(text) = String::from_utf8(content)
|
|
231
|
-
{
|
|
232
|
-
contents.insert(path, text);
|
|
233
|
-
}
|
|
234
|
-
}
|
|
235
|
-
Ok(true)
|
|
236
|
-
})
|
|
237
|
-
.map_err(|e| KreuzbergError::parsing(format!("Failed to read 7z entries: {}", e)))?;
|
|
238
|
-
|
|
239
|
-
Ok(contents)
|
|
240
|
-
}
|
|
241
|
-
|
|
242
|
-
#[cfg(test)]
|
|
243
|
-
mod tests {
|
|
244
|
-
use super::*;
|
|
245
|
-
use std::io::Write;
|
|
246
|
-
use tar::Builder as TarBuilder;
|
|
247
|
-
use zip::write::{FileOptions, ZipWriter};
|
|
248
|
-
|
|
249
|
-
#[test]
|
|
250
|
-
fn test_extract_zip_metadata() {
|
|
251
|
-
let mut cursor = Cursor::new(Vec::new());
|
|
252
|
-
{
|
|
253
|
-
let mut zip = ZipWriter::new(&mut cursor);
|
|
254
|
-
let options = FileOptions::<'_, ()>::default();
|
|
255
|
-
|
|
256
|
-
zip.start_file("test.txt", options).unwrap();
|
|
257
|
-
zip.write_all(b"Hello, World!").unwrap();
|
|
258
|
-
|
|
259
|
-
zip.start_file("dir/file.md", options).unwrap();
|
|
260
|
-
zip.write_all(b"# Header").unwrap();
|
|
261
|
-
|
|
262
|
-
zip.finish().unwrap();
|
|
263
|
-
}
|
|
264
|
-
|
|
265
|
-
let bytes = cursor.into_inner();
|
|
266
|
-
let metadata = extract_zip_metadata(&bytes).unwrap();
|
|
267
|
-
|
|
268
|
-
assert_eq!(metadata.format, "ZIP");
|
|
269
|
-
assert_eq!(metadata.file_count, 2);
|
|
270
|
-
assert_eq!(metadata.file_list.len(), 2);
|
|
271
|
-
assert!(metadata.total_size > 0);
|
|
272
|
-
}
|
|
273
|
-
|
|
274
|
-
#[test]
|
|
275
|
-
fn test_extract_tar_metadata() {
|
|
276
|
-
let mut cursor = Cursor::new(Vec::new());
|
|
277
|
-
{
|
|
278
|
-
let mut tar = TarBuilder::new(&mut cursor);
|
|
279
|
-
|
|
280
|
-
let data1 = b"Hello, World!";
|
|
281
|
-
let mut header1 = tar::Header::new_gnu();
|
|
282
|
-
header1.set_path("test.txt").unwrap();
|
|
283
|
-
header1.set_size(data1.len() as u64);
|
|
284
|
-
header1.set_cksum();
|
|
285
|
-
tar.append(&header1, &data1[..]).unwrap();
|
|
286
|
-
|
|
287
|
-
let data2 = b"# Header";
|
|
288
|
-
let mut header2 = tar::Header::new_gnu();
|
|
289
|
-
header2.set_path("dir/file.md").unwrap();
|
|
290
|
-
header2.set_size(data2.len() as u64);
|
|
291
|
-
header2.set_cksum();
|
|
292
|
-
tar.append(&header2, &data2[..]).unwrap();
|
|
293
|
-
|
|
294
|
-
tar.finish().unwrap();
|
|
295
|
-
}
|
|
296
|
-
|
|
297
|
-
let bytes = cursor.into_inner();
|
|
298
|
-
let metadata = extract_tar_metadata(&bytes).unwrap();
|
|
299
|
-
|
|
300
|
-
assert_eq!(metadata.format, "TAR");
|
|
301
|
-
assert_eq!(metadata.file_count, 2);
|
|
302
|
-
assert_eq!(metadata.file_list.len(), 2);
|
|
303
|
-
assert!(metadata.total_size > 0);
|
|
304
|
-
}
|
|
305
|
-
|
|
306
|
-
#[test]
|
|
307
|
-
fn test_extract_zip_text_content() {
|
|
308
|
-
let mut cursor = Cursor::new(Vec::new());
|
|
309
|
-
{
|
|
310
|
-
let mut zip = ZipWriter::new(&mut cursor);
|
|
311
|
-
let options = FileOptions::<'_, ()>::default();
|
|
312
|
-
|
|
313
|
-
zip.start_file("test.txt", options).unwrap();
|
|
314
|
-
zip.write_all(b"Hello, World!").unwrap();
|
|
315
|
-
|
|
316
|
-
zip.start_file("readme.md", options).unwrap();
|
|
317
|
-
zip.write_all(b"# README").unwrap();
|
|
318
|
-
|
|
319
|
-
zip.finish().unwrap();
|
|
320
|
-
}
|
|
321
|
-
|
|
322
|
-
let bytes = cursor.into_inner();
|
|
323
|
-
let contents = extract_zip_text_content(&bytes).unwrap();
|
|
324
|
-
|
|
325
|
-
assert_eq!(contents.len(), 2);
|
|
326
|
-
assert_eq!(contents.get("test.txt").unwrap(), "Hello, World!");
|
|
327
|
-
assert_eq!(contents.get("readme.md").unwrap(), "# README");
|
|
328
|
-
}
|
|
329
|
-
|
|
330
|
-
#[test]
|
|
331
|
-
fn test_extract_tar_text_content() {
|
|
332
|
-
let mut cursor = Cursor::new(Vec::new());
|
|
333
|
-
{
|
|
334
|
-
let mut tar = TarBuilder::new(&mut cursor);
|
|
335
|
-
|
|
336
|
-
let data1 = b"Hello, World!";
|
|
337
|
-
let mut header1 = tar::Header::new_gnu();
|
|
338
|
-
header1.set_path("test.txt").unwrap();
|
|
339
|
-
header1.set_size(data1.len() as u64);
|
|
340
|
-
header1.set_cksum();
|
|
341
|
-
tar.append(&header1, &data1[..]).unwrap();
|
|
342
|
-
|
|
343
|
-
let data2 = b"# README";
|
|
344
|
-
let mut header2 = tar::Header::new_gnu();
|
|
345
|
-
header2.set_path("readme.md").unwrap();
|
|
346
|
-
header2.set_size(data2.len() as u64);
|
|
347
|
-
header2.set_cksum();
|
|
348
|
-
tar.append(&header2, &data2[..]).unwrap();
|
|
349
|
-
|
|
350
|
-
tar.finish().unwrap();
|
|
351
|
-
}
|
|
352
|
-
|
|
353
|
-
let bytes = cursor.into_inner();
|
|
354
|
-
let contents = extract_tar_text_content(&bytes).unwrap();
|
|
355
|
-
|
|
356
|
-
assert_eq!(contents.len(), 2);
|
|
357
|
-
assert_eq!(contents.get("test.txt").unwrap(), "Hello, World!");
|
|
358
|
-
assert_eq!(contents.get("readme.md").unwrap(), "# README");
|
|
359
|
-
}
|
|
360
|
-
|
|
361
|
-
#[test]
|
|
362
|
-
fn test_extract_zip_metadata_invalid() {
|
|
363
|
-
let invalid_bytes = vec![0, 1, 2, 3, 4, 5];
|
|
364
|
-
let result = extract_zip_metadata(&invalid_bytes);
|
|
365
|
-
assert!(result.is_err());
|
|
366
|
-
}
|
|
367
|
-
|
|
368
|
-
#[test]
|
|
369
|
-
fn test_extract_tar_metadata_invalid() {
|
|
370
|
-
let invalid_bytes = vec![0, 1, 2, 3, 4, 5];
|
|
371
|
-
let result = extract_tar_metadata(&invalid_bytes);
|
|
372
|
-
assert!(result.is_err());
|
|
373
|
-
}
|
|
374
|
-
|
|
375
|
-
#[test]
|
|
376
|
-
fn test_extract_zip_metadata_with_directories() {
|
|
377
|
-
let mut cursor = Cursor::new(Vec::new());
|
|
378
|
-
{
|
|
379
|
-
let mut zip = ZipWriter::new(&mut cursor);
|
|
380
|
-
let options = FileOptions::<'_, ()>::default();
|
|
381
|
-
|
|
382
|
-
zip.add_directory("dir1/", options).unwrap();
|
|
383
|
-
zip.add_directory("dir1/subdir/", options).unwrap();
|
|
384
|
-
|
|
385
|
-
zip.start_file("dir1/file1.txt", options).unwrap();
|
|
386
|
-
zip.write_all(b"content1").unwrap();
|
|
387
|
-
|
|
388
|
-
zip.start_file("dir1/subdir/file2.txt", options).unwrap();
|
|
389
|
-
zip.write_all(b"content2").unwrap();
|
|
390
|
-
|
|
391
|
-
zip.finish().unwrap();
|
|
392
|
-
}
|
|
393
|
-
|
|
394
|
-
let bytes = cursor.into_inner();
|
|
395
|
-
let metadata = extract_zip_metadata(&bytes).unwrap();
|
|
396
|
-
|
|
397
|
-
assert_eq!(metadata.format, "ZIP");
|
|
398
|
-
assert_eq!(metadata.file_count, 4);
|
|
399
|
-
assert_eq!(metadata.total_size, 16);
|
|
400
|
-
|
|
401
|
-
let dir_count = metadata.file_list.iter().filter(|e| e.is_dir).count();
|
|
402
|
-
assert_eq!(dir_count, 2);
|
|
403
|
-
}
|
|
404
|
-
|
|
405
|
-
#[test]
|
|
406
|
-
fn test_extract_tar_metadata_with_directories() {
|
|
407
|
-
let mut cursor = Cursor::new(Vec::new());
|
|
408
|
-
{
|
|
409
|
-
let mut tar = TarBuilder::new(&mut cursor);
|
|
410
|
-
|
|
411
|
-
let mut header_dir = tar::Header::new_gnu();
|
|
412
|
-
header_dir.set_path("dir1/").unwrap();
|
|
413
|
-
header_dir.set_size(0);
|
|
414
|
-
header_dir.set_entry_type(tar::EntryType::Directory);
|
|
415
|
-
header_dir.set_cksum();
|
|
416
|
-
tar.append(&header_dir, &[][..]).unwrap();
|
|
417
|
-
|
|
418
|
-
let data = b"content1";
|
|
419
|
-
let mut header1 = tar::Header::new_gnu();
|
|
420
|
-
header1.set_path("dir1/file1.txt").unwrap();
|
|
421
|
-
header1.set_size(data.len() as u64);
|
|
422
|
-
header1.set_cksum();
|
|
423
|
-
tar.append(&header1, &data[..]).unwrap();
|
|
424
|
-
|
|
425
|
-
tar.finish().unwrap();
|
|
426
|
-
}
|
|
427
|
-
|
|
428
|
-
let bytes = cursor.into_inner();
|
|
429
|
-
let metadata = extract_tar_metadata(&bytes).unwrap();
|
|
430
|
-
|
|
431
|
-
assert_eq!(metadata.format, "TAR");
|
|
432
|
-
assert_eq!(metadata.file_count, 2);
|
|
433
|
-
|
|
434
|
-
let dir_count = metadata.file_list.iter().filter(|e| e.is_dir).count();
|
|
435
|
-
assert_eq!(dir_count, 1);
|
|
436
|
-
}
|
|
437
|
-
|
|
438
|
-
#[test]
|
|
439
|
-
fn test_extract_tar_gz_metadata() {
|
|
440
|
-
let mut tar_data = Vec::new();
|
|
441
|
-
{
|
|
442
|
-
let mut tar = TarBuilder::new(&mut tar_data);
|
|
443
|
-
|
|
444
|
-
let data = b"Hello from gzip!";
|
|
445
|
-
let mut header = tar::Header::new_gnu();
|
|
446
|
-
header.set_path("test.txt").unwrap();
|
|
447
|
-
header.set_size(data.len() as u64);
|
|
448
|
-
header.set_cksum();
|
|
449
|
-
tar.append(&header, &data[..]).unwrap();
|
|
450
|
-
|
|
451
|
-
tar.finish().unwrap();
|
|
452
|
-
}
|
|
453
|
-
|
|
454
|
-
let metadata = extract_tar_metadata(&tar_data).unwrap();
|
|
455
|
-
assert_eq!(metadata.format, "TAR");
|
|
456
|
-
assert_eq!(metadata.file_count, 1);
|
|
457
|
-
assert_eq!(metadata.file_list[0].path, "test.txt");
|
|
458
|
-
}
|
|
459
|
-
|
|
460
|
-
#[test]
|
|
461
|
-
fn test_extract_7z_metadata_with_files() {
|
|
462
|
-
use sevenz_rust::SevenZWriter;
|
|
463
|
-
|
|
464
|
-
let mut cursor = Cursor::new(Vec::new());
|
|
465
|
-
{
|
|
466
|
-
let mut sz = SevenZWriter::new(&mut cursor).unwrap();
|
|
467
|
-
|
|
468
|
-
sz.push_archive_entry(
|
|
469
|
-
sevenz_rust::SevenZArchiveEntry::from_path("test.txt", "test.txt".to_string()),
|
|
470
|
-
Some(Cursor::new(b"Hello 7z!".to_vec())),
|
|
471
|
-
)
|
|
472
|
-
.unwrap();
|
|
473
|
-
|
|
474
|
-
sz.push_archive_entry(
|
|
475
|
-
sevenz_rust::SevenZArchiveEntry::from_path("data.json", "data.json".to_string()),
|
|
476
|
-
Some(Cursor::new(b"{\"key\":\"value\"}".to_vec())),
|
|
477
|
-
)
|
|
478
|
-
.unwrap();
|
|
479
|
-
|
|
480
|
-
sz.finish().unwrap();
|
|
481
|
-
}
|
|
482
|
-
|
|
483
|
-
let bytes = cursor.into_inner();
|
|
484
|
-
let metadata = extract_7z_metadata(&bytes).unwrap();
|
|
485
|
-
|
|
486
|
-
assert_eq!(metadata.format, "7Z");
|
|
487
|
-
assert_eq!(metadata.file_count, 2);
|
|
488
|
-
assert!(metadata.total_size > 0);
|
|
489
|
-
}
|
|
490
|
-
|
|
491
|
-
#[test]
|
|
492
|
-
fn test_extract_zip_within_zip() {
|
|
493
|
-
let mut inner_cursor = Cursor::new(Vec::new());
|
|
494
|
-
{
|
|
495
|
-
let mut inner_zip = ZipWriter::new(&mut inner_cursor);
|
|
496
|
-
let options = FileOptions::<'_, ()>::default();
|
|
497
|
-
|
|
498
|
-
inner_zip.start_file("inner.txt", options).unwrap();
|
|
499
|
-
inner_zip.write_all(b"Nested content").unwrap();
|
|
500
|
-
|
|
501
|
-
inner_zip.finish().unwrap();
|
|
502
|
-
}
|
|
503
|
-
let inner_bytes = inner_cursor.into_inner();
|
|
504
|
-
|
|
505
|
-
let mut outer_cursor = Cursor::new(Vec::new());
|
|
506
|
-
{
|
|
507
|
-
let mut outer_zip = ZipWriter::new(&mut outer_cursor);
|
|
508
|
-
let options = FileOptions::<'_, ()>::default();
|
|
509
|
-
|
|
510
|
-
outer_zip.start_file("archive.zip", options).unwrap();
|
|
511
|
-
outer_zip.write_all(&inner_bytes).unwrap();
|
|
512
|
-
|
|
513
|
-
outer_zip.start_file("readme.txt", options).unwrap();
|
|
514
|
-
outer_zip.write_all(b"Outer content").unwrap();
|
|
515
|
-
|
|
516
|
-
outer_zip.finish().unwrap();
|
|
517
|
-
}
|
|
518
|
-
|
|
519
|
-
let outer_bytes = outer_cursor.into_inner();
|
|
520
|
-
let metadata = extract_zip_metadata(&outer_bytes).unwrap();
|
|
521
|
-
|
|
522
|
-
assert_eq!(metadata.file_count, 2);
|
|
523
|
-
|
|
524
|
-
let archive_entry = metadata.file_list.iter().find(|e| e.path == "archive.zip");
|
|
525
|
-
assert!(archive_entry.is_some());
|
|
526
|
-
assert!(archive_entry.unwrap().size > 0);
|
|
527
|
-
}
|
|
528
|
-
|
|
529
|
-
#[test]
|
|
530
|
-
fn test_extract_tar_within_tar() {
|
|
531
|
-
let mut inner_cursor = Cursor::new(Vec::new());
|
|
532
|
-
{
|
|
533
|
-
let mut inner_tar = TarBuilder::new(&mut inner_cursor);
|
|
534
|
-
|
|
535
|
-
let data = b"Nested content";
|
|
536
|
-
let mut header = tar::Header::new_gnu();
|
|
537
|
-
header.set_path("inner.txt").unwrap();
|
|
538
|
-
header.set_size(data.len() as u64);
|
|
539
|
-
header.set_cksum();
|
|
540
|
-
inner_tar.append(&header, &data[..]).unwrap();
|
|
541
|
-
|
|
542
|
-
inner_tar.finish().unwrap();
|
|
543
|
-
}
|
|
544
|
-
let inner_bytes = inner_cursor.into_inner();
|
|
545
|
-
|
|
546
|
-
let mut outer_cursor = Cursor::new(Vec::new());
|
|
547
|
-
{
|
|
548
|
-
let mut outer_tar = TarBuilder::new(&mut outer_cursor);
|
|
549
|
-
|
|
550
|
-
let mut header1 = tar::Header::new_gnu();
|
|
551
|
-
header1.set_path("archive.tar").unwrap();
|
|
552
|
-
header1.set_size(inner_bytes.len() as u64);
|
|
553
|
-
header1.set_cksum();
|
|
554
|
-
outer_tar.append(&header1, &inner_bytes[..]).unwrap();
|
|
555
|
-
|
|
556
|
-
let data = b"Outer content";
|
|
557
|
-
let mut header2 = tar::Header::new_gnu();
|
|
558
|
-
header2.set_path("readme.txt").unwrap();
|
|
559
|
-
header2.set_size(data.len() as u64);
|
|
560
|
-
header2.set_cksum();
|
|
561
|
-
outer_tar.append(&header2, &data[..]).unwrap();
|
|
562
|
-
|
|
563
|
-
outer_tar.finish().unwrap();
|
|
564
|
-
}
|
|
565
|
-
|
|
566
|
-
let outer_bytes = outer_cursor.into_inner();
|
|
567
|
-
let metadata = extract_tar_metadata(&outer_bytes).unwrap();
|
|
568
|
-
|
|
569
|
-
assert_eq!(metadata.file_count, 2);
|
|
570
|
-
|
|
571
|
-
let archive_entry = metadata.file_list.iter().find(|e| e.path == "archive.tar");
|
|
572
|
-
assert!(archive_entry.is_some());
|
|
573
|
-
}
|
|
574
|
-
|
|
575
|
-
#[test]
|
|
576
|
-
fn test_extract_zip_corrupted_data() {
|
|
577
|
-
let mut valid_cursor = Cursor::new(Vec::new());
|
|
578
|
-
{
|
|
579
|
-
let mut zip = ZipWriter::new(&mut valid_cursor);
|
|
580
|
-
let options = FileOptions::<'_, ()>::default();
|
|
581
|
-
|
|
582
|
-
zip.start_file("test.txt", options).unwrap();
|
|
583
|
-
zip.write_all(b"content").unwrap();
|
|
584
|
-
|
|
585
|
-
zip.finish().unwrap();
|
|
586
|
-
}
|
|
587
|
-
|
|
588
|
-
let mut corrupted = valid_cursor.into_inner();
|
|
589
|
-
corrupted.truncate(corrupted.len() / 2);
|
|
590
|
-
|
|
591
|
-
let result = extract_zip_metadata(&corrupted);
|
|
592
|
-
assert!(result.is_err());
|
|
593
|
-
|
|
594
|
-
if let Err(e) = result {
|
|
595
|
-
assert!(matches!(e, KreuzbergError::Parsing { .. }));
|
|
596
|
-
}
|
|
597
|
-
}
|
|
598
|
-
|
|
599
|
-
#[test]
|
|
600
|
-
fn test_extract_tar_corrupted_data() {
|
|
601
|
-
let mut valid_cursor = Cursor::new(Vec::new());
|
|
602
|
-
{
|
|
603
|
-
let mut tar = TarBuilder::new(&mut valid_cursor);
|
|
604
|
-
|
|
605
|
-
let data = b"content";
|
|
606
|
-
let mut header = tar::Header::new_gnu();
|
|
607
|
-
header.set_path("test.txt").unwrap();
|
|
608
|
-
header.set_size(data.len() as u64);
|
|
609
|
-
header.set_cksum();
|
|
610
|
-
tar.append(&header, &data[..]).unwrap();
|
|
611
|
-
|
|
612
|
-
tar.finish().unwrap();
|
|
613
|
-
}
|
|
614
|
-
|
|
615
|
-
let mut corrupted = valid_cursor.into_inner();
|
|
616
|
-
corrupted[100] = 0xFF;
|
|
617
|
-
|
|
618
|
-
let result = extract_tar_metadata(&corrupted);
|
|
619
|
-
assert!(result.is_err());
|
|
620
|
-
}
|
|
621
|
-
|
|
622
|
-
#[test]
|
|
623
|
-
fn test_extract_zip_empty_archive() {
|
|
624
|
-
let mut cursor = Cursor::new(Vec::new());
|
|
625
|
-
{
|
|
626
|
-
let zip = ZipWriter::new(&mut cursor);
|
|
627
|
-
zip.finish().unwrap();
|
|
628
|
-
}
|
|
629
|
-
|
|
630
|
-
let bytes = cursor.into_inner();
|
|
631
|
-
let metadata = extract_zip_metadata(&bytes).unwrap();
|
|
632
|
-
|
|
633
|
-
assert_eq!(metadata.format, "ZIP");
|
|
634
|
-
assert_eq!(metadata.file_count, 0);
|
|
635
|
-
assert_eq!(metadata.total_size, 0);
|
|
636
|
-
assert_eq!(metadata.file_list.len(), 0);
|
|
637
|
-
}
|
|
638
|
-
|
|
639
|
-
#[test]
|
|
640
|
-
fn test_extract_tar_empty_archive() {
|
|
641
|
-
let mut cursor = Cursor::new(Vec::new());
|
|
642
|
-
{
|
|
643
|
-
let mut tar = TarBuilder::new(&mut cursor);
|
|
644
|
-
tar.finish().unwrap();
|
|
645
|
-
}
|
|
646
|
-
|
|
647
|
-
let bytes = cursor.into_inner();
|
|
648
|
-
let metadata = extract_tar_metadata(&bytes).unwrap();
|
|
649
|
-
|
|
650
|
-
assert_eq!(metadata.format, "TAR");
|
|
651
|
-
assert_eq!(metadata.file_count, 0);
|
|
652
|
-
assert_eq!(metadata.total_size, 0);
|
|
653
|
-
assert_eq!(metadata.file_list.len(), 0);
|
|
654
|
-
}
|
|
655
|
-
|
|
656
|
-
#[test]
|
|
657
|
-
fn test_extract_zip_multiple_text_files() {
|
|
658
|
-
let mut cursor = Cursor::new(Vec::new());
|
|
659
|
-
{
|
|
660
|
-
let mut zip = ZipWriter::new(&mut cursor);
|
|
661
|
-
let options = FileOptions::<'_, ()>::default();
|
|
662
|
-
|
|
663
|
-
zip.start_file("file1.txt", options).unwrap();
|
|
664
|
-
zip.write_all(b"Content 1").unwrap();
|
|
665
|
-
|
|
666
|
-
zip.start_file("file2.md", options).unwrap();
|
|
667
|
-
zip.write_all(b"# Markdown").unwrap();
|
|
668
|
-
|
|
669
|
-
zip.start_file("data.json", options).unwrap();
|
|
670
|
-
zip.write_all(b"{\"key\":\"value\"}").unwrap();
|
|
671
|
-
|
|
672
|
-
zip.start_file("binary.bin", options).unwrap();
|
|
673
|
-
zip.write_all(&[0xFF, 0xFE, 0xFD]).unwrap();
|
|
674
|
-
|
|
675
|
-
zip.finish().unwrap();
|
|
676
|
-
}
|
|
677
|
-
|
|
678
|
-
let bytes = cursor.into_inner();
|
|
679
|
-
let contents = extract_zip_text_content(&bytes).unwrap();
|
|
680
|
-
|
|
681
|
-
assert_eq!(contents.len(), 3);
|
|
682
|
-
assert_eq!(contents.get("file1.txt").unwrap(), "Content 1");
|
|
683
|
-
assert_eq!(contents.get("file2.md").unwrap(), "# Markdown");
|
|
684
|
-
assert_eq!(contents.get("data.json").unwrap(), "{\"key\":\"value\"}");
|
|
685
|
-
assert!(!contents.contains_key("binary.bin"));
|
|
686
|
-
}
|
|
687
|
-
|
|
688
|
-
#[test]
|
|
689
|
-
fn test_extract_tar_multiple_text_files() {
|
|
690
|
-
let mut cursor = Cursor::new(Vec::new());
|
|
691
|
-
{
|
|
692
|
-
let mut tar = TarBuilder::new(&mut cursor);
|
|
693
|
-
|
|
694
|
-
let files = vec![
|
|
695
|
-
("file1.txt", b"Content 1" as &[u8]),
|
|
696
|
-
("file2.md", b"# Markdown"),
|
|
697
|
-
("data.xml", b"<root>data</root>"),
|
|
698
|
-
("config.yaml", b"key: value"),
|
|
699
|
-
];
|
|
700
|
-
|
|
701
|
-
for (path, data) in files {
|
|
702
|
-
let mut header = tar::Header::new_gnu();
|
|
703
|
-
header.set_path(path).unwrap();
|
|
704
|
-
header.set_size(data.len() as u64);
|
|
705
|
-
header.set_cksum();
|
|
706
|
-
tar.append(&header, data).unwrap();
|
|
707
|
-
}
|
|
708
|
-
|
|
709
|
-
tar.finish().unwrap();
|
|
710
|
-
}
|
|
711
|
-
|
|
712
|
-
let bytes = cursor.into_inner();
|
|
713
|
-
let contents = extract_tar_text_content(&bytes).unwrap();
|
|
714
|
-
|
|
715
|
-
assert_eq!(contents.len(), 4);
|
|
716
|
-
assert_eq!(contents.get("file1.txt").unwrap(), "Content 1");
|
|
717
|
-
assert_eq!(contents.get("file2.md").unwrap(), "# Markdown");
|
|
718
|
-
assert_eq!(contents.get("data.xml").unwrap(), "<root>data</root>");
|
|
719
|
-
assert_eq!(contents.get("config.yaml").unwrap(), "key: value");
|
|
720
|
-
}
|
|
721
|
-
|
|
722
|
-
#[test]
|
|
723
|
-
fn test_extract_zip_preserves_directory_structure() {
|
|
724
|
-
let mut cursor = Cursor::new(Vec::new());
|
|
725
|
-
{
|
|
726
|
-
let mut zip = ZipWriter::new(&mut cursor);
|
|
727
|
-
let options = FileOptions::<'_, ()>::default();
|
|
728
|
-
|
|
729
|
-
zip.add_directory("root/", options).unwrap();
|
|
730
|
-
zip.add_directory("root/sub1/", options).unwrap();
|
|
731
|
-
zip.add_directory("root/sub2/", options).unwrap();
|
|
732
|
-
|
|
733
|
-
zip.start_file("root/file1.txt", options).unwrap();
|
|
734
|
-
zip.write_all(b"Root file").unwrap();
|
|
735
|
-
|
|
736
|
-
zip.start_file("root/sub1/file2.txt", options).unwrap();
|
|
737
|
-
zip.write_all(b"Sub1 file").unwrap();
|
|
738
|
-
|
|
739
|
-
zip.start_file("root/sub2/file3.txt", options).unwrap();
|
|
740
|
-
zip.write_all(b"Sub2 file").unwrap();
|
|
741
|
-
|
|
742
|
-
zip.finish().unwrap();
|
|
743
|
-
}
|
|
744
|
-
|
|
745
|
-
let bytes = cursor.into_inner();
|
|
746
|
-
let metadata = extract_zip_metadata(&bytes).unwrap();
|
|
747
|
-
|
|
748
|
-
let paths: Vec<&str> = metadata.file_list.iter().map(|e| e.path.as_str()).collect();
|
|
749
|
-
assert!(paths.contains(&"root/"));
|
|
750
|
-
assert!(paths.contains(&"root/sub1/"));
|
|
751
|
-
assert!(paths.contains(&"root/sub2/"));
|
|
752
|
-
assert!(paths.contains(&"root/file1.txt"));
|
|
753
|
-
assert!(paths.contains(&"root/sub1/file2.txt"));
|
|
754
|
-
assert!(paths.contains(&"root/sub2/file3.txt"));
|
|
755
|
-
}
|
|
756
|
-
|
|
757
|
-
#[test]
|
|
758
|
-
fn test_extract_zip_with_large_file() {
|
|
759
|
-
let mut cursor = Cursor::new(Vec::new());
|
|
760
|
-
{
|
|
761
|
-
let mut zip = ZipWriter::new(&mut cursor);
|
|
762
|
-
let options = FileOptions::<'_, ()>::default();
|
|
763
|
-
|
|
764
|
-
let large_content = "x".repeat(10_000);
|
|
765
|
-
|
|
766
|
-
zip.start_file("large.txt", options).unwrap();
|
|
767
|
-
zip.write_all(large_content.as_bytes()).unwrap();
|
|
768
|
-
|
|
769
|
-
zip.finish().unwrap();
|
|
770
|
-
}
|
|
771
|
-
|
|
772
|
-
let bytes = cursor.into_inner();
|
|
773
|
-
let metadata = extract_zip_metadata(&bytes).unwrap();
|
|
774
|
-
|
|
775
|
-
assert_eq!(metadata.file_count, 1);
|
|
776
|
-
assert_eq!(metadata.total_size, 10_000);
|
|
777
|
-
|
|
778
|
-
let contents = extract_zip_text_content(&bytes).unwrap();
|
|
779
|
-
assert_eq!(contents.get("large.txt").unwrap().len(), 10_000);
|
|
780
|
-
}
|
|
781
|
-
|
|
782
|
-
#[test]
|
|
783
|
-
fn test_extract_zip_with_many_files() {
|
|
784
|
-
let mut cursor = Cursor::new(Vec::new());
|
|
785
|
-
{
|
|
786
|
-
let mut zip = ZipWriter::new(&mut cursor);
|
|
787
|
-
let options = FileOptions::<'_, ()>::default();
|
|
788
|
-
|
|
789
|
-
for i in 0..100 {
|
|
790
|
-
let filename = format!("file_{}.txt", i);
|
|
791
|
-
let content = format!("Content {}", i);
|
|
792
|
-
|
|
793
|
-
zip.start_file(&filename, options).unwrap();
|
|
794
|
-
zip.write_all(content.as_bytes()).unwrap();
|
|
795
|
-
}
|
|
796
|
-
|
|
797
|
-
zip.finish().unwrap();
|
|
798
|
-
}
|
|
799
|
-
|
|
800
|
-
let bytes = cursor.into_inner();
|
|
801
|
-
let metadata = extract_zip_metadata(&bytes).unwrap();
|
|
802
|
-
|
|
803
|
-
assert_eq!(metadata.file_count, 100);
|
|
804
|
-
assert_eq!(metadata.file_list.len(), 100);
|
|
805
|
-
|
|
806
|
-
let contents = extract_zip_text_content(&bytes).unwrap();
|
|
807
|
-
assert_eq!(contents.len(), 100);
|
|
808
|
-
}
|
|
809
|
-
|
|
810
|
-
#[test]
|
|
811
|
-
fn test_extract_zip_with_long_paths() {
|
|
812
|
-
let mut cursor = Cursor::new(Vec::new());
|
|
813
|
-
{
|
|
814
|
-
let mut zip = ZipWriter::new(&mut cursor);
|
|
815
|
-
let options = FileOptions::<'_, ()>::default();
|
|
816
|
-
|
|
817
|
-
let long_path = format!("{}/file.txt", "a".repeat(200));
|
|
818
|
-
|
|
819
|
-
zip.start_file(&long_path, options).unwrap();
|
|
820
|
-
zip.write_all(b"Deep file").unwrap();
|
|
821
|
-
|
|
822
|
-
zip.finish().unwrap();
|
|
823
|
-
}
|
|
824
|
-
|
|
825
|
-
let bytes = cursor.into_inner();
|
|
826
|
-
let metadata = extract_zip_metadata(&bytes).unwrap();
|
|
827
|
-
|
|
828
|
-
assert_eq!(metadata.file_count, 1);
|
|
829
|
-
assert!(metadata.file_list[0].path.len() > 200);
|
|
830
|
-
|
|
831
|
-
let contents = extract_zip_text_content(&bytes).unwrap();
|
|
832
|
-
assert_eq!(contents.len(), 1);
|
|
833
|
-
}
|
|
834
|
-
|
|
835
|
-
#[test]
|
|
836
|
-
fn test_extract_7z_text_content() {
|
|
837
|
-
use sevenz_rust::SevenZWriter;
|
|
838
|
-
|
|
839
|
-
let mut cursor = Cursor::new(Vec::new());
|
|
840
|
-
{
|
|
841
|
-
let mut sz = SevenZWriter::new(&mut cursor).unwrap();
|
|
842
|
-
|
|
843
|
-
sz.push_archive_entry(
|
|
844
|
-
sevenz_rust::SevenZArchiveEntry::from_path("test.txt", "test.txt".to_string()),
|
|
845
|
-
Some(Cursor::new(b"Hello 7z text!".to_vec())),
|
|
846
|
-
)
|
|
847
|
-
.unwrap();
|
|
848
|
-
|
|
849
|
-
sz.push_archive_entry(
|
|
850
|
-
sevenz_rust::SevenZArchiveEntry::from_path("readme.md", "readme.md".to_string()),
|
|
851
|
-
Some(Cursor::new(b"# 7z README".to_vec())),
|
|
852
|
-
)
|
|
853
|
-
.unwrap();
|
|
854
|
-
|
|
855
|
-
sz.finish().unwrap();
|
|
856
|
-
}
|
|
857
|
-
|
|
858
|
-
let bytes = cursor.into_inner();
|
|
859
|
-
let contents = extract_7z_text_content(&bytes).unwrap();
|
|
860
|
-
|
|
861
|
-
assert_eq!(contents.len(), 2);
|
|
862
|
-
assert_eq!(contents.get("test.txt").unwrap(), "Hello 7z text!");
|
|
863
|
-
assert_eq!(contents.get("readme.md").unwrap(), "# 7z README");
|
|
864
|
-
}
|
|
865
|
-
|
|
866
|
-
#[test]
|
|
867
|
-
fn test_extract_7z_empty_archive() {
|
|
868
|
-
use sevenz_rust::SevenZWriter;
|
|
869
|
-
|
|
870
|
-
let mut cursor = Cursor::new(Vec::new());
|
|
871
|
-
{
|
|
872
|
-
let sz = SevenZWriter::new(&mut cursor).unwrap();
|
|
873
|
-
sz.finish().unwrap();
|
|
874
|
-
}
|
|
875
|
-
|
|
876
|
-
let bytes = cursor.into_inner();
|
|
877
|
-
let metadata = extract_7z_metadata(&bytes).unwrap();
|
|
878
|
-
|
|
879
|
-
assert_eq!(metadata.format, "7Z");
|
|
880
|
-
assert_eq!(metadata.file_count, 0);
|
|
881
|
-
assert_eq!(metadata.total_size, 0);
|
|
882
|
-
}
|
|
883
|
-
|
|
884
|
-
#[test]
|
|
885
|
-
fn test_extract_tar_with_large_file() {
|
|
886
|
-
let mut cursor = Cursor::new(Vec::new());
|
|
887
|
-
{
|
|
888
|
-
let mut tar = TarBuilder::new(&mut cursor);
|
|
889
|
-
|
|
890
|
-
let large_content = "y".repeat(50_000);
|
|
891
|
-
|
|
892
|
-
let mut header = tar::Header::new_gnu();
|
|
893
|
-
header.set_path("large.txt").unwrap();
|
|
894
|
-
header.set_size(large_content.len() as u64);
|
|
895
|
-
header.set_cksum();
|
|
896
|
-
tar.append(&header, large_content.as_bytes()).unwrap();
|
|
897
|
-
|
|
898
|
-
tar.finish().unwrap();
|
|
899
|
-
}
|
|
900
|
-
|
|
901
|
-
let bytes = cursor.into_inner();
|
|
902
|
-
let metadata = extract_tar_metadata(&bytes).unwrap();
|
|
903
|
-
|
|
904
|
-
assert_eq!(metadata.file_count, 1);
|
|
905
|
-
assert_eq!(metadata.total_size, 50_000);
|
|
906
|
-
|
|
907
|
-
let contents = extract_tar_text_content(&bytes).unwrap();
|
|
908
|
-
assert_eq!(contents.get("large.txt").unwrap().len(), 50_000);
|
|
909
|
-
}
|
|
910
|
-
|
|
911
|
-
#[test]
|
|
912
|
-
fn test_extract_zip_text_content_filters_non_text_extensions() {
|
|
913
|
-
let mut cursor = Cursor::new(Vec::new());
|
|
914
|
-
{
|
|
915
|
-
let mut zip = ZipWriter::new(&mut cursor);
|
|
916
|
-
let options = FileOptions::<'_, ()>::default();
|
|
917
|
-
|
|
918
|
-
zip.start_file("document.txt", options).unwrap();
|
|
919
|
-
zip.write_all(b"Text file").unwrap();
|
|
920
|
-
|
|
921
|
-
zip.start_file("image.png", options).unwrap();
|
|
922
|
-
zip.write_all(&[0x89, 0x50, 0x4E, 0x47]).unwrap();
|
|
923
|
-
|
|
924
|
-
zip.start_file("binary.exe", options).unwrap();
|
|
925
|
-
zip.write_all(&[0x4D, 0x5A]).unwrap();
|
|
926
|
-
|
|
927
|
-
zip.start_file("config.toml", options).unwrap();
|
|
928
|
-
zip.write_all(b"[section]").unwrap();
|
|
929
|
-
|
|
930
|
-
zip.finish().unwrap();
|
|
931
|
-
}
|
|
932
|
-
|
|
933
|
-
let bytes = cursor.into_inner();
|
|
934
|
-
let contents = extract_zip_text_content(&bytes).unwrap();
|
|
935
|
-
|
|
936
|
-
assert_eq!(contents.len(), 2);
|
|
937
|
-
assert!(contents.contains_key("document.txt"));
|
|
938
|
-
assert!(contents.contains_key("config.toml"));
|
|
939
|
-
assert!(!contents.contains_key("image.png"));
|
|
940
|
-
assert!(!contents.contains_key("binary.exe"));
|
|
941
|
-
}
|
|
942
|
-
|
|
943
|
-
#[test]
|
|
944
|
-
fn test_extract_7z_corrupted_data() {
|
|
945
|
-
let invalid_7z_data = vec![0x37, 0x7A, 0xBC, 0xAF, 0x27, 0x1C, 0x00];
|
|
946
|
-
|
|
947
|
-
let result = extract_7z_metadata(&invalid_7z_data);
|
|
948
|
-
assert!(result.is_err());
|
|
949
|
-
|
|
950
|
-
if let Err(e) = result {
|
|
951
|
-
assert!(matches!(e, KreuzbergError::Parsing { .. }));
|
|
952
|
-
}
|
|
953
|
-
}
|
|
954
|
-
}
|
|
1
|
+
//! Archive extraction functionality.
|
|
2
|
+
//!
|
|
3
|
+
//! This module provides functions for extracting file lists and contents from archives.
|
|
4
|
+
|
|
5
|
+
use crate::error::{KreuzbergError, Result};
|
|
6
|
+
use sevenz_rust::SevenZReader;
|
|
7
|
+
use std::collections::HashMap;
|
|
8
|
+
use std::io::{Cursor, Read};
|
|
9
|
+
use tar::Archive as TarArchive;
|
|
10
|
+
use zip::ZipArchive;
|
|
11
|
+
|
|
12
|
+
/// Archive metadata extracted from an archive file.
|
|
13
|
+
#[derive(Debug, Clone)]
|
|
14
|
+
pub struct ArchiveMetadata {
|
|
15
|
+
/// Archive format (e.g., "ZIP", "TAR")
|
|
16
|
+
pub format: String,
|
|
17
|
+
/// List of files in the archive
|
|
18
|
+
pub file_list: Vec<ArchiveEntry>,
|
|
19
|
+
/// Total number of files
|
|
20
|
+
pub file_count: usize,
|
|
21
|
+
/// Total uncompressed size in bytes
|
|
22
|
+
pub total_size: u64,
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/// Information about a single file in an archive.
|
|
26
|
+
#[derive(Debug, Clone)]
|
|
27
|
+
pub struct ArchiveEntry {
|
|
28
|
+
/// File path within the archive
|
|
29
|
+
pub path: String,
|
|
30
|
+
/// File size in bytes
|
|
31
|
+
pub size: u64,
|
|
32
|
+
/// Whether this is a directory
|
|
33
|
+
pub is_dir: bool,
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/// Extract metadata from a ZIP archive.
|
|
37
|
+
pub fn extract_zip_metadata(bytes: &[u8]) -> Result<ArchiveMetadata> {
|
|
38
|
+
let cursor = Cursor::new(bytes);
|
|
39
|
+
let mut archive =
|
|
40
|
+
ZipArchive::new(cursor).map_err(|e| KreuzbergError::parsing(format!("Failed to read ZIP archive: {}", e)))?;
|
|
41
|
+
|
|
42
|
+
let mut file_list = Vec::new();
|
|
43
|
+
let mut total_size = 0u64;
|
|
44
|
+
|
|
45
|
+
for i in 0..archive.len() {
|
|
46
|
+
let file = archive
|
|
47
|
+
.by_index(i)
|
|
48
|
+
.map_err(|e| KreuzbergError::parsing(format!("Failed to read ZIP entry: {}", e)))?;
|
|
49
|
+
|
|
50
|
+
let path = file.name().to_string();
|
|
51
|
+
let size = file.size();
|
|
52
|
+
let is_dir = file.is_dir();
|
|
53
|
+
|
|
54
|
+
if !is_dir {
|
|
55
|
+
total_size += size;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
file_list.push(ArchiveEntry { path, size, is_dir });
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
Ok(ArchiveMetadata {
|
|
62
|
+
format: "ZIP".to_string(),
|
|
63
|
+
file_list,
|
|
64
|
+
file_count: archive.len(),
|
|
65
|
+
total_size,
|
|
66
|
+
})
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/// Extract metadata from a TAR archive.
|
|
70
|
+
pub fn extract_tar_metadata(bytes: &[u8]) -> Result<ArchiveMetadata> {
|
|
71
|
+
let cursor = Cursor::new(bytes);
|
|
72
|
+
let mut archive = TarArchive::new(cursor);
|
|
73
|
+
|
|
74
|
+
let mut file_list = Vec::new();
|
|
75
|
+
let mut total_size = 0u64;
|
|
76
|
+
let mut file_count = 0;
|
|
77
|
+
|
|
78
|
+
let entries = archive
|
|
79
|
+
.entries()
|
|
80
|
+
.map_err(|e| KreuzbergError::parsing(format!("Failed to read TAR archive: {}", e)))?;
|
|
81
|
+
|
|
82
|
+
for entry_result in entries {
|
|
83
|
+
let entry = entry_result.map_err(|e| KreuzbergError::parsing(format!("Failed to read TAR entry: {}", e)))?;
|
|
84
|
+
|
|
85
|
+
let path = entry
|
|
86
|
+
.path()
|
|
87
|
+
.map_err(|e| KreuzbergError::parsing(format!("Failed to read TAR entry path: {}", e)))?
|
|
88
|
+
.to_string_lossy()
|
|
89
|
+
.to_string();
|
|
90
|
+
|
|
91
|
+
let size = entry.size();
|
|
92
|
+
let is_dir = entry.header().entry_type().is_dir();
|
|
93
|
+
|
|
94
|
+
if !is_dir {
|
|
95
|
+
total_size += size;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
file_count += 1;
|
|
99
|
+
file_list.push(ArchiveEntry { path, size, is_dir });
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
Ok(ArchiveMetadata {
|
|
103
|
+
format: "TAR".to_string(),
|
|
104
|
+
file_list,
|
|
105
|
+
file_count,
|
|
106
|
+
total_size,
|
|
107
|
+
})
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/// Extract text content from files within a ZIP archive.
|
|
111
|
+
///
|
|
112
|
+
/// Only extracts files with common text extensions: .txt, .md, .json, .xml, .html, .csv, .log
|
|
113
|
+
pub fn extract_zip_text_content(bytes: &[u8]) -> Result<HashMap<String, String>> {
|
|
114
|
+
let cursor = Cursor::new(bytes);
|
|
115
|
+
let mut archive =
|
|
116
|
+
ZipArchive::new(cursor).map_err(|e| KreuzbergError::parsing(format!("Failed to read ZIP archive: {}", e)))?;
|
|
117
|
+
|
|
118
|
+
let mut contents = HashMap::new();
|
|
119
|
+
let text_extensions = [
|
|
120
|
+
".txt", ".md", ".json", ".xml", ".html", ".csv", ".log", ".yaml", ".toml",
|
|
121
|
+
];
|
|
122
|
+
|
|
123
|
+
for i in 0..archive.len() {
|
|
124
|
+
let mut file = archive
|
|
125
|
+
.by_index(i)
|
|
126
|
+
.map_err(|e| KreuzbergError::parsing(format!("Failed to read ZIP entry: {}", e)))?;
|
|
127
|
+
|
|
128
|
+
let path = file.name().to_string();
|
|
129
|
+
|
|
130
|
+
if !file.is_dir() && text_extensions.iter().any(|ext| path.to_lowercase().ends_with(ext)) {
|
|
131
|
+
let mut content = String::new();
|
|
132
|
+
if file.read_to_string(&mut content).is_ok() {
|
|
133
|
+
contents.insert(path, content);
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
Ok(contents)
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/// Extract text content from files within a TAR archive.
|
|
142
|
+
///
|
|
143
|
+
/// Only extracts files with common text extensions: .txt, .md, .json, .xml, .html, .csv, .log
|
|
144
|
+
pub fn extract_tar_text_content(bytes: &[u8]) -> Result<HashMap<String, String>> {
|
|
145
|
+
let cursor = Cursor::new(bytes);
|
|
146
|
+
let mut archive = TarArchive::new(cursor);
|
|
147
|
+
|
|
148
|
+
let mut contents = HashMap::new();
|
|
149
|
+
let text_extensions = [
|
|
150
|
+
".txt", ".md", ".json", ".xml", ".html", ".csv", ".log", ".yaml", ".toml",
|
|
151
|
+
];
|
|
152
|
+
|
|
153
|
+
let entries = archive
|
|
154
|
+
.entries()
|
|
155
|
+
.map_err(|e| KreuzbergError::parsing(format!("Failed to read TAR archive: {}", e)))?;
|
|
156
|
+
|
|
157
|
+
for entry_result in entries {
|
|
158
|
+
let mut entry =
|
|
159
|
+
entry_result.map_err(|e| KreuzbergError::parsing(format!("Failed to read TAR entry: {}", e)))?;
|
|
160
|
+
|
|
161
|
+
let path = entry
|
|
162
|
+
.path()
|
|
163
|
+
.map_err(|e| KreuzbergError::parsing(format!("Failed to read TAR entry path: {}", e)))?
|
|
164
|
+
.to_string_lossy()
|
|
165
|
+
.to_string();
|
|
166
|
+
|
|
167
|
+
if !entry.header().entry_type().is_dir() && text_extensions.iter().any(|ext| path.to_lowercase().ends_with(ext))
|
|
168
|
+
{
|
|
169
|
+
let mut content = String::new();
|
|
170
|
+
if entry.read_to_string(&mut content).is_ok() {
|
|
171
|
+
contents.insert(path, content);
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
Ok(contents)
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/// Extract metadata from a 7z archive.
|
|
180
|
+
pub fn extract_7z_metadata(bytes: &[u8]) -> Result<ArchiveMetadata> {
|
|
181
|
+
let cursor = Cursor::new(bytes);
|
|
182
|
+
let archive = SevenZReader::new(cursor, bytes.len() as u64, "".into())
|
|
183
|
+
.map_err(|e| KreuzbergError::parsing(format!("Failed to read 7z archive: {}", e)))?;
|
|
184
|
+
|
|
185
|
+
let mut file_list = Vec::new();
|
|
186
|
+
let mut total_size = 0u64;
|
|
187
|
+
|
|
188
|
+
for entry in &archive.archive().files {
|
|
189
|
+
let path = entry.name().to_string();
|
|
190
|
+
let size = entry.size();
|
|
191
|
+
let is_dir = entry.is_directory();
|
|
192
|
+
|
|
193
|
+
if !is_dir {
|
|
194
|
+
total_size += size;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
file_list.push(ArchiveEntry { path, size, is_dir });
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
let file_count = file_list.len();
|
|
201
|
+
|
|
202
|
+
Ok(ArchiveMetadata {
|
|
203
|
+
format: "7Z".to_string(),
|
|
204
|
+
file_list,
|
|
205
|
+
file_count,
|
|
206
|
+
total_size,
|
|
207
|
+
})
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
/// Extract text content from files within a 7z archive.
|
|
211
|
+
///
|
|
212
|
+
/// Only extracts files with common text extensions: .txt, .md, .json, .xml, .html, .csv, .log
|
|
213
|
+
pub fn extract_7z_text_content(bytes: &[u8]) -> Result<HashMap<String, String>> {
|
|
214
|
+
let cursor = Cursor::new(bytes);
|
|
215
|
+
let mut archive = SevenZReader::new(cursor, bytes.len() as u64, "".into())
|
|
216
|
+
.map_err(|e| KreuzbergError::parsing(format!("Failed to read 7z archive: {}", e)))?;
|
|
217
|
+
|
|
218
|
+
let mut contents = HashMap::new();
|
|
219
|
+
let text_extensions = [
|
|
220
|
+
".txt", ".md", ".json", ".xml", ".html", ".csv", ".log", ".yaml", ".toml",
|
|
221
|
+
];
|
|
222
|
+
|
|
223
|
+
archive
|
|
224
|
+
.for_each_entries(|entry, reader| {
|
|
225
|
+
let path = entry.name().to_string();
|
|
226
|
+
|
|
227
|
+
if !entry.is_directory() && text_extensions.iter().any(|ext| path.to_lowercase().ends_with(ext)) {
|
|
228
|
+
let mut content = Vec::new();
|
|
229
|
+
if let Ok(_) = reader.read_to_end(&mut content)
|
|
230
|
+
&& let Ok(text) = String::from_utf8(content)
|
|
231
|
+
{
|
|
232
|
+
contents.insert(path, text);
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
Ok(true)
|
|
236
|
+
})
|
|
237
|
+
.map_err(|e| KreuzbergError::parsing(format!("Failed to read 7z entries: {}", e)))?;
|
|
238
|
+
|
|
239
|
+
Ok(contents)
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
#[cfg(test)]
|
|
243
|
+
mod tests {
|
|
244
|
+
use super::*;
|
|
245
|
+
use std::io::Write;
|
|
246
|
+
use tar::Builder as TarBuilder;
|
|
247
|
+
use zip::write::{FileOptions, ZipWriter};
|
|
248
|
+
|
|
249
|
+
#[test]
|
|
250
|
+
fn test_extract_zip_metadata() {
|
|
251
|
+
let mut cursor = Cursor::new(Vec::new());
|
|
252
|
+
{
|
|
253
|
+
let mut zip = ZipWriter::new(&mut cursor);
|
|
254
|
+
let options = FileOptions::<'_, ()>::default();
|
|
255
|
+
|
|
256
|
+
zip.start_file("test.txt", options).unwrap();
|
|
257
|
+
zip.write_all(b"Hello, World!").unwrap();
|
|
258
|
+
|
|
259
|
+
zip.start_file("dir/file.md", options).unwrap();
|
|
260
|
+
zip.write_all(b"# Header").unwrap();
|
|
261
|
+
|
|
262
|
+
zip.finish().unwrap();
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
let bytes = cursor.into_inner();
|
|
266
|
+
let metadata = extract_zip_metadata(&bytes).unwrap();
|
|
267
|
+
|
|
268
|
+
assert_eq!(metadata.format, "ZIP");
|
|
269
|
+
assert_eq!(metadata.file_count, 2);
|
|
270
|
+
assert_eq!(metadata.file_list.len(), 2);
|
|
271
|
+
assert!(metadata.total_size > 0);
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
#[test]
|
|
275
|
+
fn test_extract_tar_metadata() {
|
|
276
|
+
let mut cursor = Cursor::new(Vec::new());
|
|
277
|
+
{
|
|
278
|
+
let mut tar = TarBuilder::new(&mut cursor);
|
|
279
|
+
|
|
280
|
+
let data1 = b"Hello, World!";
|
|
281
|
+
let mut header1 = tar::Header::new_gnu();
|
|
282
|
+
header1.set_path("test.txt").unwrap();
|
|
283
|
+
header1.set_size(data1.len() as u64);
|
|
284
|
+
header1.set_cksum();
|
|
285
|
+
tar.append(&header1, &data1[..]).unwrap();
|
|
286
|
+
|
|
287
|
+
let data2 = b"# Header";
|
|
288
|
+
let mut header2 = tar::Header::new_gnu();
|
|
289
|
+
header2.set_path("dir/file.md").unwrap();
|
|
290
|
+
header2.set_size(data2.len() as u64);
|
|
291
|
+
header2.set_cksum();
|
|
292
|
+
tar.append(&header2, &data2[..]).unwrap();
|
|
293
|
+
|
|
294
|
+
tar.finish().unwrap();
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
let bytes = cursor.into_inner();
|
|
298
|
+
let metadata = extract_tar_metadata(&bytes).unwrap();
|
|
299
|
+
|
|
300
|
+
assert_eq!(metadata.format, "TAR");
|
|
301
|
+
assert_eq!(metadata.file_count, 2);
|
|
302
|
+
assert_eq!(metadata.file_list.len(), 2);
|
|
303
|
+
assert!(metadata.total_size > 0);
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
#[test]
|
|
307
|
+
fn test_extract_zip_text_content() {
|
|
308
|
+
let mut cursor = Cursor::new(Vec::new());
|
|
309
|
+
{
|
|
310
|
+
let mut zip = ZipWriter::new(&mut cursor);
|
|
311
|
+
let options = FileOptions::<'_, ()>::default();
|
|
312
|
+
|
|
313
|
+
zip.start_file("test.txt", options).unwrap();
|
|
314
|
+
zip.write_all(b"Hello, World!").unwrap();
|
|
315
|
+
|
|
316
|
+
zip.start_file("readme.md", options).unwrap();
|
|
317
|
+
zip.write_all(b"# README").unwrap();
|
|
318
|
+
|
|
319
|
+
zip.finish().unwrap();
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
let bytes = cursor.into_inner();
|
|
323
|
+
let contents = extract_zip_text_content(&bytes).unwrap();
|
|
324
|
+
|
|
325
|
+
assert_eq!(contents.len(), 2);
|
|
326
|
+
assert_eq!(contents.get("test.txt").unwrap(), "Hello, World!");
|
|
327
|
+
assert_eq!(contents.get("readme.md").unwrap(), "# README");
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
#[test]
|
|
331
|
+
fn test_extract_tar_text_content() {
|
|
332
|
+
let mut cursor = Cursor::new(Vec::new());
|
|
333
|
+
{
|
|
334
|
+
let mut tar = TarBuilder::new(&mut cursor);
|
|
335
|
+
|
|
336
|
+
let data1 = b"Hello, World!";
|
|
337
|
+
let mut header1 = tar::Header::new_gnu();
|
|
338
|
+
header1.set_path("test.txt").unwrap();
|
|
339
|
+
header1.set_size(data1.len() as u64);
|
|
340
|
+
header1.set_cksum();
|
|
341
|
+
tar.append(&header1, &data1[..]).unwrap();
|
|
342
|
+
|
|
343
|
+
let data2 = b"# README";
|
|
344
|
+
let mut header2 = tar::Header::new_gnu();
|
|
345
|
+
header2.set_path("readme.md").unwrap();
|
|
346
|
+
header2.set_size(data2.len() as u64);
|
|
347
|
+
header2.set_cksum();
|
|
348
|
+
tar.append(&header2, &data2[..]).unwrap();
|
|
349
|
+
|
|
350
|
+
tar.finish().unwrap();
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
let bytes = cursor.into_inner();
|
|
354
|
+
let contents = extract_tar_text_content(&bytes).unwrap();
|
|
355
|
+
|
|
356
|
+
assert_eq!(contents.len(), 2);
|
|
357
|
+
assert_eq!(contents.get("test.txt").unwrap(), "Hello, World!");
|
|
358
|
+
assert_eq!(contents.get("readme.md").unwrap(), "# README");
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
#[test]
|
|
362
|
+
fn test_extract_zip_metadata_invalid() {
|
|
363
|
+
let invalid_bytes = vec![0, 1, 2, 3, 4, 5];
|
|
364
|
+
let result = extract_zip_metadata(&invalid_bytes);
|
|
365
|
+
assert!(result.is_err());
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
#[test]
|
|
369
|
+
fn test_extract_tar_metadata_invalid() {
|
|
370
|
+
let invalid_bytes = vec![0, 1, 2, 3, 4, 5];
|
|
371
|
+
let result = extract_tar_metadata(&invalid_bytes);
|
|
372
|
+
assert!(result.is_err());
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
#[test]
|
|
376
|
+
fn test_extract_zip_metadata_with_directories() {
|
|
377
|
+
let mut cursor = Cursor::new(Vec::new());
|
|
378
|
+
{
|
|
379
|
+
let mut zip = ZipWriter::new(&mut cursor);
|
|
380
|
+
let options = FileOptions::<'_, ()>::default();
|
|
381
|
+
|
|
382
|
+
zip.add_directory("dir1/", options).unwrap();
|
|
383
|
+
zip.add_directory("dir1/subdir/", options).unwrap();
|
|
384
|
+
|
|
385
|
+
zip.start_file("dir1/file1.txt", options).unwrap();
|
|
386
|
+
zip.write_all(b"content1").unwrap();
|
|
387
|
+
|
|
388
|
+
zip.start_file("dir1/subdir/file2.txt", options).unwrap();
|
|
389
|
+
zip.write_all(b"content2").unwrap();
|
|
390
|
+
|
|
391
|
+
zip.finish().unwrap();
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
let bytes = cursor.into_inner();
|
|
395
|
+
let metadata = extract_zip_metadata(&bytes).unwrap();
|
|
396
|
+
|
|
397
|
+
assert_eq!(metadata.format, "ZIP");
|
|
398
|
+
assert_eq!(metadata.file_count, 4);
|
|
399
|
+
assert_eq!(metadata.total_size, 16);
|
|
400
|
+
|
|
401
|
+
let dir_count = metadata.file_list.iter().filter(|e| e.is_dir).count();
|
|
402
|
+
assert_eq!(dir_count, 2);
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
#[test]
|
|
406
|
+
fn test_extract_tar_metadata_with_directories() {
|
|
407
|
+
let mut cursor = Cursor::new(Vec::new());
|
|
408
|
+
{
|
|
409
|
+
let mut tar = TarBuilder::new(&mut cursor);
|
|
410
|
+
|
|
411
|
+
let mut header_dir = tar::Header::new_gnu();
|
|
412
|
+
header_dir.set_path("dir1/").unwrap();
|
|
413
|
+
header_dir.set_size(0);
|
|
414
|
+
header_dir.set_entry_type(tar::EntryType::Directory);
|
|
415
|
+
header_dir.set_cksum();
|
|
416
|
+
tar.append(&header_dir, &[][..]).unwrap();
|
|
417
|
+
|
|
418
|
+
let data = b"content1";
|
|
419
|
+
let mut header1 = tar::Header::new_gnu();
|
|
420
|
+
header1.set_path("dir1/file1.txt").unwrap();
|
|
421
|
+
header1.set_size(data.len() as u64);
|
|
422
|
+
header1.set_cksum();
|
|
423
|
+
tar.append(&header1, &data[..]).unwrap();
|
|
424
|
+
|
|
425
|
+
tar.finish().unwrap();
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
let bytes = cursor.into_inner();
|
|
429
|
+
let metadata = extract_tar_metadata(&bytes).unwrap();
|
|
430
|
+
|
|
431
|
+
assert_eq!(metadata.format, "TAR");
|
|
432
|
+
assert_eq!(metadata.file_count, 2);
|
|
433
|
+
|
|
434
|
+
let dir_count = metadata.file_list.iter().filter(|e| e.is_dir).count();
|
|
435
|
+
assert_eq!(dir_count, 1);
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
#[test]
|
|
439
|
+
fn test_extract_tar_gz_metadata() {
|
|
440
|
+
let mut tar_data = Vec::new();
|
|
441
|
+
{
|
|
442
|
+
let mut tar = TarBuilder::new(&mut tar_data);
|
|
443
|
+
|
|
444
|
+
let data = b"Hello from gzip!";
|
|
445
|
+
let mut header = tar::Header::new_gnu();
|
|
446
|
+
header.set_path("test.txt").unwrap();
|
|
447
|
+
header.set_size(data.len() as u64);
|
|
448
|
+
header.set_cksum();
|
|
449
|
+
tar.append(&header, &data[..]).unwrap();
|
|
450
|
+
|
|
451
|
+
tar.finish().unwrap();
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
let metadata = extract_tar_metadata(&tar_data).unwrap();
|
|
455
|
+
assert_eq!(metadata.format, "TAR");
|
|
456
|
+
assert_eq!(metadata.file_count, 1);
|
|
457
|
+
assert_eq!(metadata.file_list[0].path, "test.txt");
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
#[test]
|
|
461
|
+
fn test_extract_7z_metadata_with_files() {
|
|
462
|
+
use sevenz_rust::SevenZWriter;
|
|
463
|
+
|
|
464
|
+
let mut cursor = Cursor::new(Vec::new());
|
|
465
|
+
{
|
|
466
|
+
let mut sz = SevenZWriter::new(&mut cursor).unwrap();
|
|
467
|
+
|
|
468
|
+
sz.push_archive_entry(
|
|
469
|
+
sevenz_rust::SevenZArchiveEntry::from_path("test.txt", "test.txt".to_string()),
|
|
470
|
+
Some(Cursor::new(b"Hello 7z!".to_vec())),
|
|
471
|
+
)
|
|
472
|
+
.unwrap();
|
|
473
|
+
|
|
474
|
+
sz.push_archive_entry(
|
|
475
|
+
sevenz_rust::SevenZArchiveEntry::from_path("data.json", "data.json".to_string()),
|
|
476
|
+
Some(Cursor::new(b"{\"key\":\"value\"}".to_vec())),
|
|
477
|
+
)
|
|
478
|
+
.unwrap();
|
|
479
|
+
|
|
480
|
+
sz.finish().unwrap();
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
let bytes = cursor.into_inner();
|
|
484
|
+
let metadata = extract_7z_metadata(&bytes).unwrap();
|
|
485
|
+
|
|
486
|
+
assert_eq!(metadata.format, "7Z");
|
|
487
|
+
assert_eq!(metadata.file_count, 2);
|
|
488
|
+
assert!(metadata.total_size > 0);
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
#[test]
|
|
492
|
+
fn test_extract_zip_within_zip() {
|
|
493
|
+
let mut inner_cursor = Cursor::new(Vec::new());
|
|
494
|
+
{
|
|
495
|
+
let mut inner_zip = ZipWriter::new(&mut inner_cursor);
|
|
496
|
+
let options = FileOptions::<'_, ()>::default();
|
|
497
|
+
|
|
498
|
+
inner_zip.start_file("inner.txt", options).unwrap();
|
|
499
|
+
inner_zip.write_all(b"Nested content").unwrap();
|
|
500
|
+
|
|
501
|
+
inner_zip.finish().unwrap();
|
|
502
|
+
}
|
|
503
|
+
let inner_bytes = inner_cursor.into_inner();
|
|
504
|
+
|
|
505
|
+
let mut outer_cursor = Cursor::new(Vec::new());
|
|
506
|
+
{
|
|
507
|
+
let mut outer_zip = ZipWriter::new(&mut outer_cursor);
|
|
508
|
+
let options = FileOptions::<'_, ()>::default();
|
|
509
|
+
|
|
510
|
+
outer_zip.start_file("archive.zip", options).unwrap();
|
|
511
|
+
outer_zip.write_all(&inner_bytes).unwrap();
|
|
512
|
+
|
|
513
|
+
outer_zip.start_file("readme.txt", options).unwrap();
|
|
514
|
+
outer_zip.write_all(b"Outer content").unwrap();
|
|
515
|
+
|
|
516
|
+
outer_zip.finish().unwrap();
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
let outer_bytes = outer_cursor.into_inner();
|
|
520
|
+
let metadata = extract_zip_metadata(&outer_bytes).unwrap();
|
|
521
|
+
|
|
522
|
+
assert_eq!(metadata.file_count, 2);
|
|
523
|
+
|
|
524
|
+
let archive_entry = metadata.file_list.iter().find(|e| e.path == "archive.zip");
|
|
525
|
+
assert!(archive_entry.is_some());
|
|
526
|
+
assert!(archive_entry.unwrap().size > 0);
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
#[test]
|
|
530
|
+
fn test_extract_tar_within_tar() {
|
|
531
|
+
let mut inner_cursor = Cursor::new(Vec::new());
|
|
532
|
+
{
|
|
533
|
+
let mut inner_tar = TarBuilder::new(&mut inner_cursor);
|
|
534
|
+
|
|
535
|
+
let data = b"Nested content";
|
|
536
|
+
let mut header = tar::Header::new_gnu();
|
|
537
|
+
header.set_path("inner.txt").unwrap();
|
|
538
|
+
header.set_size(data.len() as u64);
|
|
539
|
+
header.set_cksum();
|
|
540
|
+
inner_tar.append(&header, &data[..]).unwrap();
|
|
541
|
+
|
|
542
|
+
inner_tar.finish().unwrap();
|
|
543
|
+
}
|
|
544
|
+
let inner_bytes = inner_cursor.into_inner();
|
|
545
|
+
|
|
546
|
+
let mut outer_cursor = Cursor::new(Vec::new());
|
|
547
|
+
{
|
|
548
|
+
let mut outer_tar = TarBuilder::new(&mut outer_cursor);
|
|
549
|
+
|
|
550
|
+
let mut header1 = tar::Header::new_gnu();
|
|
551
|
+
header1.set_path("archive.tar").unwrap();
|
|
552
|
+
header1.set_size(inner_bytes.len() as u64);
|
|
553
|
+
header1.set_cksum();
|
|
554
|
+
outer_tar.append(&header1, &inner_bytes[..]).unwrap();
|
|
555
|
+
|
|
556
|
+
let data = b"Outer content";
|
|
557
|
+
let mut header2 = tar::Header::new_gnu();
|
|
558
|
+
header2.set_path("readme.txt").unwrap();
|
|
559
|
+
header2.set_size(data.len() as u64);
|
|
560
|
+
header2.set_cksum();
|
|
561
|
+
outer_tar.append(&header2, &data[..]).unwrap();
|
|
562
|
+
|
|
563
|
+
outer_tar.finish().unwrap();
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
let outer_bytes = outer_cursor.into_inner();
|
|
567
|
+
let metadata = extract_tar_metadata(&outer_bytes).unwrap();
|
|
568
|
+
|
|
569
|
+
assert_eq!(metadata.file_count, 2);
|
|
570
|
+
|
|
571
|
+
let archive_entry = metadata.file_list.iter().find(|e| e.path == "archive.tar");
|
|
572
|
+
assert!(archive_entry.is_some());
|
|
573
|
+
}
|
|
574
|
+
|
|
575
|
+
#[test]
|
|
576
|
+
fn test_extract_zip_corrupted_data() {
|
|
577
|
+
let mut valid_cursor = Cursor::new(Vec::new());
|
|
578
|
+
{
|
|
579
|
+
let mut zip = ZipWriter::new(&mut valid_cursor);
|
|
580
|
+
let options = FileOptions::<'_, ()>::default();
|
|
581
|
+
|
|
582
|
+
zip.start_file("test.txt", options).unwrap();
|
|
583
|
+
zip.write_all(b"content").unwrap();
|
|
584
|
+
|
|
585
|
+
zip.finish().unwrap();
|
|
586
|
+
}
|
|
587
|
+
|
|
588
|
+
let mut corrupted = valid_cursor.into_inner();
|
|
589
|
+
corrupted.truncate(corrupted.len() / 2);
|
|
590
|
+
|
|
591
|
+
let result = extract_zip_metadata(&corrupted);
|
|
592
|
+
assert!(result.is_err());
|
|
593
|
+
|
|
594
|
+
if let Err(e) = result {
|
|
595
|
+
assert!(matches!(e, KreuzbergError::Parsing { .. }));
|
|
596
|
+
}
|
|
597
|
+
}
|
|
598
|
+
|
|
599
|
+
#[test]
|
|
600
|
+
fn test_extract_tar_corrupted_data() {
|
|
601
|
+
let mut valid_cursor = Cursor::new(Vec::new());
|
|
602
|
+
{
|
|
603
|
+
let mut tar = TarBuilder::new(&mut valid_cursor);
|
|
604
|
+
|
|
605
|
+
let data = b"content";
|
|
606
|
+
let mut header = tar::Header::new_gnu();
|
|
607
|
+
header.set_path("test.txt").unwrap();
|
|
608
|
+
header.set_size(data.len() as u64);
|
|
609
|
+
header.set_cksum();
|
|
610
|
+
tar.append(&header, &data[..]).unwrap();
|
|
611
|
+
|
|
612
|
+
tar.finish().unwrap();
|
|
613
|
+
}
|
|
614
|
+
|
|
615
|
+
let mut corrupted = valid_cursor.into_inner();
|
|
616
|
+
corrupted[100] = 0xFF;
|
|
617
|
+
|
|
618
|
+
let result = extract_tar_metadata(&corrupted);
|
|
619
|
+
assert!(result.is_err());
|
|
620
|
+
}
|
|
621
|
+
|
|
622
|
+
#[test]
|
|
623
|
+
fn test_extract_zip_empty_archive() {
|
|
624
|
+
let mut cursor = Cursor::new(Vec::new());
|
|
625
|
+
{
|
|
626
|
+
let zip = ZipWriter::new(&mut cursor);
|
|
627
|
+
zip.finish().unwrap();
|
|
628
|
+
}
|
|
629
|
+
|
|
630
|
+
let bytes = cursor.into_inner();
|
|
631
|
+
let metadata = extract_zip_metadata(&bytes).unwrap();
|
|
632
|
+
|
|
633
|
+
assert_eq!(metadata.format, "ZIP");
|
|
634
|
+
assert_eq!(metadata.file_count, 0);
|
|
635
|
+
assert_eq!(metadata.total_size, 0);
|
|
636
|
+
assert_eq!(metadata.file_list.len(), 0);
|
|
637
|
+
}
|
|
638
|
+
|
|
639
|
+
#[test]
|
|
640
|
+
fn test_extract_tar_empty_archive() {
|
|
641
|
+
let mut cursor = Cursor::new(Vec::new());
|
|
642
|
+
{
|
|
643
|
+
let mut tar = TarBuilder::new(&mut cursor);
|
|
644
|
+
tar.finish().unwrap();
|
|
645
|
+
}
|
|
646
|
+
|
|
647
|
+
let bytes = cursor.into_inner();
|
|
648
|
+
let metadata = extract_tar_metadata(&bytes).unwrap();
|
|
649
|
+
|
|
650
|
+
assert_eq!(metadata.format, "TAR");
|
|
651
|
+
assert_eq!(metadata.file_count, 0);
|
|
652
|
+
assert_eq!(metadata.total_size, 0);
|
|
653
|
+
assert_eq!(metadata.file_list.len(), 0);
|
|
654
|
+
}
|
|
655
|
+
|
|
656
|
+
#[test]
|
|
657
|
+
fn test_extract_zip_multiple_text_files() {
|
|
658
|
+
let mut cursor = Cursor::new(Vec::new());
|
|
659
|
+
{
|
|
660
|
+
let mut zip = ZipWriter::new(&mut cursor);
|
|
661
|
+
let options = FileOptions::<'_, ()>::default();
|
|
662
|
+
|
|
663
|
+
zip.start_file("file1.txt", options).unwrap();
|
|
664
|
+
zip.write_all(b"Content 1").unwrap();
|
|
665
|
+
|
|
666
|
+
zip.start_file("file2.md", options).unwrap();
|
|
667
|
+
zip.write_all(b"# Markdown").unwrap();
|
|
668
|
+
|
|
669
|
+
zip.start_file("data.json", options).unwrap();
|
|
670
|
+
zip.write_all(b"{\"key\":\"value\"}").unwrap();
|
|
671
|
+
|
|
672
|
+
zip.start_file("binary.bin", options).unwrap();
|
|
673
|
+
zip.write_all(&[0xFF, 0xFE, 0xFD]).unwrap();
|
|
674
|
+
|
|
675
|
+
zip.finish().unwrap();
|
|
676
|
+
}
|
|
677
|
+
|
|
678
|
+
let bytes = cursor.into_inner();
|
|
679
|
+
let contents = extract_zip_text_content(&bytes).unwrap();
|
|
680
|
+
|
|
681
|
+
assert_eq!(contents.len(), 3);
|
|
682
|
+
assert_eq!(contents.get("file1.txt").unwrap(), "Content 1");
|
|
683
|
+
assert_eq!(contents.get("file2.md").unwrap(), "# Markdown");
|
|
684
|
+
assert_eq!(contents.get("data.json").unwrap(), "{\"key\":\"value\"}");
|
|
685
|
+
assert!(!contents.contains_key("binary.bin"));
|
|
686
|
+
}
|
|
687
|
+
|
|
688
|
+
#[test]
|
|
689
|
+
fn test_extract_tar_multiple_text_files() {
|
|
690
|
+
let mut cursor = Cursor::new(Vec::new());
|
|
691
|
+
{
|
|
692
|
+
let mut tar = TarBuilder::new(&mut cursor);
|
|
693
|
+
|
|
694
|
+
let files = vec![
|
|
695
|
+
("file1.txt", b"Content 1" as &[u8]),
|
|
696
|
+
("file2.md", b"# Markdown"),
|
|
697
|
+
("data.xml", b"<root>data</root>"),
|
|
698
|
+
("config.yaml", b"key: value"),
|
|
699
|
+
];
|
|
700
|
+
|
|
701
|
+
for (path, data) in files {
|
|
702
|
+
let mut header = tar::Header::new_gnu();
|
|
703
|
+
header.set_path(path).unwrap();
|
|
704
|
+
header.set_size(data.len() as u64);
|
|
705
|
+
header.set_cksum();
|
|
706
|
+
tar.append(&header, data).unwrap();
|
|
707
|
+
}
|
|
708
|
+
|
|
709
|
+
tar.finish().unwrap();
|
|
710
|
+
}
|
|
711
|
+
|
|
712
|
+
let bytes = cursor.into_inner();
|
|
713
|
+
let contents = extract_tar_text_content(&bytes).unwrap();
|
|
714
|
+
|
|
715
|
+
assert_eq!(contents.len(), 4);
|
|
716
|
+
assert_eq!(contents.get("file1.txt").unwrap(), "Content 1");
|
|
717
|
+
assert_eq!(contents.get("file2.md").unwrap(), "# Markdown");
|
|
718
|
+
assert_eq!(contents.get("data.xml").unwrap(), "<root>data</root>");
|
|
719
|
+
assert_eq!(contents.get("config.yaml").unwrap(), "key: value");
|
|
720
|
+
}
|
|
721
|
+
|
|
722
|
+
#[test]
|
|
723
|
+
fn test_extract_zip_preserves_directory_structure() {
|
|
724
|
+
let mut cursor = Cursor::new(Vec::new());
|
|
725
|
+
{
|
|
726
|
+
let mut zip = ZipWriter::new(&mut cursor);
|
|
727
|
+
let options = FileOptions::<'_, ()>::default();
|
|
728
|
+
|
|
729
|
+
zip.add_directory("root/", options).unwrap();
|
|
730
|
+
zip.add_directory("root/sub1/", options).unwrap();
|
|
731
|
+
zip.add_directory("root/sub2/", options).unwrap();
|
|
732
|
+
|
|
733
|
+
zip.start_file("root/file1.txt", options).unwrap();
|
|
734
|
+
zip.write_all(b"Root file").unwrap();
|
|
735
|
+
|
|
736
|
+
zip.start_file("root/sub1/file2.txt", options).unwrap();
|
|
737
|
+
zip.write_all(b"Sub1 file").unwrap();
|
|
738
|
+
|
|
739
|
+
zip.start_file("root/sub2/file3.txt", options).unwrap();
|
|
740
|
+
zip.write_all(b"Sub2 file").unwrap();
|
|
741
|
+
|
|
742
|
+
zip.finish().unwrap();
|
|
743
|
+
}
|
|
744
|
+
|
|
745
|
+
let bytes = cursor.into_inner();
|
|
746
|
+
let metadata = extract_zip_metadata(&bytes).unwrap();
|
|
747
|
+
|
|
748
|
+
let paths: Vec<&str> = metadata.file_list.iter().map(|e| e.path.as_str()).collect();
|
|
749
|
+
assert!(paths.contains(&"root/"));
|
|
750
|
+
assert!(paths.contains(&"root/sub1/"));
|
|
751
|
+
assert!(paths.contains(&"root/sub2/"));
|
|
752
|
+
assert!(paths.contains(&"root/file1.txt"));
|
|
753
|
+
assert!(paths.contains(&"root/sub1/file2.txt"));
|
|
754
|
+
assert!(paths.contains(&"root/sub2/file3.txt"));
|
|
755
|
+
}
|
|
756
|
+
|
|
757
|
+
#[test]
|
|
758
|
+
fn test_extract_zip_with_large_file() {
|
|
759
|
+
let mut cursor = Cursor::new(Vec::new());
|
|
760
|
+
{
|
|
761
|
+
let mut zip = ZipWriter::new(&mut cursor);
|
|
762
|
+
let options = FileOptions::<'_, ()>::default();
|
|
763
|
+
|
|
764
|
+
let large_content = "x".repeat(10_000);
|
|
765
|
+
|
|
766
|
+
zip.start_file("large.txt", options).unwrap();
|
|
767
|
+
zip.write_all(large_content.as_bytes()).unwrap();
|
|
768
|
+
|
|
769
|
+
zip.finish().unwrap();
|
|
770
|
+
}
|
|
771
|
+
|
|
772
|
+
let bytes = cursor.into_inner();
|
|
773
|
+
let metadata = extract_zip_metadata(&bytes).unwrap();
|
|
774
|
+
|
|
775
|
+
assert_eq!(metadata.file_count, 1);
|
|
776
|
+
assert_eq!(metadata.total_size, 10_000);
|
|
777
|
+
|
|
778
|
+
let contents = extract_zip_text_content(&bytes).unwrap();
|
|
779
|
+
assert_eq!(contents.get("large.txt").unwrap().len(), 10_000);
|
|
780
|
+
}
|
|
781
|
+
|
|
782
|
+
#[test]
|
|
783
|
+
fn test_extract_zip_with_many_files() {
|
|
784
|
+
let mut cursor = Cursor::new(Vec::new());
|
|
785
|
+
{
|
|
786
|
+
let mut zip = ZipWriter::new(&mut cursor);
|
|
787
|
+
let options = FileOptions::<'_, ()>::default();
|
|
788
|
+
|
|
789
|
+
for i in 0..100 {
|
|
790
|
+
let filename = format!("file_{}.txt", i);
|
|
791
|
+
let content = format!("Content {}", i);
|
|
792
|
+
|
|
793
|
+
zip.start_file(&filename, options).unwrap();
|
|
794
|
+
zip.write_all(content.as_bytes()).unwrap();
|
|
795
|
+
}
|
|
796
|
+
|
|
797
|
+
zip.finish().unwrap();
|
|
798
|
+
}
|
|
799
|
+
|
|
800
|
+
let bytes = cursor.into_inner();
|
|
801
|
+
let metadata = extract_zip_metadata(&bytes).unwrap();
|
|
802
|
+
|
|
803
|
+
assert_eq!(metadata.file_count, 100);
|
|
804
|
+
assert_eq!(metadata.file_list.len(), 100);
|
|
805
|
+
|
|
806
|
+
let contents = extract_zip_text_content(&bytes).unwrap();
|
|
807
|
+
assert_eq!(contents.len(), 100);
|
|
808
|
+
}
|
|
809
|
+
|
|
810
|
+
#[test]
|
|
811
|
+
fn test_extract_zip_with_long_paths() {
|
|
812
|
+
let mut cursor = Cursor::new(Vec::new());
|
|
813
|
+
{
|
|
814
|
+
let mut zip = ZipWriter::new(&mut cursor);
|
|
815
|
+
let options = FileOptions::<'_, ()>::default();
|
|
816
|
+
|
|
817
|
+
let long_path = format!("{}/file.txt", "a".repeat(200));
|
|
818
|
+
|
|
819
|
+
zip.start_file(&long_path, options).unwrap();
|
|
820
|
+
zip.write_all(b"Deep file").unwrap();
|
|
821
|
+
|
|
822
|
+
zip.finish().unwrap();
|
|
823
|
+
}
|
|
824
|
+
|
|
825
|
+
let bytes = cursor.into_inner();
|
|
826
|
+
let metadata = extract_zip_metadata(&bytes).unwrap();
|
|
827
|
+
|
|
828
|
+
assert_eq!(metadata.file_count, 1);
|
|
829
|
+
assert!(metadata.file_list[0].path.len() > 200);
|
|
830
|
+
|
|
831
|
+
let contents = extract_zip_text_content(&bytes).unwrap();
|
|
832
|
+
assert_eq!(contents.len(), 1);
|
|
833
|
+
}
|
|
834
|
+
|
|
835
|
+
#[test]
|
|
836
|
+
fn test_extract_7z_text_content() {
|
|
837
|
+
use sevenz_rust::SevenZWriter;
|
|
838
|
+
|
|
839
|
+
let mut cursor = Cursor::new(Vec::new());
|
|
840
|
+
{
|
|
841
|
+
let mut sz = SevenZWriter::new(&mut cursor).unwrap();
|
|
842
|
+
|
|
843
|
+
sz.push_archive_entry(
|
|
844
|
+
sevenz_rust::SevenZArchiveEntry::from_path("test.txt", "test.txt".to_string()),
|
|
845
|
+
Some(Cursor::new(b"Hello 7z text!".to_vec())),
|
|
846
|
+
)
|
|
847
|
+
.unwrap();
|
|
848
|
+
|
|
849
|
+
sz.push_archive_entry(
|
|
850
|
+
sevenz_rust::SevenZArchiveEntry::from_path("readme.md", "readme.md".to_string()),
|
|
851
|
+
Some(Cursor::new(b"# 7z README".to_vec())),
|
|
852
|
+
)
|
|
853
|
+
.unwrap();
|
|
854
|
+
|
|
855
|
+
sz.finish().unwrap();
|
|
856
|
+
}
|
|
857
|
+
|
|
858
|
+
let bytes = cursor.into_inner();
|
|
859
|
+
let contents = extract_7z_text_content(&bytes).unwrap();
|
|
860
|
+
|
|
861
|
+
assert_eq!(contents.len(), 2);
|
|
862
|
+
assert_eq!(contents.get("test.txt").unwrap(), "Hello 7z text!");
|
|
863
|
+
assert_eq!(contents.get("readme.md").unwrap(), "# 7z README");
|
|
864
|
+
}
|
|
865
|
+
|
|
866
|
+
#[test]
|
|
867
|
+
fn test_extract_7z_empty_archive() {
|
|
868
|
+
use sevenz_rust::SevenZWriter;
|
|
869
|
+
|
|
870
|
+
let mut cursor = Cursor::new(Vec::new());
|
|
871
|
+
{
|
|
872
|
+
let sz = SevenZWriter::new(&mut cursor).unwrap();
|
|
873
|
+
sz.finish().unwrap();
|
|
874
|
+
}
|
|
875
|
+
|
|
876
|
+
let bytes = cursor.into_inner();
|
|
877
|
+
let metadata = extract_7z_metadata(&bytes).unwrap();
|
|
878
|
+
|
|
879
|
+
assert_eq!(metadata.format, "7Z");
|
|
880
|
+
assert_eq!(metadata.file_count, 0);
|
|
881
|
+
assert_eq!(metadata.total_size, 0);
|
|
882
|
+
}
|
|
883
|
+
|
|
884
|
+
#[test]
|
|
885
|
+
fn test_extract_tar_with_large_file() {
|
|
886
|
+
let mut cursor = Cursor::new(Vec::new());
|
|
887
|
+
{
|
|
888
|
+
let mut tar = TarBuilder::new(&mut cursor);
|
|
889
|
+
|
|
890
|
+
let large_content = "y".repeat(50_000);
|
|
891
|
+
|
|
892
|
+
let mut header = tar::Header::new_gnu();
|
|
893
|
+
header.set_path("large.txt").unwrap();
|
|
894
|
+
header.set_size(large_content.len() as u64);
|
|
895
|
+
header.set_cksum();
|
|
896
|
+
tar.append(&header, large_content.as_bytes()).unwrap();
|
|
897
|
+
|
|
898
|
+
tar.finish().unwrap();
|
|
899
|
+
}
|
|
900
|
+
|
|
901
|
+
let bytes = cursor.into_inner();
|
|
902
|
+
let metadata = extract_tar_metadata(&bytes).unwrap();
|
|
903
|
+
|
|
904
|
+
assert_eq!(metadata.file_count, 1);
|
|
905
|
+
assert_eq!(metadata.total_size, 50_000);
|
|
906
|
+
|
|
907
|
+
let contents = extract_tar_text_content(&bytes).unwrap();
|
|
908
|
+
assert_eq!(contents.get("large.txt").unwrap().len(), 50_000);
|
|
909
|
+
}
|
|
910
|
+
|
|
911
|
+
#[test]
|
|
912
|
+
fn test_extract_zip_text_content_filters_non_text_extensions() {
|
|
913
|
+
let mut cursor = Cursor::new(Vec::new());
|
|
914
|
+
{
|
|
915
|
+
let mut zip = ZipWriter::new(&mut cursor);
|
|
916
|
+
let options = FileOptions::<'_, ()>::default();
|
|
917
|
+
|
|
918
|
+
zip.start_file("document.txt", options).unwrap();
|
|
919
|
+
zip.write_all(b"Text file").unwrap();
|
|
920
|
+
|
|
921
|
+
zip.start_file("image.png", options).unwrap();
|
|
922
|
+
zip.write_all(&[0x89, 0x50, 0x4E, 0x47]).unwrap();
|
|
923
|
+
|
|
924
|
+
zip.start_file("binary.exe", options).unwrap();
|
|
925
|
+
zip.write_all(&[0x4D, 0x5A]).unwrap();
|
|
926
|
+
|
|
927
|
+
zip.start_file("config.toml", options).unwrap();
|
|
928
|
+
zip.write_all(b"[section]").unwrap();
|
|
929
|
+
|
|
930
|
+
zip.finish().unwrap();
|
|
931
|
+
}
|
|
932
|
+
|
|
933
|
+
let bytes = cursor.into_inner();
|
|
934
|
+
let contents = extract_zip_text_content(&bytes).unwrap();
|
|
935
|
+
|
|
936
|
+
assert_eq!(contents.len(), 2);
|
|
937
|
+
assert!(contents.contains_key("document.txt"));
|
|
938
|
+
assert!(contents.contains_key("config.toml"));
|
|
939
|
+
assert!(!contents.contains_key("image.png"));
|
|
940
|
+
assert!(!contents.contains_key("binary.exe"));
|
|
941
|
+
}
|
|
942
|
+
|
|
943
|
+
#[test]
|
|
944
|
+
fn test_extract_7z_corrupted_data() {
|
|
945
|
+
let invalid_7z_data = vec![0x37, 0x7A, 0xBC, 0xAF, 0x27, 0x1C, 0x00];
|
|
946
|
+
|
|
947
|
+
let result = extract_7z_metadata(&invalid_7z_data);
|
|
948
|
+
assert!(result.is_err());
|
|
949
|
+
|
|
950
|
+
if let Err(e) = result {
|
|
951
|
+
assert!(matches!(e, KreuzbergError::Parsing { .. }));
|
|
952
|
+
}
|
|
953
|
+
}
|
|
954
|
+
}
|