omnizip 0.3.35 → 0.3.37
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +67 -0
- data/docs/getting-started/basic-usage.adoc +14 -22
- data/docs/guides/advanced-features/encryption.adoc +16 -39
- data/docs/guides/advanced-features/index.adoc +16 -6
- data/docs/guides/advanced-features/parallel-processing.adoc +63 -165
- data/docs/guides/advanced-features/progress-tracking.adoc +65 -261
- data/docs/guides/advanced-features/streaming.adoc +43 -113
- data/docs/guides/compression-algorithms/bzip2.adoc +2 -1
- data/docs/guides/compression-algorithms/deflate.adoc +1 -7
- data/docs/guides/compression-algorithms/lzma.adoc +10 -14
- data/docs/guides/compression-algorithms/lzma2.adoc +34 -117
- data/docs/guides/compression-algorithms/ppmd.adoc +24 -27
- data/docs/guides/compression-algorithms/zstandard.adoc +85 -150
- data/docs/troubleshooting/index.adoc +7 -5
- data/lib/omnizip/algorithms/zstandard.rb +2 -3
- data/lib/omnizip/archive/builder.rb +48 -0
- data/lib/omnizip/archive/reader_session.rb +67 -0
- data/lib/omnizip/archive.rb +86 -0
- data/lib/omnizip/archive_handler.rb +1 -0
- data/lib/omnizip/archive_handlers/rar_handler.rb +50 -0
- data/lib/omnizip/archive_handlers/seven_zip_handler.rb +6 -2
- data/lib/omnizip/archive_handlers/tar_handler.rb +4 -0
- data/lib/omnizip/archive_handlers/zip_handler.rb +25 -2
- data/lib/omnizip/archive_handlers.rb +1 -0
- data/lib/omnizip/convenience.rb +91 -11
- data/lib/omnizip/formats/rar/block_parser.rb +8 -12
- data/lib/omnizip/formats/rar/constants.rb +3 -1
- data/lib/omnizip/formats/rar/header.rb +16 -6
- data/lib/omnizip/formats/rar/reader.rb +6 -2
- data/lib/omnizip/version.rb +1 -1
- data/lib/omnizip.rb +1 -0
- metadata +6 -2
|
@@ -65,9 +65,8 @@ The encoder is fully production-ready for creating LZMA-compressed streams. The
|
|
|
65
65
|
[source,ruby]
|
|
66
66
|
----
|
|
67
67
|
# Create LZMA-compressed archive
|
|
68
|
-
Omnizip::Archive.create('backup.7z', format: :seven_zip
|
|
69
|
-
|
|
70
|
-
archive.level = 7
|
|
68
|
+
Omnizip::Archive.create('backup.7z', format: :seven_zip,
|
|
69
|
+
algorithm: :lzma, level: 7) do |archive|
|
|
71
70
|
archive.add_directory('project/')
|
|
72
71
|
end
|
|
73
72
|
----
|
|
@@ -290,10 +289,9 @@ Improve compression of executables:
|
|
|
290
289
|
[source,ruby]
|
|
291
290
|
----
|
|
292
291
|
# Compress executable with BCJ filter
|
|
293
|
-
Omnizip::Archive.create('app.7z', format: :seven_zip
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
archive.filter = :bcj_x86 # For x86 executables
|
|
292
|
+
Omnizip::Archive.create('app.7z', format: :seven_zip,
|
|
293
|
+
algorithm: :lzma, level: 9,
|
|
294
|
+
filters: [:bcj_x86]) do |archive| # For x86 executables
|
|
297
295
|
archive.add_file('application.exe')
|
|
298
296
|
end
|
|
299
297
|
|
|
@@ -344,10 +342,9 @@ end
|
|
|
344
342
|
[source,ruby]
|
|
345
343
|
----
|
|
346
344
|
# Package software release with maximum compression
|
|
347
|
-
Omnizip::Archive.create("app-v1.0.7z", format: :seven_zip
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
archive.solid = true # Even better compression
|
|
345
|
+
Omnizip::Archive.create("app-v1.0.7z", format: :seven_zip,
|
|
346
|
+
algorithm: :lzma, level: 9,
|
|
347
|
+
solid: true) do |archive| # Even better compression
|
|
351
348
|
archive.add_directory('release/')
|
|
352
349
|
end
|
|
353
350
|
----
|
|
@@ -374,9 +371,8 @@ Omnizip.compress_directory(
|
|
|
374
371
|
Omnizip.compress_directory(
|
|
375
372
|
'/var/www/',
|
|
376
373
|
"backup_#{Date.today}.7z",
|
|
377
|
-
|
|
378
|
-
level: 6
|
|
379
|
-
parallel: true # Speed up with parallel processing
|
|
374
|
+
algorithm: :lzma,
|
|
375
|
+
level: 6 # Balanced
|
|
380
376
|
)
|
|
381
377
|
----
|
|
382
378
|
|
|
@@ -19,7 +19,7 @@ LZMA2 is an enhanced version of LZMA designed for better multi-threading support
|
|
|
19
19
|
|Excellent (same as LZMA)
|
|
20
20
|
|
|
21
21
|
|Compression Speed
|
|
22
|
-
|Slow to medium (
|
|
22
|
+
|Slow to medium (single-threaded)
|
|
23
23
|
|
|
24
24
|
|Decompression Speed
|
|
25
25
|
|Fast
|
|
@@ -48,7 +48,7 @@ LZMA2 is an enhanced version of LZMA designed for better multi-threading support
|
|
|
48
48
|
|Multi-Threading
|
|
49
49
|
|Limited
|
|
50
50
|
|Excellent
|
|
51
|
-
|LZMA2 is
|
|
51
|
+
|LZMA2 chunks are independent, but Omnizip's encoder is single-threaded
|
|
52
52
|
|
|
53
53
|
|Incompressible Data
|
|
54
54
|
|Poor
|
|
@@ -75,7 +75,6 @@ LZMA2 is an enhanced version of LZMA designed for better multi-threading support
|
|
|
75
75
|
|
|
76
76
|
**Choose LZMA2 when**:
|
|
77
77
|
|
|
78
|
-
* You have multi-core CPU and want parallel compression
|
|
79
78
|
* Working with mixed content (text + binary + compressed)
|
|
80
79
|
* Creating modern 7z archives (it's the default)
|
|
81
80
|
* Need better streaming support
|
|
@@ -94,34 +93,24 @@ LZMA2 is an enhanced version of LZMA designed for better multi-threading support
|
|
|
94
93
|
[source,ruby]
|
|
95
94
|
----
|
|
96
95
|
# Create with LZMA2 (modern default)
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
96
|
+
# :lzma2 is the default algorithm for 7z
|
|
97
|
+
Omnizip::Archive.create('backup.7z', format: :seven_zip,
|
|
98
|
+
algorithm: :lzma2, level: 9) do |archive|
|
|
100
99
|
archive.add_directory('project/')
|
|
101
100
|
end
|
|
102
101
|
----
|
|
103
102
|
|
|
104
103
|
=== With Parallel Processing
|
|
105
104
|
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
Omnizip::Archive.create('backup.7z', format: :seven_zip) do |archive|
|
|
112
|
-
archive.compression = :lzma2
|
|
113
|
-
archive.level = 7
|
|
114
|
-
archive.parallel = true
|
|
115
|
-
archive.threads = 4 # Use 4 cores
|
|
116
|
-
archive.add_directory('large_project/')
|
|
117
|
-
end
|
|
118
|
-
|
|
119
|
-
# Can achieve 2-3x speedup on multi-core systems
|
|
120
|
-
----
|
|
105
|
+
Omnizip's codecs are pure Ruby and single-threaded: there is no
|
|
106
|
+
+parallel:+ or +threads:+ option. To use multiple cores, run
|
|
107
|
+
several archive operations in separate processes instead (see the
|
|
108
|
+
link:../advanced-features/parallel-processing.html[Parallel
|
|
109
|
+
Processing guide]).
|
|
121
110
|
|
|
122
111
|
== Compression Levels
|
|
123
112
|
|
|
124
|
-
Same levels as LZMA (1-9)
|
|
113
|
+
Same levels as LZMA (1-9):
|
|
125
114
|
|
|
126
115
|
[cols="1,2,2,2"]
|
|
127
116
|
|===
|
|
@@ -145,34 +134,17 @@ Same levels as LZMA (1-9) but with better parallelization:
|
|
|
145
134
|
|
|
146
135
|
== Advanced Configuration
|
|
147
136
|
|
|
148
|
-
===
|
|
149
|
-
|
|
150
|
-
Control parallelization granularity:
|
|
151
|
-
|
|
152
|
-
[source,ruby]
|
|
153
|
-
----
|
|
154
|
-
# Smaller chunks = better parallelization, more overhead
|
|
155
|
-
Omnizip::Archive.create('backup.7z', format: :seven_zip) do |archive|
|
|
156
|
-
archive.compression = :lzma2
|
|
157
|
-
archive.chunk_size = 8.megabytes # Process 8 MB chunks
|
|
158
|
-
archive.parallel = true
|
|
159
|
-
archive.add_directory('data/')
|
|
160
|
-
end
|
|
161
|
-
|
|
162
|
-
# Larger chunks = less overhead, less parallel benefit
|
|
163
|
-
# Default: 64 MB (good balance)
|
|
164
|
-
----
|
|
165
|
-
|
|
166
|
-
=== Dictionary Reset
|
|
137
|
+
=== Dictionary Size
|
|
167
138
|
|
|
168
|
-
LZMA2
|
|
139
|
+
LZMA2's dictionary size follows the compression +level:+. It can
|
|
140
|
+
also be set explicitly:
|
|
169
141
|
|
|
170
142
|
[source,ruby]
|
|
171
143
|
----
|
|
172
|
-
#
|
|
173
|
-
Omnizip::Archive.create('
|
|
174
|
-
|
|
175
|
-
|
|
144
|
+
# 64 MB dictionary for large homogeneous datasets
|
|
145
|
+
Omnizip::Archive.create('dataset.7z', format: :seven_zip,
|
|
146
|
+
algorithm: :lzma2,
|
|
147
|
+
dict_size: 64 * 1024 * 1024) do |archive|
|
|
176
148
|
archive.add_directory('mixed_data/') # Text + images + binaries
|
|
177
149
|
end
|
|
178
150
|
----
|
|
@@ -181,34 +153,9 @@ end
|
|
|
181
153
|
|
|
182
154
|
=== Parallel Speedup
|
|
183
155
|
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|All text
|
|
189
|
-
|60s
|
|
190
|
-
|35s
|
|
191
|
-
|20s
|
|
192
|
-
|15s
|
|
193
|
-
|
|
194
|
-
|Mixed content
|
|
195
|
-
|45s
|
|
196
|
-
|28s
|
|
197
|
-
|18s
|
|
198
|
-
|14s
|
|
199
|
-
|
|
200
|
-
|Many small files
|
|
201
|
-
|30s
|
|
202
|
-
|18s
|
|
203
|
-
|12s
|
|
204
|
-
|10s
|
|
205
|
-
|
|
206
|
-
|Few large files
|
|
207
|
-
|90s
|
|
208
|
-
|52s
|
|
209
|
-
|30s
|
|
210
|
-
|22s
|
|
211
|
-
|===
|
|
156
|
+
Not applicable: Omnizip's codecs are pure Ruby and single-threaded.
|
|
157
|
+
Compression of independent archives can be parallelized at the
|
|
158
|
+
process level (see the link:../advanced-features/parallel-processing.html[Parallel Processing guide]).
|
|
212
159
|
|
|
213
160
|
=== Incompressible Data Handling
|
|
214
161
|
|
|
@@ -226,23 +173,16 @@ Pure text 30s 30s Same
|
|
|
226
173
|
|
|
227
174
|
== Use Cases
|
|
228
175
|
|
|
229
|
-
===
|
|
230
|
-
|
|
231
|
-
Maximize server CPU utilization:
|
|
176
|
+
=== Server Backups
|
|
232
177
|
|
|
233
178
|
[source,ruby]
|
|
234
179
|
----
|
|
235
|
-
# Server with 16 cores
|
|
236
180
|
Omnizip.compress_directory(
|
|
237
181
|
'/var/www/',
|
|
238
182
|
'www-backup.7z',
|
|
239
|
-
|
|
240
|
-
level: 9
|
|
241
|
-
|
|
242
|
-
threads: 16 # Use all cores
|
|
243
|
-
) do |progress|
|
|
244
|
-
puts "#{progress.percentage}% - #{progress.active_threads} active threads"
|
|
245
|
-
end
|
|
183
|
+
algorithm: :lzma2,
|
|
184
|
+
level: 9
|
|
185
|
+
)
|
|
246
186
|
----
|
|
247
187
|
|
|
248
188
|
=== Large Dataset Compression
|
|
@@ -252,12 +192,9 @@ Process large datasets efficiently:
|
|
|
252
192
|
[source,ruby]
|
|
253
193
|
----
|
|
254
194
|
# Compress large research dataset
|
|
255
|
-
Omnizip::Archive.create('dataset.7z', format: :seven_zip
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
archive.solid = true # Better compression
|
|
259
|
-
archive.parallel = true
|
|
260
|
-
|
|
195
|
+
Omnizip::Archive.create('dataset.7z', format: :seven_zip,
|
|
196
|
+
algorithm: :lzma2, level: 9,
|
|
197
|
+
solid: true) do |archive| # Better compression
|
|
261
198
|
# Add gigabytes of data
|
|
262
199
|
archive.add_directory('research_data/')
|
|
263
200
|
end
|
|
@@ -265,7 +202,7 @@ end
|
|
|
265
202
|
|
|
266
203
|
=== Software Build Artifacts
|
|
267
204
|
|
|
268
|
-
Package build outputs
|
|
205
|
+
Package build outputs (BCJ filter for executables):
|
|
269
206
|
|
|
270
207
|
[source,ruby]
|
|
271
208
|
----
|
|
@@ -273,10 +210,9 @@ Package build outputs with parallel compression:
|
|
|
273
210
|
Omnizip.compress_directory(
|
|
274
211
|
'build/release/',
|
|
275
212
|
"app-#{version}.7z",
|
|
276
|
-
|
|
213
|
+
algorithm: :lzma2,
|
|
277
214
|
level: 7,
|
|
278
|
-
|
|
279
|
-
parallel: true
|
|
215
|
+
filters: [:bcj_x86]
|
|
280
216
|
)
|
|
281
217
|
----
|
|
282
218
|
|
|
@@ -286,34 +222,15 @@ Omnizip.compress_directory(
|
|
|
286
222
|
----
|
|
287
223
|
# Create LZMA2 archive
|
|
288
224
|
$ omnizip archive create backup.7z files/ \
|
|
289
|
-
--
|
|
225
|
+
--algorithm lzma2 \
|
|
290
226
|
--level 9
|
|
291
|
-
|
|
292
|
-
# With parallel processing
|
|
293
|
-
$ omnizip archive create backup.7z files/ \
|
|
294
|
-
--compression lzma2 \
|
|
295
|
-
--level 9 \
|
|
296
|
-
--parallel \
|
|
297
|
-
--threads 8
|
|
298
|
-
|
|
299
|
-
# Show compression statistics
|
|
300
|
-
$ omnizip archive create backup.7z files/ \
|
|
301
|
-
--compression lzma2 \
|
|
302
|
-
--level 9 \
|
|
303
|
-
--stats
|
|
304
|
-
Algorithm: LZMA2, Level: 9
|
|
305
|
-
Dictionary: 64 MB
|
|
306
|
-
Threads: 8
|
|
307
|
-
Original: 1.2 GB
|
|
308
|
-
Compressed: 456 MB (62% reduction)
|
|
309
|
-
Time: 3m 45s (2.7x speedup from parallelization)
|
|
310
227
|
----
|
|
311
228
|
|
|
312
229
|
== Technical Details
|
|
313
230
|
|
|
314
231
|
**LZMA2 Improvements over LZMA**:
|
|
315
232
|
|
|
316
|
-
. **Chunk-Based**: Data divided into independent chunks
|
|
233
|
+
. **Chunk-Based**: Data divided into independent chunks (which is what allows multi-threaded encoders; Omnizip's is single-threaded)
|
|
317
234
|
. **Dictionary Reset**: Can reset dictionary between chunks
|
|
318
235
|
. **Uncompressed Chunks**: Stores incompressible data without overhead
|
|
319
236
|
. **Better Streaming**: Designed for better streaming support
|
|
@@ -323,5 +240,5 @@ Time: 3m 45s (2.7x speedup from parallelization)
|
|
|
323
240
|
|
|
324
241
|
* link:lzma.html[LZMA] - Original algorithm
|
|
325
242
|
* link:zstandard.html[Zstandard] - Fast alternative
|
|
326
|
-
* link:../advanced-features/parallel-processing.html[Parallel Processing] -
|
|
243
|
+
* link:../advanced-features/parallel-processing.html[Parallel Processing] - Process-level parallelism for multiple archives
|
|
327
244
|
* link:../../compatibility.html[Compatibility] - LZMA2 support across tools
|
|
@@ -55,30 +55,33 @@ PPMd7 (Prediction by Partial Matching, variant H revision 7) is a statistical co
|
|
|
55
55
|
[source,ruby]
|
|
56
56
|
----
|
|
57
57
|
# Compress source code with PPMd7
|
|
58
|
-
Omnizip::Archive.create('code.7z', format: :seven_zip
|
|
59
|
-
|
|
60
|
-
archive.level = 6 # Default
|
|
58
|
+
Omnizip::Archive.create('code.7z', format: :seven_zip,
|
|
59
|
+
algorithm: :ppmd7) do |archive|
|
|
61
60
|
archive.add_directory('src/')
|
|
62
61
|
end
|
|
63
62
|
----
|
|
64
63
|
|
|
65
|
-
===
|
|
64
|
+
=== Memory and Order
|
|
66
65
|
|
|
67
|
-
|
|
66
|
+
PPMd memory (2^mem_size bytes) and context order are encoder
|
|
67
|
+
parameters; the writer currently emits PPMd with default settings
|
|
68
|
+
(empty properties, which readers interpret as defaults), so there
|
|
69
|
+
are no +mem_size:/+order: knobs yet:
|
|
68
70
|
|
|
69
71
|
[source,ruby]
|
|
70
72
|
----
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
archive.compression = :ppmd7
|
|
74
|
-
archive.mem_size = 24 # 2^24 = 16 MB
|
|
75
|
-
archive.order = 6 # Context order (default)
|
|
73
|
+
Omnizip::Archive.create('text.7z', format: :seven_zip,
|
|
74
|
+
algorithm: :ppmd7) do |archive|
|
|
76
75
|
archive.add_directory('documents/')
|
|
77
76
|
end
|
|
78
77
|
----
|
|
79
78
|
|
|
80
79
|
== Configuration Parameters
|
|
81
80
|
|
|
81
|
+
The writer currently emits PPMd with default memory/order settings;
|
|
82
|
+
the tables below describe the algorithm's parameters for reference
|
|
83
|
+
when choosing PPMd.
|
|
84
|
+
|
|
82
85
|
=== Memory Size
|
|
83
86
|
|
|
84
87
|
Controls context tree size (2^mem_size bytes):
|
|
@@ -142,10 +145,8 @@ Number of previous characters used for prediction:
|
|
|
142
145
|
[source,ruby]
|
|
143
146
|
----
|
|
144
147
|
# For large source code archive
|
|
145
|
-
Omnizip::Archive.create('linux-kernel.7z', format: :seven_zip
|
|
146
|
-
|
|
147
|
-
archive.mem_size = 26 # 64 MB memory
|
|
148
|
-
archive.order = 8 # Deep context
|
|
148
|
+
Omnizip::Archive.create('linux-kernel.7z', format: :seven_zip,
|
|
149
|
+
algorithm: :ppmd7) do |archive|
|
|
149
150
|
archive.add_directory('linux-6.0/')
|
|
150
151
|
end
|
|
151
152
|
----
|
|
@@ -215,10 +216,8 @@ For 10 MB text file:
|
|
|
215
216
|
[source,ruby]
|
|
216
217
|
----
|
|
217
218
|
# Archive project source code
|
|
218
|
-
Omnizip::Archive.create("project-v1.0.7z", format: :seven_zip
|
|
219
|
-
|
|
220
|
-
archive.mem_size = 24 # 16 MB
|
|
221
|
-
archive.order = 6
|
|
219
|
+
Omnizip::Archive.create("project-v1.0.7z", format: :seven_zip,
|
|
220
|
+
algorithm: :ppmd7) do |archive|
|
|
222
221
|
archive.add_directory('src/')
|
|
223
222
|
archive.add_directory('tests/')
|
|
224
223
|
archive.add_file('README.md')
|
|
@@ -232,13 +231,11 @@ end
|
|
|
232
231
|
# Compress application logs with maximum ratio
|
|
233
232
|
log_files = Dir.glob('/var/log/app/*.log')
|
|
234
233
|
|
|
235
|
-
Omnizip.
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
order: 8
|
|
241
|
-
)
|
|
234
|
+
Omnizip::Archive.create("logs_#{Date.today}.7z",
|
|
235
|
+
format: :seven_zip,
|
|
236
|
+
algorithm: :ppmd7) do |archive|
|
|
237
|
+
log_files.each { |log| archive.add_file(log) }
|
|
238
|
+
end
|
|
242
239
|
----
|
|
243
240
|
|
|
244
241
|
=== Documentation Archives
|
|
@@ -246,8 +243,8 @@ Omnizip.compress_files(
|
|
|
246
243
|
[source,ruby]
|
|
247
244
|
----
|
|
248
245
|
# Compress documentation and markup files
|
|
249
|
-
Omnizip::Archive.create('docs.7z', format: :seven_zip
|
|
250
|
-
|
|
246
|
+
Omnizip::Archive.create('docs.7z', format: :seven_zip,
|
|
247
|
+
algorithm: :ppmd7) do |archive|
|
|
251
248
|
archive.add_directory('docs/')
|
|
252
249
|
archive.add_directory('wiki/')
|
|
253
250
|
archive.add_directory('examples/')
|