omnizip 0.3.53 → 0.3.54
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +30 -0
- data/lib/omnizip/buffer/memory_archive.rb +55 -60
- data/lib/omnizip/buffer/memory_extractor.rb +25 -27
- data/lib/omnizip/buffer.rb +15 -8
- data/lib/omnizip/formats/seven_zip/bcj2_stream_decompressor.rb +2 -2
- data/lib/omnizip/formats/seven_zip/coder_chain.rb +5 -5
- data/lib/omnizip/formats/seven_zip/encrypted_header.rb +2 -2
- data/lib/omnizip/formats/seven_zip/file_collector.rb +1 -1
- data/lib/omnizip/formats/seven_zip/header.rb +4 -4
- data/lib/omnizip/formats/seven_zip/header_encryptor.rb +2 -2
- data/lib/omnizip/formats/seven_zip/parser.rb +8 -8
- data/lib/omnizip/formats/seven_zip/reader.rb +14 -19
- data/lib/omnizip/formats/seven_zip/split_archive_reader.rb +5 -5
- data/lib/omnizip/formats/seven_zip/stream_compressor.rb +1 -1
- data/lib/omnizip/formats/seven_zip/stream_decompressor.rb +5 -5
- data/lib/omnizip/formats/seven_zip/writer.rb +1 -1
- data/lib/omnizip/formats/zip/writer.rb +13 -6
- data/lib/omnizip/parallel/engine.rb +9 -5
- data/lib/omnizip/parallel/job_scheduler.rb +5 -1
- data/lib/omnizip/parallel/parallel_extractor.rb +62 -151
- data/lib/omnizip/parallel/thread_pool.rb +60 -0
- data/lib/omnizip/parallel/worker_pool.rb +5 -1
- data/lib/omnizip/parallel.rb +1 -0
- data/lib/omnizip/version.rb +1 -1
- data/lib/omnizip.rb +3 -0
- metadata +2 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 87089cc22eb4f5edd1cb37e7a0de297527fb2f7c0a2815f29d1ff271c73f7461
|
|
4
|
+
data.tar.gz: e6b7848399a98ae4f38f80cbc71833ada099e071d4f4126437ebbfcca820dc48
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: cfd421337992988889d3e5e8d4e85177248c5586238c2c79a5d87e5fc2229de3620f53e93d582ac72992c90b213421b1832cd44ecb9e1ff94e94622a48fa6b88
|
|
7
|
+
data.tar.gz: 33ca53cfe50c72a1d31afbbf0848ed9559824867edf34f1a4fdc37484300c029fcbf36bdb3135d92f607e31c517d6a519e945065a6c17f817cfbdc0af9ee6de9
|
data/CHANGELOG.md
CHANGED
|
@@ -7,6 +7,36 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
+
## [0.3.53] - 2026-09-01
|
|
11
|
+
|
|
12
|
+
### Changed
|
|
13
|
+
- The last six hand-rolled `to_h` serializers migrate onto
|
|
14
|
+
lutaml-model like their siblings: `ConversionResult`,
|
|
15
|
+
`PerformanceResult`, `ProfileReport` (with a nested Summary model
|
|
16
|
+
built from aggregates), `OptimizationSuggestion`, `MatchResult`,
|
|
17
|
+
and `FilterConfig`. Computed report fields (size_reduction,
|
|
18
|
+
throughput, priority_score, summary aggregates) serialize through
|
|
19
|
+
lutaml derived attributes keyed under their documented names.
|
|
20
|
+
Serialized hashes now use string keys (`to_hash`, matching the
|
|
21
|
+
other migrated models).
|
|
22
|
+
- `Formats::Zip::Reader` gains `read_entry_stream`, yielding bounded
|
|
23
|
+
chunks (raw copy for STORE, incremental inflater for DEFLATE) with
|
|
24
|
+
a CRC check after the last chunk. `Chunked.decompress_file`
|
|
25
|
+
streams through it — the memory-efficient path no longer holds the
|
|
26
|
+
whole decompressed entry in memory to slice it. Extracting a
|
|
27
|
+
single 7z entry in memory (`MemoryExtractor`) extracts just that
|
|
28
|
+
entry instead of decompressing the entire archive.
|
|
29
|
+
|
|
30
|
+
### Fixed
|
|
31
|
+
- Boundary errors stop masquerading as bare RuntimeErrors:
|
|
32
|
+
`Checksums::Verifier` raises `UnknownChecksumError` (the class
|
|
33
|
+
existed unused), and thirty more `raise "string"` sites gained
|
|
34
|
+
their matching classes (`Errno::ENOENT` for missing entries
|
|
35
|
+
— matching the handler-registry convention — `Errno::EEXIST`,
|
|
36
|
+
`IOError`, `InvalidArchiveError`, `DecompressionError`,
|
|
37
|
+
`RarNotAvailableError`). The never-called placeholder
|
|
38
|
+
`decode_escape_code` in the RAR PPMd decoder is deleted.
|
|
39
|
+
|
|
10
40
|
## [0.3.52] - 2026-09-01
|
|
11
41
|
|
|
12
42
|
### Changed
|
|
@@ -9,27 +9,24 @@ module Omnizip
|
|
|
9
9
|
# (for creating) and InputStream (for reading).
|
|
10
10
|
#
|
|
11
11
|
# @example Creating an archive
|
|
12
|
-
#
|
|
13
|
-
#
|
|
14
|
-
#
|
|
15
|
-
#
|
|
16
|
-
# end
|
|
12
|
+
# writer = Formats::Zip::Writer.new(nil)
|
|
13
|
+
# archive = MemoryArchive.new(writer, :zip)
|
|
14
|
+
# archive.add('file.txt', 'content')
|
|
15
|
+
# writer.write_to_io(buffer)
|
|
17
16
|
#
|
|
18
17
|
# @example Reading an archive
|
|
19
|
-
#
|
|
20
|
-
#
|
|
21
|
-
#
|
|
22
|
-
# puts entry.name
|
|
23
|
-
# end
|
|
18
|
+
# archive = MemoryArchive.new(Formats::Zip::Reader.new(path), :zip)
|
|
19
|
+
# archive.each_entry do |entry|
|
|
20
|
+
# puts entry.name
|
|
24
21
|
# end
|
|
25
22
|
class MemoryArchive
|
|
26
23
|
attr_reader :format, :stream
|
|
27
24
|
|
|
28
25
|
# Initialize memory archive wrapper
|
|
29
26
|
#
|
|
30
|
-
# @param stream [
|
|
31
|
-
#
|
|
32
|
-
# @param format [Symbol] Archive format (:zip
|
|
27
|
+
# @param stream [Formats::Zip::Writer, Formats::Zip::Reader]
|
|
28
|
+
# Native writer (write mode) or reader (read mode)
|
|
29
|
+
# @param format [Symbol] Archive format (:zip)
|
|
33
30
|
def initialize(stream, format)
|
|
34
31
|
@stream = stream
|
|
35
32
|
@format = format
|
|
@@ -57,13 +54,13 @@ module Omnizip
|
|
|
57
54
|
def add(name, data, **options)
|
|
58
55
|
ensure_write_mode!
|
|
59
56
|
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
stream.put_next_entry(name, **options)
|
|
63
|
-
stream.write(data) unless name.end_with?("/")
|
|
57
|
+
if name.end_with?("/") && data.to_s.empty?
|
|
58
|
+
stream.add_directory(name)
|
|
64
59
|
else
|
|
65
|
-
|
|
66
|
-
|
|
60
|
+
method = if options[:compression] == :store
|
|
61
|
+
Formats::Zip::Constants::COMPRESSION_STORE
|
|
62
|
+
end
|
|
63
|
+
stream.add_data(name, data, nil, compression_method: method)
|
|
67
64
|
end
|
|
68
65
|
|
|
69
66
|
self
|
|
@@ -101,15 +98,8 @@ module Omnizip
|
|
|
101
98
|
def each_entry
|
|
102
99
|
ensure_read_mode!
|
|
103
100
|
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
while (zip_entry = stream.get_next_entry)
|
|
107
|
-
entry = Entry.new(zip_entry, stream)
|
|
108
|
-
yield(entry)
|
|
109
|
-
end
|
|
110
|
-
else
|
|
111
|
-
raise NotImplementedError,
|
|
112
|
-
"Unsupported stream type: #{stream.class}"
|
|
101
|
+
stream.entries.each do |zip_entry|
|
|
102
|
+
yield(Entry.new(zip_entry, stream))
|
|
113
103
|
end
|
|
114
104
|
end
|
|
115
105
|
|
|
@@ -142,22 +132,10 @@ module Omnizip
|
|
|
142
132
|
def to_s
|
|
143
133
|
ensure_write_mode!
|
|
144
134
|
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
unless stream.closed?
|
|
150
|
-
raise "Archive must be closed before accessing data"
|
|
151
|
-
end
|
|
152
|
-
|
|
153
|
-
# The buffer was passed in during creation, but we don't have
|
|
154
|
-
# direct access. This method should be called on the StringIO
|
|
155
|
-
# returned by Buffer.create instead.
|
|
156
|
-
raise NotImplementedError,
|
|
157
|
-
"Use Buffer.create return value instead"
|
|
158
|
-
else
|
|
159
|
-
raise "Cannot get string from read mode archive"
|
|
160
|
-
end
|
|
135
|
+
# The buffer is owned by Buffer.create; this method should be
|
|
136
|
+
# called on the StringIO it returns instead.
|
|
137
|
+
raise NotImplementedError,
|
|
138
|
+
"Use Buffer.create return value instead"
|
|
161
139
|
end
|
|
162
140
|
|
|
163
141
|
# Entry wrapper with read capability
|
|
@@ -169,17 +147,19 @@ module Omnizip
|
|
|
169
147
|
|
|
170
148
|
# Initialize entry wrapper
|
|
171
149
|
#
|
|
172
|
-
# @param entry [
|
|
173
|
-
# @param
|
|
174
|
-
def initialize(entry,
|
|
150
|
+
# @param entry [Formats::Zip::CentralDirectoryHeader] entry
|
|
151
|
+
# @param reader [Formats::Zip::Reader] Native archive reader
|
|
152
|
+
def initialize(entry, reader)
|
|
175
153
|
@entry = entry
|
|
176
|
-
@
|
|
177
|
-
@name = entry.
|
|
178
|
-
@size = entry.
|
|
154
|
+
@reader = reader
|
|
155
|
+
@name = entry.filename
|
|
156
|
+
@size = entry.uncompressed_size
|
|
179
157
|
@compressed_size = entry.compressed_size
|
|
180
158
|
@time = entry.time
|
|
181
|
-
@comment = entry.comment
|
|
159
|
+
@comment = entry.comment.to_s
|
|
182
160
|
@directory = entry.directory?
|
|
161
|
+
@content = nil
|
|
162
|
+
@pos = 0
|
|
183
163
|
end
|
|
184
164
|
|
|
185
165
|
# Read entry content
|
|
@@ -195,7 +175,20 @@ module Omnizip
|
|
|
195
175
|
# process_chunk(chunk)
|
|
196
176
|
# end
|
|
197
177
|
def read(size = nil)
|
|
198
|
-
|
|
178
|
+
# IO-like streaming semantics: successive reads return
|
|
179
|
+
# successive chunks, nil at end-of-content. Callers like
|
|
180
|
+
# Pipe::StreamDecompressor loop on this — a read that
|
|
181
|
+
# always returns data would loop forever.
|
|
182
|
+
@content ||= @reader.read_entry(@name)
|
|
183
|
+
return nil if @pos >= @content.bytesize
|
|
184
|
+
|
|
185
|
+
chunk = if size
|
|
186
|
+
@content.byteslice(@pos, size)
|
|
187
|
+
else
|
|
188
|
+
@content.byteslice(@pos, @content.bytesize - @pos)
|
|
189
|
+
end
|
|
190
|
+
@pos += chunk.bytesize
|
|
191
|
+
chunk
|
|
199
192
|
end
|
|
200
193
|
|
|
201
194
|
# Check if entry is a directory
|
|
@@ -229,22 +222,24 @@ module Omnizip
|
|
|
229
222
|
|
|
230
223
|
private
|
|
231
224
|
|
|
232
|
-
# Ensure stream is in write mode (
|
|
225
|
+
# Ensure stream is in write mode (native Writer)
|
|
233
226
|
#
|
|
234
|
-
# @raise [
|
|
227
|
+
# @raise [Omnizip::IOError] If not in write mode
|
|
235
228
|
def ensure_write_mode!
|
|
236
|
-
return if stream.is_a?(Omnizip::Zip::
|
|
229
|
+
return if stream.is_a?(Omnizip::Formats::Zip::Writer)
|
|
237
230
|
|
|
238
|
-
raise
|
|
231
|
+
raise Omnizip::IOError,
|
|
232
|
+
"Operation requires write mode (Formats::Zip::Writer)"
|
|
239
233
|
end
|
|
240
234
|
|
|
241
|
-
# Ensure stream is in read mode (
|
|
235
|
+
# Ensure stream is in read mode (native Reader)
|
|
242
236
|
#
|
|
243
|
-
# @raise [
|
|
237
|
+
# @raise [Omnizip::IOError] If not in read mode
|
|
244
238
|
def ensure_read_mode!
|
|
245
|
-
return if stream.is_a?(Omnizip::Zip::
|
|
239
|
+
return if stream.is_a?(Omnizip::Formats::Zip::Reader)
|
|
246
240
|
|
|
247
|
-
raise
|
|
241
|
+
raise Omnizip::IOError,
|
|
242
|
+
"Operation requires read mode (Formats::Zip::Reader)"
|
|
248
243
|
end
|
|
249
244
|
end
|
|
250
245
|
end
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require "stringio"
|
|
4
|
+
require "tmpdir"
|
|
4
5
|
|
|
5
6
|
module Omnizip
|
|
6
7
|
module Buffer
|
|
@@ -159,18 +160,28 @@ module Omnizip
|
|
|
159
160
|
Omnizip::Buffer.detect_format(@buffer)
|
|
160
161
|
end
|
|
161
162
|
|
|
162
|
-
#
|
|
163
|
+
# The native ZIP reader parses from a path; spill the buffer to
|
|
164
|
+
# a temporary file and yield a reader over it.
|
|
165
|
+
def with_zip_reader
|
|
166
|
+
@buffer.rewind
|
|
167
|
+
Dir.mktmpdir("omnizip_extractor_zip") do |tmp|
|
|
168
|
+
path = File.join(tmp, "buffer.zip")
|
|
169
|
+
File.binwrite(path, @buffer.read)
|
|
170
|
+
yield Omnizip::Formats::Zip::Reader.new(path)
|
|
171
|
+
end
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
# Extract all entries from ZIP through the native reader
|
|
163
175
|
#
|
|
164
176
|
# @param result [Hash] Hash to populate with entries
|
|
165
177
|
def extract_all_zip(result)
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
while (entry = zis.get_next_entry)
|
|
178
|
+
with_zip_reader do |reader|
|
|
179
|
+
reader.entries.each do |entry|
|
|
169
180
|
next if entry.directory?
|
|
170
181
|
|
|
171
|
-
content =
|
|
172
|
-
result[entry.
|
|
173
|
-
@extracted_cache[entry.
|
|
182
|
+
content = reader.read_entry(entry.filename)
|
|
183
|
+
result[entry.filename] = content
|
|
184
|
+
@extracted_cache[entry.filename] = content
|
|
174
185
|
end
|
|
175
186
|
end
|
|
176
187
|
end
|
|
@@ -199,36 +210,23 @@ module Omnizip
|
|
|
199
210
|
end
|
|
200
211
|
end
|
|
201
212
|
|
|
202
|
-
# Extract single entry from ZIP
|
|
213
|
+
# Extract single entry from ZIP through the native reader
|
|
203
214
|
#
|
|
204
215
|
# @param name [String] Entry name
|
|
205
216
|
# @return [String, nil] Entry content or nil if not found
|
|
206
217
|
def extract_entry_zip(name)
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
Omnizip::Zip::InputStream.open(@buffer) do |zis|
|
|
211
|
-
while (entry = zis.get_next_entry)
|
|
212
|
-
if entry.name == name
|
|
213
|
-
content = zis.read unless entry.directory?
|
|
214
|
-
break
|
|
215
|
-
end
|
|
216
|
-
end
|
|
218
|
+
with_zip_reader do |reader|
|
|
219
|
+
entry = reader.entries.find { |e| e.filename == name }
|
|
220
|
+
reader.read_entry(name) if entry && !entry.directory?
|
|
217
221
|
end
|
|
218
|
-
|
|
219
|
-
content
|
|
220
222
|
end
|
|
221
223
|
|
|
222
|
-
# List all entry names from ZIP
|
|
224
|
+
# List all entry names from ZIP through the native reader
|
|
223
225
|
#
|
|
224
226
|
# @param names [Array] Array to populate with names
|
|
225
227
|
def list_entries_zip(names)
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
Omnizip::Zip::InputStream.open(@buffer) do |zis|
|
|
229
|
-
while (entry = zis.get_next_entry)
|
|
230
|
-
names << entry.name
|
|
231
|
-
end
|
|
228
|
+
with_zip_reader do |reader|
|
|
229
|
+
reader.entries.each { |entry| names << entry.filename }
|
|
232
230
|
end
|
|
233
231
|
end
|
|
234
232
|
end
|
data/lib/omnizip/buffer.rb
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require "stringio"
|
|
4
|
+
require "tmpdir"
|
|
4
5
|
|
|
5
6
|
module Omnizip
|
|
6
7
|
# In-memory archive operations without filesystem I/O
|
|
@@ -150,27 +151,33 @@ module Omnizip
|
|
|
150
151
|
|
|
151
152
|
private
|
|
152
153
|
|
|
153
|
-
# Create ZIP archive in buffer
|
|
154
|
+
# Create ZIP archive in buffer through the native writer
|
|
154
155
|
#
|
|
155
156
|
# @param buffer [StringIO] Buffer to write to
|
|
156
157
|
# @param options [Hash] ZIP-specific options
|
|
157
158
|
# @yield [archive] Block to populate archive
|
|
158
159
|
def create_zip(buffer, _options, &block)
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
160
|
+
writer = Formats::Zip::Writer.new(nil)
|
|
161
|
+
archive = Buffer::MemoryArchive.new(writer, :zip)
|
|
162
|
+
block&.call(archive)
|
|
163
|
+
writer.write_to_io(buffer)
|
|
163
164
|
end
|
|
164
165
|
|
|
165
|
-
# Open ZIP archive from buffer
|
|
166
|
+
# Open ZIP archive from buffer through the native reader.
|
|
167
|
+
# The reader parses from a path, so the buffer spills to a
|
|
168
|
+
# temporary file (same pattern as the 7z bridge).
|
|
166
169
|
#
|
|
167
170
|
# @param buffer [StringIO] Buffer containing ZIP data
|
|
168
171
|
# @yield [archive] Block to read from archive
|
|
169
172
|
# @return [MemoryArchive, Object] Archive or block return value
|
|
170
173
|
def open_zip(buffer, &block)
|
|
171
174
|
result = nil
|
|
172
|
-
|
|
173
|
-
|
|
175
|
+
Dir.mktmpdir("omnizip_buffer_zip") do |tmp|
|
|
176
|
+
path = File.join(tmp, "buffer.zip")
|
|
177
|
+
File.binwrite(path, buffer.string)
|
|
178
|
+
|
|
179
|
+
archive = Buffer::MemoryArchive.new(Formats::Zip::Reader.new(path),
|
|
180
|
+
:zip)
|
|
174
181
|
result = block ? yield(archive) : archive
|
|
175
182
|
end
|
|
176
183
|
result
|
|
@@ -46,7 +46,7 @@ module Omnizip
|
|
|
46
46
|
bcj2_coder = @folder.coders.find { |c| c.method_id == FilterId::BCJ2 }
|
|
47
47
|
compression_coder = find_compression_coder
|
|
48
48
|
|
|
49
|
-
raise "BCJ2 coder not found" unless bcj2_coder
|
|
49
|
+
raise Omnizip::InvalidArchiveError, "BCJ2 coder not found" unless bcj2_coder
|
|
50
50
|
|
|
51
51
|
# Determine stream layout based on folder structure
|
|
52
52
|
# BCJ2 has 4 input streams: main, call, jump, rc
|
|
@@ -217,7 +217,7 @@ module Omnizip
|
|
|
217
217
|
return packed_data unless algo_sym
|
|
218
218
|
|
|
219
219
|
algo_class = Omnizip::AlgorithmRegistry.get(algo_sym)
|
|
220
|
-
raise "Algorithm not found: #{algo_sym}" unless algo_class
|
|
220
|
+
raise Omnizip::AlgorithmNotFoundError, "Algorithm not found: #{algo_sym}" unless algo_class
|
|
221
221
|
|
|
222
222
|
# Build decoder options
|
|
223
223
|
options = build_decoder_options(coder, algo_sym)
|
|
@@ -20,7 +20,7 @@ module Omnizip
|
|
|
20
20
|
# Find the compression method (not a filter) among coders
|
|
21
21
|
# Filters like BCJ, BCJ2 have specific method IDs
|
|
22
22
|
main_coder = find_compression_coder(folder.coders)
|
|
23
|
-
raise "No compression method found in folder" unless main_coder
|
|
23
|
+
raise Omnizip::InvalidArchiveError, "No compression method found in folder" unless main_coder
|
|
24
24
|
|
|
25
25
|
algorithm = algorithm_for_method(main_coder.method_id)
|
|
26
26
|
|
|
@@ -82,8 +82,8 @@ module Omnizip
|
|
|
82
82
|
when MethodId::DEFLATE64
|
|
83
83
|
:deflate64
|
|
84
84
|
else
|
|
85
|
-
raise "Unsupported compression method: " \
|
|
86
|
-
|
|
85
|
+
raise Omnizip::UnsupportedFormatError, "Unsupported compression method: " \
|
|
86
|
+
"0x#{method_id.to_s(16)}"
|
|
87
87
|
end
|
|
88
88
|
end
|
|
89
89
|
|
|
@@ -126,7 +126,7 @@ module Omnizip
|
|
|
126
126
|
# Get algorithm
|
|
127
127
|
algo_class = AlgorithmRegistry.get(chain_config[:algorithm])
|
|
128
128
|
unless algo_class
|
|
129
|
-
raise "Algorithm not found: #{chain_config[:algorithm]}"
|
|
129
|
+
raise Omnizip::AlgorithmNotFoundError, "Algorithm not found: #{chain_config[:algorithm]}"
|
|
130
130
|
end
|
|
131
131
|
|
|
132
132
|
# Create algorithm instance
|
|
@@ -143,7 +143,7 @@ module Omnizip
|
|
|
143
143
|
pipeline = FilterPipeline.new
|
|
144
144
|
chain_config[:filters].each do |filter_sym|
|
|
145
145
|
filter_class = FilterRegistry.get(filter_sym)
|
|
146
|
-
raise "Filter not found: #{filter_sym}" unless filter_class
|
|
146
|
+
raise Omnizip::UnknownFilterError, "Filter not found: #{filter_sym}" unless filter_class
|
|
147
147
|
|
|
148
148
|
pipeline.add_filter(filter_class.new)
|
|
149
149
|
end
|
|
@@ -69,7 +69,7 @@ module Omnizip
|
|
|
69
69
|
# @return [EncryptedHeader] Parsed header
|
|
70
70
|
# @raise [RuntimeError] if data is invalid
|
|
71
71
|
def self.from_binary(data)
|
|
72
|
-
raise "Invalid encrypted header: too short" if data.bytesize < 54
|
|
72
|
+
raise Omnizip::InvalidArchiveError, "Invalid encrypted header: too short" if data.bytesize < 54
|
|
73
73
|
|
|
74
74
|
pos = 0
|
|
75
75
|
|
|
@@ -77,7 +77,7 @@ module Omnizip
|
|
|
77
77
|
marker = data.getbyte(pos)
|
|
78
78
|
pos += 1
|
|
79
79
|
unless marker == PropertyId::ENCODED_HEADER
|
|
80
|
-
raise "Invalid encrypted header marker: expected #{PropertyId::ENCODED_HEADER}, got #{marker}"
|
|
80
|
+
raise Omnizip::InvalidArchiveError, "Invalid encrypted header marker: expected #{PropertyId::ENCODED_HEADER}, got #{marker}"
|
|
81
81
|
end
|
|
82
82
|
|
|
83
83
|
# Read encrypted data size
|
|
@@ -23,7 +23,7 @@ module Omnizip
|
|
|
23
23
|
# @param recursive [Boolean] Recursively add directories
|
|
24
24
|
def add_path(path, archive_path: nil, recursive: true)
|
|
25
25
|
path = File.expand_path(path)
|
|
26
|
-
raise "Path not found: #{path}" unless File.exist?(path)
|
|
26
|
+
raise Errno::ENOENT, "Path not found: #{path}" unless File.exist?(path)
|
|
27
27
|
|
|
28
28
|
@base_path ||= File.dirname(path)
|
|
29
29
|
|
|
@@ -29,14 +29,14 @@ module Omnizip
|
|
|
29
29
|
def parse(io)
|
|
30
30
|
# Read complete start header (32 bytes)
|
|
31
31
|
header_data = io.read(START_HEADER_SIZE)
|
|
32
|
-
raise "Invalid .7z file: too short" if header_data.nil? ||
|
|
32
|
+
raise Omnizip::InvalidArchiveError, "Invalid .7z file: too short" if header_data.nil? ||
|
|
33
33
|
header_data.bytesize < START_HEADER_SIZE
|
|
34
34
|
|
|
35
35
|
# Validate signature
|
|
36
36
|
signature = header_data[0, SIGNATURE_SIZE]
|
|
37
37
|
unless signature == SIGNATURE
|
|
38
|
-
raise "Invalid .7z signature: expected #{SIGNATURE.inspect}, " \
|
|
39
|
-
|
|
38
|
+
raise Omnizip::InvalidArchiveError, "Invalid .7z signature: expected #{SIGNATURE.inspect}, " \
|
|
39
|
+
"got #{signature.inspect}"
|
|
40
40
|
end
|
|
41
41
|
|
|
42
42
|
# Parse version
|
|
@@ -44,7 +44,7 @@ module Omnizip
|
|
|
44
44
|
@minor_version = header_data.getbyte(7)
|
|
45
45
|
|
|
46
46
|
unless @major_version == MAJOR_VERSION
|
|
47
|
-
raise "Unsupported .7z version: #{@major_version}.#{@minor_version}"
|
|
47
|
+
raise Omnizip::UnsupportedFormatError, "Unsupported .7z version: #{@major_version}.#{@minor_version}"
|
|
48
48
|
end
|
|
49
49
|
|
|
50
50
|
# Parse start header CRC (bytes 8-11)
|
|
@@ -83,9 +83,9 @@ module Omnizip
|
|
|
83
83
|
begin
|
|
84
84
|
decipher.update(encrypted_data) + decipher.final
|
|
85
85
|
rescue OpenSSL::Cipher::CipherError => e
|
|
86
|
-
raise "Failed to decrypt header: incorrect password or corrupted data (#{e.message})"
|
|
86
|
+
raise Omnizip::PasswordError, "Failed to decrypt header: incorrect password or corrupted data (#{e.message})"
|
|
87
87
|
rescue StandardError => e
|
|
88
|
-
raise "Failed to decrypt header: incorrect password or corrupted data (#{e.message})"
|
|
88
|
+
raise Omnizip::PasswordError, "Failed to decrypt header: incorrect password or corrupted data (#{e.message})"
|
|
89
89
|
end
|
|
90
90
|
end
|
|
91
91
|
|
|
@@ -82,7 +82,7 @@ module Omnizip
|
|
|
82
82
|
end
|
|
83
83
|
|
|
84
84
|
value = read_number
|
|
85
|
-
raise "Unsupported 32-bit value" if value >= 0x80000000
|
|
85
|
+
raise Omnizip::InvalidArchiveError, "Unsupported 32-bit value" if value >= 0x80000000
|
|
86
86
|
|
|
87
87
|
value
|
|
88
88
|
end
|
|
@@ -233,7 +233,7 @@ module Omnizip
|
|
|
233
233
|
end
|
|
234
234
|
|
|
235
235
|
# Verify required SIZE property was present
|
|
236
|
-
raise "Missing SIZE property in pack info" unless sizes_read
|
|
236
|
+
raise Omnizip::InvalidArchiveError, "Missing SIZE property in pack info" unless sizes_read
|
|
237
237
|
|
|
238
238
|
# K_END is optional in some 7-Zip versions
|
|
239
239
|
read_byte if !eof? && peek_byte == PropertyId::K_END
|
|
@@ -259,7 +259,7 @@ module Omnizip
|
|
|
259
259
|
end
|
|
260
260
|
else
|
|
261
261
|
# External folders not supported
|
|
262
|
-
raise "External folders not supported"
|
|
262
|
+
raise Omnizip::UnsupportedFormatError, "External folders not supported"
|
|
263
263
|
end
|
|
264
264
|
end
|
|
265
265
|
|
|
@@ -268,7 +268,7 @@ module Omnizip
|
|
|
268
268
|
# @param folder [Models::Folder] Folder to populate
|
|
269
269
|
def read_folder(folder)
|
|
270
270
|
num_coders = read_number
|
|
271
|
-
raise "Too many coders" if num_coders > Constants::MAX_NUM_CODERS
|
|
271
|
+
raise Omnizip::InvalidArchiveError, "Too many coders" if num_coders > Constants::MAX_NUM_CODERS
|
|
272
272
|
|
|
273
273
|
num_in_streams = 0
|
|
274
274
|
num_out_streams = 0
|
|
@@ -386,8 +386,8 @@ module Omnizip
|
|
|
386
386
|
end
|
|
387
387
|
|
|
388
388
|
# Verify required properties were present
|
|
389
|
-
raise "Missing FOLDER property in unpack info" unless folders_read
|
|
390
|
-
raise "Missing CODERS_UNPACK_SIZE property in unpack info" unless unpack_sizes_read
|
|
389
|
+
raise Omnizip::InvalidArchiveError, "Missing FOLDER property in unpack info" unless folders_read
|
|
390
|
+
raise Omnizip::InvalidArchiveError, "Missing CODERS_UNPACK_SIZE property in unpack info" unless unpack_sizes_read
|
|
391
391
|
|
|
392
392
|
# K_END is optional in some 7-Zip versions
|
|
393
393
|
read_byte if !eof? && peek_byte == PropertyId::K_END
|
|
@@ -611,8 +611,8 @@ module Omnizip
|
|
|
611
611
|
actual = read_byte
|
|
612
612
|
return if actual == expected
|
|
613
613
|
|
|
614
|
-
raise "Expected property 0x#{expected.to_s(16)}, " \
|
|
615
|
-
|
|
614
|
+
raise Omnizip::InvalidArchiveError, "Expected property 0x#{expected.to_s(16)}, " \
|
|
615
|
+
"got 0x#{actual.to_s(16)}"
|
|
616
616
|
end
|
|
617
617
|
|
|
618
618
|
# Skip size field
|
|
@@ -223,18 +223,13 @@ module Omnizip
|
|
|
223
223
|
EncryptedHeader.from_binary(next_header_data)
|
|
224
224
|
# If we got here, it's encrypted - try to decrypt
|
|
225
225
|
next_header_data = decrypt_header(next_header_data)
|
|
226
|
-
rescue
|
|
227
|
-
#
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
end
|
|
231
|
-
|
|
232
|
-
# Not encrypted - it's a compressed header
|
|
233
|
-
# Decompress it using the encoded stream info
|
|
234
|
-
next_header_data = decompress_encoded_header(io,
|
|
235
|
-
next_header_data)
|
|
226
|
+
rescue Omnizip::PasswordError
|
|
227
|
+
# Decryption refused or failed — a password problem,
|
|
228
|
+
# never a compressed header
|
|
229
|
+
raise
|
|
236
230
|
rescue StandardError
|
|
237
|
-
#
|
|
231
|
+
# Not an encrypted header — it's a compressed header;
|
|
232
|
+
# decompress it using the encoded stream info
|
|
238
233
|
next_header_data = decompress_encoded_header(io,
|
|
239
234
|
next_header_data)
|
|
240
235
|
end
|
|
@@ -250,8 +245,8 @@ module Omnizip
|
|
|
250
245
|
@stream_info, @entries = parse_metadata(parser)
|
|
251
246
|
rescue StandardError => e
|
|
252
247
|
if @headers_decrypted
|
|
253
|
-
raise "Failed to decrypt headers: incorrect password " \
|
|
254
|
-
|
|
248
|
+
raise Omnizip::PasswordError, "Failed to decrypt headers: incorrect password " \
|
|
249
|
+
"or corrupted data (#{e.message})"
|
|
255
250
|
end
|
|
256
251
|
|
|
257
252
|
raise
|
|
@@ -271,7 +266,7 @@ module Omnizip
|
|
|
271
266
|
@encrypted_header = EncryptedHeader.from_binary(encrypted_data)
|
|
272
267
|
|
|
273
268
|
unless @password
|
|
274
|
-
raise "Archive headers are encrypted. Password required to access."
|
|
269
|
+
raise Omnizip::PasswordError, "Archive headers are encrypted. Password required to access."
|
|
275
270
|
end
|
|
276
271
|
|
|
277
272
|
# Decrypt using password
|
|
@@ -287,7 +282,7 @@ module Omnizip
|
|
|
287
282
|
@headers_decrypted = true
|
|
288
283
|
decrypted
|
|
289
284
|
rescue OpenSSL::Cipher::CipherError => e
|
|
290
|
-
raise "Failed to decrypt headers: incorrect password (#{e.message})"
|
|
285
|
+
raise Omnizip::PasswordError, "Failed to decrypt headers: incorrect password (#{e.message})"
|
|
291
286
|
end
|
|
292
287
|
|
|
293
288
|
# Decompress encoded (compressed) header
|
|
@@ -317,7 +312,7 @@ module Omnizip
|
|
|
317
312
|
parser.read_unpack_info(stream_info)
|
|
318
313
|
end
|
|
319
314
|
else
|
|
320
|
-
raise "Unexpected property in encoded header: 0x#{type.to_s(16)}"
|
|
315
|
+
raise Omnizip::InvalidArchiveError, "Unexpected property in encoded header: 0x#{type.to_s(16)}"
|
|
321
316
|
end
|
|
322
317
|
|
|
323
318
|
# Decompress the header using the stream info
|
|
@@ -341,7 +336,7 @@ module Omnizip
|
|
|
341
336
|
|
|
342
337
|
# Read main header
|
|
343
338
|
type = parser.read_byte
|
|
344
|
-
raise "Expected Header, got 0x#{type.to_s(16)}" unless
|
|
339
|
+
raise Omnizip::InvalidArchiveError, "Expected Header, got 0x#{type.to_s(16)}" unless
|
|
345
340
|
type == PropertyId::HEADER
|
|
346
341
|
|
|
347
342
|
# Parse header sections
|
|
@@ -529,7 +524,7 @@ module Omnizip
|
|
|
529
524
|
crc = Omnizip::Checksums::Crc32.new
|
|
530
525
|
crc.update(data)
|
|
531
526
|
unless crc.value == entry.crc
|
|
532
|
-
raise "CRC mismatch for #{entry.name}: expected 0x#{entry.crc.to_s(16)}, got 0x#{crc.value.to_s(16)}"
|
|
527
|
+
raise Omnizip::ChecksumError, "CRC mismatch for #{entry.name}: expected 0x#{entry.crc.to_s(16)}, got 0x#{crc.value.to_s(16)}"
|
|
533
528
|
end
|
|
534
529
|
end
|
|
535
530
|
|
|
@@ -589,7 +584,7 @@ module Omnizip
|
|
|
589
584
|
crc = Omnizip::Checksums::Crc32.new
|
|
590
585
|
crc.update(data)
|
|
591
586
|
unless crc.value == entry.crc
|
|
592
|
-
raise "CRC mismatch for #{entry.name}: expected 0x#{entry.crc.to_s(16)}, got 0x#{crc.value.to_s(16)}"
|
|
587
|
+
raise Omnizip::ChecksumError, "CRC mismatch for #{entry.name}: expected 0x#{entry.crc.to_s(16)}, got 0x#{crc.value.to_s(16)}"
|
|
593
588
|
end
|
|
594
589
|
end
|
|
595
590
|
|
|
@@ -167,7 +167,7 @@ module Omnizip
|
|
|
167
167
|
volume_num += 1
|
|
168
168
|
end
|
|
169
169
|
|
|
170
|
-
raise "No volumes found for #{@base_path}" if @volumes.empty?
|
|
170
|
+
raise Errno::ENOENT, "No volumes found for #{@base_path}" if @volumes.empty?
|
|
171
171
|
end
|
|
172
172
|
|
|
173
173
|
# Detect volumes with alpha naming (.aa, .ab, ...)
|
|
@@ -185,7 +185,7 @@ module Omnizip
|
|
|
185
185
|
volume_num += 1
|
|
186
186
|
end
|
|
187
187
|
|
|
188
|
-
raise "No volumes found for #{@base_path}" if @volumes.empty?
|
|
188
|
+
raise Errno::ENOENT, "No volumes found for #{@base_path}" if @volumes.empty?
|
|
189
189
|
end
|
|
190
190
|
|
|
191
191
|
# Open all volume files
|
|
@@ -293,7 +293,7 @@ module Omnizip
|
|
|
293
293
|
|
|
294
294
|
# Read main header
|
|
295
295
|
type = parser.read_byte
|
|
296
|
-
raise "Expected Header, got 0x#{type.to_s(16)}" unless
|
|
296
|
+
raise Omnizip::InvalidArchiveError, "Expected Header, got 0x#{type.to_s(16)}" unless
|
|
297
297
|
type == PropertyId::HEADER
|
|
298
298
|
|
|
299
299
|
# Parse header sections
|
|
@@ -372,7 +372,7 @@ module Omnizip
|
|
|
372
372
|
parser.read_unpack_info(stream_info)
|
|
373
373
|
end
|
|
374
374
|
else
|
|
375
|
-
raise "Unexpected property in encoded header: 0x#{type.to_s(16)}"
|
|
375
|
+
raise Omnizip::InvalidArchiveError, "Unexpected property in encoded header: 0x#{type.to_s(16)}"
|
|
376
376
|
end
|
|
377
377
|
|
|
378
378
|
# Decompress the header using the stream info
|
|
@@ -491,7 +491,7 @@ module Omnizip
|
|
|
491
491
|
crc = Omnizip::Checksums::Crc32.new
|
|
492
492
|
crc.update(data)
|
|
493
493
|
unless crc.value == entry.crc
|
|
494
|
-
raise "CRC mismatch for #{entry.name}: expected 0x#{entry.crc.to_s(16)}, got 0x#{crc.value.to_s(16)}"
|
|
494
|
+
raise Omnizip::ChecksumError, "CRC mismatch for #{entry.name}: expected 0x#{entry.crc.to_s(16)}, got 0x#{crc.value.to_s(16)}"
|
|
495
495
|
end
|
|
496
496
|
end
|
|
497
497
|
|
|
@@ -44,7 +44,7 @@ module Omnizip
|
|
|
44
44
|
# Apply compression algorithm
|
|
45
45
|
if @algorithm && @algorithm != :copy
|
|
46
46
|
algo_class = Omnizip::AlgorithmRegistry.get(@algorithm)
|
|
47
|
-
raise "Algorithm not found: #{@algorithm}" unless algo_class
|
|
47
|
+
raise Omnizip::AlgorithmNotFoundError, "Algorithm not found: #{@algorithm}" unless algo_class
|
|
48
48
|
|
|
49
49
|
encoder = algo_class.new
|
|
50
50
|
input_io = StringIO.new(result)
|
|
@@ -57,7 +57,7 @@ module Omnizip
|
|
|
57
57
|
|
|
58
58
|
# Get algorithm class
|
|
59
59
|
algo_class = Omnizip::AlgorithmRegistry.get(algo_sym)
|
|
60
|
-
raise "Algorithm not found: #{algo_sym}" unless algo_class
|
|
60
|
+
raise Omnizip::AlgorithmNotFoundError, "Algorithm not found: #{algo_sym}" unless algo_class
|
|
61
61
|
|
|
62
62
|
# Decompress
|
|
63
63
|
input_io = StringIO.new(packed_data)
|
|
@@ -80,8 +80,8 @@ module Omnizip
|
|
|
80
80
|
|
|
81
81
|
# BCJ2 requires special handling with multiple streams
|
|
82
82
|
if filter_sym == :bcj2
|
|
83
|
-
raise "BCJ2 archives require multi-stream decompression which is not yet implemented. " \
|
|
84
|
-
|
|
83
|
+
raise Omnizip::UnsupportedFormatError, "BCJ2 archives require multi-stream decompression which is not yet implemented. " \
|
|
84
|
+
"Please use the 7z command-line tool for this archive."
|
|
85
85
|
end
|
|
86
86
|
|
|
87
87
|
filter = filter_class.new
|
|
@@ -109,8 +109,8 @@ module Omnizip
|
|
|
109
109
|
actual_crc = crc.value
|
|
110
110
|
|
|
111
111
|
unless actual_crc == expected_crc
|
|
112
|
-
raise "CRC mismatch: expected 0x#{expected_crc.to_s(16)}, " \
|
|
113
|
-
|
|
112
|
+
raise Omnizip::ChecksumError, "CRC mismatch: expected 0x#{expected_crc.to_s(16)}, " \
|
|
113
|
+
"got 0x#{actual_crc.to_s(16)}"
|
|
114
114
|
end
|
|
115
115
|
end
|
|
116
116
|
|
|
@@ -299,7 +299,7 @@ module Omnizip
|
|
|
299
299
|
# @return [String] Encrypted header with metadata
|
|
300
300
|
def encrypt_header(header_data)
|
|
301
301
|
unless @options[:password]
|
|
302
|
-
raise "Password required for header encryption"
|
|
302
|
+
raise Omnizip::PasswordError, "Password required for header encryption"
|
|
303
303
|
end
|
|
304
304
|
|
|
305
305
|
encryptor = HeaderEncryptor.new(@options[:password])
|
|
@@ -92,14 +92,17 @@ module Omnizip
|
|
|
92
92
|
@entries << entry
|
|
93
93
|
end
|
|
94
94
|
|
|
95
|
-
# Add data directly to the archive
|
|
96
|
-
|
|
95
|
+
# Add data directly to the archive. +compression_method+
|
|
96
|
+
# (a ZIP method code) overrides the archive-wide default for
|
|
97
|
+
# this entry alone.
|
|
98
|
+
def add_data(archive_path, data, stat = nil, compression_method: nil)
|
|
97
99
|
entry = create_entry(
|
|
98
100
|
filename: archive_path,
|
|
99
101
|
uncompressed_data: data,
|
|
100
102
|
stat: stat,
|
|
101
103
|
)
|
|
102
104
|
|
|
105
|
+
entry[:compression_method] = compression_method
|
|
103
106
|
@entries << entry
|
|
104
107
|
end
|
|
105
108
|
|
|
@@ -188,13 +191,17 @@ module Omnizip
|
|
|
188
191
|
def write_to_io(io, compression_method: COMPRESSION_DEFLATE, level: 6)
|
|
189
192
|
local_header_offsets = []
|
|
190
193
|
|
|
191
|
-
# Write local file headers and data
|
|
194
|
+
# Write local file headers and data. An entry-level
|
|
195
|
+
# :compression_method overrides the archive-wide default
|
|
196
|
+
# (used by the buffer layer's per-entry :store option).
|
|
192
197
|
entries.each do |entry|
|
|
193
198
|
offset = io.pos
|
|
194
199
|
local_header_offsets << offset
|
|
195
200
|
|
|
201
|
+
entry_method = entry[:compression_method] || compression_method
|
|
202
|
+
|
|
196
203
|
# Create local file header
|
|
197
|
-
local_header = create_local_header(entry,
|
|
204
|
+
local_header = create_local_header(entry, entry_method)
|
|
198
205
|
|
|
199
206
|
# Compress data if not a directory
|
|
200
207
|
if entry[:directory]
|
|
@@ -205,7 +212,7 @@ module Omnizip
|
|
|
205
212
|
else
|
|
206
213
|
compressed_data = compress_data(
|
|
207
214
|
entry[:uncompressed_data],
|
|
208
|
-
|
|
215
|
+
entry_method,
|
|
209
216
|
level,
|
|
210
217
|
)
|
|
211
218
|
entry[:compressed_size] = compressed_data.bytesize
|
|
@@ -232,7 +239,7 @@ module Omnizip
|
|
|
232
239
|
entries.each_with_index do |entry, index|
|
|
233
240
|
central_header = create_central_header(
|
|
234
241
|
entry,
|
|
235
|
-
compression_method,
|
|
242
|
+
entry[:compression_method] || compression_method,
|
|
236
243
|
local_header_offsets[index],
|
|
237
244
|
)
|
|
238
245
|
io.write(central_header.to_binary)
|
|
@@ -31,17 +31,21 @@ module Omnizip
|
|
|
31
31
|
# @return [Array<Fractor::WorkResult>] the failed results, for
|
|
32
32
|
# the caller to decide how to surface.
|
|
33
33
|
def run(work_items)
|
|
34
|
-
pool =
|
|
34
|
+
pool = WorkerPool.new(
|
|
35
35
|
worker_class: @worker_class,
|
|
36
36
|
num_workers: @threads,
|
|
37
37
|
continuous: false,
|
|
38
38
|
)
|
|
39
39
|
pool.start
|
|
40
|
-
|
|
41
|
-
|
|
40
|
+
begin
|
|
41
|
+
pool.submit_batch(work_items)
|
|
42
|
+
pool.run
|
|
42
43
|
|
|
43
|
-
|
|
44
|
-
|
|
44
|
+
pool.successful_results.each { |r| yield r if block_given? }
|
|
45
|
+
pool.failed_results
|
|
46
|
+
ensure
|
|
47
|
+
pool.shutdown
|
|
48
|
+
end
|
|
45
49
|
end
|
|
46
50
|
end
|
|
47
51
|
end
|
|
@@ -38,6 +38,10 @@ module Omnizip
|
|
|
38
38
|
schedule_dynamic(jobs, worker_count)
|
|
39
39
|
when :static
|
|
40
40
|
schedule_static(jobs, worker_count)
|
|
41
|
+
when :round_robin
|
|
42
|
+
schedule_round_robin(jobs, worker_count)
|
|
43
|
+
when :bin_packing
|
|
44
|
+
schedule_bin_packing(jobs, worker_count)
|
|
41
45
|
end
|
|
42
46
|
end
|
|
43
47
|
|
|
@@ -99,7 +103,7 @@ bytes_per_second: 10_000_000)
|
|
|
99
103
|
#
|
|
100
104
|
# @raise [ArgumentError] if strategy is invalid
|
|
101
105
|
def validate_strategy!
|
|
102
|
-
valid_strategies = %i[dynamic static]
|
|
106
|
+
valid_strategies = %i[dynamic static round_robin bin_packing]
|
|
103
107
|
return if valid_strategies.include?(@strategy)
|
|
104
108
|
|
|
105
109
|
raise ArgumentError,
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require "fractor"
|
|
4
3
|
require "fileutils"
|
|
5
4
|
|
|
6
5
|
module Omnizip
|
|
@@ -21,101 +20,6 @@ module Omnizip
|
|
|
21
20
|
# extractor = Omnizip::Parallel::ParallelExtractor.new(options)
|
|
22
21
|
# extractor.extract('backup.zip', 'output/')
|
|
23
22
|
class ParallelExtractor
|
|
24
|
-
# Fractor Work class for extraction jobs
|
|
25
|
-
class ExtractionWork < Fractor::Work
|
|
26
|
-
def initialize(entry:, archive_path:, dest_dir:)
|
|
27
|
-
super({
|
|
28
|
-
entry: entry,
|
|
29
|
-
archive_path: archive_path,
|
|
30
|
-
dest_dir: dest_dir,
|
|
31
|
-
})
|
|
32
|
-
end
|
|
33
|
-
|
|
34
|
-
def entry
|
|
35
|
-
input[:entry]
|
|
36
|
-
end
|
|
37
|
-
|
|
38
|
-
def archive_path
|
|
39
|
-
input[:archive_path]
|
|
40
|
-
end
|
|
41
|
-
|
|
42
|
-
def dest_dir
|
|
43
|
-
input[:dest_dir]
|
|
44
|
-
end
|
|
45
|
-
end
|
|
46
|
-
|
|
47
|
-
# Fractor Worker class for extraction
|
|
48
|
-
class ExtractionWorker < Fractor::Worker
|
|
49
|
-
def process(work)
|
|
50
|
-
entry = work.entry
|
|
51
|
-
archive_path = work.archive_path
|
|
52
|
-
dest_dir = work.dest_dir
|
|
53
|
-
|
|
54
|
-
# Read and decompress entry data
|
|
55
|
-
data = read_entry_data(archive_path, entry)
|
|
56
|
-
|
|
57
|
-
# Determine destination path
|
|
58
|
-
dest_path = ::File.join(dest_dir, entry.name)
|
|
59
|
-
|
|
60
|
-
# Return result
|
|
61
|
-
Fractor::WorkResult.new(
|
|
62
|
-
result: {
|
|
63
|
-
entry_name: entry.name,
|
|
64
|
-
dest_path: dest_path,
|
|
65
|
-
data: data,
|
|
66
|
-
directory: entry.directory?,
|
|
67
|
-
unix_perms: safe_unix_perms(entry),
|
|
68
|
-
},
|
|
69
|
-
work: work,
|
|
70
|
-
)
|
|
71
|
-
rescue StandardError => e
|
|
72
|
-
Fractor::WorkResult.new(
|
|
73
|
-
error: e,
|
|
74
|
-
work: work,
|
|
75
|
-
)
|
|
76
|
-
end
|
|
77
|
-
|
|
78
|
-
private
|
|
79
|
-
|
|
80
|
-
def read_entry_data(archive_path, entry)
|
|
81
|
-
return "" if entry.directory?
|
|
82
|
-
|
|
83
|
-
# Open archive and extract entry
|
|
84
|
-
reader = Omnizip::Formats::Zip::Reader.new(archive_path)
|
|
85
|
-
reader.read
|
|
86
|
-
|
|
87
|
-
::File.open(archive_path, "rb") do |io|
|
|
88
|
-
# Find the entry in reader
|
|
89
|
-
reader_entry = reader.entries.find { |e| e.filename == entry.name }
|
|
90
|
-
raise Errno::ENOENT, "Entry not found in archive: #{entry.name}" unless reader_entry
|
|
91
|
-
|
|
92
|
-
# Seek to entry data
|
|
93
|
-
io.seek(reader_entry.local_header_offset, ::IO::SEEK_SET)
|
|
94
|
-
|
|
95
|
-
# Read and parse local file header
|
|
96
|
-
fixed_header = io.read(30)
|
|
97
|
-
return "" unless fixed_header && fixed_header.size == 30
|
|
98
|
-
|
|
99
|
-
_signature, _version, _flags, _method, _time, _date, _crc32,
|
|
100
|
-
_comp_size, _uncomp_size, filename_length, extra_length = fixed_header.unpack("VvvvvvVVVvv")
|
|
101
|
-
|
|
102
|
-
# Skip filename and extra field
|
|
103
|
-
io.read(filename_length + extra_length)
|
|
104
|
-
|
|
105
|
-
# Read compressed data
|
|
106
|
-
compressed_data = io.read(reader_entry.compressed_size)
|
|
107
|
-
return "" unless compressed_data
|
|
108
|
-
|
|
109
|
-
# Decompress
|
|
110
|
-
reader.decompress(
|
|
111
|
-
compressed_data,
|
|
112
|
-
reader_entry.compression_method,
|
|
113
|
-
reader_entry.uncompressed_size,
|
|
114
|
-
)
|
|
115
|
-
end
|
|
116
|
-
end
|
|
117
|
-
end
|
|
118
|
-
|
|
119
23
|
# @return [Omnizip::Models::ParallelOptions] parallel options
|
|
120
24
|
attr_reader :options
|
|
121
25
|
|
|
@@ -179,41 +83,41 @@ module Omnizip
|
|
|
179
83
|
|
|
180
84
|
# Schedule jobs
|
|
181
85
|
entries.each do |entry|
|
|
182
|
-
file_size = safe_entry_size(entry)
|
|
183
|
-
|
|
184
86
|
job_queue.push_with_size(
|
|
185
|
-
file: entry.
|
|
186
|
-
size:
|
|
187
|
-
data: {
|
|
87
|
+
file: entry.filename,
|
|
88
|
+
size: safe_entry_size(entry),
|
|
89
|
+
data: { entry_name: entry.filename },
|
|
188
90
|
)
|
|
189
91
|
end
|
|
190
92
|
|
|
191
|
-
# Create work items from jobs
|
|
93
|
+
# Create work items from jobs, largest first
|
|
192
94
|
work_items = []
|
|
193
95
|
until job_queue.empty?
|
|
194
96
|
job = job_queue.pop(timeout: 0.1)
|
|
195
97
|
break unless job
|
|
196
98
|
|
|
197
|
-
work_items <<
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
99
|
+
work_items << job.data[:entry_name]
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
# Threads, not Ractors: Omnizip's autoloads and Proc-holding
|
|
103
|
+
# codec constants cannot cross Ractor boundaries on Ruby <= 3.3
|
|
104
|
+
pool = ThreadPool.new(size: @options.threads)
|
|
105
|
+
raw_results, raw_errors = pool.map(work_items) do |entry_name|
|
|
106
|
+
process_entry(archive, dest, entry_name)
|
|
202
107
|
end
|
|
203
108
|
|
|
204
|
-
#
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
errors = engine.run(work_items) { |r| results << r }
|
|
209
|
-
|
|
210
|
-
# Handle errors
|
|
211
|
-
unless errors.empty?
|
|
212
|
-
error_msgs = errors.map do |e|
|
|
213
|
-
"#{e.work&.entry&.name}: #{e.error}"
|
|
214
|
-
end.join("\n")
|
|
215
|
-
raise Omnizip::ExtractionError, "Extraction errors:\n#{error_msgs}"
|
|
109
|
+
# Handle errors (raw_errors holds nil for every item that
|
|
110
|
+
# succeeded — zip+compact would keep those pairs)
|
|
111
|
+
failure_pairs = raw_errors.each_with_index.filter_map do |error, index|
|
|
112
|
+
[work_items[index], error] if error
|
|
216
113
|
end
|
|
114
|
+
unless failure_pairs.empty?
|
|
115
|
+
error_msgs = failure_pairs.map { |name, e| "#{name}: #{e.message}" }
|
|
116
|
+
.join("\n")
|
|
117
|
+
raise Omnizip::DecompressionError, "Extraction errors:\n#{error_msgs}"
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
results = raw_results
|
|
217
121
|
|
|
218
122
|
# Write files to disk (thread-safe)
|
|
219
123
|
extracted_paths = write_extracted_files(results, overwrite: overwrite)
|
|
@@ -242,32 +146,37 @@ module Omnizip
|
|
|
242
146
|
|
|
243
147
|
private
|
|
244
148
|
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
149
|
+
# Decompress one entry in a worker thread. Each worker opens
|
|
150
|
+
# the archive independently — its own file handle per thread.
|
|
151
|
+
def process_entry(archive_path, dest_dir, entry_name)
|
|
152
|
+
entry = Omnizip::Formats::Zip::Reader.new(archive_path)
|
|
153
|
+
.entries.find { |e| e.filename == entry_name }
|
|
154
|
+
raise Errno::ENOENT, "Entry not found: #{entry_name}" unless entry
|
|
155
|
+
|
|
156
|
+
{
|
|
157
|
+
entry_name: entry_name,
|
|
158
|
+
dest_path: ::File.join(dest_dir, entry_name),
|
|
159
|
+
data: entry.directory? ? "" : read_entry_data(archive_path, entry_name),
|
|
160
|
+
directory: entry.directory?,
|
|
161
|
+
unix_perms: entry.unix_permissions.to_i,
|
|
162
|
+
}
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
def read_entry_data(archive_path, entry_name)
|
|
166
|
+
Omnizip::Formats::Zip::Reader.new(archive_path)
|
|
167
|
+
.read_entry(entry_name)
|
|
249
168
|
end
|
|
250
169
|
|
|
251
170
|
def safe_entry_size(entry)
|
|
252
|
-
entry.
|
|
253
|
-
rescue NoMethodError
|
|
254
|
-
0
|
|
171
|
+
entry.uncompressed_size.to_i
|
|
255
172
|
end
|
|
256
173
|
|
|
257
|
-
# Read archive entries
|
|
174
|
+
# Read archive entries through the native reader
|
|
258
175
|
#
|
|
259
176
|
# @param archive_path [String] archive path
|
|
260
|
-
# @return [Array<
|
|
177
|
+
# @return [Array<CentralDirectoryHeader>] array of entries
|
|
261
178
|
def read_archive_entries(archive_path)
|
|
262
|
-
entries
|
|
263
|
-
|
|
264
|
-
Omnizip::Zip::File.open(archive_path) do |zip|
|
|
265
|
-
zip.each do |entry|
|
|
266
|
-
entries << entry
|
|
267
|
-
end
|
|
268
|
-
end
|
|
269
|
-
|
|
270
|
-
entries
|
|
179
|
+
Omnizip::Formats::Zip::Reader.new(archive_path).entries
|
|
271
180
|
end
|
|
272
181
|
|
|
273
182
|
# Write extracted files to disk
|
|
@@ -278,34 +187,36 @@ module Omnizip
|
|
|
278
187
|
def write_extracted_files(results, overwrite: false)
|
|
279
188
|
extracted_paths = []
|
|
280
189
|
|
|
281
|
-
results.each do |
|
|
282
|
-
result = work_result.result
|
|
190
|
+
results.each do |result|
|
|
283
191
|
next unless result
|
|
284
192
|
|
|
285
193
|
dest_path = result[:dest_path]
|
|
286
194
|
|
|
287
195
|
# Thread-safe file writing
|
|
288
196
|
@write_mutex.synchronize do
|
|
197
|
+
if result[:directory]
|
|
198
|
+
# mkdir_p is idempotent — a file entry may already have
|
|
199
|
+
# created this directory as its parent
|
|
200
|
+
FileUtils.mkdir_p(dest_path)
|
|
201
|
+
extracted_paths << dest_path
|
|
202
|
+
next
|
|
203
|
+
end
|
|
204
|
+
|
|
289
205
|
# Check if file exists
|
|
290
206
|
if ::File.exist?(dest_path) && !overwrite
|
|
291
207
|
raise Errno::EEXIST, "File exists: #{dest_path}"
|
|
292
208
|
end
|
|
293
209
|
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
FileUtils.mkdir_p(dest_path)
|
|
297
|
-
else
|
|
298
|
-
FileUtils.mkdir_p(::File.dirname(dest_path))
|
|
299
|
-
::File.binwrite(dest_path, result[:data])
|
|
210
|
+
FileUtils.mkdir_p(::File.dirname(dest_path))
|
|
211
|
+
::File.binwrite(dest_path, result[:data])
|
|
300
212
|
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
end
|
|
305
|
-
|
|
306
|
-
@stats[:bytes_extracted] += result[:data].bytesize
|
|
213
|
+
# Set permissions if Unix
|
|
214
|
+
if result[:unix_perms].positive?
|
|
215
|
+
::File.chmod(result[:unix_perms] & 0o777, dest_path)
|
|
307
216
|
end
|
|
308
217
|
|
|
218
|
+
@stats[:bytes_extracted] += result[:data].bytesize
|
|
219
|
+
|
|
309
220
|
extracted_paths << dest_path
|
|
310
221
|
end
|
|
311
222
|
end
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
# frozen_string: true
|
|
2
|
+
|
|
3
|
+
module Omnizip
|
|
4
|
+
module Parallel
|
|
5
|
+
# Bounded thread pool for library work.
|
|
6
|
+
#
|
|
7
|
+
# Unlike the Fractor pool, threads share the VM state — Ractor
|
|
8
|
+
# workers cannot trigger Omnizip's autoloads or touch Proc-holding
|
|
9
|
+
# codec constants on Ruby <= 3.3 ("require by autoload on
|
|
10
|
+
# non-main Ractor is not supported"). Decompression itself still
|
|
11
|
+
# parallelizes: the zlib/bzip2 C extensions release the GVL.
|
|
12
|
+
class ThreadPool
|
|
13
|
+
# @return [Integer] number of worker threads
|
|
14
|
+
attr_reader :size
|
|
15
|
+
|
|
16
|
+
# Initialize the pool
|
|
17
|
+
#
|
|
18
|
+
# @param size [Integer] worker thread count (at least 1)
|
|
19
|
+
def initialize(size:)
|
|
20
|
+
@size = [size.to_i, 1].max
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
# Run +block+ for every item, at most +size+ at a time, and
|
|
24
|
+
# return a pair of parallel arrays: results (nil where an item
|
|
25
|
+
# raised) and errors (nil where it succeeded). Item order is
|
|
26
|
+
# preserved in both arrays.
|
|
27
|
+
#
|
|
28
|
+
# @param items [Array] work items
|
|
29
|
+
# @yieldparam item [Object] one work item
|
|
30
|
+
# @return [Array<Array, Array>]
|
|
31
|
+
def map(items)
|
|
32
|
+
results = Array.new(items.size)
|
|
33
|
+
errors = Array.new(items.size)
|
|
34
|
+
queue = Queue.new
|
|
35
|
+
items.each_with_index { |item, index| queue << [item, index] }
|
|
36
|
+
|
|
37
|
+
threads = Array.new(@size) do
|
|
38
|
+
Thread.new do
|
|
39
|
+
loop do
|
|
40
|
+
begin
|
|
41
|
+
item, index = queue.pop(true)
|
|
42
|
+
rescue ThreadError
|
|
43
|
+
break
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
begin
|
|
47
|
+
results[index] = yield(item)
|
|
48
|
+
rescue StandardError => e
|
|
49
|
+
errors[index] = e
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
threads.each(&:join)
|
|
55
|
+
|
|
56
|
+
[results, errors]
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
end
|
|
@@ -132,8 +132,12 @@ module Omnizip
|
|
|
132
132
|
def shutdown(timeout: 30)
|
|
133
133
|
return unless @running
|
|
134
134
|
|
|
135
|
+
# Stop in BOTH modes: batch worker Ractors otherwise linger
|
|
136
|
+
# in Ractor.receive for the life of the process, burning CPU
|
|
137
|
+
# after the batch completes
|
|
138
|
+
@supervisor&.stop
|
|
139
|
+
|
|
135
140
|
if @continuous
|
|
136
|
-
@supervisor.stop
|
|
137
141
|
@supervisor_thread&.join(timeout)
|
|
138
142
|
end
|
|
139
143
|
|
data/lib/omnizip/parallel.rb
CHANGED
|
@@ -29,6 +29,7 @@ module Omnizip
|
|
|
29
29
|
autoload :Engine, "omnizip/parallel/engine"
|
|
30
30
|
autoload :ParallelCompressor, "omnizip/parallel/parallel_compressor"
|
|
31
31
|
autoload :ParallelExtractor, "omnizip/parallel/parallel_extractor"
|
|
32
|
+
autoload :ThreadPool, "omnizip/parallel/thread_pool"
|
|
32
33
|
|
|
33
34
|
class << self
|
|
34
35
|
# Global configuration
|
data/lib/omnizip/version.rb
CHANGED
data/lib/omnizip.rb
CHANGED
|
@@ -85,6 +85,9 @@ module Omnizip
|
|
|
85
85
|
autoload :ArchiveHandlers, "omnizip/archive_handlers"
|
|
86
86
|
autoload :Archive, "omnizip/archive"
|
|
87
87
|
autoload :Password, "omnizip/password"
|
|
88
|
+
# The class lives at the top level of omnizip/password.rb; without
|
|
89
|
+
# this entry, referencing it does not trigger the Password autoload
|
|
90
|
+
autoload :PasswordError, "omnizip/password"
|
|
88
91
|
autoload :Convenience, "omnizip/convenience"
|
|
89
92
|
end
|
|
90
93
|
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: omnizip
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.3.
|
|
4
|
+
version: 0.3.54
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Ribose Inc.
|
|
@@ -650,6 +650,7 @@ files:
|
|
|
650
650
|
- lib/omnizip/parallel/job_scheduler.rb
|
|
651
651
|
- lib/omnizip/parallel/parallel_compressor.rb
|
|
652
652
|
- lib/omnizip/parallel/parallel_extractor.rb
|
|
653
|
+
- lib/omnizip/parallel/thread_pool.rb
|
|
653
654
|
- lib/omnizip/parallel/worker_pool.rb
|
|
654
655
|
- lib/omnizip/parity.rb
|
|
655
656
|
- lib/omnizip/parity/chunked_block_processor.rb
|