zip_tricks 5.1.0 → 5.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +96 -0
- data/.gitignore +2 -1
- data/.rubocop.yml +2 -0
- data/CHANGELOG.md +49 -3
- data/CONTRIBUTING.md +4 -4
- data/README.md +28 -17
- data/lib/zip_tricks/block_write.rb +26 -21
- data/lib/zip_tricks/file_reader.rb +8 -6
- data/lib/zip_tricks/null_writer.rb +1 -1
- data/lib/zip_tricks/output_enumerator.rb +48 -27
- data/lib/zip_tricks/path_set.rb +4 -0
- data/lib/zip_tricks/rails_streaming.rb +7 -9
- data/lib/zip_tricks/size_estimator.rb +0 -3
- data/lib/zip_tricks/stream_crc32.rb +2 -2
- data/lib/zip_tricks/streamer/deflated_writer.rb +9 -28
- data/lib/zip_tricks/streamer/entry.rb +5 -1
- data/lib/zip_tricks/streamer/stored_writer.rb +4 -3
- data/lib/zip_tricks/streamer.rb +69 -28
- data/lib/zip_tricks/version.rb +1 -1
- data/lib/zip_tricks/write_and_tell.rb +2 -12
- data/lib/zip_tricks/write_buffer.rb +37 -17
- data/lib/zip_tricks/zip_writer.rb +37 -30
- data/zip_tricks.gemspec +4 -6
- metadata +9 -51
- data/.travis.yml +0 -11
- data/qa/README_QA.md +0 -16
- data/qa/generate_test_files.rb +0 -126
- data/qa/in/VTYL8830.jpg +0 -0
- data/qa/in/war-and-peace.txt +0 -10810
- data/qa/support.rb +0 -88
- data/qa/test-report-2016-07-28.txt +0 -156
- data/qa/test-report-2016-12-12.txt +0 -156
- data/qa/test-report-2017-04-2.txt +0 -168
- data/qa/test-report.txt +0 -28
|
@@ -4,13 +4,6 @@
|
|
|
4
4
|
# registers data passing through it in a CRC32 checksum calculator. Is made to be completely
|
|
5
5
|
# interchangeable with the StoredWriter in terms of interface.
|
|
6
6
|
class ZipTricks::Streamer::DeflatedWriter
|
|
7
|
-
# After how many bytes of incoming data the deflater for the
|
|
8
|
-
# contents must be flushed. This is done to prevent unreasonable
|
|
9
|
-
# memory use when archiving large files, and to ensure we write to
|
|
10
|
-
# the socket often enough while still maintaining acceptable
|
|
11
|
-
# compression
|
|
12
|
-
FLUSH_EVERY_N_BYTES = 1024 * 1024 * 5
|
|
13
|
-
|
|
14
7
|
# The amount of bytes we will buffer before computing the intermediate
|
|
15
8
|
# CRC32 checksums. Benchmarks show that the optimum is 64KB (see
|
|
16
9
|
# `bench/buffered_crc32_bench.rb), if that is exceeded Zlib is going
|
|
@@ -18,11 +11,10 @@ class ZipTricks::Streamer::DeflatedWriter
|
|
|
18
11
|
CRC32_BUFFER_SIZE = 64 * 1024
|
|
19
12
|
|
|
20
13
|
def initialize(io)
|
|
21
|
-
@compressed_io =
|
|
22
|
-
@uncompressed_size = 0
|
|
14
|
+
@compressed_io = io
|
|
23
15
|
@deflater = ::Zlib::Deflate.new(Zlib::DEFAULT_COMPRESSION, -::Zlib::MAX_WBITS)
|
|
24
|
-
@crc = ZipTricks::
|
|
25
|
-
@
|
|
16
|
+
@crc = ZipTricks::StreamCRC32.new
|
|
17
|
+
@crc_buf = ZipTricks::WriteBuffer.new(@crc, CRC32_BUFFER_SIZE)
|
|
26
18
|
end
|
|
27
19
|
|
|
28
20
|
# Writes the given data into the deflater, and flushes the deflater
|
|
@@ -31,13 +23,8 @@ class ZipTricks::Streamer::DeflatedWriter
|
|
|
31
23
|
# @param data[String] data to be written
|
|
32
24
|
# @return self
|
|
33
25
|
def <<(data)
|
|
34
|
-
@
|
|
35
|
-
@
|
|
36
|
-
@compressed_io << @deflater.deflate(data)
|
|
37
|
-
@crc << data
|
|
38
|
-
|
|
39
|
-
interim_flush
|
|
40
|
-
|
|
26
|
+
@deflater.deflate(data) { |chunk| @compressed_io << chunk }
|
|
27
|
+
@crc_buf << data
|
|
41
28
|
self
|
|
42
29
|
end
|
|
43
30
|
|
|
@@ -45,18 +32,12 @@ class ZipTricks::Streamer::DeflatedWriter
|
|
|
45
32
|
# compressed data written and the CRC32 checksum. The return value
|
|
46
33
|
# can be directly used as the argument to {Streamer#update_last_entry_and_write_data_descriptor}
|
|
47
34
|
#
|
|
48
|
-
# @param data[String] data to be written
|
|
49
35
|
# @return [Hash] a hash of `{crc32, compressed_size, uncompressed_size}`
|
|
50
36
|
def finish
|
|
51
37
|
@compressed_io << @deflater.finish until @deflater.finished?
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
def interim_flush
|
|
58
|
-
return if @bytes_since_last_flush < FLUSH_EVERY_N_BYTES
|
|
59
|
-
@compressed_io << @deflater.flush
|
|
60
|
-
@bytes_since_last_flush = 0
|
|
38
|
+
@crc_buf.flush
|
|
39
|
+
{crc32: @crc.to_i, compressed_size: @deflater.total_out, uncompressed_size: @deflater.total_in}
|
|
40
|
+
ensure
|
|
41
|
+
@deflater.close
|
|
61
42
|
end
|
|
62
43
|
end
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
# Normally you will not have to use this class directly
|
|
5
5
|
class ZipTricks::Streamer::Entry < Struct.new(:filename, :crc32, :compressed_size,
|
|
6
6
|
:uncompressed_size, :storage_mode, :mtime,
|
|
7
|
-
:use_data_descriptor)
|
|
7
|
+
:use_data_descriptor, :local_header_offset, :bytes_used_for_local_header, :bytes_used_for_data_descriptor, :unix_permissions)
|
|
8
8
|
def initialize(*)
|
|
9
9
|
super
|
|
10
10
|
filename.force_encoding(Encoding::UTF_8)
|
|
@@ -15,6 +15,10 @@ class ZipTricks::Streamer::Entry < Struct.new(:filename, :crc32, :compressed_siz
|
|
|
15
15
|
end)
|
|
16
16
|
end
|
|
17
17
|
|
|
18
|
+
def total_bytes_used
|
|
19
|
+
bytes_used_for_local_header + compressed_size + bytes_used_for_data_descriptor
|
|
20
|
+
end
|
|
21
|
+
|
|
18
22
|
# Set the general purpose flags for the entry. We care about is the EFS
|
|
19
23
|
# bit (bit 11) which should be set if the filename is UTF8. If it is, we need to set the
|
|
20
24
|
# bit so that the unarchiving application knows that the filename in the archive is UTF-8
|
|
@@ -12,7 +12,8 @@ class ZipTricks::Streamer::StoredWriter
|
|
|
12
12
|
|
|
13
13
|
def initialize(io)
|
|
14
14
|
@io = ZipTricks::WriteAndTell.new(io)
|
|
15
|
-
@
|
|
15
|
+
@crc_compute = ZipTricks::StreamCRC32.new
|
|
16
|
+
@crc = ZipTricks::WriteBuffer.new(@crc_compute, CRC32_BUFFER_SIZE)
|
|
16
17
|
end
|
|
17
18
|
|
|
18
19
|
# Writes the given data to the contained IO object.
|
|
@@ -28,9 +29,9 @@ class ZipTricks::Streamer::StoredWriter
|
|
|
28
29
|
# Returns the amount of data written and the CRC32 checksum. The return value
|
|
29
30
|
# can be directly used as the argument to {Streamer#update_last_entry_and_write_data_descriptor}
|
|
30
31
|
#
|
|
31
|
-
# @param data[String] data to be written
|
|
32
32
|
# @return [Hash] a hash of `{crc32, compressed_size, uncompressed_size}`
|
|
33
33
|
def finish
|
|
34
|
-
|
|
34
|
+
@crc.flush
|
|
35
|
+
{crc32: @crc_compute.to_i, compressed_size: @io.tell, uncompressed_size: @io.tell}
|
|
35
36
|
end
|
|
36
37
|
end
|
data/lib/zip_tricks/streamer.rb
CHANGED
|
@@ -91,8 +91,9 @@ class ZipTricks::Streamer
|
|
|
91
91
|
InvalidOutput = Class.new(ArgumentError)
|
|
92
92
|
Overflow = Class.new(StandardError)
|
|
93
93
|
UnknownMode = Class.new(StandardError)
|
|
94
|
+
OffsetOutOfSync = Class.new(StandardError)
|
|
94
95
|
|
|
95
|
-
private_constant :
|
|
96
|
+
private_constant :STORED, :DEFLATED
|
|
96
97
|
|
|
97
98
|
# Creates a new Streamer on top of the given IO-ish object and yields it. Once the given block
|
|
98
99
|
# returns, the Streamer will have it's `close` method called, which will write out the central
|
|
@@ -130,28 +131,26 @@ class ZipTricks::Streamer
|
|
|
130
131
|
# end
|
|
131
132
|
#
|
|
132
133
|
# @param kwargs_for_new [Hash] keyword arguments for {Streamer.new}
|
|
133
|
-
# @return [
|
|
134
|
+
# @return [ZipTricks::OutputEnumerator] the enumerator you can read bytestrings of the ZIP from by calling `each`
|
|
134
135
|
def self.output_enum(**kwargs_for_new, &zip_streamer_block)
|
|
135
136
|
ZipTricks::OutputEnumerator.new(**kwargs_for_new, &zip_streamer_block)
|
|
136
137
|
end
|
|
137
138
|
|
|
138
139
|
# Creates a new Streamer on top of the given IO-ish object.
|
|
139
140
|
#
|
|
140
|
-
# @param
|
|
141
|
+
# @param writable[#<<] the destination IO for the ZIP. Anything that responds to `<<` can be used.
|
|
141
142
|
# @param writer[ZipTricks::ZipWriter] the object to be used as the writer.
|
|
142
143
|
# Defaults to an instance of ZipTricks::ZipWriter, normally you won't need to override it
|
|
143
144
|
# @param auto_rename_duplicate_filenames[Boolean] whether duplicate filenames, when encountered,
|
|
144
145
|
# should be suffixed with (1), (2) etc. Default value is `false` - if
|
|
145
146
|
# dupliate names are used an exception will be raised
|
|
146
|
-
def initialize(
|
|
147
|
-
raise InvalidOutput, 'The
|
|
148
|
-
|
|
149
|
-
@dedupe_filenames = auto_rename_duplicate_filenames
|
|
150
|
-
@out = ZipTricks::WriteAndTell.new(stream)
|
|
147
|
+
def initialize(writable, writer: create_writer, auto_rename_duplicate_filenames: false)
|
|
148
|
+
raise InvalidOutput, 'The writable must respond to #<<' unless writable.respond_to?(:<<)
|
|
149
|
+
@out = ZipTricks::WriteAndTell.new(writable)
|
|
151
150
|
@files = []
|
|
152
|
-
@local_header_offsets = []
|
|
153
151
|
@path_set = ZipTricks::PathSet.new
|
|
154
152
|
@writer = writer
|
|
153
|
+
@dedupe_filenames = auto_rename_duplicate_filenames
|
|
155
154
|
end
|
|
156
155
|
|
|
157
156
|
# Writes a part of a zip entry body (actual binary data of the entry) into the output stream.
|
|
@@ -201,21 +200,20 @@ class ZipTricks::Streamer
|
|
|
201
200
|
# @param uncompressed_size [Integer] the size of the entry when uncompressed, in bytes
|
|
202
201
|
# @param crc32 [Integer] the CRC32 checksum of the entry when uncompressed
|
|
203
202
|
# @param use_data_descriptor [Boolean] whether the entry body will be followed by a data descriptor
|
|
203
|
+
# @param unix_permissions[Fixnum?] which UNIX permissions to set, normally the default should be used
|
|
204
204
|
# @return [Integer] the offset the output IO is at after writing the entry header
|
|
205
|
-
def add_deflated_entry(filename:, modification_time: Time.now.utc, compressed_size: 0, uncompressed_size: 0, crc32: 0, use_data_descriptor: false)
|
|
205
|
+
def add_deflated_entry(filename:, modification_time: Time.now.utc, compressed_size: 0, uncompressed_size: 0, crc32: 0, unix_permissions: nil, use_data_descriptor: false)
|
|
206
206
|
add_file_and_write_local_header(filename: filename,
|
|
207
207
|
modification_time: modification_time,
|
|
208
208
|
crc32: crc32,
|
|
209
209
|
storage_mode: DEFLATED,
|
|
210
210
|
compressed_size: compressed_size,
|
|
211
211
|
uncompressed_size: uncompressed_size,
|
|
212
|
+
unix_permissions: unix_permissions,
|
|
212
213
|
use_data_descriptor: use_data_descriptor)
|
|
213
214
|
@out.tell
|
|
214
215
|
end
|
|
215
216
|
|
|
216
|
-
# Will be phased out in ZipTricks 5.x
|
|
217
|
-
alias_method :add_compressed_entry, :add_deflated_entry
|
|
218
|
-
|
|
219
217
|
# Writes out the local header for an entry (file in the ZIP) that is using
|
|
220
218
|
# the stored storage model (is stored as-is).
|
|
221
219
|
# Once this method is called, the `<<` method has to be called one or more
|
|
@@ -226,14 +224,16 @@ class ZipTricks::Streamer
|
|
|
226
224
|
# @param size [Integer] the size of the file when uncompressed, in bytes
|
|
227
225
|
# @param crc32 [Integer] the CRC32 checksum of the entry when uncompressed
|
|
228
226
|
# @param use_data_descriptor [Boolean] whether the entry body will be followed by a data descriptor. When in use
|
|
227
|
+
# @param unix_permissions[Fixnum?] which UNIX permissions to set, normally the default should be used
|
|
229
228
|
# @return [Integer] the offset the output IO is at after writing the entry header
|
|
230
|
-
def add_stored_entry(filename:, modification_time: Time.now.utc, size: 0, crc32: 0, use_data_descriptor: false)
|
|
229
|
+
def add_stored_entry(filename:, modification_time: Time.now.utc, size: 0, crc32: 0, unix_permissions: nil, use_data_descriptor: false)
|
|
231
230
|
add_file_and_write_local_header(filename: filename,
|
|
232
231
|
modification_time: modification_time,
|
|
233
232
|
crc32: crc32,
|
|
234
233
|
storage_mode: STORED,
|
|
235
234
|
compressed_size: size,
|
|
236
235
|
uncompressed_size: size,
|
|
236
|
+
unix_permissions: unix_permissions,
|
|
237
237
|
use_data_descriptor: use_data_descriptor)
|
|
238
238
|
@out.tell
|
|
239
239
|
end
|
|
@@ -242,14 +242,16 @@ class ZipTricks::Streamer
|
|
|
242
242
|
#
|
|
243
243
|
# @param dirname [String] the name of the directory in the archive
|
|
244
244
|
# @param modification_time [Time] the modification time of the directory in the archive
|
|
245
|
+
# @param unix_permissions[Fixnum?] which UNIX permissions to set, normally the default should be used
|
|
245
246
|
# @return [Integer] the offset the output IO is at after writing the entry header
|
|
246
|
-
def add_empty_directory(dirname:, modification_time: Time.now.utc)
|
|
247
|
+
def add_empty_directory(dirname:, modification_time: Time.now.utc, unix_permissions: nil)
|
|
247
248
|
add_file_and_write_local_header(filename: dirname.to_s + '/',
|
|
248
249
|
modification_time: modification_time,
|
|
249
250
|
crc32: 0,
|
|
250
251
|
storage_mode: STORED,
|
|
251
252
|
compressed_size: 0,
|
|
252
253
|
uncompressed_size: 0,
|
|
254
|
+
unix_permissions: unix_permissions,
|
|
253
255
|
use_data_descriptor: false)
|
|
254
256
|
@out.tell
|
|
255
257
|
end
|
|
@@ -287,14 +289,16 @@ class ZipTricks::Streamer
|
|
|
287
289
|
#
|
|
288
290
|
# @param filename[String] the name of the file in the archive
|
|
289
291
|
# @param modification_time [Time] the modification time of the file in the archive
|
|
292
|
+
# @param unix_permissions[Fixnum?] which UNIX permissions to set, normally the default should be used
|
|
290
293
|
# @yield [#<<, #write] an object that the file contents must be written to that will be automatically closed
|
|
291
294
|
# @return [#<<, #write, #close] an object that the file contents must be written to, has to be closed manually
|
|
292
|
-
def write_stored_file(filename, modification_time: Time.now.utc)
|
|
295
|
+
def write_stored_file(filename, modification_time: Time.now.utc, unix_permissions: nil)
|
|
293
296
|
add_stored_entry(filename: filename,
|
|
294
297
|
modification_time: modification_time,
|
|
295
298
|
use_data_descriptor: true,
|
|
296
299
|
crc32: 0,
|
|
297
|
-
size: 0
|
|
300
|
+
size: 0,
|
|
301
|
+
unix_permissions: unix_permissions)
|
|
298
302
|
|
|
299
303
|
writable = Writable.new(self, StoredWriter.new(@out))
|
|
300
304
|
if block_given?
|
|
@@ -339,14 +343,16 @@ class ZipTricks::Streamer
|
|
|
339
343
|
#
|
|
340
344
|
# @param filename[String] the name of the file in the archive
|
|
341
345
|
# @param modification_time [Time] the modification time of the file in the archive
|
|
346
|
+
# @param unix_permissions[Fixnum?] which UNIX permissions to set, normally the default should be used
|
|
342
347
|
# @yield [#<<, #write] an object that the file contents must be written to
|
|
343
|
-
def write_deflated_file(filename, modification_time: Time.now.utc)
|
|
348
|
+
def write_deflated_file(filename, modification_time: Time.now.utc, unix_permissions: nil)
|
|
344
349
|
add_deflated_entry(filename: filename,
|
|
345
350
|
modification_time: modification_time,
|
|
346
351
|
use_data_descriptor: true,
|
|
347
352
|
crc32: 0,
|
|
348
353
|
compressed_size: 0,
|
|
349
|
-
uncompressed_size: 0
|
|
354
|
+
uncompressed_size: 0,
|
|
355
|
+
unix_permissions: unix_permissions)
|
|
350
356
|
|
|
351
357
|
writable = Writable.new(self, DeflatedWriter.new(@out))
|
|
352
358
|
if block_given?
|
|
@@ -363,21 +369,24 @@ class ZipTricks::Streamer
|
|
|
363
369
|
#
|
|
364
370
|
# @return [Integer] the offset the output IO is at after closing the archive
|
|
365
371
|
def close
|
|
372
|
+
# Make sure offsets are in order
|
|
373
|
+
verify_offsets!
|
|
374
|
+
|
|
366
375
|
# Record the central directory offset, so that it can be written into the EOCD record
|
|
367
376
|
cdir_starts_at = @out.tell
|
|
368
377
|
|
|
369
378
|
# Write out the central directory entries, one for each file
|
|
370
|
-
@files.
|
|
371
|
-
header_loc = @local_header_offsets.fetch(i)
|
|
379
|
+
@files.each do |entry|
|
|
372
380
|
@writer.write_central_directory_file_header(io: @out,
|
|
373
|
-
local_file_header_location:
|
|
381
|
+
local_file_header_location: entry.local_header_offset,
|
|
374
382
|
gp_flags: entry.gp_flags,
|
|
375
383
|
storage_mode: entry.storage_mode,
|
|
376
384
|
compressed_size: entry.compressed_size,
|
|
377
385
|
uncompressed_size: entry.uncompressed_size,
|
|
378
386
|
mtime: entry.mtime,
|
|
379
387
|
crc32: entry.crc32,
|
|
380
|
-
filename: entry.filename
|
|
388
|
+
filename: entry.filename,
|
|
389
|
+
unix_permissions: entry.unix_permissions)
|
|
381
390
|
end
|
|
382
391
|
|
|
383
392
|
# Record the central directory size, for the EOCDR
|
|
@@ -423,15 +432,40 @@ class ZipTricks::Streamer
|
|
|
423
432
|
last_entry.compressed_size = compressed_size
|
|
424
433
|
last_entry.uncompressed_size = uncompressed_size
|
|
425
434
|
|
|
435
|
+
offset_before_data_descriptor = @out.tell
|
|
426
436
|
@writer.write_data_descriptor(io: @out,
|
|
427
437
|
crc32: last_entry.crc32,
|
|
428
438
|
compressed_size: last_entry.compressed_size,
|
|
429
439
|
uncompressed_size: last_entry.uncompressed_size)
|
|
440
|
+
last_entry.bytes_used_for_data_descriptor = @out.tell - offset_before_data_descriptor
|
|
441
|
+
|
|
430
442
|
@out.tell
|
|
431
443
|
end
|
|
432
444
|
|
|
433
445
|
private
|
|
434
446
|
|
|
447
|
+
def verify_offsets!
|
|
448
|
+
# We need to check whether the offsets noted for the entries actually make sense
|
|
449
|
+
computed_offset = @files.map(&:total_bytes_used).inject(0, &:+)
|
|
450
|
+
actual_offset = @out.tell
|
|
451
|
+
if computed_offset != actual_offset
|
|
452
|
+
message = <<-EMS
|
|
453
|
+
The offset of the Streamer output IO is out of sync with the expected value. All entries written so far,
|
|
454
|
+
including their compressed bodies, local headers and data descriptors, add up to a certain offset,
|
|
455
|
+
but this offset does not match the actual offset of the IO.
|
|
456
|
+
|
|
457
|
+
Entries add up to #{computed_offset} bytes and the IO is at #{actual_offset} bytes.
|
|
458
|
+
|
|
459
|
+
This can happen if you write local headers for an entry, write the "body" of the entry directly to the IO
|
|
460
|
+
object which is your destination, but do not adjust the offset known to the Streamer object. To adjust
|
|
461
|
+
the offfset you need to call `Streamer#simulate_write(body_size)` after outputting the entry. Otherwise
|
|
462
|
+
the local header offsets of the entries you write are going to be incorrect and some ZIP applications
|
|
463
|
+
are going to have problems opening your archive.
|
|
464
|
+
EMS
|
|
465
|
+
raise OffsetOutOfSync, message
|
|
466
|
+
end
|
|
467
|
+
end
|
|
468
|
+
|
|
435
469
|
def add_file_and_write_local_header(
|
|
436
470
|
filename:,
|
|
437
471
|
modification_time:,
|
|
@@ -439,7 +473,8 @@ class ZipTricks::Streamer
|
|
|
439
473
|
storage_mode:,
|
|
440
474
|
compressed_size:,
|
|
441
475
|
uncompressed_size:,
|
|
442
|
-
use_data_descriptor
|
|
476
|
+
use_data_descriptor:,
|
|
477
|
+
unix_permissions:)
|
|
443
478
|
|
|
444
479
|
# Clean backslashes
|
|
445
480
|
filename = remove_backslash(filename)
|
|
@@ -464,16 +499,19 @@ class ZipTricks::Streamer
|
|
|
464
499
|
uncompressed_size = 0
|
|
465
500
|
end
|
|
466
501
|
|
|
502
|
+
local_header_starts_at = @out.tell
|
|
503
|
+
|
|
467
504
|
e = Entry.new(filename,
|
|
468
505
|
crc32,
|
|
469
506
|
compressed_size,
|
|
470
507
|
uncompressed_size,
|
|
471
508
|
storage_mode,
|
|
472
509
|
modification_time,
|
|
473
|
-
use_data_descriptor
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
510
|
+
use_data_descriptor,
|
|
511
|
+
_local_file_header_offset = local_header_starts_at,
|
|
512
|
+
_bytes_used_for_local_header = 0,
|
|
513
|
+
_bytes_used_for_data_descriptor = 0,
|
|
514
|
+
unix_permissions)
|
|
477
515
|
|
|
478
516
|
@writer.write_local_file_header(io: @out,
|
|
479
517
|
gp_flags: e.gp_flags,
|
|
@@ -483,6 +521,9 @@ class ZipTricks::Streamer
|
|
|
483
521
|
mtime: e.mtime,
|
|
484
522
|
filename: e.filename,
|
|
485
523
|
storage_mode: e.storage_mode)
|
|
524
|
+
e.bytes_used_for_local_header = @out.tell - e.local_header_offset
|
|
525
|
+
|
|
526
|
+
@files << e
|
|
486
527
|
end
|
|
487
528
|
|
|
488
529
|
def remove_backslash(filename)
|
data/lib/zip_tricks/version.rb
CHANGED
|
@@ -10,9 +10,8 @@ class ZipTricks::WriteAndTell
|
|
|
10
10
|
|
|
11
11
|
def <<(bytes)
|
|
12
12
|
return self if bytes.nil?
|
|
13
|
-
|
|
14
|
-
@
|
|
15
|
-
@pos += binary_bytes.bytesize
|
|
13
|
+
@io << bytes.b
|
|
14
|
+
@pos += bytes.bytesize
|
|
16
15
|
self
|
|
17
16
|
end
|
|
18
17
|
|
|
@@ -23,13 +22,4 @@ class ZipTricks::WriteAndTell
|
|
|
23
22
|
def tell
|
|
24
23
|
@pos
|
|
25
24
|
end
|
|
26
|
-
|
|
27
|
-
private
|
|
28
|
-
|
|
29
|
-
def binary(str)
|
|
30
|
-
return str if str.encoding == Encoding::BINARY
|
|
31
|
-
str.force_encoding(Encoding::BINARY)
|
|
32
|
-
rescue RuntimeError # the string is frozen
|
|
33
|
-
str.dup.force_encoding(Encoding::BINARY)
|
|
34
|
-
end
|
|
35
25
|
end
|
|
@@ -7,13 +7,34 @@
|
|
|
7
7
|
# CRC32 combine operations - and this adds up. Since the CRC32 value
|
|
8
8
|
# is usually not needed until the complete output has completed
|
|
9
9
|
# we can buffer at least some amount of data before computing CRC32 over it.
|
|
10
|
+
# We also use this buffer for output via Rack, where some amount of buffering
|
|
11
|
+
# helps reduce the number of syscalls made by the webserver. ZipTricks performs
|
|
12
|
+
# lots of very small writes, and some degree of speedup (about 20%) can be achieved
|
|
13
|
+
# with a buffer of a few KB.
|
|
14
|
+
#
|
|
15
|
+
# Note that there is no guarantee that the write buffer is going to flush at or above
|
|
16
|
+
# the given `buffer_size`, because for writes which exceed the buffer size it will
|
|
17
|
+
# first `flush` and then write through the oversized chunk, without buffering it. This
|
|
18
|
+
# helps conserve memory. Also note that the buffer will *not* duplicate strings for you
|
|
19
|
+
# and *will* yield the same buffer String over and over, so if you are storing it in an
|
|
20
|
+
# Array you might need to duplicate it.
|
|
21
|
+
#
|
|
22
|
+
# Note also that the WriteBuffer assumes that the object it `<<`-writes into is going
|
|
23
|
+
# to **consume** in some way the string that it passes in. After the `<<` method returns,
|
|
24
|
+
# the WriteBuffer will be cleared, and it passes the same String reference on every call
|
|
25
|
+
# to `<<`. Therefore, if you need to retain the output of the WriteBuffer in, say, an Array,
|
|
26
|
+
# you might need to `.dup` the `String` it gives you.
|
|
10
27
|
class ZipTricks::WriteBuffer
|
|
11
28
|
# Creates a new WriteBuffer bypassing into a given writable object
|
|
12
29
|
#
|
|
13
|
-
# @param writable[#<<] An object that responds to `#<<` with
|
|
30
|
+
# @param writable[#<<] An object that responds to `#<<` with a String as argument
|
|
14
31
|
# @param buffer_size[Integer] How many bytes to buffer
|
|
15
32
|
def initialize(writable, buffer_size)
|
|
16
|
-
|
|
33
|
+
# Allocating the buffer using a zero-padded String as a variation
|
|
34
|
+
# on using capacity:, which JRuby apparently does not like very much. The
|
|
35
|
+
# desire here is that the buffer doesn't have to be resized during the lifetime
|
|
36
|
+
# of the object.
|
|
37
|
+
@buf = ("\0".b * (buffer_size * 2)).clear
|
|
17
38
|
@buffer_size = buffer_size
|
|
18
39
|
@writable = writable
|
|
19
40
|
end
|
|
@@ -24,28 +45,27 @@ class ZipTricks::WriteBuffer
|
|
|
24
45
|
# @param data[String] data to be written
|
|
25
46
|
# @return self
|
|
26
47
|
def <<(data)
|
|
27
|
-
|
|
28
|
-
|
|
48
|
+
if data.bytesize >= @buffer_size
|
|
49
|
+
flush unless @buf.empty? # <- this is were we can output less than @buffer_size
|
|
50
|
+
@writable << data
|
|
51
|
+
else
|
|
52
|
+
@buf << data
|
|
53
|
+
flush if @buf.bytesize >= @buffer_size
|
|
54
|
+
end
|
|
29
55
|
self
|
|
30
56
|
end
|
|
31
57
|
|
|
32
58
|
# Explicitly flushes the buffer if it contains anything
|
|
33
59
|
#
|
|
34
60
|
# @return self
|
|
35
|
-
def flush
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
61
|
+
def flush
|
|
62
|
+
unless @buf.empty?
|
|
63
|
+
@writable << @buf
|
|
64
|
+
@buf.clear
|
|
65
|
+
end
|
|
39
66
|
self
|
|
40
67
|
end
|
|
41
68
|
|
|
42
|
-
#
|
|
43
|
-
|
|
44
|
-
# computation by retrieving the CRC as an integer
|
|
45
|
-
#
|
|
46
|
-
# @return [Integer] the return value of `writable#to_i`
|
|
47
|
-
def to_i
|
|
48
|
-
flush!
|
|
49
|
-
@writable.to_i
|
|
50
|
-
end
|
|
69
|
+
# `flush!` was renamed to `flush` but we preserve this method for backwards compatibility
|
|
70
|
+
alias_method :flush!, :flush
|
|
51
71
|
end
|
|
@@ -31,22 +31,10 @@ class ZipTricks::ZipWriter
|
|
|
31
31
|
VERSION_MADE_BY = 52
|
|
32
32
|
VERSION_NEEDED_TO_EXTRACT = 20
|
|
33
33
|
VERSION_NEEDED_TO_EXTRACT_ZIP64 = 45
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
# zipinfo.external_attr = 0644 << 16L # permissions -r-wr--r--.
|
|
39
|
-
# We snatch the incantations from Rubyzip for this.
|
|
40
|
-
unix_perms = 0o644
|
|
41
|
-
file_type_file = 0o10
|
|
42
|
-
(file_type_file << 12 | (unix_perms & 0o7777)) << 16
|
|
43
|
-
end
|
|
44
|
-
EMPTY_DIRECTORY_EXTERNAL_ATTRS = begin
|
|
45
|
-
# Applies permissions to an empty directory.
|
|
46
|
-
unix_perms = 0o755
|
|
47
|
-
file_type_dir = 0o04
|
|
48
|
-
(file_type_dir << 12 | (unix_perms & 0o7777)) << 16
|
|
49
|
-
end
|
|
34
|
+
DEFAULT_FILE_UNIX_PERMISSIONS = 0o644
|
|
35
|
+
DEFAULT_DIRECTORY_UNIX_PERMISSIONS = 0o755
|
|
36
|
+
FILE_TYPE_FILE = 0o10
|
|
37
|
+
FILE_TYPE_DIRECTORY = 0o04
|
|
50
38
|
MADE_BY_SIGNATURE = begin
|
|
51
39
|
# A combination of the VERSION_MADE_BY low byte and the OS type high byte
|
|
52
40
|
os_type = 3 # UNIX
|
|
@@ -57,14 +45,15 @@ class ZipTricks::ZipWriter
|
|
|
57
45
|
C_UINT2 = 'v' # Encode a 2-byte unsigned little-endian uint
|
|
58
46
|
C_UINT8 = 'Q<' # Encode an 8-byte unsigned little-endian uint
|
|
59
47
|
C_CHAR = 'C' # For bit-encoded strings
|
|
60
|
-
C_INT4 = '
|
|
48
|
+
C_INT4 = 'l<' # Encode a 4-byte signed little-endian int
|
|
61
49
|
|
|
62
50
|
private_constant :FOUR_BYTE_MAX_UINT,
|
|
63
51
|
:TWO_BYTE_MAX_UINT,
|
|
64
52
|
:VERSION_MADE_BY,
|
|
65
53
|
:VERSION_NEEDED_TO_EXTRACT,
|
|
66
54
|
:VERSION_NEEDED_TO_EXTRACT_ZIP64,
|
|
67
|
-
:
|
|
55
|
+
:FILE_TYPE_FILE,
|
|
56
|
+
:FILE_TYPE_DIRECTORY,
|
|
68
57
|
:MADE_BY_SIGNATURE,
|
|
69
58
|
:C_UINT4,
|
|
70
59
|
:C_UINT2,
|
|
@@ -118,7 +107,7 @@ class ZipTricks::ZipWriter
|
|
|
118
107
|
if requires_zip64
|
|
119
108
|
extra_fields << zip_64_extra_for_local_file_header(compressed_size: compressed_size, uncompressed_size: uncompressed_size)
|
|
120
109
|
end
|
|
121
|
-
extra_fields <<
|
|
110
|
+
extra_fields << timestamp_extra_for_local_file_header(mtime)
|
|
122
111
|
|
|
123
112
|
io << [extra_fields.size].pack(C_UINT2) # extra field length 2 bytes
|
|
124
113
|
|
|
@@ -136,6 +125,7 @@ class ZipTricks::ZipWriter
|
|
|
136
125
|
# @param crc32[Fixnum] The CRC32 checksum of the file
|
|
137
126
|
# @param mtime[Time] the modification time to be recorded in the ZIP
|
|
138
127
|
# @param gp_flags[Fixnum] bit-packed general purpose flags
|
|
128
|
+
# @param unix_permissions[Fixnum?] the permissions for the file, or nil for the default to be used
|
|
139
129
|
# @return [void]
|
|
140
130
|
def write_central_directory_file_header(io:,
|
|
141
131
|
local_file_header_location:,
|
|
@@ -145,7 +135,9 @@ class ZipTricks::ZipWriter
|
|
|
145
135
|
uncompressed_size:,
|
|
146
136
|
mtime:,
|
|
147
137
|
crc32:,
|
|
148
|
-
filename
|
|
138
|
+
filename:,
|
|
139
|
+
unix_permissions: nil
|
|
140
|
+
)
|
|
149
141
|
# At this point if the header begins somewhere beyound 0xFFFFFFFF we _have_ to record the offset
|
|
150
142
|
# of the local file header as a zip64 extra field, so we give up, give in, you loose, love will always win...
|
|
151
143
|
add_zip64 = (local_file_header_location > FOUR_BYTE_MAX_UINT) ||
|
|
@@ -182,7 +174,7 @@ class ZipTricks::ZipWriter
|
|
|
182
174
|
compressed_size: compressed_size,
|
|
183
175
|
uncompressed_size: uncompressed_size)
|
|
184
176
|
end
|
|
185
|
-
extra_fields <<
|
|
177
|
+
extra_fields << timestamp_extra_for_central_directory_entry(mtime)
|
|
186
178
|
|
|
187
179
|
io << [extra_fields.size].pack(C_UINT2) # extra field length 2 bytes
|
|
188
180
|
|
|
@@ -195,16 +187,20 @@ class ZipTricks::ZipWriter
|
|
|
195
187
|
[TWO_BYTE_MAX_UINT].pack(C_UINT2)
|
|
196
188
|
else
|
|
197
189
|
[0].pack(C_UINT2)
|
|
198
|
-
|
|
190
|
+
end
|
|
199
191
|
io << [0].pack(C_UINT2) # internal file attributes 2 bytes
|
|
200
192
|
|
|
201
193
|
# Because the add_empty_directory method will create a directory with a trailing "/",
|
|
202
194
|
# this check can be used to assign proper permissions to the created directory.
|
|
203
|
-
|
|
204
|
-
|
|
195
|
+
# external file attributes 4 bytes
|
|
196
|
+
external_attrs = if filename.end_with?('/')
|
|
197
|
+
unix_permissions ||= DEFAULT_DIRECTORY_UNIX_PERMISSIONS
|
|
198
|
+
generate_external_attrs(unix_permissions, FILE_TYPE_DIRECTORY)
|
|
205
199
|
else
|
|
206
|
-
|
|
200
|
+
unix_permissions ||= DEFAULT_FILE_UNIX_PERMISSIONS
|
|
201
|
+
generate_external_attrs(unix_permissions, FILE_TYPE_FILE)
|
|
207
202
|
end
|
|
203
|
+
io << [external_attrs].pack(C_UINT4)
|
|
208
204
|
|
|
209
205
|
io << if add_zip64 # relative offset of local header 4 bytes
|
|
210
206
|
[FOUR_BYTE_MAX_UINT].pack(C_UINT4)
|
|
@@ -345,12 +341,14 @@ class ZipTricks::ZipWriter
|
|
|
345
341
|
pack_array(data_and_packspecs)
|
|
346
342
|
end
|
|
347
343
|
|
|
348
|
-
# Writes the extended timestamp information field
|
|
344
|
+
# Writes the extended timestamp information field for local headers.
|
|
345
|
+
#
|
|
346
|
+
# The spec defines 2
|
|
349
347
|
# different formats - the one for the local file header can also accomodate the
|
|
350
348
|
# atime and ctime, whereas the one for the central directory can only take
|
|
351
349
|
# the mtime - and refers the reader to the local header extra to obtain the
|
|
352
350
|
# remaining times
|
|
353
|
-
def
|
|
351
|
+
def timestamp_extra_for_local_file_header(mtime)
|
|
354
352
|
# Local-header version:
|
|
355
353
|
#
|
|
356
354
|
# Value Size Description
|
|
@@ -378,16 +376,21 @@ class ZipTricks::ZipWriter
|
|
|
378
376
|
# bit 1 if set, access time is present
|
|
379
377
|
# bit 2 if set, creation time is present
|
|
380
378
|
# bits 3-7 reserved for additional timestamps; not set
|
|
381
|
-
flags =
|
|
379
|
+
flags = 0b00000001 # Set the lowest bit only, to indicate that only mtime is present
|
|
382
380
|
data_and_packspecs = [
|
|
383
381
|
0x5455, C_UINT2, # tag for this extra block type ("UT")
|
|
384
|
-
(1 + 4), C_UINT2, # the size of this block (1 byte used for the Flag +
|
|
382
|
+
(1 + 4), C_UINT2, # the size of this block (1 byte used for the Flag + 3 longs used for the timestamp)
|
|
385
383
|
flags, C_CHAR, # encode a single byte
|
|
386
|
-
mtime.utc.to_i, C_INT4, # Use a signed
|
|
384
|
+
mtime.utc.to_i, C_INT4, # Use a signed int, not the unsigned one used by the rest of the ZIP spec.
|
|
387
385
|
]
|
|
386
|
+
# The atime and ctime can be omitted if not present
|
|
388
387
|
pack_array(data_and_packspecs)
|
|
389
388
|
end
|
|
390
389
|
|
|
390
|
+
# Since we do not supply atime or ctime, the contents of the two extra fields (central dir and local header)
|
|
391
|
+
# is exactly the same, so we can use a method alias.
|
|
392
|
+
alias_method :timestamp_extra_for_central_directory_entry, :timestamp_extra_for_local_file_header
|
|
393
|
+
|
|
391
394
|
# Writes the Zip64 extra field for the central directory header.It differs from the extra used in the local file header because it
|
|
392
395
|
# also contains the location of the local file header in the ZIP as an 8-byte int.
|
|
393
396
|
#
|
|
@@ -427,4 +430,8 @@ class ZipTricks::ZipWriter
|
|
|
427
430
|
values, packspecs = values_to_packspecs.partition.each_with_index { |_, i| i.even? }
|
|
428
431
|
values.pack(packspecs.join)
|
|
429
432
|
end
|
|
433
|
+
|
|
434
|
+
def generate_external_attrs(unix_permissions_int, file_type_int)
|
|
435
|
+
(file_type_int << 12 | (unix_permissions_int & 0o7777)) << 16
|
|
436
|
+
end
|
|
430
437
|
end
|