zip_tricks 2.8.1 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile +3 -3
  3. data/IMPLEMENTATION_DETAILS.md +2 -10
  4. data/README.md +62 -59
  5. data/examples/archive_size_estimate.rb +4 -4
  6. data/examples/rack_application.rb +3 -5
  7. data/lib/zip_tricks/block_deflate.rb +21 -0
  8. data/lib/zip_tricks/file_reader.rb +491 -0
  9. data/lib/zip_tricks/null_writer.rb +7 -2
  10. data/lib/zip_tricks/rack_body.rb +3 -3
  11. data/lib/zip_tricks/remote_io.rb +30 -20
  12. data/lib/zip_tricks/remote_uncap.rb +10 -10
  13. data/lib/zip_tricks/size_estimator.rb +64 -0
  14. data/lib/zip_tricks/stream_crc32.rb +2 -2
  15. data/lib/zip_tricks/streamer/deflated_writer.rb +26 -0
  16. data/lib/zip_tricks/streamer/entry.rb +21 -0
  17. data/lib/zip_tricks/streamer/stored_writer.rb +25 -0
  18. data/lib/zip_tricks/streamer/writable.rb +20 -0
  19. data/lib/zip_tricks/streamer.rb +172 -66
  20. data/lib/zip_tricks/zip_writer.rb +346 -0
  21. data/lib/zip_tricks.rb +1 -4
  22. data/spec/spec_helper.rb +1 -38
  23. data/spec/zip_tricks/file_reader_spec.rb +47 -0
  24. data/spec/zip_tricks/rack_body_spec.rb +2 -2
  25. data/spec/zip_tricks/remote_io_spec.rb +8 -20
  26. data/spec/zip_tricks/remote_uncap_spec.rb +4 -4
  27. data/spec/zip_tricks/size_estimator_spec.rb +31 -0
  28. data/spec/zip_tricks/streamer_spec.rb +59 -36
  29. data/spec/zip_tricks/zip_writer_spec.rb +408 -0
  30. data/zip_tricks.gemspec +20 -14
  31. metadata +33 -16
  32. data/lib/zip_tricks/manifest.rb +0 -85
  33. data/lib/zip_tricks/microzip.rb +0 -339
  34. data/lib/zip_tricks/stored_size_estimator.rb +0 -44
  35. data/spec/zip_tricks/manifest_spec.rb +0 -60
  36. data/spec/zip_tricks/microzip_interop_spec.rb +0 -48
  37. data/spec/zip_tricks/microzip_spec.rb +0 -546
  38. data/spec/zip_tricks/stored_size_estimator_spec.rb +0 -22
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: zip_tricks
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.8.1
4
+ version: 3.0.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Julik Tarkhanov
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-07-22 00:00:00.000000000 Z
11
+ date: 2016-08-03 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rubyzip
@@ -20,7 +20,7 @@ dependencies:
20
20
  - - "~>"
21
21
  - !ruby/object:Gem::Version
22
22
  version: '1.1'
23
- type: :runtime
23
+ type: :development
24
24
  prerelease: false
25
25
  version_requirements: !ruby/object:Gem::Requirement
26
26
  requirements:
@@ -31,19 +31,19 @@ dependencies:
31
31
  - !ruby/object:Gem::Version
32
32
  version: '1.1'
33
33
  - !ruby/object:Gem::Dependency
34
- name: very_tiny_state_machine
34
+ name: terminal-table
35
35
  requirement: !ruby/object:Gem::Requirement
36
36
  requirements:
37
- - - "~>"
37
+ - - ">="
38
38
  - !ruby/object:Gem::Version
39
- version: '2'
40
- type: :runtime
39
+ version: '0'
40
+ type: :development
41
41
  prerelease: false
42
42
  version_requirements: !ruby/object:Gem::Requirement
43
43
  requirements:
44
- - - "~>"
44
+ - - ">="
45
45
  - !ruby/object:Gem::Version
46
- version: '2'
46
+ version: '0'
47
47
  - !ruby/object:Gem::Dependency
48
48
  name: range_utils
49
49
  requirement: !ruby/object:Gem::Requirement
@@ -106,6 +106,20 @@ dependencies:
106
106
  - - "~>"
107
107
  - !ruby/object:Gem::Version
108
108
  version: 3.2.0
109
+ - !ruby/object:Gem::Dependency
110
+ name: coderay
111
+ requirement: !ruby/object:Gem::Requirement
112
+ requirements:
113
+ - - ">="
114
+ - !ruby/object:Gem::Version
115
+ version: '0'
116
+ type: :development
117
+ prerelease: false
118
+ version_requirements: !ruby/object:Gem::Requirement
119
+ requirements:
120
+ - - ">="
121
+ - !ruby/object:Gem::Version
122
+ version: '0'
109
123
  - !ruby/object:Gem::Dependency
110
124
  name: yard
111
125
  requirement: !ruby/object:Gem::Requirement
@@ -172,30 +186,33 @@ files:
172
186
  - lib/zip_tricks.rb
173
187
  - lib/zip_tricks/block_deflate.rb
174
188
  - lib/zip_tricks/block_write.rb
175
- - lib/zip_tricks/manifest.rb
176
- - lib/zip_tricks/microzip.rb
189
+ - lib/zip_tricks/file_reader.rb
177
190
  - lib/zip_tricks/null_writer.rb
178
191
  - lib/zip_tricks/rack_body.rb
179
192
  - lib/zip_tricks/remote_io.rb
180
193
  - lib/zip_tricks/remote_uncap.rb
181
- - lib/zip_tricks/stored_size_estimator.rb
194
+ - lib/zip_tricks/size_estimator.rb
182
195
  - lib/zip_tricks/stream_crc32.rb
183
196
  - lib/zip_tricks/streamer.rb
197
+ - lib/zip_tricks/streamer/deflated_writer.rb
198
+ - lib/zip_tricks/streamer/entry.rb
199
+ - lib/zip_tricks/streamer/stored_writer.rb
200
+ - lib/zip_tricks/streamer/writable.rb
184
201
  - lib/zip_tricks/write_and_tell.rb
202
+ - lib/zip_tricks/zip_writer.rb
185
203
  - spec/spec_helper.rb
186
204
  - spec/zip_tricks/block_deflate_spec.rb
187
205
  - spec/zip_tricks/block_write_spec.rb
188
- - spec/zip_tricks/manifest_spec.rb
189
- - spec/zip_tricks/microzip_interop_spec.rb
190
- - spec/zip_tricks/microzip_spec.rb
206
+ - spec/zip_tricks/file_reader_spec.rb
191
207
  - spec/zip_tricks/rack_body_spec.rb
192
208
  - spec/zip_tricks/remote_io_spec.rb
193
209
  - spec/zip_tricks/remote_uncap_spec.rb
194
- - spec/zip_tricks/stored_size_estimator_spec.rb
210
+ - spec/zip_tricks/size_estimator_spec.rb
195
211
  - spec/zip_tricks/stream_crc32_spec.rb
196
212
  - spec/zip_tricks/streamer_spec.rb
197
213
  - spec/zip_tricks/war-and-peace.txt
198
214
  - spec/zip_tricks/write_and_tell_spec.rb
215
+ - spec/zip_tricks/zip_writer_spec.rb
199
216
  - zip_tricks.gemspec
200
217
  homepage: http://github.com/wetransfer/zip_tricks
201
218
  licenses:
@@ -1,85 +0,0 @@
1
- # Helps to estimate archive sizes
2
- class ZipTricks::Manifest < Struct.new(:zip_streamer, :io, :part_list)
3
-
4
- # Describes a span within the ZIP bytestream
5
- class ZipSpan < Struct.new(:part_type, :byte_range_in_zip, :filename, :additional_metadata)
6
- end
7
-
8
- # Builds an array of spans within the ZIP file and computes the size of the resulting archive in bytes.
9
- #
10
- # zip_spans, bytesize = Manifest.build do | b |
11
- # b.add_stored_entry(name: "file.doc", size: 898291)
12
- # b.add_compressed_entry(name: "family.tif", size: 89281911, compressed_size: 121908)
13
- # end
14
- # bytesize #=> ... (Fixnum or Bignum)
15
- # zip_spans[0] #=> Manifest::ZipSpan(part_type: :entry_header, byte_range_in_zip: 0..44, ...)
16
- # zip_spans[-1] #=> Manifest::ZipSpan(part_type: :central_directory, byte_range_in_zip: 776721..898921, ...)
17
- #
18
- # @return [Array<ZipSpan>, Fixnum] an array of byte spans within the final ZIP, and the total size of the archive
19
- # @yield [Manifest] the manifest object you can add entries to
20
- def self.build
21
- output_io = ZipTricks::WriteAndTell.new(ZipTricks::NullWriter)
22
- part_list = []
23
- last_range_end = 0
24
- ZipTricks::Streamer.open(output_io) do | zip_streamer |
25
- manifest = new(zip_streamer, output_io, part_list)
26
- yield(manifest)
27
- last_range_end = part_list[-1].byte_range_in_zip.end
28
- end
29
-
30
- # Record the position of the central directory
31
- directory_location = (last_range_end + 1)..(output_io.tell - 1)
32
- part_list << ZipSpan.new(:central_directory, directory_location, :central_directory, nil)
33
-
34
- [part_list, output_io.tell]
35
- end
36
-
37
- # Add a fake entry to the archive, to see how big it is going to be in the end.
38
- #
39
- # @param name [String] the name of the file (filenames are variable-width in the ZIP)
40
- # @param size_uncompressed [Fixnum] size of the uncompressed entry
41
- # @param segment_info[Object] if you need to save anything to retrieve later from the Manifest,
42
- # pass it here (like the URL of the file)
43
- # @return self
44
- def add_stored_entry(name:, size_uncompressed:, segment_info: nil)
45
- register_part(:entry_header, name, segment_info) do
46
- zip_streamer.add_stored_entry(name, size_uncompressed, C_fake_crc)
47
- end
48
-
49
- register_part(:entry_body, name, segment_info) do
50
- zip_streamer.simulate_write(size_uncompressed)
51
- end
52
-
53
- self
54
- end
55
-
56
- # Add a fake entry to the archive, to see how big it is going to be in the end.
57
- #
58
- # @param name [String] the name of the file (filenames are variable-width in the ZIP)
59
- # @param size_uncompressed [Fixnum] size of the uncompressed entry
60
- # @param size_compressed [Fixnum] size of the compressed entry
61
- # @param segment_info[Object] if you need to save anything to retrieve later from the Manifest,
62
- # pass it here (like the URL of the file)
63
- # @return self
64
- def add_compressed_entry(name:, size_uncompressed:, size_compressed:, segment_info: nil)
65
- register_part(:entry_header, name, segment_info) do
66
- zip_streamer.add_compressed_entry(name, size_uncompressed, C_fake_crc, size_compressed)
67
- end
68
-
69
- register_part(:entry_body, name, segment_info) do
70
- zip_streamer.simulate_write(size_compressed)
71
- end
72
-
73
- self
74
- end
75
-
76
- private
77
-
78
- C_fake_crc = Zlib.crc32('Mary had a little lamb')
79
- private_constant :C_fake_crc
80
-
81
- def register_part(span_type, filename, metadata)
82
- before, _, after = io.tell, yield, (io.tell - 1)
83
- part_list << ZipSpan.new(span_type, (before..after), filename, metadata)
84
- end
85
- end
@@ -1,339 +0,0 @@
1
- # A replacement for RubyZip for streaming, with a couple of small differences.
2
- # The first difference is that it is verbosely-written-to-the-spec and you can actually
3
- # follow what is happening. It does not support quite a few fancy features of Rubyzip,
4
- # but instead it can be digested in one reading, and has solid Zip64 support. It also does
5
- # not attempt any tricks with Zip64 placeholder extra fields because the ZipTricks streaming
6
- # engine assumes you _know_ how large your file is (both compressed and uncompressed) _and_
7
- # you have the file's CRC32 checksum upfront.
8
- #
9
- # Just like Rubyzip it will switch to Zip64 automatically if required, but there is no global
10
- # setting to enable that behavior - it is always on.
11
- class ZipTricks::Microzip
12
- STORED = 0
13
- DEFLATED = 8
14
-
15
- TooMuch = Class.new(StandardError)
16
- PathError = Class.new(StandardError)
17
- DuplicateFilenames = Class.new(StandardError)
18
- UnknownMode = Class.new(StandardError)
19
-
20
- FOUR_BYTE_MAX_UINT = 0xFFFFFFFF
21
- TWO_BYTE_MAX_UINT = 0xFFFF
22
-
23
- VERSION_MADE_BY = 52
24
- VERSION_NEEDED_TO_EXTRACT = 20
25
- VERSION_NEEDED_TO_EXTRACT_ZIP64 = 45
26
- DEFAULT_EXTERNAL_ATTRS = begin
27
- # These need to be set so that the unarchived files do not become executable on UNIX, for
28
- # security purposes. Strictly speaking we would want to make this user-customizable,
29
- # but for now just putting in sane defaults will do. For example, Trac with zipinfo does this:
30
- # zipinfo.external_attr = 0644 << 16L # permissions -r-wr--r--.
31
- # We snatch the incantations from Rubyzip for this.
32
- unix_perms = 0644
33
- file_type_file = 010
34
- external_attrs = (file_type_file << 12 | (unix_perms & 07777)) << 16
35
- end
36
- MADE_BY_SIGNATURE = begin
37
- # A combination of the VERSION_MADE_BY low byte and the OS type high byte
38
- os_type = 3 # UNIX
39
- [VERSION_MADE_BY, os_type].pack('CC')
40
- end
41
-
42
- C_V = 'V'.freeze
43
- C_v = 'v'.freeze
44
- C_Qe = 'Q<'.freeze
45
-
46
- class Entry < Struct.new(:filename, :crc32, :compressed_size, :uncompressed_size, :storage_mode, :mtime)
47
- def initialize(*)
48
- super
49
- filename.force_encoding(Encoding::UTF_8)
50
- @requires_efs_flag = !(filename.encode(Encoding::ASCII) rescue false)
51
- @requires_zip64 = (compressed_size > FOUR_BYTE_MAX_UINT || uncompressed_size > FOUR_BYTE_MAX_UINT)
52
- raise TooMuch, "Filename is too long" if filename.bytesize > TWO_BYTE_MAX_UINT
53
- raise PathError, "Paths in ZIP may only contain forward slashes (UNIX separators)" if filename.include?('\\')
54
- end
55
-
56
- def requires_zip64?
57
- @requires_zip64
58
- end
59
-
60
- # Set the general purpose flags for the entry. The only flag we care about is the EFS
61
- # bit (bit 11) which should be set if the filename is UTF8. If it is, we need to set the
62
- # bit so that the unarchiving application knows that the filename in the archive is UTF-8
63
- # encoded, and not some DOS default. For ASCII entries it does not matter.
64
- def gp_flags_based_on_filename
65
- @requires_efs_flag ? (0b00000000000 | 0b100000000000) : 0b00000000000
66
- end
67
-
68
- def write_local_file_header(io)
69
- # TBD: caveat. If this entry _does_ fit into a standard zip segment (both compressed and
70
- # uncompressed size at or below 0xFFFF etc), but it is _located_ at an offset that requires
71
- # Zip64 to be used (beyound 4GB), we are going to be omitting the Zip64 extras in the local
72
- # file header, but we will be enabling them when writing the central directory. Then the
73
- # CD record for the file _will_ have Zip64 extra, but the local file header won't. In theory,
74
- # this should not pose a problem, but then again... life in this world can be harsh.
75
- #
76
- # If it turns out that it _does_ pose a problem, we can always do:
77
- #
78
- # @requires_zip64 = true if io.tell > FOUR_BYTE_MAX_UINT
79
- #
80
- # right here, and have the data written regardless even if the file fits.
81
- io << [0x04034b50].pack(C_V) # local file header signature 4 bytes (0x04034b50)
82
-
83
- if @requires_zip64 # version needed to extract 2 bytes
84
- io << [VERSION_NEEDED_TO_EXTRACT_ZIP64].pack(C_v)
85
- else
86
- io << [VERSION_NEEDED_TO_EXTRACT].pack(C_v)
87
- end
88
-
89
- io << [gp_flags_based_on_filename].pack("v") # general purpose bit flag 2 bytes
90
- io << [storage_mode].pack("v") # compression method 2 bytes
91
- io << [to_binary_dos_time(mtime)].pack(C_v) # last mod file time 2 bytes
92
- io << [to_binary_dos_date(mtime)].pack(C_v) # last mod file date 2 bytes
93
- io << [crc32].pack(C_V) # crc-32 4 bytes
94
-
95
- if @requires_zip64
96
- io << [FOUR_BYTE_MAX_UINT].pack(C_V) # compressed size 4 bytes
97
- io << [FOUR_BYTE_MAX_UINT].pack(C_V) # uncompressed size 4 bytes
98
- else
99
- io << [compressed_size].pack(C_V) # compressed size 4 bytes
100
- io << [uncompressed_size].pack(C_V) # uncompressed size 4 bytes
101
- end
102
-
103
- # Filename should not be longer than 0xFFFF otherwise this wont fit here
104
- io << [filename.bytesize].pack(C_v) # file name length 2 bytes
105
-
106
- extra_size = 0
107
- if @requires_zip64
108
- extra_size += bytesize_of {|buf| write_zip_64_extra_for_local_file_header(buf) }
109
- end
110
- io << [extra_size].pack(C_v) # extra field length 2 bytes
111
-
112
- io << filename # file name (variable size)
113
-
114
- # Interesting tidbit:
115
- # https://social.technet.microsoft.com/Forums/windows/en-US/6a60399f-2879-4859-b7ab-6ddd08a70948
116
- # TL;DR of it is: Windows 7 Explorer _will_ open Zip64 entries. However, it desires to have the
117
- # Zip64 extra field as _the first_ extra field. If we decide to add the Info-ZIP UTF-8 field...
118
- write_zip_64_extra_for_local_file_header(io) if @requires_zip64
119
- end
120
-
121
- def write_zip_64_extra_for_local_file_header(io)
122
- io << [0x0001].pack(C_v) # 2 bytes Tag for this "extra" block type
123
- io << [16].pack(C_v) # 2 bytes Size of this "extra" block. For us it will always be 16 (2x8)
124
- io << [uncompressed_size].pack(C_Qe) # 8 bytes Original uncompressed file size
125
- io << [compressed_size].pack(C_Qe) # 8 bytes Size of compressed data
126
- end
127
-
128
- def write_zip_64_extra_for_central_directory_file_header(io, local_file_header_location)
129
- io << [0x0001].pack(C_v) # 2 bytes Tag for this "extra" block type
130
- io << [28].pack(C_v) # 2 bytes Size of this "extra" block. For us it will always be 28
131
- io << [uncompressed_size].pack(C_Qe) # 8 bytes Original uncompressed file size
132
- io << [compressed_size].pack(C_Qe) # 8 bytes Size of compressed data
133
- io << [local_file_header_location].pack(C_Qe) # 8 bytes Offset of local header record
134
- io << [0].pack(C_V) # 4 bytes Number of the disk on which this file starts
135
- end
136
-
137
- def write_central_directory_file_header(io, local_file_header_location)
138
- # At this point if the header begins somewhere beyound 0xFFFFFFFF we _have_ to record the offset
139
- # of the local file header as a zip64 extra field, so we give up, give in, you loose, love will always win...
140
- @requires_zip64 = true if local_file_header_location > FOUR_BYTE_MAX_UINT
141
-
142
- io << [0x02014b50].pack(C_V) # central file header signature 4 bytes (0x02014b50)
143
- io << MADE_BY_SIGNATURE # version made by 2 bytes
144
- if @requires_zip64
145
- io << [VERSION_NEEDED_TO_EXTRACT_ZIP64].pack(C_v) # version needed to extract 2 bytes
146
- else
147
- io << [VERSION_NEEDED_TO_EXTRACT].pack(C_v) # version needed to extract 2 bytes
148
- end
149
-
150
- io << [gp_flags_based_on_filename].pack(C_v) # general purpose bit flag 2 bytes
151
- io << [storage_mode].pack(C_v) # compression method 2 bytes
152
- io << [to_binary_dos_time(mtime)].pack(C_v) # last mod file time 2 bytes
153
- io << [to_binary_dos_date(mtime)].pack(C_v) # last mod file date 2 bytes
154
- io << [crc32].pack(C_V) # crc-32 4 bytes
155
-
156
- if @requires_zip64
157
- io << [FOUR_BYTE_MAX_UINT].pack(C_V) # compressed size 4 bytes
158
- io << [FOUR_BYTE_MAX_UINT].pack(C_V) # uncompressed size 4 bytes
159
- else
160
- io << [compressed_size].pack(C_V) # compressed size 4 bytes
161
- io << [uncompressed_size].pack(C_V) # uncompressed size 4 bytes
162
- end
163
-
164
- # Filename should not be longer than 0xFFFF otherwise this wont fit here
165
- io << [filename.bytesize].pack(C_v) # file name length 2 bytes
166
-
167
- extra_size = 0
168
- if @requires_zip64
169
- extra_size += bytesize_of {|buf|
170
- write_zip_64_extra_for_central_directory_file_header(buf, local_file_header_location)
171
- }
172
- end
173
- io << [extra_size].pack(C_v) # extra field length 2 bytes
174
-
175
- io << [0].pack(C_v) # file comment length 2 bytes
176
-
177
- # For The Unarchiver < 3.11.1 this field has to be set to the overflow value if zip64 is used
178
- # because otherwise it does not properly advance the pointer when reading the Zip64 extra field
179
- # https://bitbucket.org/WAHa_06x36/theunarchiver/pull-requests/2/bug-fix-for-zip64-extra-field-parser/diff
180
- if @requires_zip64
181
- io << [TWO_BYTE_MAX_UINT].pack(C_v) # disk number start 2 bytes
182
- else
183
- io << [0].pack(C_v) # disk number start 2 bytes
184
- end
185
- io << [0].pack(C_v) # internal file attributes 2 bytes
186
- io << [DEFAULT_EXTERNAL_ATTRS].pack(C_V) # external file attributes 4 bytes
187
-
188
- if @requires_zip64
189
- io << [FOUR_BYTE_MAX_UINT].pack(C_V) # relative offset of local header 4 bytes
190
- else
191
- io << [local_file_header_location].pack(C_V) # relative offset of local header 4 bytes
192
- end
193
- io << filename # file name (variable size)
194
-
195
- if @requires_zip64 # extra field (variable size)
196
- write_zip_64_extra_for_central_directory_file_header(io, local_file_header_location)
197
- end
198
- # file comment (variable size)
199
- end
200
-
201
- private
202
-
203
- def bytesize_of
204
- ''.force_encoding(Encoding::BINARY).tap {|b| yield(b) }.bytesize
205
- end
206
-
207
- def to_binary_dos_time(t)
208
- (t.sec/2) + (t.min << 5) + (t.hour << 11)
209
- end
210
-
211
- def to_binary_dos_date(t)
212
- (t.day) + (t.month << 5) + ((t.year - 1980) << 9)
213
- end
214
- end
215
-
216
- # Creates a new streaming writer.
217
- # The writer is stateful and knows it's list of ZIP file entries as they are being added.
218
- def initialize
219
- @files = []
220
- @local_header_offsets = []
221
- end
222
-
223
- # Adds a file to the entry list and immediately writes out it's local file header into the
224
- # output stream.
225
- #
226
- # @param io[#<<, #tell] the buffer to write the local file header to
227
- # @param filename[String] The name of the file
228
- # @param crc32[Fixnum] The CRC32 checksum of the file
229
- # @param compressed_size[Fixnum] The size of the compressed (or stored) data - how much space it uses in the ZIP
230
- # @param uncompressed_size[Fixnum] The size of the file once extracted
231
- # @param storage_mode[Fixnum] Either 0 for "stored" or 8 for "deflated"
232
- # @param mtime[Time] What modification time to record for the file
233
- # @return [void]
234
- def add_local_file_header(io:, filename:, crc32:, compressed_size:, uncompressed_size:, storage_mode:, mtime: Time.now.utc)
235
- if @files.any?{|e| e.filename == filename }
236
- raise DuplicateFilenames, "Filename #{filename.inspect} already used in the archive"
237
- end
238
- raise UnknownMode, "Unknown compression mode #{storage_mode}" unless [STORED, DEFLATED].include?(storage_mode)
239
- e = Entry.new(filename, crc32, compressed_size, uncompressed_size, storage_mode, mtime)
240
- @files << e
241
- @local_header_offsets << io.tell
242
- e.write_local_file_header(io)
243
- end
244
-
245
- # Writes the central directory (including the Zip6 salient bits if necessary)
246
- #
247
- # @param io[#<<, #tell] the buffer to write the central directory to.
248
- # The method will use `tell` on the buffer since it has to know where the central directory is located
249
- # @return [void]
250
- def write_central_directory(io)
251
- start_of_central_directory = io.tell
252
-
253
- # Central directory file headers, per file in order
254
- @files.each_with_index do |file, i|
255
- local_file_header_offset_from_start_of_file = @local_header_offsets.fetch(i)
256
- file.write_central_directory_file_header(io, local_file_header_offset_from_start_of_file)
257
- end
258
- central_dir_size = io.tell - start_of_central_directory
259
-
260
- zip64_required = central_dir_size > FOUR_BYTE_MAX_UINT ||
261
- start_of_central_directory > FOUR_BYTE_MAX_UINT ||
262
- @files.length > TWO_BYTE_MAX_UINT ||
263
- @files.any?(&:requires_zip64?)
264
-
265
- # Then, if zip64 is used
266
- if zip64_required
267
- # [zip64 end of central directory record]
268
- zip64_eocdr_offset = io.tell
269
- # zip64 end of central dir
270
- io << [0x06064b50].pack(C_V) # signature 4 bytes (0x06064b50)
271
- io << [44].pack(C_Qe) # size of zip64 end of central
272
- # directory record 8 bytes
273
- # (this is ex. the 12 bytes of the signature and the size value itself).
274
- # Without the extensible data sector it is always 44.
275
- io << MADE_BY_SIGNATURE # version made by 2 bytes
276
- io << [VERSION_NEEDED_TO_EXTRACT_ZIP64].pack(C_v) # version needed to extract 2 bytes
277
- io << [0].pack(C_V) # number of this disk 4 bytes
278
- io << [0].pack(C_V) # number of the disk with the
279
- # start of the central directory 4 bytes
280
- io << [@files.length].pack(C_Qe) # total number of entries in the
281
- # central directory on this disk 8 bytes
282
- io << [@files.length].pack(C_Qe) # total number of entries in the
283
- # central directory 8 bytes
284
- io << [central_dir_size].pack(C_Qe) # size of the central directory 8 bytes
285
- # offset of start of central
286
- # directory with respect to
287
- io << [start_of_central_directory].pack(C_Qe) # the starting disk number 8 bytes
288
- # zip64 extensible data sector (variable size), blank for us
289
-
290
- # [zip64 end of central directory locator]
291
- io << [0x07064b50].pack(C_V) # zip64 end of central dir locator
292
- # signature 4 bytes (0x07064b50)
293
- io << [0].pack(C_V) # number of the disk with the
294
- # start of the zip64 end of
295
- # central directory 4 bytes
296
- io << [zip64_eocdr_offset].pack(C_Qe) # relative offset of the zip64
297
- # end of central directory record 8 bytes
298
- # (note: "relative" is actually "from the start of the file")
299
- io << [1].pack(C_V) # total number of disks 4 bytes
300
- end
301
-
302
- # Then the end of central directory record:
303
- io << [0x06054b50].pack(C_V) # end of central dir signature 4 bytes (0x06054b50)
304
- io << [0].pack(C_v) # number of this disk 2 bytes
305
- io << [0].pack(C_v) # number of the disk with the
306
- # start of the central directory 2 bytes
307
-
308
- if zip64_required # the number of entries will be read from the zip64 part of the central directory
309
- io << [TWO_BYTE_MAX_UINT].pack(C_v) # total number of entries in the
310
- # central directory on this disk 2 bytes
311
- io << [TWO_BYTE_MAX_UINT].pack(C_v) # total number of entries in
312
- # the central directory 2 bytes
313
- else
314
- io << [@files.length].pack(C_v) # total number of entries in the
315
- # central directory on this disk 2 bytes
316
- io << [@files.length].pack(C_v) # total number of entries in
317
- # the central directory 2 bytes
318
- end
319
-
320
- if zip64_required
321
- io << [FOUR_BYTE_MAX_UINT].pack(C_V) # size of the central directory 4 bytes
322
- io << [FOUR_BYTE_MAX_UINT].pack(C_V) # offset of start of central
323
- # directory with respect to
324
- # the starting disk number 4 bytes
325
- else
326
- io << [central_dir_size].pack(C_V) # size of the central directory 4 bytes
327
- io << [start_of_central_directory].pack(C_V) # offset of start of central
328
- # directory with respect to
329
- # the starting disk number 4 bytes
330
- end
331
- io << [0].pack(C_v) # .ZIP file comment length 2 bytes
332
- # .ZIP file comment (variable size)
333
- end
334
-
335
- private_constant :FOUR_BYTE_MAX_UINT, :TWO_BYTE_MAX_UINT,
336
- :VERSION_MADE_BY, :VERSION_NEEDED_TO_EXTRACT, :VERSION_NEEDED_TO_EXTRACT_ZIP64,
337
- :DEFAULT_EXTERNAL_ATTRS, :MADE_BY_SIGNATURE,
338
- :Entry, :C_V, :C_v, :C_Qe
339
- end
@@ -1,44 +0,0 @@
1
- # Helps to estimate archive sizes
2
- class ZipTricks::StoredSizeEstimator < Struct.new(:manifest)
3
-
4
- # Performs the estimate using fake archiving. It needs to know the sizes of the
5
- # entries upfront. Usage:
6
- #
7
- # expected_zip_size = StoredSizeEstimator.perform_fake_archiving do | estimator |
8
- # estimator.add_stored_entry("file.doc", size=898291)
9
- # estimator.add_compressed_entry("family.tif", size=89281911, compressed_size=121908)
10
- # end
11
- #
12
- # @return [Fixnum] the size of the resulting archive, in bytes
13
- # @yield [StoredSizeEstimator] the estimator
14
- def self.perform_fake_archiving
15
- _, bytes = ZipTricks::Manifest.build do |manifest|
16
- # The API for this class uses positional arguments. The Manifest API
17
- # uses keyword arguments.
18
- call_adapter = new(manifest)
19
- yield(call_adapter)
20
- end
21
- bytes
22
- end
23
-
24
- # Add a fake entry to the archive, to see how big it is going to be in the end.
25
- #
26
- # @param name [String] the name of the file (filenames are variable-width in the ZIP)
27
- # @param size_uncompressed [Fixnum] size of the uncompressed entry
28
- # @return self
29
- def add_stored_entry(name, size_uncompressed)
30
- manifest.add_stored_entry(name: name, size_uncompressed: size_uncompressed)
31
- self
32
- end
33
-
34
- # Add a fake entry to the archive, to see how big it is going to be in the end.
35
- #
36
- # @param name [String] the name of the file (filenames are variable-width in the ZIP)
37
- # @param size_uncompressed [Fixnum] size of the uncompressed entry
38
- # @param size_compressed [Fixnum] size of the compressed entry
39
- # @return self
40
- def add_compressed_entry(name, size_uncompressed, size_compressed)
41
- manifest.add_compressed_entry(name: name, size_uncompressed: size_uncompressed, size_compressed: size_compressed)
42
- self
43
- end
44
- end
@@ -1,60 +0,0 @@
1
- require_relative '../spec_helper'
2
-
3
- describe ZipTricks::Manifest do
4
- it 'builds a map of the contained ranges, and has its cumulative size match the predicted archive size exactly' do
5
- # Generate a couple of random files
6
- raw_file_1 = SecureRandom.random_bytes(1024 * 20)
7
- raw_file_2 = SecureRandom.random_bytes(1024 * 128)
8
- raw_file_3 = SecureRandom.random_bytes(1258695)
9
-
10
- manifest, bytesize = described_class.build do | builder |
11
- r = builder.add_stored_entry(name: "first-file.bin", size_uncompressed: raw_file_1.size)
12
- expect(r).to eq(builder), "add_stored_entry should return self"
13
-
14
- builder.add_stored_entry(name: "second-file.bin", size_uncompressed: raw_file_2.size)
15
-
16
- r = builder.add_compressed_entry(name: "second-file-comp.bin", size_uncompressed: raw_file_2.size,
17
- size_compressed: raw_file_3.size, segment_info: 'http://example.com/second-file-deflated-segment.bin')
18
- expect(r).to eq(builder), "add_compressed_entry should return self"
19
- end
20
-
21
- require 'range_utils'
22
-
23
- expect(manifest).to be_kind_of(Array)
24
- total_size_of_all_parts = manifest.inject(0) do | total_bytes, span |
25
- total_bytes + RangeUtils.size_from_range(span.byte_range_in_zip)
26
- end
27
- expect(total_size_of_all_parts).to eq(1410595)
28
- expect(bytesize).to eq(1410595)
29
-
30
- expect(manifest.length).to eq(7)
31
-
32
- first_header = manifest[0]
33
- expect(first_header.part_type).to eq(:entry_header)
34
- expect(first_header.byte_range_in_zip).to eq(0..43)
35
- expect(first_header.filename).to eq("first-file.bin")
36
- expect(first_header.additional_metadata).to be_nil
37
-
38
- first_body = manifest[1]
39
- expect(first_body.part_type).to eq(:entry_body)
40
- expect(first_body.byte_range_in_zip).to eq(44..20523)
41
- expect(first_body.filename).to eq("first-file.bin")
42
- expect(first_body.additional_metadata).to be_nil
43
-
44
- third_header = manifest[4]
45
- expect(third_header.part_type).to eq(:entry_header)
46
- expect(third_header.byte_range_in_zip).to eq(151641..151690)
47
- expect(third_header.filename).to eq("second-file-comp.bin")
48
- expect(third_header.additional_metadata).to eq("http://example.com/second-file-deflated-segment.bin")
49
-
50
- third_body = manifest[5]
51
- expect(third_body.part_type).to eq(:entry_body)
52
- expect(third_body.byte_range_in_zip).to eq(151691..1410385)
53
- expect(third_body.filename).to eq("second-file-comp.bin")
54
- expect(third_body.additional_metadata).to eq("http://example.com/second-file-deflated-segment.bin")
55
-
56
- cd = manifest[-1]
57
- expect(cd.part_type).to eq(:central_directory)
58
- expect(cd.byte_range_in_zip).to eq(1410386..1410594)
59
- end
60
- end