zip_tricks 4.2.3 → 4.2.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: cc23010c353f0d1118cdbb38649c1363ffe34b7c
4
- data.tar.gz: 8be7f24974d25870fd0b9c6852e0ad8fcff1ff0b
3
+ metadata.gz: 359804bad07357aa54564d6e5325e13cd4256b5f
4
+ data.tar.gz: df97c2d477569adc8885a477527376d91be7b767
5
5
  SHA512:
6
- metadata.gz: 15d547d8f049bd7eb71470632c43ccd836d24908d5e98061421caa0af28f8e1747bcbc993906e112fc2874af8b423d879510be2b6a49d1c346f820058f715e11
7
- data.tar.gz: b16df9dcbb072b215f866f83cfd073157d21ca2d7e5f336e0226c509b858925e12149f831d816dbf159d041f0cfab27d16bb934692c66b7eefa5bdc048cbb5e1
6
+ metadata.gz: 74119e2e85d89f7b5254be4585734896fd27d67db5a8b77a1db97a4dae3891f3e9c99027d7bea0d389e864758cd172cd83bbc6b6358b3b25d00504a0e3113fe7
7
+ data.tar.gz: f1b381a1f5a7cba65e991e17a77bf46139889a10282747bcb5a622b43b04026f783cf562fdadfc97e8e953103890b0a05ce676128e53e77235506e40a85e1027
data/.gitignore ADDED
@@ -0,0 +1,53 @@
1
+ # rcov generated
2
+ coverage
3
+ coverage.data
4
+
5
+ # rdoc generated
6
+ rdoc
7
+
8
+ # yard generated
9
+ doc
10
+ .yardoc
11
+
12
+ # bundler
13
+ .bundle
14
+ Gemfile.lock
15
+
16
+ # jeweler generated
17
+ pkg
18
+
19
+ # Have editor/IDE/OS specific files you need to ignore? Consider using a global gitignore:
20
+ #
21
+ # * Create a file at ~/.gitignore
22
+ # * Include files you want ignored
23
+ # * Run: git config --global core.excludesfile ~/.gitignore
24
+ #
25
+ # After doing this, these files will be ignored in all your git projects,
26
+ # saving you from having to 'pollute' every project you touch with them
27
+ #
28
+ # Not sure what to needs to be ignored for particular editors/OSes? Here's some ideas to get you started. (Remember, remove the leading # of the line)
29
+ #
30
+ # For MacOS:
31
+ #
32
+ #.DS_Store
33
+
34
+ tmp
35
+ testing/*.zip
36
+
37
+ # For TextMate
38
+ #*.tmproj
39
+ #tmtags
40
+
41
+ # For emacs:
42
+ #*~
43
+ #\#*
44
+ #.\#*
45
+
46
+ # For vim:
47
+ #*.swp
48
+
49
+ # For redcar:
50
+ #.redcar
51
+
52
+ # For rubinius:
53
+ #*.rbc
data/Gemfile CHANGED
@@ -1,14 +1,4 @@
1
- source "http://rubygems.org"
1
+ source 'https://rubygems.org'
2
2
 
3
- group :development do
4
- gem 'rubyzip', '~> 1.1'
5
- gem 'terminal-table'
6
- gem 'range_utils'
7
- gem 'rack', '~> 1.6' # For Jeweler
8
- gem 'rake', '~> 10.4'
9
- gem "rspec", "~> 3.2.0", '< 3.3'
10
- gem 'coderay'
11
- gem "yard", "~> 0.8"
12
- gem "bundler", "~> 1.0"
13
- gem "jeweler", "~> 2", '>= 2.1.2'
14
- end
3
+ # Specify your gem's dependencies in zip_tricks.gemspec
4
+ gemspec
data/README.md CHANGED
@@ -20,6 +20,42 @@ Ruby 2.1+ syntax support (keyword arguments with defaults) and a working zlib (a
20
20
  jRuby might experience problems when using the reader methods due to the argument of `IO#seek` being limited
21
21
  to [32 bit sizes.](https://github.com/jruby/jruby/issues/3817)
22
22
 
23
+
24
+ ## Diving in: send some large CSV reports from Rails
25
+
26
+ The easiest is to use the Rails' built-in streaming feature:
27
+
28
+ ```ruby
29
+ class ZipsController < ActionController::Base
30
+ include ActionController::Live
31
+
32
+ def download
33
+ response.headers['Content-Type'] = 'application/zip'
34
+
35
+ # Create a wrapper for the write call that quacks like something you
36
+ # can << to, used by ZipTricks
37
+ w = ZipTricks::BlockWrite.new { |chunk| response.stream.write(chunk) }
38
+
39
+ # Send out the archive of some Substantially Large CSV Files (tm)
40
+ ZipTricks::Streamer.open(w) do |zip|
41
+ zip.write_deflated_file('report1.csv') do |sink|
42
+ CSV(sink) do |csv_write|
43
+ csv << Person.column_names
44
+ Person.all.find_each do |person|
45
+ csv << person.attributes.values
46
+ end
47
+ end
48
+ end
49
+ zip.write_deflated_file('report2.csv') do |sink|
50
+ ...
51
+ end
52
+ end
53
+ ensure
54
+ response.stream.close
55
+ end
56
+ end
57
+ ```
58
+
23
59
  ## Create a ZIP file without size estimation, compress on-the-fly during writes
24
60
 
25
61
  Basic use case is compressing on the fly. Some data will be buffered by the Zlib deflater, but
data/Rakefile CHANGED
@@ -1,51 +1,6 @@
1
- # encoding: utf-8
1
+ require "bundler/gem_tasks"
2
+ require "rspec/core/rake_task"
2
3
 
3
- require 'rubygems'
4
- require 'bundler'
5
- begin
6
- Bundler.setup(:default, :development)
7
- rescue Bundler::BundlerError => e
8
- $stderr.puts e.message
9
- $stderr.puts "Run `bundle install` to install missing gems"
10
- exit e.status_code
11
- end
12
- require 'rake'
13
- require_relative 'lib/zip_tricks'
14
- require 'jeweler'
15
-
16
- Jeweler::Tasks.new do |gem|
17
- # gem is a Gem::Specification... see http://guides.rubygems.org/specification-reference/ for more options
18
- gem.name = "zip_tricks"
19
- gem.homepage = "http://github.com/wetransfer/zip_tricks"
20
- gem.license = "MIT"
21
- gem.version = ZipTricks::VERSION
22
- gem.summary = 'Stream out ZIP files from Ruby'
23
- gem.description = 'Stream out ZIP files from Ruby'
24
- gem.email = "me@julik.nl"
25
- gem.authors = ["Julik Tarkhanov"]
26
- gem.files.exclude "testing/**/*"
27
- # dependencies defined in Gemfile
28
- end
29
- Jeweler::RubygemsDotOrgTasks.new
30
-
31
- require 'rspec/core'
32
- require 'rspec/core/rake_task'
33
- RSpec::Core::RakeTask.new(:spec) do |spec|
34
- spec.pattern = FileList['spec/**/*_spec.rb']
35
- end
36
-
37
- desc "Code coverage detail"
38
- task :simplecov do
39
- ENV['COVERAGE'] = "true"
40
- Rake::Task['spec'].execute
41
- end
4
+ RSpec::Core::RakeTask.new(:spec)
42
5
 
43
6
  task :default => :spec
44
-
45
- require 'yard'
46
- desc "Generate YARD documentation"
47
- YARD::Rake::YardocTask.new do |t|
48
- t.files = ['lib/**/*.rb', 'ext/**/*.c' ]
49
- t.options = ['--markup markdown']
50
- t.stats_options = ['--list-undoc']
51
- end
data/lib/zip_tricks.rb CHANGED
@@ -1,6 +1,4 @@
1
1
  module ZipTricks
2
- VERSION = '4.2.3'
3
-
4
2
  # Require all the sub-components except myself
5
3
  Dir.glob(__dir__ + '/**/*.rb').sort.each {|p| require p unless p == __FILE__ }
6
4
  end
@@ -1,3 +1,5 @@
1
+ require 'zlib'
2
+
1
3
  # Permits Deflate compression in independent blocks. The workflow is as follows:
2
4
  #
3
5
  # * Run every block to compress through deflate_chunk, remove the header, footer and adler32 from the result
@@ -0,0 +1,3 @@
1
+ module ZipTricks
2
+ VERSION = '4.2.4'
3
+ end
@@ -45,15 +45,16 @@ class ZipTricks::ZipWriter
45
45
  [VERSION_MADE_BY, os_type].pack('CC')
46
46
  end
47
47
 
48
- C_V = 'V'.freeze # Encode a 4-byte little-endian uint
49
- C_v = 'v'.freeze # Encode a 2-byte little-endian uint
50
- C_Qe = 'Q<'.freeze # Encode an 8-byte little-endian uint
51
- C_es = ''.freeze
48
+ C_V = 'V'.freeze # Encode a 4-byte unsigned little-endian uint
49
+ C_v = 'v'.freeze # Encode a 2-byte unsigned little-endian uint
50
+ C_Qe = 'Q<'.freeze # Encode an 8-byte unsigned little-endian uint
51
+ C_C = 'C'.freeze # For bit-encoded strings
52
+ C_N = 'N'.freeze # Encode a 4-byte signed little-endian int
52
53
 
53
54
  private_constant :FOUR_BYTE_MAX_UINT, :TWO_BYTE_MAX_UINT,
54
55
  :VERSION_MADE_BY, :VERSION_NEEDED_TO_EXTRACT, :VERSION_NEEDED_TO_EXTRACT_ZIP64,
55
56
  :DEFAULT_EXTERNAL_ATTRS, :MADE_BY_SIGNATURE,
56
- :C_V, :C_v, :C_Qe, :C_es, :ZIP_TRICKS_COMMENT
57
+ :C_V, :C_v, :C_Qe, :ZIP_TRICKS_COMMENT
57
58
 
58
59
  # Writes the local file header, that precedes the actual file _data_.
59
60
  #
@@ -93,21 +94,21 @@ class ZipTricks::ZipWriter
93
94
  # Filename should not be longer than 0xFFFF otherwise this wont fit here
94
95
  io << [filename.bytesize].pack(C_v) # file name length 2 bytes
95
96
 
97
+ # Interesting tidbit:
98
+ # https://social.technet.microsoft.com/Forums/windows/en-US/6a60399f-2879-4859-b7ab-6ddd08a70948
99
+ # TL;DR of it is: Windows 7 Explorer _will_ open Zip64 entries. However, it desires to have the
100
+ # Zip64 extra field as _the first_ extra field.
96
101
  extra_fields = if requires_zip64
97
102
  zip_64_extra_for_local_file_header(compressed_size: compressed_size, uncompressed_size: uncompressed_size)
98
103
  else
99
- C_es
104
+ ''
100
105
  end
106
+ extra_fields << timestamp_extra(mtime)
101
107
 
102
108
  io << [extra_fields.bytesize].pack(C_v) # extra field length 2 bytes
103
109
 
104
110
  io << filename # file name (variable size)
105
-
106
- # Interesting tidbit:
107
- # https://social.technet.microsoft.com/Forums/windows/en-US/6a60399f-2879-4859-b7ab-6ddd08a70948
108
- # TL;DR of it is: Windows 7 Explorer _will_ open Zip64 entries. However, it desires to have the
109
- # Zip64 extra field as _the first_ extra field. If we decide to add the Info-ZIP UTF-8 field...
110
- io << extra_fields if requires_zip64
111
+ io << extra_fields
111
112
  end
112
113
 
113
114
  # Writes the file header for the central directory, for a particular file in the archive. When writing out this data,
@@ -158,8 +159,10 @@ class ZipTricks::ZipWriter
158
159
  zip_64_extra_for_central_directory_file_header(local_file_header_location: local_file_header_location,
159
160
  compressed_size: compressed_size, uncompressed_size: uncompressed_size)
160
161
  else
161
- C_es
162
+ ''
162
163
  end
164
+ extra_fields << timestamp_extra(mtime)
165
+
163
166
  io << [extra_fields.bytesize].pack(C_v) # extra field length 2 bytes
164
167
 
165
168
  io << [0].pack(C_v) # file comment length 2 bytes
@@ -313,6 +316,49 @@ class ZipTricks::ZipWriter
313
316
  ]
314
317
  pack_array(data_and_packspecs)
315
318
  end
319
+
320
+ # Writes the extended timestamp information field. The spec defines 2
321
+ # different formats - the one for the local file header can also accomodate the
322
+ # atime and ctime, whereas the one for the central directory can only take
323
+ # the mtime - and refers the reader to the local header extra to obtain the
324
+ # remaining times
325
+ def timestamp_extra(mtime)
326
+ # Local-header version:
327
+ #
328
+ # Value Size Description
329
+ # ----- ---- -----------
330
+ # (time) 0x5455 Short tag for this extra block type ("UT")
331
+ # TSize Short total data size for this block
332
+ # Flags Byte info bits
333
+ # (ModTime) Long time of last modification (UTC/GMT)
334
+ # (AcTime) Long time of last access (UTC/GMT)
335
+ # (CrTime) Long time of original creation (UTC/GMT)
336
+ #
337
+ # Central-header version:
338
+ #
339
+ # Value Size Description
340
+ # ----- ---- -----------
341
+ # (time) 0x5455 Short tag for this extra block type ("UT")
342
+ # TSize Short total data size for this block
343
+ # Flags Byte info bits (refers to local header!)
344
+ # (ModTime) Long time of last modification (UTC/GMT)
345
+ #
346
+ # The lower three bits of Flags in both headers indicate which time-
347
+ # stamps are present in the LOCAL extra field:
348
+ #
349
+ # bit 0 if set, modification time is present
350
+ # bit 1 if set, access time is present
351
+ # bit 2 if set, creation time is present
352
+ # bits 3-7 reserved for additional timestamps; not set
353
+ flags = 0b10000000 # Set bit 1 only to indicate only mtime is present
354
+ data_and_packspecs = [
355
+ 0x5455, C_v, # tag for this extra block type ("UT")
356
+ (1 + 4), C_v, # the size of this block (1 byte used for the Flag + 1 long used for the timestamp)
357
+ flags, C_C, # encode a single byte
358
+ mtime.utc.to_i, C_N, # Use a signed long, not the unsigned one used by the rest of the ZIP spec.
359
+ ]
360
+ pack_array(data_and_packspecs)
361
+ end
316
362
 
317
363
  # Writes the Zip64 extra field for the central directory header.It differs from the extra used in the local file header because it
318
364
  # also contains the location of the local file header in the ZIP as an 8-byte int.
@@ -0,0 +1,12 @@
1
+ ## The _actual_ testing for ZipTricks
2
+
3
+ Consists of a fairly straightforward procedure.
4
+
5
+ 1. Run `generate_test_files.rb`. This will take some time and produce a number of ZIP files.
6
+ 2. Open them with the following ZIP unarchivers:
7
+ * A recent version of `zipinfo` with the `-tlhvz` flags - to see the information about the file
8
+ * ArchiveUtility on OSX
9
+ * The Unarchiver on OSX
10
+ * Built-in Explorer on Windows 7
11
+ * 7Zip 9.20 on Windows 7
12
+ * Write down your observations in `test-report.txt` and, when cutting a release, timestamp a copy of that file.
@@ -0,0 +1,86 @@
1
+ require_relative 'support'
2
+
3
+ build_test "Two small stored files" do |zip|
4
+ zip.add_stored_entry(filename: 'text.txt', size: $war_and_peace.bytesize, crc32: $war_and_peace_crc)
5
+ zip << $war_and_peace
6
+
7
+ zip.add_stored_entry(filename: 'image.jpg', size: $image_file.bytesize, crc32: $image_file_crc)
8
+ zip << $image_file
9
+ end
10
+
11
+ build_test "Filename with diacritics" do |zip|
12
+ zip.add_stored_entry(filename: 'Kungälv.txt', size: $war_and_peace.bytesize, crc32: $war_and_peace_crc)
13
+ zip << $war_and_peace
14
+ end
15
+
16
+ build_test "Purely UTF-8 filename" do |zip|
17
+ zip.add_stored_entry(filename: 'Война и мир.txt', size: $war_and_peace.bytesize, crc32: $war_and_peace_crc)
18
+ zip << $war_and_peace
19
+ end
20
+
21
+ # The trick of this test is that each file of the 2, on it's own, does _not_ exceed the
22
+ # size threshold for Zip64. Together, however, they do.
23
+ build_test "Two entries larger than the overall Zip64 offset" do |zip|
24
+ big = generate_big_entry((0xFFFFFFFF / 2) + 1024)
25
+ zip.add_stored_entry(filename: 'repeated-A.txt', size: big.size, crc32: big.crc32)
26
+ big.write_to(zip)
27
+
28
+ zip.add_stored_entry(filename: 'repeated-B.txt', size: big.size, crc32: big.crc32)
29
+ big.write_to(zip)
30
+ end
31
+
32
+ build_test "One entry that requires Zip64 and a tiny entry following it" do |zip|
33
+ big = generate_big_entry(0xFFFFFFFF + 2048)
34
+ zip.add_stored_entry(filename: 'large-requires-zip64.bin', size: big.size, crc32: big.crc32)
35
+ big.write_to(zip)
36
+
37
+ zip.add_stored_entry(filename: 'tiny-after.txt', size: $war_and_peace.bytesize, crc32: $war_and_peace_crc)
38
+ zip << $war_and_peace
39
+ end
40
+
41
+ build_test "One tiny entry followed by second that requires Zip64" do |zip|
42
+ zip.add_stored_entry(filename: 'tiny-at-start.txt', size: $war_and_peace.bytesize, crc32: $war_and_peace_crc)
43
+ zip << $war_and_peace
44
+
45
+ big = generate_big_entry(0xFFFFFFFF + 2048)
46
+ zip.add_stored_entry(filename: 'large-requires-zip64.bin', size: big.size, crc32: big.crc32)
47
+ big.write_to(zip)
48
+ end
49
+
50
+ build_test "Two entries both requiring Zip64" do |zip|
51
+ big = generate_big_entry(0xFFFFFFFF + 2048)
52
+ zip.add_stored_entry(filename: 'huge-file-1.bin', size: big.size, crc32: big.crc32)
53
+ big.write_to(zip)
54
+
55
+ zip.add_stored_entry(filename: 'huge-file-2.bin', size: big.size, crc32: big.crc32)
56
+ big.write_to(zip)
57
+ end
58
+
59
+ build_test "Two stored entries using data descriptors" do |zip|
60
+ zip.write_stored_file('stored.1.bin') do |sink|
61
+ sink << Random.new.bytes(1024 * 1024 * 4)
62
+ end
63
+ zip.write_stored_file('stored.2.bin') do |sink|
64
+ sink << Random.new.bytes(1024 * 1024 * 3)
65
+ end
66
+ end
67
+
68
+ build_test "One entry deflated using data descriptors" do |zip|
69
+ big = generate_big_entry(0xFFFFFFFF / 64)
70
+ zip.write_deflated_file('war-and-peace-repeated-compressed.txt') do |sink|
71
+ big.write_to(sink)
72
+ end
73
+ end
74
+
75
+ build_test "Two entries larger than the overall Zip64 offset using data descriptors" do |zip|
76
+ big = generate_big_entry((0xFFFFFFFF / 2) + 1024)
77
+
78
+ zip.write_stored_file('repeated-A.txt') { |sink| big.write_to(sink) }
79
+ zip.write_stored_file('repeated-B.txt') { |sink| big.write_to(sink) }
80
+ end
81
+
82
+ build_test "One stored entry larger than Zip64 threshold using data descriptors" do |zip|
83
+ big = generate_big_entry(0xFFFFFFFF + 64000)
84
+
85
+ zip.write_stored_file('repeated-A.txt') { |sink| big.write_to(sink) }
86
+ end
Binary file
File without changes
@@ -0,0 +1,83 @@
1
+ require 'rubygems'
2
+ require 'bundler'
3
+ Bundler.setup
4
+ require_relative '../lib/zip_tricks'
5
+ require 'terminal-table'
6
+
7
+ $war_and_peace = File.open(__dir__ + '/in/war-and-peace.txt', 'rb'){|f| f.read }.freeze
8
+ $war_and_peace_crc = Zlib.crc32($war_and_peace)
9
+
10
+ $image_file = File.open(__dir__ + '/in/VTYL8830.jpg', 'rb'){|f| f.read }.freeze
11
+ $image_file_crc = Zlib.crc32($image_file)
12
+
13
+ class BigEntry < Struct.new(:crc32, :size, :iterations)
14
+ def write_to(io)
15
+ iterations.times { io << $war_and_peace }
16
+ end
17
+ end
18
+
19
+ def generate_big_entry(desired_minimum_size)
20
+ repeats = (desired_minimum_size.to_f / $war_and_peace.bytesize).ceil
21
+ crc_stream = ZipTricks::StreamCRC32.new
22
+ repeats.times { crc_stream << $war_and_peace }
23
+ entry_size = $war_and_peace.bytesize * repeats
24
+ raise "Ooops" if entry_size < desired_minimum_size
25
+ BigEntry.new(crc_stream.to_i, entry_size, repeats)
26
+ end
27
+
28
+ TestDesc = Struct.new(:title, :filename)
29
+
30
+ $tests_performed = 0
31
+ $test_descs = []
32
+ $builder_threads = []
33
+ at_exit { $builder_threads.map(&:join) }
34
+
35
+ def build_test(test_description)
36
+ $tests_performed += 1
37
+
38
+ test_file_base = test_description.downcase.gsub(/\-/, '').gsub(/[\s\:]+/, '_')
39
+ filename = '%02d-%s.zip' % [$tests_performed, test_file_base]
40
+
41
+ puts 'Test %02d: %s' % [$tests_performed, test_description]
42
+ puts filename
43
+ puts ""
44
+
45
+ $test_descs << TestDesc.new(test_description, filename)
46
+ $builder_threads << Thread.new do
47
+ File.open(File.join(__dir__, filename + '.tmp'), 'wb') do |of|
48
+ ZipTricks::Streamer.open(of) do |zip|
49
+ yield(zip)
50
+ end
51
+ end
52
+ File.rename(File.join(__dir__, filename + '.tmp'), File.join(__dir__, filename))
53
+ end
54
+ end
55
+
56
+ # For quickly disabling them by prepending "x" (like RSpec)
57
+ def xbuild_test(*)
58
+ end
59
+
60
+ # Prints a text file that you can then fill in
61
+ def prepare_test_protocol
62
+ File.open(__dir__ + '/test-report.txt', 'wb') do |f|
63
+ platforms = [
64
+ "OSX 10.11 - Archive Utility (builtin)",
65
+ "OSX - The Unarchiver 3.10",
66
+ "Windows7 x64 - Builtin Explorer ZIP opener",
67
+ "Windows7 x64 - 7Zip 9.20",
68
+ ]
69
+ platforms.each do |platform_name|
70
+ f.puts ''
71
+ table = Terminal::Table.new title: platform_name, headings: ['Test', 'Outcome']
72
+ $test_descs.each_with_index do |desc, i|
73
+ test_name = [desc.filename, '%s' % desc.title].join("\n")
74
+ outcome = ' ' * 64
75
+ table << [test_name, outcome]
76
+ table << :separator if i < ($test_descs.length - 1)
77
+ end
78
+ f.puts table
79
+ end
80
+ end
81
+ end
82
+
83
+ at_exit { prepare_test_protocol }