zip_tricks 4.2.3 → 4.2.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cc23010c353f0d1118cdbb38649c1363ffe34b7c
-  data.tar.gz: 8be7f24974d25870fd0b9c6852e0ad8fcff1ff0b
+  metadata.gz: 359804bad07357aa54564d6e5325e13cd4256b5f
+  data.tar.gz: df97c2d477569adc8885a477527376d91be7b767
 SHA512:
-  metadata.gz: 15d547d8f049bd7eb71470632c43ccd836d24908d5e98061421caa0af28f8e1747bcbc993906e112fc2874af8b423d879510be2b6a49d1c346f820058f715e11
-  data.tar.gz: b16df9dcbb072b215f866f83cfd073157d21ca2d7e5f336e0226c509b858925e12149f831d816dbf159d041f0cfab27d16bb934692c66b7eefa5bdc048cbb5e1
+  metadata.gz: 74119e2e85d89f7b5254be4585734896fd27d67db5a8b77a1db97a4dae3891f3e9c99027d7bea0d389e864758cd172cd83bbc6b6358b3b25d00504a0e3113fe7
+  data.tar.gz: f1b381a1f5a7cba65e991e17a77bf46139889a10282747bcb5a622b43b04026f783cf562fdadfc97e8e953103890b0a05ce676128e53e77235506e40a85e1027

data/.gitignore

@@ -0,0 +1,53 @@
+# rcov generated
+coverage
+coverage.data
+
+# rdoc generated
+rdoc
+
+# yard generated
+doc
+.yardoc
+
+# bundler
+.bundle
+Gemfile.lock
+
+# jeweler generated
+pkg
+
+# Have editor/IDE/OS specific files you need to ignore? Consider using a global gitignore: 
+#
+# * Create a file at ~/.gitignore
+# * Include files you want ignored
+# * Run: git config --global core.excludesfile ~/.gitignore
+#
+# After doing this, these files will be ignored in all your git projects,
+# saving you from having to 'pollute' every project you touch with them
+#
+# Not sure what to needs to be ignored for particular editors/OSes? Here's some ideas to get you started. (Remember, remove the leading # of the line)
+#
+# For MacOS:
+#
+#.DS_Store
+
+tmp
+testing/*.zip
+
+# For TextMate
+#*.tmproj
+#tmtags
+
+# For emacs:
+#*~
+#\#*
+#.\#*
+
+# For vim:
+#*.swp
+
+# For redcar:
+#.redcar
+
+# For rubinius:
+#*.rbc

data/Gemfile

@@ -1,14 +1,4 @@
-source "http://rubygems.org"
+source 'https://rubygems.org'
 
-group :development do
-  gem 'rubyzip', '~> 1.1'
-  gem 'terminal-table'
-  gem 'range_utils'
-  gem 'rack', '~> 1.6' # For Jeweler
-  gem 'rake', '~> 10.4'
-  gem "rspec", "~> 3.2.0", '< 3.3'
-  gem 'coderay'
-  gem "yard", "~> 0.8"
-  gem "bundler", "~> 1.0"
-  gem "jeweler", "~> 2", '>= 2.1.2'
-end
+# Specify your gem's dependencies in zip_tricks.gemspec
+gemspec

data/README.md

@@ -20,6 +20,42 @@ Ruby 2.1+ syntax support (keyword arguments with defaults) and a working zlib (a
 jRuby might experience problems when using the reader methods due to the argument of `IO#seek` being limited
 to [32 bit sizes.](https://github.com/jruby/jruby/issues/3817)
 
+
+## Diving in: send some large CSV reports from Rails
+
+The easiest is to use the Rails' built-in streaming feature:
+
+```ruby
+class ZipsController < ActionController::Base
+  include ActionController::Live
+
+  def download
+    response.headers['Content-Type'] = 'application/zip'
+    
+    # Create a wrapper for the write call that quacks like something you
+    # can << to, used by ZipTricks
+    w = ZipTricks::BlockWrite.new { |chunk| response.stream.write(chunk) }
+    
+    # Send out the archive of some Substantially Large CSV Files (tm)
+    ZipTricks::Streamer.open(w) do |zip|
+      zip.write_deflated_file('report1.csv') do |sink|
+        CSV(sink) do |csv_write|
+          csv << Person.column_names
+          Person.all.find_each do |person|
+            csv << person.attributes.values
+          end
+        end
+      end
+      zip.write_deflated_file('report2.csv') do |sink|
+        ...
+      end
+    end
+  ensure
+    response.stream.close
+  end
+end
+```
+
 ## Create a ZIP file without size estimation, compress on-the-fly during writes
 
 Basic use case is compressing on the fly. Some data will be buffered by the Zlib deflater, but

data/Rakefile

@@ -1,51 +1,6 @@
-# encoding: utf-8
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
 
-require 'rubygems'
-require 'bundler'
-begin
-  Bundler.setup(:default, :development)
-rescue Bundler::BundlerError => e
-  $stderr.puts e.message
-  $stderr.puts "Run `bundle install` to install missing gems"
-  exit e.status_code
-end
-require 'rake'
-require_relative 'lib/zip_tricks'
-require 'jeweler'
-
-Jeweler::Tasks.new do |gem|
-  # gem is a Gem::Specification... see http://guides.rubygems.org/specification-reference/ for more options
-  gem.name = "zip_tricks"
-  gem.homepage = "http://github.com/wetransfer/zip_tricks"
-  gem.license = "MIT"
-  gem.version = ZipTricks::VERSION
-  gem.summary = 'Stream out ZIP files from Ruby'
-  gem.description = 'Stream out ZIP files from Ruby'
-  gem.email = "me@julik.nl"
-  gem.authors = ["Julik Tarkhanov"]
-  gem.files.exclude "testing/**/*"
-  # dependencies defined in Gemfile
-end
-Jeweler::RubygemsDotOrgTasks.new
-
-require 'rspec/core'
-require 'rspec/core/rake_task'
-RSpec::Core::RakeTask.new(:spec) do |spec|
-  spec.pattern = FileList['spec/**/*_spec.rb']
-end
-
-desc "Code coverage detail"
-task :simplecov do
-  ENV['COVERAGE'] = "true"
-  Rake::Task['spec'].execute
-end
+RSpec::Core::RakeTask.new(:spec)
 
 task :default => :spec
-
-require 'yard'
-desc "Generate YARD documentation"
-YARD::Rake::YardocTask.new do |t|
-  t.files   = ['lib/**/*.rb', 'ext/**/*.c' ]
-  t.options = ['--markup markdown']
-  t.stats_options = ['--list-undoc']
-end

data/lib/zip_tricks.rb

@@ -1,6 +1,4 @@
 module ZipTricks
-  VERSION = '4.2.3'
-  
   # Require all the sub-components except myself
   Dir.glob(__dir__ + '/**/*.rb').sort.each {|p| require p unless p == __FILE__ }
 end

data/lib/zip_tricks/block_deflate.rb

@@ -1,3 +1,5 @@
+require 'zlib'
+
 # Permits Deflate compression in independent blocks. The workflow is as follows:
 #
 # * Run every block to compress through deflate_chunk, remove the header, footer and adler32 from the result

data/lib/zip_tricks/version.rb

@@ -0,0 +1,3 @@
+module ZipTricks
+  VERSION = '4.2.4'
+end

data/lib/zip_tricks/zip_writer.rb

@@ -45,15 +45,16 @@ class ZipTricks::ZipWriter
     [VERSION_MADE_BY, os_type].pack('CC')
   end
 
-  C_V = 'V'.freeze    # Encode a 4-byte little-endian uint
-  C_v = 'v'.freeze    # Encode a 2-byte little-endian uint
-  C_Qe = 'Q<'.freeze  # Encode an 8-byte little-endian uint
-  C_es = ''.freeze
+  C_V = 'V'.freeze    # Encode a 4-byte unsigned little-endian uint
+  C_v = 'v'.freeze    # Encode a 2-byte unsigned little-endian uint
+  C_Qe = 'Q<'.freeze  # Encode an 8-byte unsigned little-endian uint
+  C_C = 'C'.freeze # For bit-encoded strings
+  C_N = 'N'.freeze # Encode a 4-byte signed little-endian int
   
   private_constant :FOUR_BYTE_MAX_UINT, :TWO_BYTE_MAX_UINT,
     :VERSION_MADE_BY, :VERSION_NEEDED_TO_EXTRACT, :VERSION_NEEDED_TO_EXTRACT_ZIP64,
     :DEFAULT_EXTERNAL_ATTRS, :MADE_BY_SIGNATURE,
-    :C_V, :C_v, :C_Qe, :C_es, :ZIP_TRICKS_COMMENT
+    :C_V, :C_v, :C_Qe, :ZIP_TRICKS_COMMENT
 
   # Writes the local file header, that precedes the actual file _data_. 
   # 
@@ -93,21 +94,21 @@ class ZipTricks::ZipWriter
     # Filename should not be longer than 0xFFFF otherwise this wont fit here
     io << [filename.bytesize].pack(C_v)                 # file name length             2 bytes
 
+    # Interesting tidbit:
+    # https://social.technet.microsoft.com/Forums/windows/en-US/6a60399f-2879-4859-b7ab-6ddd08a70948
+    # TL;DR of it is: Windows 7 Explorer _will_ open Zip64 entries. However, it desires to have the
+    # Zip64 extra field as _the first_ extra field.
     extra_fields = if requires_zip64
       zip_64_extra_for_local_file_header(compressed_size: compressed_size, uncompressed_size: uncompressed_size)
     else
-      C_es
+      ''
     end
+    extra_fields << timestamp_extra(mtime)
 
     io << [extra_fields.bytesize].pack(C_v)            # extra field length              2 bytes
 
     io << filename                                     # file name (variable size)
-
-    # Interesting tidbit:
-    # https://social.technet.microsoft.com/Forums/windows/en-US/6a60399f-2879-4859-b7ab-6ddd08a70948
-    # TL;DR of it is: Windows 7 Explorer _will_ open Zip64 entries. However, it desires to have the
-    # Zip64 extra field as _the first_ extra field. If we decide to add the Info-ZIP UTF-8 field...
-    io << extra_fields if requires_zip64
+    io << extra_fields
   end
 
   # Writes the file header for the central directory, for a particular file in the archive. When writing out this data,
@@ -158,8 +159,10 @@ class ZipTricks::ZipWriter
       zip_64_extra_for_central_directory_file_header(local_file_header_location: local_file_header_location,
           compressed_size: compressed_size, uncompressed_size: uncompressed_size)
     else
-      C_es
+      ''
     end
+    extra_fields << timestamp_extra(mtime)
+    
     io << [extra_fields.bytesize].pack(C_v)             # extra field length              2 bytes
 
     io << [0].pack(C_v)                                 # file comment length             2 bytes
@@ -313,6 +316,49 @@ class ZipTricks::ZipWriter
     ]
     pack_array(data_and_packspecs)
   end
+
+  # Writes the extended timestamp information field. The spec defines 2
+  # different formats - the one for the local file header can also accomodate the
+  # atime and ctime, whereas the one for the central directory can only take
+  # the mtime - and refers the reader to the local header extra to obtain the
+  # remaining times
+  def timestamp_extra(mtime)
+    #         Local-header version:
+    # 
+    #         Value         Size        Description
+    #         -----         ----        -----------
+    # (time)  0x5455        Short       tag for this extra block type ("UT")
+    #         TSize         Short       total data size for this block
+    #         Flags         Byte        info bits
+    #         (ModTime)     Long        time of last modification (UTC/GMT)
+    #         (AcTime)      Long        time of last access (UTC/GMT)
+    #         (CrTime)      Long        time of original creation (UTC/GMT)
+    # 
+    #         Central-header version:
+    # 
+    #         Value         Size        Description
+    #         -----         ----        -----------
+    # (time)  0x5455        Short       tag for this extra block type ("UT")
+    #         TSize         Short       total data size for this block
+    #         Flags         Byte        info bits (refers to local header!)
+    #         (ModTime)     Long        time of last modification (UTC/GMT)
+    #
+    # The lower three bits of Flags in both headers indicate which time-
+    #       stamps are present in the LOCAL extra field:
+    # 
+    #       bit 0           if set, modification time is present
+    #       bit 1           if set, access time is present
+    #       bit 2           if set, creation time is present
+    #       bits 3-7        reserved for additional timestamps; not set
+    flags = 0b10000000 # Set bit 1 only to indicate only mtime is present
+    data_and_packspecs = [
+      0x5455, C_v,  # tag for this extra block type ("UT")
+      (1 + 4), C_v, # the size of this block (1 byte used for the Flag + 1 long used for the timestamp)
+      flags, C_C,   # encode a single byte
+      mtime.utc.to_i, C_N, # Use a signed long, not the unsigned one used by the rest of the ZIP spec.
+    ]
+    pack_array(data_and_packspecs)
+  end
   
   # Writes the Zip64 extra field for the central directory header.It differs from the extra used in the local file header because it
   # also contains the location of the local file header in the ZIP as an 8-byte int.

data/testing/README_TESTING.md

@@ -0,0 +1,12 @@
+## The _actual_ testing for ZipTricks
+
+Consists of a fairly straightforward procedure.
+
+1. Run `generate_test_files.rb`. This will take some time and produce a number of ZIP files.
+2. Open them with the following ZIP unarchivers:
+  * A recent version of `zipinfo` with the `-tlhvz` flags - to see the information about the file
+  * ArchiveUtility on OSX
+  * The Unarchiver on OSX
+  * Built-in Explorer on Windows 7
+  * 7Zip 9.20 on Windows 7
+* Write down your observations in `test-report.txt` and, when cutting a release, timestamp a copy of that file.

data/testing/generate_test_files.rb

@@ -0,0 +1,86 @@
+require_relative 'support'
+
+build_test "Two small stored files" do |zip|
+  zip.add_stored_entry(filename: 'text.txt', size: $war_and_peace.bytesize, crc32: $war_and_peace_crc)
+  zip << $war_and_peace
+
+  zip.add_stored_entry(filename: 'image.jpg', size: $image_file.bytesize, crc32: $image_file_crc)
+  zip << $image_file
+end
+
+build_test "Filename with diacritics" do |zip|
+  zip.add_stored_entry(filename: 'Kungälv.txt', size: $war_and_peace.bytesize, crc32: $war_and_peace_crc)
+  zip << $war_and_peace
+end
+
+build_test "Purely UTF-8 filename" do |zip|
+  zip.add_stored_entry(filename: 'Война и мир.txt', size: $war_and_peace.bytesize, crc32: $war_and_peace_crc)
+  zip << $war_and_peace
+end
+
+# The trick of this test is that each file of the 2, on it's own, does _not_ exceed the
+# size threshold for Zip64. Together, however, they do.
+build_test "Two entries larger than the overall Zip64 offset" do |zip|
+  big = generate_big_entry((0xFFFFFFFF / 2) + 1024)
+  zip.add_stored_entry(filename: 'repeated-A.txt', size: big.size, crc32: big.crc32)
+  big.write_to(zip)
+
+  zip.add_stored_entry(filename: 'repeated-B.txt', size: big.size, crc32: big.crc32)
+  big.write_to(zip)
+end
+
+build_test "One entry that requires Zip64 and a tiny entry following it" do |zip|
+  big = generate_big_entry(0xFFFFFFFF + 2048)
+  zip.add_stored_entry(filename: 'large-requires-zip64.bin', size: big.size, crc32: big.crc32)
+  big.write_to(zip)
+
+  zip.add_stored_entry(filename: 'tiny-after.txt', size: $war_and_peace.bytesize, crc32: $war_and_peace_crc)
+  zip << $war_and_peace
+end
+
+build_test "One tiny entry followed by second that requires Zip64" do |zip|
+  zip.add_stored_entry(filename: 'tiny-at-start.txt', size: $war_and_peace.bytesize, crc32: $war_and_peace_crc)
+  zip << $war_and_peace
+
+  big = generate_big_entry(0xFFFFFFFF + 2048)
+  zip.add_stored_entry(filename: 'large-requires-zip64.bin', size: big.size, crc32: big.crc32)
+  big.write_to(zip)
+end
+
+build_test "Two entries both requiring Zip64" do |zip|
+  big = generate_big_entry(0xFFFFFFFF + 2048)
+  zip.add_stored_entry(filename: 'huge-file-1.bin', size: big.size, crc32: big.crc32)
+  big.write_to(zip)
+
+  zip.add_stored_entry(filename: 'huge-file-2.bin', size: big.size, crc32: big.crc32)
+  big.write_to(zip)
+end
+
+build_test "Two stored entries using data descriptors" do |zip|
+  zip.write_stored_file('stored.1.bin') do |sink|
+    sink << Random.new.bytes(1024 * 1024 * 4)
+  end
+  zip.write_stored_file('stored.2.bin') do |sink|
+    sink << Random.new.bytes(1024 * 1024 * 3)
+  end
+end
+
+build_test "One entry deflated using data descriptors" do |zip|
+  big = generate_big_entry(0xFFFFFFFF / 64)
+  zip.write_deflated_file('war-and-peace-repeated-compressed.txt') do |sink|
+    big.write_to(sink)
+  end
+end
+
+build_test "Two entries larger than the overall Zip64 offset using data descriptors" do |zip|
+  big = generate_big_entry((0xFFFFFFFF / 2) + 1024)
+  
+  zip.write_stored_file('repeated-A.txt') { |sink| big.write_to(sink) }
+  zip.write_stored_file('repeated-B.txt') { |sink| big.write_to(sink) }
+end
+
+build_test "One stored entry larger than Zip64 threshold using data descriptors" do |zip|
+  big = generate_big_entry(0xFFFFFFFF + 64000)
+  
+  zip.write_stored_file('repeated-A.txt') { |sink| big.write_to(sink) }
+end

data/testing/support.rb

@@ -0,0 +1,83 @@
+require 'rubygems'
+require 'bundler'
+Bundler.setup
+require_relative '../lib/zip_tricks'
+require 'terminal-table'
+
+$war_and_peace = File.open(__dir__ + '/in/war-and-peace.txt', 'rb'){|f| f.read }.freeze
+$war_and_peace_crc = Zlib.crc32($war_and_peace)
+
+$image_file     = File.open(__dir__ + '/in/VTYL8830.jpg', 'rb'){|f| f.read }.freeze
+$image_file_crc = Zlib.crc32($image_file)
+
+class BigEntry < Struct.new(:crc32, :size, :iterations)
+  def write_to(io)
+    iterations.times { io << $war_and_peace }
+  end
+end
+
+def generate_big_entry(desired_minimum_size)
+  repeats = (desired_minimum_size.to_f / $war_and_peace.bytesize).ceil
+  crc_stream = ZipTricks::StreamCRC32.new
+  repeats.times { crc_stream << $war_and_peace }
+  entry_size = $war_and_peace.bytesize * repeats
+  raise "Ooops" if entry_size < desired_minimum_size
+  BigEntry.new(crc_stream.to_i, entry_size, repeats)
+end
+
+TestDesc = Struct.new(:title, :filename)
+
+$tests_performed = 0
+$test_descs = []
+$builder_threads = []
+at_exit { $builder_threads.map(&:join) }
+
+def build_test(test_description)
+  $tests_performed += 1
+
+  test_file_base = test_description.downcase.gsub(/\-/, '').gsub(/[\s\:]+/, '_')
+  filename = '%02d-%s.zip' % [$tests_performed, test_file_base]
+
+  puts 'Test %02d: %s' % [$tests_performed, test_description]
+  puts filename
+  puts ""
+
+  $test_descs << TestDesc.new(test_description, filename)
+  $builder_threads << Thread.new do
+    File.open(File.join(__dir__, filename + '.tmp'), 'wb') do |of|
+      ZipTricks::Streamer.open(of) do |zip|
+        yield(zip)
+      end
+    end
+    File.rename(File.join(__dir__, filename + '.tmp'), File.join(__dir__, filename))
+  end
+end
+
+# For quickly disabling them by prepending "x" (like RSpec)
+def xbuild_test(*)
+end
+
+# Prints a text file that you can then fill in
+def prepare_test_protocol
+  File.open(__dir__ + '/test-report.txt', 'wb') do |f|
+    platforms = [
+      "OSX 10.11 - Archive Utility (builtin)",
+      "OSX - The Unarchiver 3.10",
+      "Windows7 x64 - Builtin Explorer ZIP opener",
+      "Windows7 x64 - 7Zip 9.20",
+    ]
+    platforms.each do |platform_name|
+      f.puts ''
+      table = Terminal::Table.new title: platform_name, headings: ['Test', 'Outcome']
+      $test_descs.each_with_index do |desc, i|
+        test_name = [desc.filename, '%s' % desc.title].join("\n")
+        outcome = ' ' * 64
+        table << [test_name, outcome]
+        table << :separator if i < ($test_descs.length - 1)
+      end
+      f.puts table
+    end
+  end
+end
+
+at_exit { prepare_test_protocol }