zip_tricks 5.0.0 → 5.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 327b846ccfc7d19442d21e145a2f767da91f0547381afe52be27db6709dee4a4
-  data.tar.gz: 4d80718a715063f1cebad32471ab83e77d38a805f3ceed82793b922d0d3a877f
+  metadata.gz: a9e1ad9dd7452add6a2f9187edecdd366699d81fba9260692ce2a96003401eca
+  data.tar.gz: 1f2665b7c3a5c99ce190b5dab76fa9334ac4c6a3307928bab94758acd98bcb56
 SHA512:
-  metadata.gz: 982bc310eb658d67262c28efb043f1ddd3957db197ef1bcbe9108f3572431211069843221d36cf9b28231617d899249ad06be5b9c756189e8f0d383b168a340f
-  data.tar.gz: 40fdde09881e02b5a816dae826554e14d76fb26a1bcf09cfcd053450668b34e2d3c63dc5bbf84417b6bdc44f021a35d38fdabd08d9ace5aabe1ff6c335b65e49
+  metadata.gz: 8864d6d56796e49e2024e46aee7cd1ac1c8ea575d757a6fbe6fd62a6e77db94eef6dc5b80bdbeae5bb932536e889fbdab224642862452f5b1acc0020e8a8c906
+  data.tar.gz: 67bc05db334c9a54804ac460f14f5919b9049913ee75532da090405f9a5b1f7e792ff4678aec0683a8b2a3136a9799ea3dd88aa3c9995491b28e4e6d88c4ac3a

data/.rubocop.yml

@@ -11,3 +11,5 @@ Style/GlobalVars:
       - qa/*.rb
       - spec/spec_helper.rb
       - spec/support/zip_inspection.rb
+Layout/IndentHeredoc:
+  Enabled: false

data/.travis.yml

@@ -1,10 +1,6 @@
 rvm:
-- 2.1
-- 2.2.10
-- 2.3.7
-- 2.4.4
-- 2.5.4
-- 2.6.2
+- 2.1.0
+- 2.6.5
 - jruby-9.0
 sudo: false
 cache: bundler

data/CHANGELOG.md

@@ -1,3 +1,32 @@
+## 5.3.1
+
+* Fix extended timestamp timestamp value encoding. Previously we would use an incorrect encoding for the timestamp value, which would output correct but nonsensical timestamps. The pack specifier is now changed to output the correct value.
+
+## 5.3.0
+
+* Raise in `Streamer#close` when the IO offset of the Streamer does not match the size of the written entries. This is a situation which
+  can occur if one adds the local headers, writes the bodies of the files to the socket/output directly, and forgets to adjust the internal
+  Streamer offset. The unadjusted offset would then produce incorrect values in both the local headers which come after the missing
+  offset adjustment _and_ in the central directory headers. Some ZIP unarchivers are able to recover from this (ones that read
+  files "straight-ahead" but others aren't - if the ZIP unarchiver uses central directory entries it would be using incorrect offsets.
+  Instead of producing an invalid ZIP, raise an exception which explains what happened and how it can be resolved.
+
+## 5.2.0
+
+* Remove `Streamer#add_compressed_entry` and `SizeEstimator#add_compressed_entry`
+
+## 5.1.1
+
+* Fix extended timestamp extra field output. The first bit of the flag would be set instead of the last bit of
+  the flag, which made it impossible for Rubyzip to read the timestamp of the entry - and it would also make
+  the extra field useless for most reading applications.
+
+## 5.1.0
+
+* Slightly rework `RemoteIO` and `RemoteUncap` and make sure they work correctly by spinning up a test webserver
+  to verify their operation. The changes to the documented API are fairly small so this is still marked as a minor
+  release.
+
 ## 5.0.0
 
 * Disable automatic filename deduplication by default, because it does not play nice with file/directory
@@ -19,16 +48,16 @@
 
 ## 4.7.4
 
-* Use a single fixed capacity string in StreamCRC32.from_io to avoid unnecessary allocations
+* Use a single fixed capacity string in `StreamCRC32.from_io` to avoid unnecessary allocations
 * Fix a few tests that were calling out to external binaries
 
 ## 4.7.3
 
-* Fix RemoteUncap#request_object_size to function correctly
+* Fix `RemoteUncap#request_object_size` to function correctly
 
 ## 4.7.2
 
-* Relax bundler dependency so that both 1.x and 2.x are supported cleanly
+* Relax bundler dependency so that both bundler 1.x and 2.x are supported cleanly
 
 ## 4.7.1
 

data/lib/zip_tricks/file_reader.rb

@@ -82,7 +82,9 @@ class ZipTricks::FileReader
 
   private_constant :StoredReader, :InflatingReader
 
-  # Represents a file within the ZIP archive being read
+  # Represents a file within the ZIP archive being read. This is different from
+  # the Entry object used in Streamer for ZIP writing, since during writing more
+  # data can be kept in memory for immediate use.
   class ZipEntry
     # @return [Fixnum] bit-packed version signature of the program that made the archive
     attr_accessor :made_by

data/lib/zip_tricks/remote_io.rb

@@ -8,14 +8,16 @@
 # actual fetches over HTTP and an object that expects a handful of IO methods to be
 # available.
 class ZipTricks::RemoteIO
-  # @param fetcher[#request_object_size, #request_range] an object that perform fetches
-  def initialize(fetcher = :NOT_SET)
+  # @param url[String, URI] the HTTP/HTTPS URL of the object to be retrieved
+  def initialize(url)
     @pos = 0
-    @fetcher = fetcher
-    @remote_size = false
+    @uri = URI(url)
+    @remote_size = nil
   end
 
   # Emulates IO#seek
+  # @param offset[Integer] absolute offset in the remote resource to seek to
+  # @param mode[Integer] The seek mode (only SEEK_SET is supported)
   def seek(offset, mode = IO::SEEK_SET)
     raise "Unsupported read mode #{mode}" unless mode == IO::SEEK_SET
     @remote_size ||= request_object_size
@@ -25,7 +27,7 @@ class ZipTricks::RemoteIO
 
   # Emulates IO#size.
   #
-  # @return [Fixnum] the size of the remote resource
+  # @return [Integer] the size of the remote resource
   def size
     @remote_size ||= request_object_size
   end
@@ -39,21 +41,20 @@ class ZipTricks::RemoteIO
   # @param n_bytes[Fixnum, nil] how many bytes to read, or `nil` to read all the way to the end
   # @return [String] the read bytes
   def read(n_bytes = nil)
-    @remote_size ||= request_object_size
-
     # If the resource is empty there is nothing to read
-    return if @remote_size.zero?
+    return if size.zero?
 
-    maximum_avaialable = @remote_size - @pos
+    maximum_avaialable = size - @pos
     n_bytes ||= maximum_avaialable # nil == read to the end of file
     return '' if n_bytes.zero?
     raise ArgumentError, "No negative reads(#{n_bytes})" if n_bytes < 0
 
     n_bytes = clamp(0, n_bytes, maximum_avaialable)
 
-    read_n_bytes_from_remote(@pos, n_bytes).tap do |data|
+    http_range = (@pos..(@pos + n_bytes - 1))
+    request_range(http_range).tap do |data|
       raise "Remote read returned #{data.bytesize} bytes instead of #{n_bytes} as requested" if data.bytesize != n_bytes
-      @pos = clamp(0, @pos + data.bytesize, @remote_size)
+      @pos = clamp(0, @pos + data.bytesize, size)
     end
   end
 
@@ -66,23 +67,41 @@ class ZipTricks::RemoteIO
 
   protected
 
+  # Only used internally when reading the remote ZIP.
+  #
+  # @param range[Range] the HTTP range of data to fetch from remote
+  # @return [String] the response body of the ranged request
   def request_range(range)
-    @fetcher.request_range(range)
+    http = Net::HTTP.start(@uri.hostname, @uri.port)
+    request = Net::HTTP::Get.new(@uri)
+    request.range = range
+    response = http.request(request)
+    case response.code
+    when "206", "200"
+      response.body
+    else
+      raise "Remote at #{@uri} replied with code #{response.code}"
+    end
   end
 
+  # For working with S3 it is a better idea to perform a GET request for one byte, since doing a HEAD
+  # request needs a different permission - and standard GET presigned URLs are not allowed to perform it
+  #
+  # @return [Integer] the size of the remote resource, parsed either from Content-Length or Content-Range header
   def request_object_size
-    @fetcher.request_object_size
-  end
-
-  # Reads N bytes at offset from remote
-  def read_n_bytes_from_remote(start_at, n_bytes)
-    range = (start_at..(start_at + n_bytes - 1))
-    request_range(range)
-  end
-
-  # Reads the Content-Length and caches it
-  def remote_size
-    @remote_size ||= request_object_size
+    http = Net::HTTP.start(@uri.hostname, @uri.port)
+    request = Net::HTTP::Get.new(@uri)
+    request.range = 0..0
+    response = http.request(request)
+    case response.code
+    when "206"
+      content_range_header_value = response['Content-Range']
+      content_range_header_value.split('/').last.to_i
+    when "200"
+      response['Content-Length'].to_i
+    else
+      raise "Remote at #{@uri} replied with code #{response.code}"
+    end
   end
 
   private

data/lib/zip_tricks/remote_uncap.rb

@@ -7,42 +7,16 @@
 #
 # Please read the security warning in `FileReader` _VERY CAREFULLY_
 # before you use this module.
-class ZipTricks::RemoteUncap
+module ZipTricks::RemoteUncap
   # @param uri[String] the HTTP(S) URL to read the ZIP footer from
   # @param reader_class[Class] which class to use for reading
   # @param options_for_zip_reader[Hash] any additional options to give to
   # {ZipTricks::FileReader} when reading
   # @return [Array<ZipTricks::FileReader::ZipEntry>] metadata about the
   # files within the remote archive
-  def self.files_within_zip_at(uri,
-                               reader_class: ZipTricks::FileReader,
-                               **options_for_zip_reader)
-    fetcher = new(uri)
-    fake_io = ZipTricks::RemoteIO.new(fetcher)
+  def self.files_within_zip_at(uri, reader_class: ZipTricks::FileReader, **options_for_zip_reader)
+    fake_io = ZipTricks::RemoteIO.new(uri)
     reader = reader_class.new
     reader.read_zip_structure(io: fake_io, **options_for_zip_reader)
   end
-
-  def initialize(uri)
-    @uri = URI(uri)
-  end
-
-  # Only used internally when reading the remote ZIP.
-  #
-  # @param range[Range] the HTTP range of data to fetch from remote
-  # @return [String] the response body of the ranged request
-  def request_range(range)
-    request = Net::HTTP::Get.new(@uri)
-    request.range = range
-    http = Net::HTTP.start(@uri.hostname, @uri.port)
-    http.request(request).body
-  end
-
-  # Only used internally when reading the remote ZIP.
-  #
-  # @return [Fixnum] the byte size of the ranged request
-  def request_object_size
-    http = Net::HTTP.start(@uri.hostname, @uri.port)
-    http.request_head(@uri)['Content-Length'].to_i
-  end
 end

data/lib/zip_tricks/size_estimator.rb

@@ -73,9 +73,6 @@ class ZipTricks::SizeEstimator
     self
   end
 
-  # Will be phased out in ZipTricks 5.x
-  alias_method :add_compressed_entry, :add_deflated_entry
-
   # Add an empty directory to the archive.
   #
   # @param dirname [String] the name of the directory

data/lib/zip_tricks/streamer.rb

@@ -91,6 +91,7 @@ class ZipTricks::Streamer
   InvalidOutput = Class.new(ArgumentError)
   Overflow = Class.new(StandardError)
   UnknownMode = Class.new(StandardError)
+  OffsetOutOfSync = Class.new(StandardError)
 
   private_constant :DeflatedWriter, :StoredWriter, :STORED, :DEFLATED
 
@@ -149,7 +150,6 @@ class ZipTricks::Streamer
     @dedupe_filenames = auto_rename_duplicate_filenames
     @out = ZipTricks::WriteAndTell.new(stream)
     @files = []
-    @local_header_offsets = []
     @path_set = ZipTricks::PathSet.new
     @writer = writer
   end
@@ -213,9 +213,6 @@ class ZipTricks::Streamer
     @out.tell
   end
 
-  # Will be phased out in ZipTricks 5.x
-  alias_method :add_compressed_entry, :add_deflated_entry
-
   # Writes out the local header for an entry (file in the ZIP) that is using
   # the stored storage model (is stored as-is).
   # Once this method is called, the `<<` method has to be called one or more
@@ -363,14 +360,16 @@ class ZipTricks::Streamer
   #
   # @return [Integer] the offset the output IO is at after closing the archive
   def close
+    # Make sure offsets are in order
+    verify_offsets!
+
     # Record the central directory offset, so that it can be written into the EOCD record
     cdir_starts_at = @out.tell
 
     # Write out the central directory entries, one for each file
-    @files.each_with_index do |entry, i|
-      header_loc = @local_header_offsets.fetch(i)
+    @files.each do |entry|
       @writer.write_central_directory_file_header(io: @out,
-                                                  local_file_header_location: header_loc,
+                                                  local_file_header_location: entry.local_header_offset,
                                                   gp_flags: entry.gp_flags,
                                                   storage_mode: entry.storage_mode,
                                                   compressed_size: entry.compressed_size,
@@ -423,15 +422,40 @@ class ZipTricks::Streamer
     last_entry.compressed_size = compressed_size
     last_entry.uncompressed_size = uncompressed_size
 
+    offset_before_data_descriptor = @out.tell
     @writer.write_data_descriptor(io: @out,
                                   crc32: last_entry.crc32,
                                   compressed_size: last_entry.compressed_size,
                                   uncompressed_size: last_entry.uncompressed_size)
+    last_entry.bytes_used_for_data_descriptor = @out.tell - offset_before_data_descriptor
+
     @out.tell
   end
 
   private
 
+  def verify_offsets!
+    # We need to check whether the offsets noted for the entries actually make sense
+    computed_offset = @files.map(&:total_bytes_used).inject(0, &:+)
+    actual_offset = @out.tell
+    if computed_offset != actual_offset
+      message = <<-EMS
+The offset of the Streamer output IO is out of sync with the expected value. All entries written so far,
+including their compressed bodies, local headers and data descriptors, add up to a certain offset,
+but this offset does not match the actual offset of the IO.
+
+Entries add up to #{computed_offset} bytes and the IO is at #{actual_offset} bytes.
+
+This can happen if you write local headers for an entry, write the "body" of the entry directly to the IO
+object which is your destination, but do not adjust the offset known to the Streamer object. To adjust
+the offfset you need to call `Streamer#simulate_write(body_size)` after outputting the entry. Otherwise
+the local header offsets of the entries you write are going to be incorrect and some ZIP applications
+are going to have problems opening your archive.
+EMS
+      raise OffsetOutOfSync, message
+    end
+  end
+
   def add_file_and_write_local_header(
       filename:,
       modification_time:,
@@ -464,16 +488,18 @@ class ZipTricks::Streamer
       uncompressed_size = 0
     end
 
+    local_header_starts_at = @out.tell
+
     e = Entry.new(filename,
                   crc32,
                   compressed_size,
                   uncompressed_size,
                   storage_mode,
                   modification_time,
-                  use_data_descriptor)
-
-    @files << e
-    @local_header_offsets << @out.tell
+                  use_data_descriptor,
+                  _local_file_header_offset = local_header_starts_at,
+                  _bytes_used_for_local_header = 0,
+                  _bytes_used_for_data_descriptor = 0)
 
     @writer.write_local_file_header(io: @out,
                                     gp_flags: e.gp_flags,
@@ -483,6 +509,9 @@ class ZipTricks::Streamer
                                     mtime: e.mtime,
                                     filename: e.filename,
                                     storage_mode: e.storage_mode)
+    e.bytes_used_for_local_header = @out.tell - e.local_header_offset
+
+    @files << e
   end
 
   def remove_backslash(filename)

data/lib/zip_tricks/streamer/entry.rb

@@ -4,7 +4,7 @@
 # Normally you will not have to use this class directly
 class ZipTricks::Streamer::Entry < Struct.new(:filename, :crc32, :compressed_size,
                                               :uncompressed_size, :storage_mode, :mtime,
-                                              :use_data_descriptor)
+                                              :use_data_descriptor, :local_header_offset, :bytes_used_for_local_header, :bytes_used_for_data_descriptor)
   def initialize(*)
     super
     filename.force_encoding(Encoding::UTF_8)
@@ -15,6 +15,10 @@ class ZipTricks::Streamer::Entry < Struct.new(:filename, :crc32, :compressed_siz
                            end)
   end
 
+  def total_bytes_used
+    bytes_used_for_local_header + compressed_size + bytes_used_for_data_descriptor
+  end
+
   # Set the general purpose flags for the entry. We care about is the EFS
   # bit (bit 11) which should be set if the filename is UTF8. If it is, we need to set the
   # bit so that the unarchiving application knows that the filename in the archive is UTF-8

data/lib/zip_tricks/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ZipTricks
-  VERSION = '5.0.0'
+  VERSION = '5.3.1'
 end

data/lib/zip_tricks/zip_writer.rb

@@ -57,7 +57,7 @@ class ZipTricks::ZipWriter
   C_UINT2 = 'v'    # Encode a 2-byte unsigned little-endian uint
   C_UINT8 = 'Q<'  # Encode an 8-byte unsigned little-endian uint
   C_CHAR = 'C' # For bit-encoded strings
-  C_INT4 = 'N' # Encode a 4-byte signed little-endian int
+  C_INT4 = 'l<' # Encode a 4-byte signed little-endian int
 
   private_constant :FOUR_BYTE_MAX_UINT,
                    :TWO_BYTE_MAX_UINT,
@@ -118,7 +118,7 @@ class ZipTricks::ZipWriter
     if requires_zip64
       extra_fields << zip_64_extra_for_local_file_header(compressed_size: compressed_size, uncompressed_size: uncompressed_size)
     end
-    extra_fields << timestamp_extra(mtime)
+    extra_fields << timestamp_extra_for_local_file_header(mtime)
 
     io << [extra_fields.size].pack(C_UINT2)                # extra field length              2 bytes
 
@@ -182,7 +182,7 @@ class ZipTricks::ZipWriter
                                                                      compressed_size: compressed_size,
                                                                      uncompressed_size: uncompressed_size)
     end
-    extra_fields << timestamp_extra(mtime)
+    extra_fields << timestamp_extra_for_central_directory_entry(mtime)
 
     io << [extra_fields.size].pack(C_UINT2)                 # extra field length              2 bytes
 
@@ -345,12 +345,14 @@ class ZipTricks::ZipWriter
     pack_array(data_and_packspecs)
   end
 
-  # Writes the extended timestamp information field. The spec defines 2
+  # Writes the extended timestamp information field for local headers.
+  #
+  # The spec defines 2
   # different formats - the one for the local file header can also accomodate the
   # atime and ctime, whereas the one for the central directory can only take
   # the mtime - and refers the reader to the local header extra to obtain the
   # remaining times
-  def timestamp_extra(mtime)
+  def timestamp_extra_for_local_file_header(mtime)
     #         Local-header version:
     #
     #         Value         Size        Description
@@ -378,16 +380,21 @@ class ZipTricks::ZipWriter
     #       bit 1           if set, access time is present
     #       bit 2           if set, creation time is present
     #       bits 3-7        reserved for additional timestamps; not set
-    flags = 0b10000000 # Set bit 1 only to indicate only mtime is present
+    flags = 0b00000001 # Set the lowest bit only, to indicate that only mtime is present
     data_and_packspecs = [
       0x5455, C_UINT2,  # tag for this extra block type ("UT")
-      (1 + 4), C_UINT2, # the size of this block (1 byte used for the Flag + 1 long used for the timestamp)
+      (1 + 4), C_UINT2, # the size of this block (1 byte used for the Flag + 3 longs used for the timestamp)
       flags, C_CHAR,   # encode a single byte
-      mtime.utc.to_i, C_INT4, # Use a signed long, not the unsigned one used by the rest of the ZIP spec.
+      mtime.utc.to_i, C_INT4, # Use a signed int, not the unsigned one used by the rest of the ZIP spec.
     ]
+    # The atime and ctime can be omitted if not present
     pack_array(data_and_packspecs)
   end
 
+  # Since we do not supply atime or ctime, the contents of the two extra fields (central dir and local header)
+  # is exactly the same, so we can use a method alias.
+  alias_method :timestamp_extra_for_central_directory_entry, :timestamp_extra_for_local_file_header
+
   # Writes the Zip64 extra field for the central directory header.It differs from the extra used in the local file header because it
   # also contains the location of the local file header in the ZIP as an 8-byte int.
   #

data/zip_tricks.gemspec

@@ -31,7 +31,7 @@ Gem::Specification.new do |spec|
   spec.require_paths  = ['lib']
 
   spec.add_development_dependency 'bundler'
-  spec.add_development_dependency 'rubyzip', '>= 1.2.2'
+  spec.add_development_dependency 'rubyzip', '~> 1'
   spec.add_development_dependency 'terminal-table'
   spec.add_development_dependency 'range_utils'
 
@@ -44,4 +44,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'allocation_stats', '~> 0.1.5'
   spec.add_development_dependency 'yard', '~> 0.9'
   spec.add_development_dependency 'wetransfer_style', '0.6.0'
+  spec.add_development_dependency 'puma'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: zip_tricks
 version: !ruby/object:Gem::Version
-  version: 5.0.0
+  version: 5.3.1
 platform: ruby
 authors:
 - Julik Tarkhanov
@@ -11,7 +11,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-11-01 00:00:00.000000000 Z
+date: 2020-06-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -31,16 +31,16 @@ dependencies:
   name: rubyzip
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2.2
+        version: '1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2.2
+        version: '1'
 - !ruby/object:Gem::Dependency
   name: terminal-table
   requirement: !ruby/object:Gem::Requirement
@@ -195,6 +195,20 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 0.6.0
+- !ruby/object:Gem::Dependency
+  name: puma
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Stream out ZIP files from Ruby
 email:
 - me@julik.nl
@@ -278,7 +292,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.6
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Stream out ZIP files from Ruby