zip_tricks 2.8.1 → 3.0.0

data/lib/zip_tricks/stream_crc32.rb

@@ -26,7 +26,7 @@ class ZipTricks::StreamCRC32
 
   # Returns the CRC32 value computed so far
   #
-  # @return crc[Fixnum] the updated CRC32 value for all the blobs so far
+  # @return [Fixnum] the updated CRC32 value for all the blobs so far
   def to_i
     @crc
   end
@@ -36,7 +36,7 @@ class ZipTricks::StreamCRC32
   #
   # @param crc32[Fixnum] the CRC32 value to append
   # @param blob_size[Fixnum] the size of the daata the `crc32` is computed from
-  # @return crc[Fixnum] the updated CRC32 value for all the blobs so far
+  # @return [Fixnum] the updated CRC32 value for all the blobs so far
   def append(crc32, blob_size)
     @crc = Zlib.crc32_combine(@crc, crc32, blob_size)
   end

data/lib/zip_tricks/streamer/deflated_writer.rb

@@ -0,0 +1,26 @@
+class ZipTricks::Streamer::DeflatedWriter
+  def initialize(io)
+    @io = io
+    @uncompressed_size = 0
+    @started_at = @io.tell
+    @crc = ZipTricks::StreamCRC32.new
+    @bytes_since_last_flush = 0
+  end
+
+  def finish
+    ZipTricks::BlockDeflate.write_terminator(@io)
+    [@crc.to_i, @io.tell - @started_at, @uncompressed_size]
+  end
+  
+  def <<(data)
+    @uncompressed_size += data.bytesize
+    @io << ZipTricks::BlockDeflate.deflate_chunk(data)
+    @crc << data
+    self
+  end
+
+  def write(data)
+    self << data
+    data.bytesize
+  end
+end

data/lib/zip_tricks/streamer/entry.rb

@@ -0,0 +1,21 @@
+# Is used internally by Streamer to keep track of entries in the archive during writing.
+# Normally you will not have to use this class directly
+class ZipTricks::Streamer::Entry < Struct.new(:filename, :crc32, :compressed_size, :uncompressed_size, :storage_mode, :mtime, :use_data_descriptor)
+  def initialize(*)
+    super
+    filename.force_encoding(Encoding::UTF_8)
+    @requires_efs_flag = !(filename.encode(Encoding::ASCII) rescue false)
+  end
+
+  # Set the general purpose flags for the entry. We care about is the EFS
+  # bit (bit 11) which should be set if the filename is UTF8. If it is, we need to set the
+  # bit so that the unarchiving application knows that the filename in the archive is UTF-8
+  # encoded, and not some DOS default. For ASCII entries it does not matter.
+  # Additionally, we care about bit 3 which toggles the use of the postfix data descriptor.
+  def gp_flags
+    flag = 0b00000000000
+    flag |= 0b100000000000 if @requires_efs_flag # bit 11
+    flag |= 0x0008 if use_data_descriptor        # bit 3
+    flag
+  end
+end

data/lib/zip_tricks/streamer/stored_writer.rb

@@ -0,0 +1,25 @@
+class ZipTricks::Streamer::StoredWriter
+  def initialize(io)
+    @io = io
+    @uncompressed_size = 0
+    @compressed_size = 0
+    @started_at = @io.tell
+    @crc = ZipTricks::StreamCRC32.new
+  end
+
+  def <<(data)
+    @io << data
+    @crc << data
+    self
+  end
+
+  def write(data)
+    self << data
+    data.bytesize
+  end
+
+  def finish
+    size = @io.tell - @started_at
+    [@crc.to_i, size, size]
+  end
+end

data/lib/zip_tricks/streamer/writable.rb

@@ -0,0 +1,20 @@
+# Gets yielded from the writing methods of the CompressingStreamer
+# and accepts the data being written into the ZIP
+class ZipTricks::Streamer::Writable
+  # Initializes a new Writable with the object it delegates the writes to.
+  # Normally you would not need to use this method directly 
+  def initialize(writer)
+    @writer = writer
+  end
+  # Writes the given data to the output stream
+  #
+  # @param d[String] the binary string to write (part of the uncompressed file)
+  # @return [self]
+  def <<(d); @writer << d; self; end
+  
+  # Writes the given data to the output stream
+  #
+  # @param d[String] the binary string to write (part of the uncompressed file)
+  # @return [Fixnum] the number of bytes written
+  def write(d); @writer << d; end
+end

data/lib/zip_tricks/streamer.rb

@@ -8,17 +8,61 @@
 # For stored entries, you need to know the CRC32 (as a uint) and the filesize upfront,
 # before the writing of the entry body starts.
 #
-# For compressed entries, you need to know the bytesize of the precompressed entry
-# as well.
+# Any object that responds to `<<` can be used as the Streamer target - you can use
+# a String, an Array, a Socket or a File, at your leisure.
+#
+# ## Using the Streamer with runtime compression
+#
+# You can use the Streamer with data descriptors (the CRC32 and the sizes will be
+# written after the file data). This allows non-rewinding on-the-fly compression.
+# If you are compressing large files, the Deflater object that the Streamer controls
+# will be regularly flushed to prevent memory inflation. 
+#
+#     ZipTricks::Streamer.open(file_socket_or_string) do |zip|
+#       zip.write_stored_file('mov.mp4') do |sink|
+#         File.open('mov.mp4', 'rb'){|source| IO.copy_stream(source, sink) }
+#       end
+#       zip.write_deflated_file('long-novel.txt') do |sink|
+#         File.open('novel.txt', 'rb'){|source| IO.copy_stream(source, sink) }
+#       end
+#     end
+#
+# The central directory will be written automatically at the end of the block.
+#
+# ## Using the Streamer with entries of known size and having a known CRC32 checksum
+#
+# Streamer allows "IO splicing" - in this mode it will only control the metadata output,
+# but you can write the data to the socket/file outside of the Streamer. For example, when
+# using the sendfile gem:
+#
+#     ZipTricks::Streamer.open(socket) do | zip |
+#       zip.add_stored_entry(filename: "myfile1.bin", size: 9090821, crc32: 12485)
+#       zip.simulate_write(tempfile1.size)
+#       socket.sendfile(tempfile1)
+#       zip.add_stored_entry(filename: "myfile2.bin", size: 458678, crc32: 89568)
+#       zip.simulate_write(tempfile2.size)
+#       socket.sendfile(tempfile2)
+#     end
+#
+# Note that you need to use `simulate_write` to let the 
+# The central directory will be written automatically at the end of the block.
 class ZipTricks::Streamer
+  require_relative 'streamer/deflated_writer'
+  require_relative 'streamer/writable'
+  require_relative 'streamer/stored_writer'
+  require_relative 'streamer/entry'
+
+  STORED = 0
+  DEFLATED = 8
+
   EntryBodySizeMismatch = Class.new(StandardError)
   InvalidOutput = Class.new(ArgumentError)
+  Overflow = Class.new(StandardError)
+  PathError = Class.new(StandardError)
+  DuplicateFilenames = Class.new(StandardError)
+  UnknownMode = Class.new(StandardError)
 
-  # Language encoding flag (EFS) bit (general purpose bit 11)
-  EFS = 0b100000000000
-
-  # Default general purpose flags for each entry.
-  DEFAULT_GP_FLAGS = 0b00000000000
+  private_constant :DeflatedWriter, :StoredWriter, :STORED, :DEFLATED
 
   # Creates a new Streamer on top of the given IO-ish object and yields it. Once the given block
   # returns, the Streamer will have it's `close` method called, which will write out the central
@@ -34,21 +78,17 @@ class ZipTricks::Streamer
 
   # Creates a new Streamer on top of the given IO-ish object.
   #
-  # @param stream [IO] the destination IO for the ZIP (should respond to `tell` and `<<`)
+  # @param stream [IO] the destination IO for the ZIP (should respond to `<<`)
   def initialize(stream)
-    raise InvalidOutput, "The stream should respond to #<<" unless stream.respond_to?(:<<)
-    stream = ZipTricks::WriteAndTell.new(stream) unless stream.respond_to?(:tell) && stream.respond_to?(:advance_position_by)
-    
-    @output_stream = stream
-    @zip = ZipTricks::Microzip.new
+    raise InvalidOutput, "The stream must respond to #<<" unless stream.respond_to?(:<<)
+    unless stream.respond_to?(:tell) && stream.respond_to?(:advance_position_by)
+      stream = ZipTricks::WriteAndTell.new(stream) 
+    end
     
-    @state_monitor = VeryTinyStateMachine.new(:before_entry, callbacks_to=self)
-    @state_monitor.permit_state :in_entry_header, :in_entry_body, :in_central_directory, :closed
-    @state_monitor.permit_transition :before_entry => :in_entry_header
-    @state_monitor.permit_transition :in_entry_header => :in_entry_body
-    @state_monitor.permit_transition :in_entry_body => :in_entry_header
-    @state_monitor.permit_transition :in_entry_body => :in_central_directory
-    @state_monitor.permit_transition :in_central_directory => :closed
+    @out = stream
+    @files = []
+    @local_header_offsets = []
+    @writer = create_writer
   end
 
   # Writes a part of a zip entry body (actual binary data of the entry) into the output stream.
@@ -56,9 +96,7 @@ class ZipTricks::Streamer
   # @param binary_data [String] a String in binary encoding
   # @return self
   def <<(binary_data)
-    @state_monitor.transition_or_maintain! :in_entry_body
-    @output_stream << binary_data
-    @bytes_written_for_entry += binary_data.bytesize
+    @out << binary_data
     self
   end
 
@@ -69,7 +107,7 @@ class ZipTricks::Streamer
   # @param binary_data [String] a String in binary encoding
   # @return [Fixnum] the number of bytes written
   def write(binary_data)
-    self << binary_data
+    @out << binary_data
     binary_data.bytesize
   end
 
@@ -80,10 +118,8 @@ class ZipTricks::Streamer
   # @param num_bytes [Numeric] how many bytes are going to be written bypassing the Streamer
   # @return [Numeric] position in the output stream / ZIP archive
   def simulate_write(num_bytes)
-    @state_monitor.transition_or_maintain! :in_entry_body
-    @output_stream.advance_position_by(num_bytes)
-    @bytes_written_for_entry += num_bytes
-    @output_stream.tell
+    @out.advance_position_by(num_bytes)
+    @out.tell
   end
 
   # Writes out the local header for an entry (file in the ZIP) that is using the deflated storage model (is compressed).
@@ -92,67 +128,137 @@ class ZipTricks::Streamer
   # Note that the deflated body that is going to be written into the output has to be _precompressed_ (pre-deflated)
   # before writing it into the Streamer, because otherwise it is impossible to know it's size upfront.
   #
-  # @param entry_name [String] the name of the file in the entry
+  # @param filename [String] the name of the file in the entry
+  # @param compressed_size [Fixnum] the size of the compressed entry that is going to be written into the archive
   # @param uncompressed_size [Fixnum] the size of the entry when uncompressed, in bytes
   # @param crc32 [Fixnum] the CRC32 checksum of the entry when uncompressed
-  # @param compressed_size [Fixnum] the size of the compressed entry that is going to be written into the archive
   # @return [Fixnum] the offset the output IO is at after writing the entry header
-  def add_compressed_entry(entry_name, uncompressed_size, crc32, compressed_size)
-    @state_monitor.transition! :in_entry_header
-    @zip.add_local_file_header(io: @output_stream, filename: entry_name, crc32: crc32,
-      compressed_size: compressed_size, uncompressed_size: uncompressed_size, storage_mode: ZipTricks::Microzip::DEFLATED)
-    @expected_bytes_for_entry = compressed_size
-    @bytes_written_for_entry = 0
-    @output_stream.tell
+  def add_compressed_entry(filename:, compressed_size:, uncompressed_size:, crc32:)
+    add_file_and_write_local_header(filename: filename, crc32: crc32, storage_mode: DEFLATED, 
+      compressed_size: compressed_size, uncompressed_size: uncompressed_size)
+    @out.tell
   end
 
   # Writes out the local header for an entry (file in the ZIP) that is using the stored storage model (is stored as-is).
   # Once this method is called, the `<<` method has to be called one or more times to write the actual contents of the body.
   #
-  # @param entry_name [String] the name of the file in the entry
-  # @param uncompressed_size [Fixnum] the size of the entry when uncompressed, in bytes
+  # @param filename [String] the name of the file in the entry
+  # @param size [Fixnum] the size of the file when uncompressed, in bytes
   # @param crc32 [Fixnum] the CRC32 checksum of the entry when uncompressed
   # @return [Fixnum] the offset the output IO is at after writing the entry header
-  def add_stored_entry(entry_name, uncompressed_size, crc32)
-    @state_monitor.transition! :in_entry_header
-    @zip.add_local_file_header(io: @output_stream, filename: entry_name, crc32: crc32,
-      compressed_size: uncompressed_size, uncompressed_size: uncompressed_size, storage_mode: ZipTricks::Microzip::STORED)
-    @bytes_written_for_entry = 0
-    @expected_bytes_for_entry = uncompressed_size
-    @output_stream.tell
+  def add_stored_entry(filename:, size:, crc32:)
+    add_file_and_write_local_header(filename: filename, crc32: crc32, storage_mode: STORED,
+      compressed_size: size, uncompressed_size: size)
+    @out.tell
   end
 
-  # Writes out the global footer and the directory entry header and the global directory of the ZIP
-  # archive using the information about the entries added using `add_stored_entry` and `add_compressed_entry`.
+  # Opens the stream for a stored file in the archive, and yields a writer for that file to the block.
+  # Once the write completes, a data descriptor will be written with the actual compressed/uncompressed
+  # sizes and the CRC32 checksum.
   #
-  # Once this method is called, the `Streamer` should be discarded (the ZIP archive is complete).
+  # @param filename[String] the name of the file in the archive
+  # @yield [#<<, #write] an object that the file contents must be written to
+  def write_stored_file(filename)
+    add_file_and_write_local_header(filename: filename, storage_mode: STORED,
+      use_data_descriptor: true, crc32: 0, compressed_size: 0, uncompressed_size: 0)
+
+    w = StoredWriter.new(@out)
+    yield(Writable.new(w))
+    crc, comp, uncomp = w.finish
+
+    # Save the information into the entry for when the time comes to write out the central directory
+    last_entry = @files[-1]
+    last_entry.crc32 = crc
+    last_entry.compressed_size = comp
+    last_entry.uncompressed_size = uncomp
+
+    @writer.write_data_descriptor(io: @out, crc32: crc, compressed_size: comp, uncompressed_size: uncomp)
+  end
+
+  # Opens the stream for a deflated file in the archive, and yields a writer for that file to the block.
+  # Once the write completes, a data descriptor will be written with the actual compressed/uncompressed
+  # sizes and the CRC32 checksum.
   #
-  # @return [Fixnum] the offset the output IO is at after writing the central directory
-  def write_central_directory!
-    @state_monitor.transition! :in_central_directory
-    @zip.write_central_directory(@output_stream)
-    @output_stream.tell
+  # @param filename[String] the name of the file in the archive
+  # @yield [#<<, #write] an object that the file contents must be written to
+  def write_deflated_file(filename)
+    add_file_and_write_local_header(filename: filename, storage_mode: DEFLATED,
+      use_data_descriptor: true, crc32: 0, compressed_size: 0, uncompressed_size: 0)
+
+    w = DeflatedWriter.new(@out)
+    yield(Writable.new(w))
+    crc, comp, uncomp = w.finish
+
+    # Save the information into the entry for when the time comes to write out the central directory
+    last_entry = @files[-1]
+    last_entry.crc32 = crc
+    last_entry.compressed_size = comp
+    last_entry.uncompressed_size = uncomp
+    write_data_descriptor_for_last_entry
   end
 
-  # Closes the archive. Writes the central directory if it has not yet been written.
-  # Switches the Streamer into a state where it can no longer be written to.
+  # Closes the archive. Writes the central directory, and switches the writer into
+  # a state where it can no longer be written to.
   #
   # Once this method is called, the `Streamer` should be discarded (the ZIP archive is complete).
   #
   # @return [Fixnum] the offset the output IO is at after closing the archive
   def close
-    write_central_directory! unless @state_monitor.in_state?(:in_central_directory)
-    @state_monitor.transition! :closed
-    @output_stream.tell
-  end
+    # Record the central directory offset, so that it can be written into the EOCD record
+    cdir_starts_at = @out.tell
+    
+    # Write out the central directory entries, one for each file
+    @files.each_with_index do |entry, i|
+      header_loc = @local_header_offsets.fetch(i)
+      @writer.write_central_directory_file_header(io: @out, local_file_header_location: header_loc,
+        gp_flags: entry.gp_flags, storage_mode: entry.storage_mode,
+        compressed_size: entry.compressed_size, uncompressed_size: entry.uncompressed_size,
+        mtime: entry.mtime, crc32: entry.crc32, filename: entry.filename) #, external_attrs: DEFAULT_EXTERNAL_ATTRS)
+    end
+    
+    # Record the central directory size, for the EOCDR
+    cdir_size = @out.tell - cdir_starts_at
 
+    # Write out the EOCDR
+    @writer. write_end_of_central_directory(io: @out, start_of_central_directory_location: cdir_starts_at,
+       central_directory_size: cdir_size, num_files_in_archive: @files.length)
+    @out.tell
+  end
+  
+  # Sets up the ZipWriter with wrappers if necessary. The method is called once, when the Streamer
+  # gets instantiated - the Writer then gets reused. This method is primarily there so that you
+  # can override it.
+  #
+  # @return [ZipTricks::ZipWriter] the writer to perform writes with
+  def create_writer
+    ZipTricks::ZipWriter.new
+  end
+  
   private
-
-  # Checks whether the number of bytes written conforms to the declared entry size
-  def leaving_in_entry_body_state
-    if @bytes_written_for_entry != @expected_bytes_for_entry
-      msg = "Wrong number of bytes written for entry (expected %d, got %d)" % [@expected_bytes_for_entry, @bytes_written_for_entry]
-      raise EntryBodySizeMismatch, msg
+  
+  def add_file_and_write_local_header(filename:, crc32:, storage_mode:, compressed_size:,
+      uncompressed_size:, use_data_descriptor: false)
+    if @files.any?{|e| e.filename == filename }
+      raise DuplicateFilenames, "Filename #{filename.inspect} already used in the archive"
     end
+    
+    raise UnknownMode, "Unknown compression mode #{storage_mode}" unless [STORED, DEFLATED].include?(storage_mode)
+    raise Overflow, "Filename is too long" if filename.bytesize > 0xFFFF
+    raise PathError, "Paths in ZIP may only contain forward slashes (UNIX separators)" if filename.include?('\\')
+
+    @check_compressed_size_after_leaving_body = !use_data_descriptor
+    @bytes_written_for_entry = 0
+    @expected_bytes_for_entry = compressed_size
+
+    e = Entry.new(filename, crc32, compressed_size, uncompressed_size, storage_mode, mtime=Time.now.utc, use_data_descriptor)
+    @files << e
+    @local_header_offsets << @out.tell
+    @writer.write_local_file_header(io: @out, gp_flags: e.gp_flags, crc32: e.crc32, compressed_size: e.compressed_size,
+      uncompressed_size: e.uncompressed_size, mtime: e.mtime, filename: e.filename, storage_mode: e.storage_mode)
+  end
+  
+  def write_data_descriptor_for_last_entry
+    e = @files.fetch(-1)
+    @writer.write_data_descriptor(io: @out, crc32: 0, compressed_size: e.compressed_size, uncompressed_size: e.uncompressed_size)
   end
 end