zippo 0.2.0

data/lib/zippo/filter/base.rb

@@ -0,0 +1,29 @@
+module Zippo
+  # Filters used to compress and decompress archive data.
+  module Filter
+    # Generic filter mixin.
+    module Base
+      # Use same block size as rubyzip for better comparison
+      BLOCK_SIZE = 131072
+      module ClassMethods
+        def filters
+          @filters_hash ||= Hash[@filters.map{|u| [u::METHOD, u]}]
+        end
+        def for(method)
+          filters[method] or raise "unknown compression method #{method}"
+        end
+
+        def inherited(klass)
+          @filters_hash = nil
+          @filters << klass
+        end
+      end
+      def self.included(base)
+        base.class_eval do
+          @filters = []
+          extend ClassMethods
+        end
+      end
+    end
+  end
+end

data/lib/zippo/filter/compressor/deflate.rb

@@ -0,0 +1,23 @@
+require 'zippo/filter/compressor'
+
+module Zippo::Filter
+  # Compresses the input data using zlib.
+  class DeflateCompressor < Compressor
+    METHOD = 8
+    DEFAULT_COMPRESSION = Zlib::DEFAULT_COMPRESSION
+
+    def initialize(io, compression_mode = DEFAULT_COMPRESSION)
+      super(io)
+      @zlib = Zlib::Deflate.new(compression_mode, -Zlib::MAX_WBITS)
+    end
+
+    private
+    def filter(buf)
+      @zlib.deflate(buf)
+    end
+
+    def tail_filter
+      @zlib.finish unless @zlib.finished?
+    end
+  end
+end

data/lib/zippo/filter/compressor/store.rb

@@ -0,0 +1,12 @@
+require 'zippo/filter/compressor'
+require 'zippo/filter/null_filters'
+
+module Zippo::Filter
+  # StoreCompressor stores the original member data directly in the zip
+  # file. No compression is performed.
+  class StoreCompressor < Compressor
+    METHOD = 0
+
+    include NullFilters
+  end
+end

data/lib/zippo/filter/compressor.rb

@@ -0,0 +1,42 @@
+require 'zippo/filter/base'
+require 'zlib'
+
+module Zippo::Filter
+  # A compressor is responsible for writing (and likely
+  # compressing) data into a zip file.
+  #
+  # @example Obtaining a compressor
+  #   compression_method = 8
+  #   compressor = Compressor.for(compression_method).new(io)
+  #   compressor.compress_to out
+  #
+  # Subclasse should include a METHOD constant to indicate
+  # which zip compression method they handle. The actual
+  # compression should be implemented in the filter and
+  # tail_filter methods.
+  class Compressor
+    include Zippo::Filter::Base
+
+    def initialize(io)
+      @io = io
+    end
+
+    def read count = nil, buf = nil
+      @io.read count, buf
+    end
+
+    def compress_to io
+      data_size = 0
+      compressed_size = 0
+      crc32 = 0
+      buf = ""
+      while (read BLOCK_SIZE, buf)
+        data_size += buf.bytesize
+        compressed_size += io.write filter(buf)
+        crc32 = Zlib.crc32(buf, crc32)
+      end
+      compressed_size += io.write tail_filter
+      return compressed_size, data_size, crc32
+    end
+  end
+end

data/lib/zippo/filter/compressors.rb

@@ -0,0 +1,3 @@
+Dir[File.join(File.dirname(__FILE__), 'compressor/*.rb')].each do |f|
+  require "zippo/filter/compressor/#{File.basename(f, '.rb')}"
+end

data/lib/zippo/filter/null_filters.rb

@@ -0,0 +1,15 @@
+module Zippo
+  module Filter
+    # Mixin for a no-op filter.
+    module NullFilters
+      protected
+      def filter(buf)
+        buf
+      end
+
+      def tail_filter
+        ""
+      end
+    end
+  end
+end

data/lib/zippo/filter/uncompressor/deflate.rb

@@ -0,0 +1,25 @@
+require 'zippo/filter/uncompressor'
+
+require 'zlib'
+require 'stringio'
+
+module Zippo::Filter
+  # Uncompresses the data using Zlib.
+  class DeflateUncompressor < Uncompressor
+    METHOD = 8
+
+    def initialize(io, compressed_size, zlib = Zlib::Inflate.new(-Zlib::MAX_WBITS)) # no header
+      super(io, compressed_size)
+      @zlib = zlib
+    end
+
+    private
+    def filter(buf)
+      @zlib.inflate(buf)
+    end
+
+    def tail_filter
+      @zlib.finish
+    end
+  end
+end

data/lib/zippo/filter/uncompressor/store.rb

@@ -0,0 +1,12 @@
+require 'zippo/filter/uncompressor'
+require 'zippo/filter/null_filters'
+
+module Zippo::Filter
+  # StoreUncompressor returns the member data as stored in the zip file.
+  # No uncompression is performed.
+  class StoreUncompressor < Uncompressor
+    METHOD = 0
+
+    include NullFilters
+  end
+end

data/lib/zippo/filter/uncompressor.rb

@@ -0,0 +1,59 @@
+require 'zippo/filter/base'
+
+module Zippo::Filter
+  # An Uncompressor is responsible for reading (and likely
+  # uncompressing) member data from a Zip file.
+  #
+  # @example Obtaining an uncompressor
+  #   compression_method = 8
+  #   uncompressor = Uncompressor.for(compression_method).new(io)
+  #   uncompressor.uncompress_to out
+  #
+  # Subclasse should include a METHOD constant to indicate
+  # which zip compression method they handle. The actual
+  # uncompression should be implemented in the filter and
+  # tail_filter methods.
+  class Uncompressor
+    include Zippo::Filter::Base
+
+    # @param [IO] io the IO the member data is located in
+    # @param [Integer] compressed_size the compressed size of the data
+    def initialize(io, compressed_size)
+      # XXX should probably capture the offset here
+      #     current usage assumes the IO has already been positioned at
+      #     the appropriate location
+      @io = io
+      @compressed_size = compressed_size
+      @remaining = @compressed_size
+    end
+
+    def read n, buf = nil
+      if @remaining >= n
+        @remaining -= n
+      elsif (n = @remaining) > 0
+        @remaining = 0
+      else
+        return nil
+      end
+
+      @io.read n, buf
+      buf.replace filter(buf)
+    end
+
+    # Uncompresses the data to the specified IO
+    #
+    # @param [IO] io the object to uncompress to, must respond to #<<
+    def uncompress_to io
+      buf = ""
+      while (read BLOCK_SIZE, buf)
+        io << buf
+      end
+      io << tail_filter
+    end
+
+    # @return [String] the uncompressed data
+    def uncompress
+      uncompress_to ""
+    end
+  end
+end

data/lib/zippo/filter/uncompressors.rb

@@ -0,0 +1,3 @@
+Dir[File.join(File.dirname(__FILE__), 'uncompressor/*.rb')].each do |f|
+  require "zippo/filter/uncompressor/#{File.basename(f, '.rb')}"
+end

data/lib/zippo/io_zip_member.rb

@@ -0,0 +1,24 @@
+require 'zippo/filter/compressor'
+require 'zippo/filter/compressor/deflate' # XXX only used for the default method
+
+module Zippo
+  # A zip member sourced from an IO object
+  class IOZipMember
+    attr_reader :name
+
+    def initialize(name, source)
+      @name = name
+      @source = source
+    end
+
+    def read
+      @source.read
+    ensure
+      @source.rewind
+    end
+
+    def write_to out, preferred_compression_method = Filter::DeflateCompressor::METHOD, recompress = nil
+      Filter::Compressor.for(preferred_compression_method).new(@source).compress_to(out)
+    end
+  end
+end

data/lib/zippo/local_file_header.rb

@@ -0,0 +1,28 @@
+require 'zippo/binary_structure'
+
+module Zippo
+  # Represents a local file header as documented in APPNOTE.TXT
+  # @see http://www.pkware.com/documents/casestudies/APPNOTE.TXT
+  class LocalFileHeader
+    SIGNATURE = 0x04034b50
+    binary_structure do
+      # @!macro [attach] bs.field
+      #   @!attribute [rw] $1
+      field :signature, 'L', :signature => SIGNATURE
+      field :version_extractable_by, 'S', :default => 20
+      field :bit_flags, 'S', :default => 0
+      field :compression_method, 'S'
+      field :last_modified_time, 'S', :default => 0 # XXX
+      field :last_modified_date, 'S', :default => 0 # XXX
+      field :crc32, 'L'
+      field :compressed_size, 'L'
+      field :uncompressed_size, 'L'
+      # set when name is set
+      field :file_name_length, 'S'
+      # set when extra_field is set
+      field :extra_field_length, 'S', :default => 0
+      field :name, 'a*', :size => :file_name_length
+      field :extra_field, 'a*', :default => '', :size => :extra_field_length
+    end
+  end
+end

data/lib/zippo/version.rb

@@ -0,0 +1,3 @@
+module Zippo
+  VERSION = "0.2.0"
+end

data/lib/zippo/zip_directory.rb

@@ -0,0 +1,80 @@
+require 'zippo/zip_member'
+require 'zippo/io_zip_member'
+require 'zippo/central_directory_reader'
+require 'forwardable'
+
+module Zippo
+  # The ZipDirectory is responsible for managing the set of ZipMembers
+  # belonging to a ZipFile.
+  class ZipDirectory
+    extend Forwardable
+    include Enumerable
+    # should delegate to entries_hash instead of entries whenever we can
+    def_delegators :entries_hash, :empty?
+    def_delegators :entries, :each, :map
+
+    def initialize io = nil
+      @io = io
+    end
+
+    # @param [String] name the name of the ZipMember
+    # @return [ZipMember] the ZipMember with the specified name
+    def [](name)
+      entries_hash[name]
+    end
+
+    # Inserts data into the ZipFile
+    #
+    # @param [String] name the name of the member to insert
+    # @param [String] data the data to insert
+    def []=(name, data)
+      insert(name, StringIO.new(data))
+    end
+
+    # Inserts data into the ZipFile
+    #
+    # - when the source is a ZipMember, the already-compressed data may
+    #   be re-used when writing out
+    # - when the source is a String, it is interpreted as a filename,
+    #   and will be used as the source of the data
+    # - otherwise, source is assumed to an already opened IO object
+    #
+    # @param [String] name the name of the member to insert
+    # @param source where to read the data from
+    #
+    # Source can be any of
+    # - an IO object
+    # - a string (path to file)
+    # - another ZipMember (allowing direct stream copy)
+    def insert(name, source)
+      set name,
+        case source
+        when ZipMember then source.with_name name
+        when String then IOZipMember.new name, File.open(source, 'r:BINARY')
+        else IOZipMember.new name, source
+        end
+    end
+
+    # @return [Hash] the hash of ZipMembers, the hash key is the
+    #   member's name
+    def entries_hash
+      @entries_hash ||= if @io
+        CentralDirectoryReader.new(@io).cd_file_headers.each_with_object({}) do |header, hash|
+          hash[header.name] = ZipMember.new @io, header
+        end
+      else
+        {}
+      end
+    end
+
+    # @return [Array] the members of the ZipFile
+    def entries
+      entries_hash.values
+    end
+
+    private
+    def set(name, member)
+      entries_hash[name] = member
+    end
+  end
+end

data/lib/zippo/zip_file.rb

@@ -0,0 +1,121 @@
+require 'zippo/zip_directory'
+require 'zippo/zip_file_writer'
+
+require 'forwardable'
+
+module Zippo
+  # ZipFile represents a Zip archive.
+  #
+  # It can be called in block form like this:
+  #
+  #   Zippo.open("file.zip") do |zip|
+  #     str = zip["file.txt"]
+  #     other = zip["other/file.txt"]
+  #     puts str
+  #   end
+  #
+  # Or without a block:
+  #
+  #   zip = Zippo.open("file.zip")
+  #   puts zip["file.txt"]
+  #   zip.close
+  #
+  # == Inserting archive members
+  #
+  # New archive members can be inserted using the insert method:
+  #
+  #   zip = Zippo.open("out.zip", "w")
+  #   zip.insert "out.txt", "something.txt"
+  #
+  # Additionally Zipfile#[]= has been overridden to support inserting
+  # strings directly:
+  #
+  #   zip["other.txt"] = "now is the time"
+  #
+  # If you already have an IO object, you can just insert it:
+  #
+  #   io = File.open("foo.bin")
+  #   zip.insert "rename.bin", io
+  #
+  # Finally, you can insert a Zippo::ZipMember from another Zip file,
+  # which can allow the compressed data to be copied directly (avoiding
+  # having to uncompress and then recompress it):
+  #
+  #   other = Zippo.open("other.zip")
+  #   zip.insert "final.bin", other["final.bin"]
+  #
+  # No data will actually be written until the ZipFile is closed:
+  #
+  #   zip.close
+  class ZipFile
+    # Opens a zip archive file
+    #
+    # @param [String] filename path to the archive
+    # @param [String] mode the mode to open in, 'r' or 'w' or 'rw'
+    #
+    # @return [Zippo::ZipFile] the opened zip file if no block is given
+    def self.open(filename, mode = 'r')
+      if block_given?
+        zippo = new(filename, mode)
+        a = yield zippo
+        zippo.close
+        return a
+      else
+        new filename, mode
+      end
+    end
+
+    def initialize(filename, mode)
+      @filename = filename
+      @mode = mode
+    end
+
+    extend Forwardable
+    def_delegators :directory, :map, :[], :[]=, :each, :insert
+
+    # Closes the ZipFile, writing it to disk if it was opened in write
+    # mode
+    def close
+      # XXX should optimize to not write anything to unchanged files
+      # In update mode, we first write the zip to a temporary zip file,
+      # then move it on top of the original file
+      out_zip = update? ? tmp_zipfile : @filename
+      ZipFileWriter.new(directory, out_zip).write if write?
+      @io.close if @io
+      File.rename out_zip, @filename if update?
+    end
+
+    # @return [ZipDirectory] the ZipDirectory
+    def directory
+      @directory ||= if read?
+        ZipDirectory.new io
+      else
+        ZipDirectory.new
+      end
+    end
+
+    private
+    def read?
+      File.exist? @filename and @mode.include? 'r'
+    end
+
+    def write?
+      @mode.include? 'w'
+    end
+
+    def update?
+      read? and write?
+    end
+
+    def io
+      @io ||= File.open(@filename, 'r:ASCII-8BIT')
+    end
+
+    def tmp_zipfile
+      # Not using Tempfile for performance
+      # Should probably throw a timestamp in there, in case multiple
+      # temps are being written at once from the one process
+      File.join File.dirname(@filename), ".zippo-tmp-#{Process.pid}-#{File.basename(@filename)}"
+    end
+  end
+end

data/lib/zippo/zip_file_writer.rb

@@ -0,0 +1,57 @@
+module Zippo
+  # ZipFileWriter writes the contents of a ZipDirectory to a Zip file.
+  class ZipFileWriter
+    # @param [ZipDirectory] directory the ZipDirectory to write
+    # @param [String] filename the filename to write to
+    def initialize(directory, filename)
+      @directory = directory
+      @filename = filename
+    end
+
+    # Writes the directory to the file.
+    def write
+      File.open(@filename,'wb:ASCII-8BIT') do |io|
+        packer = LocalFileHeader::Packer.new io
+        headers = []
+        for member in @directory
+          header = CdFileHeader.default
+          header.compression_method = 8 # XXX configurable
+          # XXX hack fix for maintaining method in zipped data copies
+          if member.is_a? ZipMember
+            header.compression_method = member.compression_method
+          end
+          header.name = member.name
+          header.extra_field = "" # XXX extra field unimplemented
+          header_size = header.file_name_length + header.extra_field_length + 30
+
+          # record header position so we can write it later
+          header.local_file_header_offset = io.pos
+
+          # move to after the header
+          io.seek header_size, IO::SEEK_CUR
+
+          # write the compressed data
+          header.compressed_size, header.uncompressed_size, header.crc32 =
+            member.write_to io, header.compression_method
+
+          # write the completed header, returning to the current position
+          io.seek header.local_file_header_offset
+          #packer.pack LocalFileHeader.from header.convert_to LocalHileHeader
+          packer.pack header.convert_to LocalFileHeader
+          io.seek header.compressed_size, IO::SEEK_CUR
+          headers << header
+        end
+
+        eocdr = EndCdRecord.default
+        eocdr.cd_offset = io.pos
+        packer = CdFileHeader::Packer.new io
+        for header in headers
+          packer.pack header
+        end
+        eocdr.cd_size = io.pos - eocdr.cd_offset
+        eocdr.records = eocdr.total_records = headers.size
+        EndCdRecord::Packer.new(io).pack eocdr
+      end
+    end
+  end
+end

data/lib/zippo/zip_member.rb

@@ -0,0 +1,85 @@
+require 'zippo/local_file_header'
+require 'zippo/filter/uncompressors'
+require 'zippo/filter/compressors'
+
+require 'forwardable'
+
+module Zippo
+  # A member of a Zip archive file.
+  class ZipMember
+    def initialize io, header
+      @io = io
+      @header = header
+    end
+
+    # @return [String] the name of the member
+    def name
+      @name ||= @header.name
+    end
+
+    # @return [Boolean] True if the member is a directory, False
+    #   otherwise
+    def directory?
+      name.end_with? '/'
+    end
+
+    extend Forwardable
+    def_delegators :@header, :crc32, :compressed_size, :uncompressed_size, :compression_method
+
+    # Reads (and possibly uncompresses) the member's data
+    #
+    # @return [String] the uncompressed member data
+    def read
+      seek_to_compressed_data
+      uncompressor.uncompress
+    end
+
+    # Duplicates this zip member and overrides the name.
+    #
+    # @param [String] name the name to use
+    # @return [ZipMember] the new ZipMember
+    def with_name name
+      dup.tap do |obj|
+        obj.instance_variable_set :@name, name
+      end
+    end
+
+    # Writes the member data to the specified IO using the specified
+    # compression method.
+    #
+    # @param [IO] out the IO to write to
+    # @param [Integer] preferred_method the compression method to use
+    # @param [Boolean] recompress whether or not to recompress the data
+    #
+    # @return [Integer, Integer, Integer] the amount written, the
+    #   original size of the data, the crc32 of the data
+    def write_to out, preferred_method = Filter::DeflateCompressor::METHOD, recompress = false
+      seek_to_compressed_data
+      if recompress
+        Filter::Compressor.for(preferred_method).new(uncompressor).compress_to(out)
+      else
+        IO.copy_stream @io, out, @header.compressed_size
+        return @header.compressed_size, @header.uncompressed_size, @header.crc32
+      end
+    end
+
+    private
+    def local_file_header
+      @io.seek @header.local_file_header_offset
+      LocalFileHeader::Unpacker.new(@io).unpack
+    end
+
+    def seek_to_compressed_data
+      @io.seek @header.local_file_header_offset + local_file_header.size
+    end
+
+    def uncompressor
+      Filter::Uncompressor.for(@header.compression_method).new(@io, @header.compressed_size)
+    end
+
+    def compressed_member_data
+      seek_to_compressed_data
+      @io.read @header.compressed_size
+    end
+  end
+end

data/lib/zippo.rb

@@ -0,0 +1,18 @@
+# Ensure binary structure optimisations are active before any structures are defined
+require 'zippo/binary_structure/meta'
+
+require "zippo/version"
+require 'zippo/zip_file'
+require 'zippo/filter/uncompressors'
+
+# Zippo is a Zip library.
+module Zippo
+  class << self
+    # Calls Zippo::ZipFile.open
+    # @see ZipFile.open
+    def open filename, mode = 'r', &block
+      Zippo::ZipFile.open filename, mode, &block
+    end
+    public :open
+  end
+end