zippo 0.2.0

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    MDM2MjkzNjVlMWI3OTlmYmU5NTJjMTVhZmExMzU2ODRjYmMwZWNiMw==
+  data.tar.gz: !binary |-
+    Zjk3OWIwMTBjZWJiMmQyOTUwNzQ1YzgzOWExODc0NThiNjMwMDhmOA==
+!binary "U0hBNTEy":
+  metadata.gz: !binary |-
+    NjI5ZDU5MTdiMWU0ZDc5NjBhNzMwMWY2YTRkNzYyZDRjZDMzNTFmOTcxMWI4
+    NDI3NGYzYWU5ZWQyOTEzYzY5N2U4YzFhNTA1ZWE2NTRlMTEzNTUzYzEyYmIx
+    ZGY4Njc5NjE1YWY5NWI4NjZjM2UxNGMxZTRiZGQ4MTFkZGM0OWU=
+  data.tar.gz: !binary |-
+    YmJkYjQ3YmQ2YWFjNjRhNGZmNWI5MTlhOGMzYjliMjg5MWQ4YzRhZTVhMGY3
+    OGIyZDJiMzM1NDNmMzY5MGVmNjFlZmFhMWM2MDY4ZjVmNDdiYzI4MmE1MDhk
+    N2I5Zjk1YzRlYTIwZWY2NDY5ZDMwYzMyNzEzZGI5NjMzZDFiZDE=

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.rspec

@@ -0,0 +1 @@
+--order random --color

data/.yardopts

@@ -0,0 +1 @@
+lib/**/*.rb

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in zippo.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2012-2013 Jonathon M. Abbott
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,83 @@
+# Zippo
+
+Zippo is a fast zip library for ruby.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'zippo'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install zippo
+
+## Usage
+
+It can be called in block form:
+
+    Zippo.open("file.zip") do |zip|
+      str = zip["file.txt"]
+      other = zip["other/file.txt"]
+      puts str
+    end
+
+Or without a block:
+
+    zip = Zippo.open("file.zip")
+    puts zip["file.txt"]
+    zip.close
+
+### Inserting archive members
+
+Files can be inserted into the zip using the
+insert method. Note that no data will be written until the
+ZipFile is closed:
+
+    zip = Zippo.open("out.zip", "w")
+    zip.insert "file1.txt", "path/to/1.txt"
+    zip.insert "file2.txt", "path/to/2.txt"
+    zip.close
+
+#### By path
+
+    zip.insert "out.txt", "something.txt"
+
+#### Directly from a string buffer
+
+    zip["other.txt"] = "now is the time"
+
+#### By IO
+
+    io = File.open("foo.dat")
+    zip.insert "data.dat", io
+
+#### From another zip file (direct stream copy)
+
+Inserting zip data from one file into another allows the
+compressed data to be reused from the original zip file
+(avoiding uncompression and recompression):
+
+    other = Zippo.open("other.zip")
+    zip.insert "final.bin", other["final.bin"]
+
+## TODO
+
+- implement date handling
+- implement unix attribute handling
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+## License
+
+MIT License. Copyright (c) 2012-2013 Jonathon M. Abbott

data/Rakefile

@@ -0,0 +1,11 @@
+#!/usr/bin/env rake
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+require 'yard'
+require './yard_extensions'
+
+task :default => :spec
+
+RSpec::Core::RakeTask.new
+
+YARD::Rake::YardocTask.new

data/lib/zippo/binary_structure/base.rb

@@ -0,0 +1,119 @@
+require 'zippo/binary_structure/structure_member'
+require 'zippo/binary_structure/structure'
+require 'zippo/binary_structure/binary_packer'
+require 'zippo/binary_structure/binary_unpacker'
+
+module Zippo
+  # BinaryStructure defines a class level DSL for
+  # implementing binary strucutures.
+  #
+  # The class will then have an ::Unpacker and ::Packer class
+  # defined underneath it that can be used to read and write
+  # the defined fields from an io.
+  #
+  # The DSL itself is fairly simple, fields are defined with
+  # a field name, "packing code" (per standard ruby
+  # Array#pack) and possibly options.
+  #
+  # - the :signature option indicates the field is a fixed
+  #   signature
+  # - the :size => <field> option indicates the field is a variable
+  #   width size field, with the size previously recorded in
+  #   the specified field
+  #
+  # @example
+  #    binary_structure do
+  #      field :foo, 'L'
+  #      field :yay, 'a4', :signature => "baz"
+  #      field :bar, 'S'
+  #      field :quux, 'a*', :size => :foo
+  #    end
+  #
+  # @see Array#pack
+  module BinaryStructure
+    module Base
+      def binary_structure &block
+        @structure = Structure.create(self, &block)
+        self.const_set :Packer, Class.new(BinaryPacker)
+        self::Packer.structure = @structure
+        self.const_set :Unpacker, Class.new(BinaryUnpacker)
+        self::Unpacker.structure = @structure
+
+        @structure.fields.each do |field|
+          attr_reader field.name
+          if @structure.dependent? field.name
+            define_method "#{field.name}=" do |value|
+              raise "can't mutate a dependent field"
+            end
+          else
+            if field.dependent
+              class_eval """
+                def #{field.name}= value
+                  @#{field.dependent} = value.bytesize
+                  @#{field.name} = value
+                end
+              """
+            else
+              attr_writer field.name
+            end
+          end
+        end
+        include InstanceMethods
+        extend ClassMethods
+        Base.after_structure_definition_hooks_for(self)
+      end
+      class << self
+        def after_structure_definition_hooks_for(klass)
+          @hooks.each do |hook|
+            hook.call(klass)
+          end if @hooks
+        end
+        def after_structure_definition &block
+          @hooks ||= []
+          @hooks << block
+        end
+      end
+      module InstanceMethods
+        def defaults
+          self.class.structure.fields.each do |field|
+            instance_variable_set "@#{field.name}", field.options[:default] if field.options[:default]
+            instance_variable_set "@#{field.name}", field.options[:signature] if field.options[:signature]
+          end
+          self
+        end
+        def size
+          self.class.structure.fields.map do |field|
+            if field.dependent
+              send field.dependent
+            else
+              field.width
+            end
+          end.inject(&:+)
+        end
+
+        def convert_to other
+          other.default.tap do |obj|
+            self.class.common_fields_with(other).each do |field|
+              obj.instance_variable_set "@#{field}", send(field)
+            end
+          end
+        end
+      end
+      module ClassMethods
+        attr_reader :structure
+        def default
+          new.defaults
+        end
+
+        # Returns the fields that this data type has in common with other.
+        #
+        # - common fields are fields with the same name
+        # - signature fields are never common
+        def common_fields_with(other)
+          structure.fields.map(&:name) &
+            other.structure.fields.reject(&:signature?).map(&:name)
+        end
+      end
+    end
+  end
+end

data/lib/zippo/binary_structure/binary_packer.rb

@@ -0,0 +1,17 @@
+module Zippo
+  module BinaryStructure
+    class BinaryPacker
+      class << self
+        attr_accessor :structure
+      end
+
+      def initialize(io)
+        @io = io
+      end
+
+      def pack obj
+        @io << self.class.structure.fields.map {|f| obj.send f.name}.pack(self.class.structure.fields.map(&:pack).join(""))
+      end
+    end
+  end
+end

data/lib/zippo/binary_structure/binary_unpacker.rb

@@ -0,0 +1,32 @@
+require 'stringio'
+
+module Zippo
+  module BinaryStructure
+    class BinaryUnpacker
+      class << self
+        attr_accessor :structure
+      end
+
+      def initialize(io)
+        @io = io
+        @io = StringIO.new @io if @io.is_a? String
+      end
+
+      # default implementation
+      # note that this will generally be overridden by
+      # define_unpack_method for optimisation
+      def unpack
+        self.class.structure.owner_class.new.tap do |obj|
+          self.class.structure.fields.each do |field|
+            if field.options[:size]
+              obj.instance_variable_set "@#{field.name}", @io.read(obj.send field.options[:size])
+            else
+              buf = @io.read field.width
+              obj.instance_variable_set "@#{field.name}", buf.unpack(field.pack).first
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/zippo/binary_structure/meta.rb

@@ -0,0 +1,146 @@
+require 'zippo/binary_structure/base'
+
+module Zippo::BinaryStructure
+  # Profiling shows most of our time is spent iterating over various fields
+  # so let's unroll those loops in advance.
+  module CodeGen
+    class << self
+      # Defines a method on the specified class that sets a bunch of fields at once.
+      def define_helper(klass, meth, fields)
+        buf = []
+        buf << "def #{meth}(#{0.upto(fields.size-1).map{|x|"a#{x}"}.join(',')})"
+        fields.each_with_index do |field, i|
+          buf << "@#{field} = a#{i}"
+        end
+        buf << "end"
+        klass.class_eval buf.join("\n")
+      end
+
+      def fields_as_args(fields)
+        fields.map {|f| "@#{f}"}.join(", ")
+      end
+
+      def call_helper(receiver, meth, fields)
+        "#{receiver}.#{meth}(#{fields_as_args(fields)})"
+      end
+
+      def define_defaults_method_for(klass)
+        buf = []
+        buf << "def defaults"
+        klass.structure.fields.each do |field|
+          buf << %{@#{field.name} = #{field.options[:default].inspect}} if field.options[:default]
+          buf << %{@#{field.name} = #{field.options[:signature].inspect}} if field.options[:signature]
+        end
+        buf << "self"
+        buf << "end"
+        klass.class_eval buf.join("\n")
+      end
+
+      # XXX - should write a spec for the "multiple helpers"
+      # implementation, none of the current binary structures would make
+      # use of it, as they all have a bunch of fixed fields,
+      # then the variable fields at the end. a test should 
+      #def self.define_unpack_method
+      def define_unpack_method_for(klass)
+        buf = "def unpack\n"
+        buf << "obj = self.class.structure.owner_class.new\n"
+        helper_num = -1 # keep track of the number of helper methods we've created
+        field_buf = []
+        # iterate over the fields, gathering up the fixed fields in a
+        # group. once a variable field is hit, unpack the current group of
+        # fixed fields, then use that to read any variable fields. repeat.
+        klass.structure.fields.each do |field|
+          if field.options[:size]
+            # unpack fixed group
+            unless field_buf.empty?
+              s = field_buf.map(&:width).inject(&:+)
+              buf << %{arr = @io.read(#{s}).unpack("#{field_buf.map(&:pack).join('')}")\n}
+              helper_name = "binary_structure_unpack_helper_#{helper_num += 1}"
+              define_helper(klass.structure.owner_class, helper_name, field_buf.map(&:name))
+              buf << "obj.#{helper_name}(*arr)\n"
+            end
+            # unpack variable-length field
+            buf << %{obj.instance_variable_set :@#{field.name}, @io.read(obj.#{field.options[:size]})\n}
+            field_buf = []
+          else
+            field_buf << field
+          end
+        end
+        buf << "obj\n"
+        buf << "end\n"
+
+        klass.class_eval(buf)
+      end
+
+      def define_converter_for(klass, meth, other)
+        # serialize the klass reference "other"
+        # we can't just use to_s, since that won't work if the class is anonymous
+        class_ref = "ObjectSpace._id2ref(#{other.object_id})"
+
+        common_fields = klass.common_fields_with(other)
+
+        helper_name = "initialize_from_#{object_id}"
+        define_helper(other, helper_name, common_fields)
+
+        default_fields = other.structure.fields.select do |field|
+          field.options[:default] || field.options[:signature]
+        end.reject do |field|
+          common_fields.include? field.name
+        end
+
+        define_helper(other, "other_fields", default_fields.map(&:name))
+        default_values = default_fields.map do |f|
+          f.options[:signature] ||
+          f.options[:default]
+        end
+
+        klass.class_eval """
+          def #{meth}
+            obj = #{class_ref}.new
+            obj.other_fields(#{default_values.map(&:inspect).join(', ')})
+            #{call_helper("obj", helper_name, common_fields)}
+            obj
+          end
+        """
+      end
+
+      def define_pack_method_for klass
+        buf = []
+
+        fields = klass.structure.fields.map(&:name)
+        packing_string = klass.structure.fields.map(&:pack).join('')
+
+        helper_method =
+          """
+            def fields_for_packing
+              [#{fields_as_args(fields)}]
+            end
+          """
+        klass.structure.owner_class.class_eval helper_method
+
+        klass.class_eval do
+          define_method :pack do |obj|
+            @io << obj.fields_for_packing.pack(packing_string)
+          end
+        end
+      end
+    end
+  end
+
+  Base.after_structure_definition do |klass|
+    # Pre-define the .defaults method
+    CodeGen.define_defaults_method_for klass
+    # Pre-define the .unpack method
+    CodeGen.define_unpack_method_for klass::Unpacker
+    # Pre-define the .pack method
+    CodeGen.define_pack_method_for klass::Packer
+  end
+
+  module InstanceMethods
+    def convert_to other
+      method = :"convert_to_#{other.object_id}"
+      ::Zippo::BinaryStructure::CodeGen.define_converter_for(self.class, method, other) unless respond_to? method
+      send method
+    end
+  end
+end

data/lib/zippo/binary_structure/structure.rb

@@ -0,0 +1,24 @@
+module Zippo
+  module BinaryStructure
+    class Structure
+      def self.create(owner_class, &block)
+        structure = new(owner_class, &block)
+        structure
+      end
+      def initialize(owner_class, &block)
+        @fields = []
+        @owner_class = owner_class
+        instance_eval &block
+      end
+      def field name, pack, options = {}
+        @fields << StructureMember.new(name, pack, options)
+      end
+      def dependent? field_name
+        fields.detect do |field|
+          field.options[:size] == field_name
+        end
+      end
+      attr_reader :fields, :owner_class
+    end
+  end
+end

data/lib/zippo/binary_structure/structure_member.rb

@@ -0,0 +1,31 @@
+module Zippo
+  module BinaryStructure
+    class StructureMember
+      def initialize(name, pack, options = {})
+        @name = name
+        @pack = pack
+        @options = options
+        @width = StructureMember.width(@pack)
+      end
+      # XXX unspec
+      def dependent
+        options[:size]
+      end
+      # XXX unspec
+      attr_reader :width
+      def self.width(pack)
+        case pack
+        when 'L' then 4
+        when 'S' then 2
+        when /^a(\d+)$/ then $1.to_i
+        when 'a*' then nil
+        end
+      end
+      attr_reader :name, :pack, :options
+
+      def signature?
+        options[:signature]
+      end
+    end
+  end
+end

data/lib/zippo/binary_structure.rb

@@ -0,0 +1,6 @@
+require 'zippo/binary_structure/base'
+require 'zippo/binary_structure/meta'
+
+Class.class_eval do
+  include Zippo::BinaryStructure::Base
+end

data/lib/zippo/cd_file_header.rb

@@ -0,0 +1,36 @@
+require 'zippo/binary_structure'
+
+module Zippo
+  # A Zip central directory file header.
+  class CdFileHeader
+    # The central file header signature from APPNOTE.TXT
+    SIGNATURE = 0x02014b50
+    binary_structure do
+      # @!macro [attach] bs.field
+      #   @!attribute [rw] $1
+      field :signature, 'L', :signature => SIGNATURE
+      field :version_made_by, 'S', :default => 0
+      field :version_extractable_by, 'S', :default => 20
+      field :bit_flags, 'S', :default => 0
+      field :compression_method, 'S'
+      field :last_modified_time, 'S', :default => 0
+      field :last_modified_date, 'S', :default => 0
+      field :crc32, 'L'
+      field :compressed_size, 'L'
+      field :uncompressed_size, 'L'
+      # set when name is set
+      field :file_name_length, 'S'
+      # set when extra_field is set
+      field :extra_field_length, 'S', :default => 0
+      # set when file comment is set
+      field :file_comment_length, 'S', :default => 0
+      field :disk_number, 'S', :default => 0
+      field :internal_file_attributes, 'S', :default => 0
+      field :external_file_attributes, 'L', :default => 0
+      field :local_file_header_offset, 'L'
+      field :name, 'a*', :size => :file_name_length
+      field :extra_field, 'a*', :size => :extra_field_length, :default => ''
+      field :comment, 'a*', :default => '', :size => :file_comment_length, :default => ''
+    end
+  end
+end

data/lib/zippo/central_directory_entries_unpacker.rb

@@ -0,0 +1,23 @@
+require 'zippo/cd_file_header'
+
+require 'stringio'
+
+module Zippo
+  # Unpacks an array of CdFileHeaders from an io stream
+  class CentralDirectoryEntriesUnpacker
+    def initialize(io, size, offset)
+      @io = io
+      @size = size
+      @offset = offset
+      @end = @offset + @size
+    end
+    def unpack
+      [].tap do |entries|
+        @io.seek @offset
+        while @io.pos < @end && entry = CdFileHeader::Unpacker.new(@io).unpack
+          entries << entry
+        end
+      end
+    end
+ end
+end

data/lib/zippo/central_directory_reader.rb

@@ -0,0 +1,44 @@
+require 'zippo/end_cd_record'
+require 'zippo/central_directory_entries_unpacker'
+
+module Zippo
+  # Reads the zip central directory from an IO stream.
+  class CentralDirectoryReader
+    def initialize(io)
+      @io = io
+    end
+
+    def end_of_cd_record
+      @end_of_cd_record ||= EndCdRecord::Unpacker.new(read_from end_of_cd_record_position).unpack
+    end
+
+    def cd_file_headers
+      @cd_file_headers ||= CentralDirectoryEntriesUnpacker.new(@io, end_of_cd_record.cd_size, end_of_cd_record.cd_offset).unpack
+    end
+
+    def end_of_cd_record_position
+      # XXX implement optimised scanning at -22 position
+      [44, 22].each do |pos|
+        next if pos > @io.size
+        return @io.size - pos if (read 4, -pos) == EndCdRecord::PACKED_SIGNATURE
+      end
+
+      scan_from = @io.size - EndCdRecord::MAX_COMMENT_LENGTH
+      scan_from = 0 if scan_from < 0
+      scan_from + (read_from(scan_from).rindex(EndCdRecord::PACKED_SIGNATURE) or raise "End of Central Directory Record not found")
+    end
+
+    private
+    # reads size from the specified offset
+    # if offset is negative, will offset from EOF
+    def read size, offset
+      @io.seek offset, (offset < 0 ? IO::SEEK_END : IO::SEEK_SET)
+      @io.read size
+    end
+
+    # reads from the specified offset until EOF
+    def read_from offset
+      read nil, offset
+    end
+  end
+end

data/lib/zippo/end_cd_record.rb

@@ -0,0 +1,21 @@
+require 'zippo/binary_structure'
+
+module Zippo
+  # A zip end of central directory record.
+  class EndCdRecord
+    SIGNATURE = 0x06054b50
+    PACKED_SIGNATURE = [SIGNATURE].pack('L')
+    MAX_COMMENT_LENGTH = 1<<16
+    binary_structure do
+      field :signature, 'L', :signature => SIGNATURE
+      field :disk, 'S', :default => 0
+      field :cd_disk, 'S', :default => 0
+      field :records, 'S'
+      field :total_records, 'S'
+      field :cd_size, 'L'
+      field :cd_offset, 'L'
+      field :comment_length, 'S', :default => 0
+      field :comment, 'a*', :default => "", :size => :comment_length
+    end
+  end
+end