jit 0.0.0 → 1.0.0

data/lib/pack/numbers.rb

@@ -0,0 +1,79 @@
+module Pack
+  module Numbers
+
+    module VarIntLE
+      def self.write(value, shift)
+        bytes = []
+        mask  = 2 ** shift - 1
+
+        until value <= mask
+          bytes.push(0x80 | value & mask)
+          value >>= shift
+
+          mask, shift = 0x7f, 7
+        end
+
+        bytes + [value]
+      end
+
+      def self.read(input, shift)
+        first = input.readbyte
+        value = first & (2 ** shift - 1)
+
+        byte = first
+
+        until byte < 0x80
+          byte   = input.readbyte
+          value |= (byte & 0x7f) << shift
+          shift += 7
+        end
+
+        [first, value]
+      end
+    end
+
+    module VarIntBE
+      def self.write(value)
+        bytes = [value & 0x7f]
+
+        until (value >>= 7) == 0
+          value -= 1
+          bytes.push(0x80 | value & 0x7f)
+        end
+
+        bytes.reverse.pack("C*")
+      end
+
+      def self.read(input)
+        byte  = input.readbyte
+        value = byte & 0x7f
+
+        until byte < 0x80
+          byte  = input.readbyte
+          value = ((value + 1) << 7) | (byte & 0x7f)
+        end
+
+        value
+      end
+    end
+
+    module PackedInt56LE
+      def self.write(value)
+        bytes = (0..6).map { |i| (value >> (8 * i)) & 0xff }
+
+        flags  = bytes.map.with_index { |b, i| b == 0 ? 0 : 1 << i }
+        header = flags.reduce(0) { |a, b| a | b }
+
+        [header] + bytes.reject { |b| b == 0 }
+      end
+
+      def self.read(input, header)
+        flags = (0..6).reject { |i| header & (1 << i) == 0 }
+        bytes = flags.map { |i| input.readbyte << (8 * i) }
+
+        bytes.reduce(0) { |a, b| a | b }
+      end
+    end
+
+  end
+end

data/lib/pack/reader.rb

@@ -0,0 +1,98 @@
+require "zlib"
+
+require_relative "./expander"
+require_relative "./numbers"
+
+module Pack
+  class Reader
+
+    attr_reader :count
+
+    def initialize(input)
+      @input = input
+    end
+
+    def read_header
+      data = @input.read(HEADER_SIZE)
+      signature, version, @count = data.unpack(HEADER_FORMAT)
+
+      unless signature == SIGNATURE
+        raise InvalidPack, "bad pack signature: #{ signature }"
+      end
+
+      unless version == VERSION
+        raise InvalidPack, "unsupported pack version: #{ version }"
+      end
+    end
+
+    def read_record
+      type, _ = read_record_header
+
+      case type
+      when COMMIT, TREE, BLOB
+        Record.new(TYPE_CODES.key(type), read_zlib_stream)
+      when OFS_DELTA
+        read_ofs_delta
+      when REF_DELTA
+        read_ref_delta
+      end
+    end
+
+    def read_info
+      type, size = read_record_header
+
+      case type
+      when COMMIT, TREE, BLOB
+        Record.new(TYPE_CODES.key(type), size)
+
+      when OFS_DELTA
+        delta = read_ofs_delta
+        size  = Expander.new(delta.delta_data).target_size
+
+        OfsDelta.new(delta.base_ofs, size)
+
+      when REF_DELTA
+        delta = read_ref_delta
+        size  = Expander.new(delta.delta_data).target_size
+
+        RefDelta.new(delta.base_oid, size)
+      end
+    end
+
+    private
+
+    def read_record_header
+      byte, size = Numbers::VarIntLE.read(@input, 4)
+      type = (byte >> 4) & 0x7
+
+      [type, size]
+    end
+
+    def read_ofs_delta
+      offset = Numbers::VarIntBE.read(@input)
+      OfsDelta.new(offset, read_zlib_stream)
+    end
+
+    def read_ref_delta
+      base_oid = @input.read(20).unpack("H40").first
+      RefDelta.new(base_oid, read_zlib_stream)
+    end
+
+    def read_zlib_stream
+      stream = Zlib::Inflate.new
+      string = ""
+      total  = 0
+
+      until stream.finished?
+        data   = @input.read_nonblock(256)
+        total += data.bytesize
+
+        string.concat(stream.inflate(data))
+      end
+      @input.seek(stream.total_in - total, IO::SEEK_CUR)
+
+      string
+    end
+
+  end
+end

data/lib/pack/stream.rb

@@ -0,0 +1,80 @@
+require "digest/sha1"
+
+module Pack
+  class Stream
+
+    attr_reader :digest, :offset
+
+    def initialize(input, buffer = "")
+      @input   = input
+      @digest  = Digest::SHA1.new
+      @offset  = 0
+      @buffer  = new_byte_string.concat(buffer)
+      @capture = nil
+    end
+
+    def capture
+      @capture = new_byte_string
+      result   = [yield, @capture]
+
+      @digest.update(@capture)
+      @capture = nil
+
+      result
+    end
+
+    def verify_checksum
+      unless read_buffered(20) == @digest.digest
+        raise InvalidPack, "Checksum does not match value read from pack"
+      end
+    end
+
+    def read(size)
+      data = read_buffered(size)
+      update_state(data)
+      data
+    end
+
+    def read_nonblock(size)
+      data = read_buffered(size, false)
+      update_state(data)
+      data
+    end
+
+    def readbyte
+      read(1).bytes.first
+    end
+
+    def seek(amount, whence = IO::SEEK_SET)
+      return unless amount < 0
+
+      data = @capture.slice!(amount .. -1)
+      @buffer.prepend(data)
+      @offset += amount
+    end
+
+    private
+
+    def new_byte_string
+      String.new("", :encoding => Encoding::ASCII_8BIT)
+    end
+
+    def read_buffered(size, block = true)
+      from_buf = @buffer.slice!(0, size)
+      needed   = size - from_buf.bytesize
+      from_io  = block ? @input.read(needed) : @input.read_nonblock(needed)
+
+      from_buf.concat(from_io.to_s)
+
+    rescue EOFError, Errno::EWOULDBLOCK
+      from_buf
+    end
+
+    def update_state(data)
+      @digest.update(data) unless @capture
+      @offset += data.bytesize
+      @capture&.concat(data)
+    end
+
+  end
+end

data/lib/pack/unpacker.rb

@@ -0,0 +1,62 @@
+require_relative "./expander"
+
+module Pack
+  class Unpacker
+
+    def initialize(database, reader, stream, progress)
+      @database = database
+      @reader   = reader
+      @stream   = stream
+      @progress = progress
+      @offsets  = {}
+    end
+
+    def process_pack
+      @progress&.start("Unpacking objects", @reader.count)
+
+      @reader.count.times do
+        process_record
+        @progress&.tick(@stream.offset)
+      end
+      @progress&.stop
+
+      @stream.verify_checksum
+    end
+
+    private
+
+    def process_record
+      offset    = @stream.offset
+      record, _ = @stream.capture { @reader.read_record }
+
+      record = resolve(record, offset)
+      @database.store(record)
+      @offsets[offset] = record.oid
+    end
+
+    def resolve(record, offset)
+      case record
+      when Record   then record
+      when OfsDelta then resolve_ofs_delta(record, offset)
+      when RefDelta then resolve_ref_delta(record)
+      end
+    end
+
+    def resolve_ofs_delta(delta, offset)
+      oid = @offsets[offset - delta.base_ofs]
+      resolve_delta(oid, delta.delta_data)
+    end
+
+    def resolve_ref_delta(delta)
+      resolve_delta(delta.base_oid, delta.delta_data)
+    end
+
+    def resolve_delta(oid, delta_data)
+      base = @database.load_raw(oid)
+      data = Expander.expand(base.data, delta_data)
+
+      Record.new(base.type, data)
+    end
+
+  end
+end

data/lib/pack/window.rb

@@ -0,0 +1,47 @@
+require "forwardable"
+
+module Pack
+  class Window
+
+    Unpacked = Struct.new(:entry, :data) do
+      extend Forwardable
+      def_delegators :entry, :type, :size, :delta, :depth
+
+      attr_accessor :delta_index
+    end
+
+    def initialize(size)
+      @objects = Array.new(size)
+      @offset  = 0
+    end
+
+    def add(entry, data)
+      unpacked = Unpacked.new(entry, data)
+      @objects[@offset] = unpacked
+      @offset = wrap(@offset + 1)
+
+      unpacked
+    end
+
+    def each
+      cursor = wrap(@offset - 2)
+      limit  = wrap(@offset - 1)
+
+      loop do
+        break if cursor == limit
+
+        unpacked = @objects[cursor]
+        yield unpacked if unpacked
+
+        cursor = wrap(cursor - 1)
+      end
+    end
+
+    private
+
+    def wrap(offset)
+      offset % @objects.size
+    end
+
+  end
+end

data/lib/pack/writer.rb

@@ -0,0 +1,92 @@
+require "digest/sha1"
+require "zlib"
+
+require_relative "./compressor"
+require_relative "./entry"
+require_relative "./numbers"
+
+module Pack
+  class Writer
+
+    def initialize(output, database, options = {})
+      @output   = output
+      @database = database
+      @digest   = Digest::SHA1.new
+      @offset   = 0
+
+      @compression = options.fetch(:compression, Zlib::DEFAULT_COMPRESSION)
+      @allow_ofs   = options[:allow_ofs]
+      @progress    = options[:progress]
+    end
+
+    def write_objects(rev_list)
+      prepare_pack_list(rev_list)
+      compress_objects
+      write_header
+      write_entries
+      @output.write(@digest.digest)
+    end
+
+    private
+
+    def write(data)
+      @output.write(data)
+      @digest.update(data)
+      @offset += data.bytesize
+    end
+
+    def prepare_pack_list(rev_list)
+      @pack_list = []
+      @progress&.start("Counting objects")
+
+      rev_list.each do |object, path|
+        add_to_pack_list(object, path)
+        @progress&.tick
+      end
+      @progress&.stop
+    end
+
+    def add_to_pack_list(object, path)
+      info = @database.load_info(object.oid)
+      @pack_list.push(Entry.new(object.oid, info, path, @allow_ofs))
+    end
+
+    def compress_objects
+      compressor = Compressor.new(@database, @progress)
+      @pack_list.each { |entry| compressor.add(entry) }
+      compressor.build_deltas
+    end
+
+    def write_header
+      header = [SIGNATURE, VERSION, @pack_list.size].pack(HEADER_FORMAT)
+      write(header)
+    end
+
+    def write_entries
+      count = @pack_list.size
+      @progress&.start("Writing objects", count) unless @output == STDOUT
+
+      @pack_list.each { |entry| write_entry(entry) }
+      @progress&.stop
+    end
+
+    def write_entry(entry)
+      write_entry(entry.delta.base) if entry.delta
+
+      return if entry.offset
+      entry.offset = @offset
+
+      object = entry.delta || @database.load_raw(entry.oid)
+
+      header = Numbers::VarIntLE.write(entry.packed_size, 4)
+      header[0] |= entry.packed_type << 4
+
+      write(header.pack("C*"))
+      write(entry.delta_prefix)
+      write(Zlib::Deflate.deflate(object.data, @compression))
+
+      @progress&.tick(@offset)
+    end
+
+  end
+end