jit 0.0.0 → 1.0.0

data/lib/pack/delta.rb

@@ -0,0 +1,58 @@
+require "forwardable"
+
+require_relative "./numbers"
+require_relative "./xdelta"
+
+module Pack
+  class Delta
+
+    Copy = Struct.new(:offset, :size) do
+      def self.parse(input, byte)
+        value  = Numbers::PackedInt56LE.read(input, byte)
+        offset = value & 0xffffffff
+        size   = value >> 32
+
+        Copy.new(offset, size)
+      end
+
+      def to_s
+        bytes = Numbers::PackedInt56LE.write((size << 32) | offset)
+        bytes[0] |= 0x80
+        bytes.pack("C*")
+      end
+    end
+
+    Insert = Struct.new(:data) do
+      def self.parse(input, byte)
+        Insert.new(input.read(byte))
+      end
+
+      def to_s
+        [data.bytesize, data].pack("Ca*")
+      end
+    end
+
+    extend Forwardable
+    def_delegator :@data, :bytesize, :size
+
+    attr_reader :base, :data
+
+    def initialize(source, target)
+      @base = source.entry
+      @data = sizeof(source) + sizeof(target)
+
+      source.delta_index ||= XDelta.create_index(source.data)
+
+      delta = source.delta_index.compress(target.data)
+      delta.each { |op| @data.concat(op.to_s) }
+    end
+
+    private
+
+    def sizeof(entry)
+      bytes = Numbers::VarIntLE.write(entry.size, 7)
+      bytes.pack("C*")
+    end
+
+  end
+end

data/lib/pack/entry.rb

@@ -0,0 +1,54 @@
+require "forwardable"
+require_relative "./numbers"
+
+module Pack
+  class Entry
+
+    extend Forwardable
+    def_delegators :@info, :type, :size
+
+    attr_accessor :offset
+    attr_reader :oid, :delta, :depth
+
+    def initialize(oid, info, path, ofs = false)
+      @oid   = oid
+      @info  = info
+      @path  = path
+      @ofs   = ofs
+      @delta = nil
+      @depth = 0
+    end
+
+    def sort_key
+      [packed_type, @path&.basename, @path&.dirname, @info.size]
+    end
+
+    def assign_delta(delta)
+      @delta = delta
+      @depth = delta.base.depth + 1
+    end
+
+    def packed_type
+      if @delta
+        @ofs ? OFS_DELTA : REF_DELTA
+      else
+        TYPE_CODES.fetch(@info.type)
+      end
+    end
+
+    def packed_size
+      @delta ? @delta.size : @info.size
+    end
+
+    def delta_prefix
+      return "" unless @delta
+
+      if @ofs
+        Numbers::VarIntBE.write(offset - @delta.base.offset)
+      else
+        [@delta.base.oid].pack("H40")
+      end
+    end
+
+  end
+end

data/lib/pack/expander.rb

@@ -0,0 +1,54 @@
+require "stringio"
+
+require_relative "./delta"
+require_relative "./numbers"
+
+module Pack
+  class Expander
+
+    attr_reader :source_size, :target_size
+
+    def self.expand(source, delta)
+      new(delta).expand(source)
+    end
+
+    def initialize(delta)
+      @delta = StringIO.new(delta)
+
+      @source_size = read_size
+      @target_size = read_size
+    end
+
+    def expand(source)
+      check_size(source, @source_size)
+      target = ""
+
+      until @delta.eof?
+        byte = @delta.readbyte
+
+        if byte < 0x80
+          insert = Delta::Insert.parse(@delta, byte)
+          target.concat(insert.data)
+        else
+          copy = Delta::Copy.parse(@delta, byte)
+          size = (copy.size == 0) ? GIT_MAX_COPY : copy.size
+          target.concat(source.byteslice(copy.offset, size))
+        end
+      end
+
+      check_size(target, @target_size)
+      target
+    end
+
+    private
+
+    def read_size
+      Numbers::VarIntLE.read(@delta, 7)[1]
+    end
+
+    def check_size(buffer, size)
+      raise "failed to apply delta" unless buffer.bytesize == size
+    end
+
+  end
+end

data/lib/pack/index.rb

@@ -0,0 +1,100 @@
+module Pack
+  class Index
+
+    HEADER_SIZE = 8
+    FANOUT_SIZE = 1024
+
+    OID_LAYER = 2
+    CRC_LAYER = 3
+    OFS_LAYER = 4
+    EXT_LAYER = 5
+
+    SIZES = {
+      OID_LAYER => 20,
+      CRC_LAYER => 4,
+      OFS_LAYER => 4,
+      EXT_LAYER => 8
+    }
+
+    def initialize(input)
+      @input = input
+      load_fanout_table
+    end
+
+    def oid_offset(oid)
+      pos = oid_position(oid)
+      return nil if pos < 0
+
+      offset = read_int32(OFS_LAYER, pos)
+
+      return offset if offset < IDX_MAX_OFFSET
+
+      pos = offset & (IDX_MAX_OFFSET - 1)
+      @input.seek(offset_for(EXT_LAYER, pos))
+      @input.read(8).unpack("Q>").first
+    end
+
+    def prefix_match(name)
+      pos = oid_position(name)
+      return [name] unless pos < 0
+
+      @input.seek(offset_for(OID_LAYER, -1 - pos))
+      oids = []
+
+      loop do
+        oid = @input.read(20).unpack("H40").first
+        return oids unless oid.start_with?(name)
+        oids << oid
+      end
+    end
+
+    private
+
+    def load_fanout_table
+      @input.seek(HEADER_SIZE)
+      @fanout = @input.read(FANOUT_SIZE).unpack("N256")
+    end
+
+    def oid_position(oid)
+      prefix = oid[0..1].to_i(16)
+      packed = [oid].pack("H40")
+
+      low  = (prefix == 0) ? 0 : @fanout[prefix - 1]
+      high = @fanout[prefix] - 1
+
+      binary_search(packed, low, high)
+    end
+
+    def read_int32(layer, pos)
+      @input.seek(offset_for(layer, pos))
+      @input.read(4).unpack("N").first
+    end
+
+    def offset_for(layer, pos)
+      offset = HEADER_SIZE + FANOUT_SIZE
+      count  = @fanout.last
+
+      SIZES.each { |n, size| offset += size * count if n < layer }
+
+      offset + pos * SIZES[layer]
+    end
+
+    def binary_search(target, low, high)
+      while low <= high
+        mid = (low + high) / 2
+
+        @input.seek(offset_for(OID_LAYER, mid))
+        oid = @input.read(20)
+
+        case oid <=> target
+        when -1 then low = mid + 1
+        when  0 then return mid
+        when  1 then high = mid - 1
+        end
+      end
+
+      -1 - low
+    end
+
+  end
+end

data/lib/pack/indexer.rb

@@ -0,0 +1,200 @@
+require "digest/sha1"
+require "zlib"
+
+require_relative "./expander"
+require_relative "./reader"
+require_relative "../temp_file"
+
+module Pack
+  class Indexer
+
+    class PackFile
+      attr_reader :digest
+
+      def initialize(pack_dir, name)
+        @file   = TempFile.new(pack_dir, name)
+        @digest = Digest::SHA1.new
+      end
+
+      def write(data)
+        @file.write(data)
+        @digest.update(data)
+      end
+
+      def move(name)
+        @file.write(@digest.digest)
+        @file.move(name)
+      end
+    end
+
+    def initialize(database, reader, stream, progress)
+      @database = database
+      @reader   = reader
+      @stream   = stream
+      @progress = progress
+
+      @index   = {}
+      @pending = Hash.new { |hash, oid| hash[oid] = [] }
+
+      @pack_file  = PackFile.new(@database.pack_path, "tmp_pack")
+      @index_file = PackFile.new(@database.pack_path, "tmp_idx")
+    end
+
+    def process_pack
+      write_header
+      write_objects
+      write_checksum
+
+      resolve_deltas
+      write_index
+    end
+
+    private
+
+    def write_header
+      header = [SIGNATURE, VERSION, @reader.count].pack(HEADER_FORMAT)
+      @pack_file.write(header)
+    end
+
+    def write_objects
+      @progress&.start("Receiving objects", @reader.count)
+
+      @reader.count.times do
+        index_object
+        @progress&.tick(@stream.offset)
+      end
+      @progress&.stop
+    end
+
+    def index_object
+      offset       = @stream.offset
+      record, data = @stream.capture { @reader.read_record }
+      crc32        = Zlib.crc32(data)
+
+      @pack_file.write(data)
+
+      case record
+      when Record
+        oid = @database.hash_object(record)
+        @index[oid] = [offset, crc32]
+      when OfsDelta
+        @pending[offset - record.base_ofs].push([offset, crc32])
+      when RefDelta
+        @pending[record.base_oid].push([offset, crc32])
+      end
+    end
+
+    def write_checksum
+      @stream.verify_checksum
+
+      filename = "pack-#{ @pack_file.digest.hexdigest }.pack"
+      @pack_file.move(filename)
+
+      path    = @database.pack_path.join(filename)
+      @pack   = File.open(path, File::RDONLY)
+      @reader = Reader.new(@pack)
+    end
+
+    def read_record_at(offset)
+      @pack.seek(offset)
+      @reader.read_record
+    end
+
+    def resolve_deltas
+      deltas = @pending.reduce(0) { |n, (_, list)| n + list.size }
+      @progress&.start("Resolving deltas", deltas)
+
+      @index.to_a.each do |oid, (offset, _)|
+        record = read_record_at(offset)
+
+        resolve_delta_base(record, offset)
+        resolve_delta_base(record, oid)
+      end
+      @progress&.stop
+    end
+
+    def resolve_delta_base(record, key)
+      pending = @pending.delete(key)
+      return unless pending
+
+      pending.each do |offset, crc32|
+        resolve_pending(record, offset, crc32)
+      end
+    end
+
+    def resolve_pending(record, offset, crc32)
+      delta  = read_record_at(offset)
+      data   = Expander.expand(record.data, delta.delta_data)
+      object = Record.new(record.type, data)
+      oid    = @database.hash_object(object)
+
+      @index[oid] = [offset, crc32]
+      @progress&.tick
+
+      resolve_delta_base(object, offset)
+      resolve_delta_base(object, oid)
+    end
+
+    def write_index
+      @object_ids = @index.keys.sort
+
+      write_object_table
+      write_crc32
+      write_offsets
+      write_index_checksum
+    end
+
+    def write_object_table
+      header = [IDX_SIGNATURE, VERSION].pack("N2")
+      @index_file.write(header)
+
+      counts = Array.new(256, 0)
+      total  = 0
+
+      @object_ids.each { |oid| counts[oid[0..1].to_i(16)] += 1 }
+
+      counts.each do |count|
+        total += count
+        @index_file.write([total].pack("N"))
+      end
+
+      @object_ids.each do |oid|
+        @index_file.write([oid].pack("H40"))
+      end
+    end
+
+    def write_crc32
+      @object_ids.each do |oid|
+        crc32 = @index[oid].last
+        @index_file.write([crc32].pack("N"))
+      end
+    end
+
+    def write_offsets
+      large_offsets = []
+
+      @object_ids.each do |oid|
+        offset = @index[oid].first
+
+        unless offset < IDX_MAX_OFFSET
+          large_offsets.push(offset)
+          offset = IDX_MAX_OFFSET | (large_offsets.size - 1)
+        end
+        @index_file.write([offset].pack("N"))
+      end
+
+      large_offsets.each do |offset|
+        @index_file.write([offset].pack("Q>"))
+      end
+    end
+
+    def write_index_checksum
+      pack_digest = @pack_file.digest
+      @index_file.write(pack_digest.digest)
+
+      filename = "pack-#{ pack_digest.hexdigest }.idx"
+      @index_file.move(filename)
+    end
+
+  end
+end