canopus 0.1.0

data/lib/canopus/git/diff.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+module Canopus
+  module Git
+    module Diff
+      Edit = Struct.new(:kind, :old_line, :new_line, :text, keyword_init: true)
+      Hunk = Struct.new(:old_start, :old_count, :new_start, :new_count, :edits, keyword_init: true) do
+        def old_text = edits.reject { |edit| edit.kind == :insert }.map(&:text).join
+        def new_text = edits.reject { |edit| edit.kind == :delete }.map(&:text).join
+      end
+      module_function
+
+      # Myers shortest edit script, with matching prefixes/suffixes removed first.
+      def edits(before, after)
+        left = before.is_a?(String) ? before.lines : before
+        right = after.is_a?(String) ? after.lines : after
+        prefix = 0
+        prefix += 1 while prefix < left.length && prefix < right.length && left[prefix] == right[prefix]
+        suffix = 0
+        suffix += 1 while suffix < left.length - prefix && suffix < right.length - prefix && left[-suffix - 1] == right[-suffix - 1]
+        a = left.slice(prefix, left.length - prefix - suffix)
+        b = right.slice(prefix, right.length - prefix - suffix)
+        middle = script(a, b)
+        pairs = left.first(prefix).map { |line| [:equal, line] } + middle + (suffix.zero? ? [] : left.last(suffix).map { |line| [:equal, line] })
+        old_line = new_line = 1
+        pairs.map do |kind, text|
+          edit = Edit.new(kind: kind, old_line: old_line, new_line: new_line, text: text)
+          old_line += 1 unless kind == :insert
+          new_line += 1 unless kind == :delete
+          edit
+        end
+      end
+
+      def hunks(before, after, context: 3)
+        raise ArgumentError, "context must be nonnegative" unless context.is_a?(Integer) && context >= 0
+        changes = edits(before, after)
+        ranges = []
+        changes.each_with_index do |edit, index|
+          next if edit.kind == :equal
+          first = [0, index - context].max
+          last = [changes.length - 1, index + context].min
+          if ranges.last && first <= ranges.last[1] + 1
+            ranges.last[1] = last
+          else
+            ranges << [first, last]
+          end
+        end
+        ranges.map do |first, last|
+          items = changes[first..last]
+          old_count = items.count { |edit| edit.kind != :insert }
+          new_count = items.count { |edit| edit.kind != :delete }
+          Hunk.new(old_start: items.first.old_line - (old_count.zero? ? 1 : 0), old_count: old_count,
+            new_start: items.first.new_line - (new_count.zero? ? 1 : 0), new_count: new_count, edits: items)
+        end
+      end
+
+      def revert(text, hunk)
+        lines = text.lines
+        start = hunk.new_count.zero? ? hunk.new_start : hunk.new_start - 1
+        raise ArgumentError, "hunk no longer applies" unless lines.slice(start, hunk.new_count)&.join == hunk.new_text
+        lines[start, hunk.new_count] = hunk.old_text.lines
+        lines.join
+      end
+
+      def unified(before, after, old_name: "a/file", new_name: "b/file", context: 3)
+        output = +"--- #{old_name}\n+++ #{new_name}\n"
+        hunks(before, after, context: context).each do |hunk|
+          output << "@@ -#{hunk.old_start},#{hunk.old_count} +#{hunk.new_start},#{hunk.new_count} @@\n"
+          hunk.edits.each do |edit|
+            output << {equal: " ", delete: "-", insert: "+"}.fetch(edit.kind) << edit.text
+            output << "\n\\n" unless edit.text.end_with?("\n")
+          end
+        end
+        output
+      end
+
+      def script(left, right)
+        return right.map { |line| [:insert, line] } if left.empty?
+        return left.map { |line| [:delete, line] } if right.empty?
+        # Entirely replaced files have no diagonals to search: avoid O(n*m).
+        return left.map { |line| [:delete, line] } + right.map { |line| [:insert, line] } unless left.intersect?(right)
+        # The linear-space bisect avoids quadratic trace storage on large edits.
+        x, y = bisect(left, right)
+        if !x || (x.zero? && y.zero?) || (x == left.length && y == right.length)
+          return left.map { |line| [:delete, line] } + right.map { |line| [:insert, line] }
+        end
+        # Reuse prefix/suffix trimming in each half through a local edit script.
+        [edits(left[0...x], right[0...y]), edits(left[x..], right[y..])].flatten.map { |edit| [edit.kind, edit.text] }
+      end
+      private_class_method :script
+
+      def bisect(left, right)
+        n, m = left.length, right.length
+        limit = (n + m + 1) / 2
+        forward = {1 => 0}
+        reverse = {1 => 0}
+        delta = n - m
+        odd = delta.odd?
+        (0..limit).each do |depth|
+          (-depth..depth).step(2) do |k|
+            x = k == -depth || (k != depth && forward.fetch(k - 1, -1) < forward.fetch(k + 1, -1)) ? forward.fetch(k + 1, 0) : forward.fetch(k - 1, 0) + 1
+            y = x - k
+            while x < n && y < m && x >= 0 && y >= 0 && left[x] == right[y]
+              x += 1
+              y += 1
+            end
+            forward[k] = x
+            if odd && reverse.key?(delta - k) && x + reverse[delta - k] >= n
+              return [x, y]
+            end
+          end
+          (-depth..depth).step(2) do |k|
+            x = k == -depth || (k != depth && reverse.fetch(k - 1, -1) < reverse.fetch(k + 1, -1)) ? reverse.fetch(k + 1, 0) : reverse.fetch(k - 1, 0) + 1
+            y = x - k
+            while x < n && y < m && x >= 0 && y >= 0 && left[n - x - 1] == right[m - y - 1]
+              x += 1
+              y += 1
+            end
+            reverse[k] = x
+            if !odd && forward.key?(delta - k) && forward[delta - k] + x >= n
+              middle_x = forward[delta - k]
+              return [middle_x, middle_x - (delta - k)]
+            end
+          end
+        end
+        nil
+      end
+      private_class_method :bisect
+    end
+  end
+end

data/lib/canopus/git/index.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require "digest/sha1"
+
+module Canopus
+  module Git
+    class Index
+      Entry = Struct.new(:path, :oid, :mode, :size, :mtime, :mtime_nsec, :ctime, :ctime_nsec,
+        :dev, :ino, :uid, :gid, :stage, :flags, :extended_flags, keyword_init: true)
+      include Enumerable
+      attr_reader :entries, :version, :path
+
+      def initialize(path)
+        @path = path
+        @entries = []
+        @version = 2
+        parse(File.binread(path)) if File.file?(path)
+      end
+
+      def each(&block) = entries.each(&block)
+      def [](path) = entries.find { |entry| entry.path == path && entry.stage.zero? }
+
+      def self.encode(entries)
+        bytes = +"DIRC".b << [2, entries.length].pack("N2")
+        entries.sort_by { |entry| [entry.path.b, entry.stage || 0] }.each do |entry|
+          name = entry.path.b
+          fields = %i[ctime ctime_nsec mtime mtime_nsec dev ino mode uid gid size].map { |field| entry.public_send(field).to_i & 0xffffffff }
+          record = fields.pack("N10") + [entry.oid].pack("H*") + [[name.bytesize, 0xfff].min | ((entry.stage || 0) << 12)].pack("n") + name + "\0"
+          record << "\0" * ((8 - record.bytesize % 8) % 8)
+          bytes << record
+        end
+        bytes + Digest::SHA1.digest(bytes)
+      end
+
+      private
+
+      def parse(bytes)
+        raise CorruptObject, "invalid Git index" unless bytes.bytesize >= 32 && bytes.start_with?("DIRC")
+        raise CorruptObject, "Git index checksum mismatch" unless Digest::SHA1.digest(bytes[0...-20]) == bytes[-20, 20]
+        @version, count = bytes[4, 8].unpack("N2")
+        raise CorruptObject, "unsupported Git index version #{version}" unless [2, 3, 4].include?(version)
+        offset = 12
+        previous = "".b
+        count.times do
+          start = offset
+          raise CorruptObject, "truncated Git index entry" if offset + 62 > bytes.bytesize - 20
+          fields = bytes[offset, 40].unpack("N10")
+          oid = bytes[offset + 40, 20].unpack1("H*")
+          flags = bytes[offset + 60, 2].unpack1("n")
+          offset += 62
+          extended = 0
+          if (flags & 0x4000).positive?
+            raise CorruptObject, "invalid index extended flags" if version == 2 || offset + 2 > bytes.bytesize - 20
+            extended = bytes[offset, 2].unpack1("n")
+            offset += 2
+          end
+          if version == 4
+            byte = bytes.getbyte(offset)
+            raise CorruptObject, "truncated index path prefix" unless byte
+            offset += 1
+            strip = byte & 0x7f
+            while (byte & 0x80).positive?
+              byte = bytes.getbyte(offset)
+              raise CorruptObject, "invalid index path prefix" unless byte && strip <= previous.bytesize
+              offset += 1
+              strip = ((strip + 1) << 7) | (byte & 0x7f)
+            end
+            raise CorruptObject, "index path prefix outside previous path" if strip > previous.bytesize
+          end
+          ending = bytes.index("\0", offset)
+          raise CorruptObject, "unterminated index path" unless ending && ending < bytes.bytesize - 20
+          name = bytes[offset...ending]
+          name = previous.byteslice(0, previous.bytesize - strip) + name if version == 4
+          raise CorruptObject, "unsafe index path" if name.empty? || name.start_with?("/") || name.split("/").any? { |part| part == ".." || part == ".git" }
+          previous = name
+          offset = ending + 1
+          offset += (8 - (offset - start) % 8) % 8 unless version == 4
+          values = %i[ctime ctime_nsec mtime mtime_nsec dev ino mode uid gid size].zip(fields).to_h
+          entries << Entry.new(**values, path: name.force_encoding(Encoding::UTF_8), oid: oid,
+            flags: flags, extended_flags: extended, stage: (flags >> 12) & 3)
+        end
+        while offset < bytes.bytesize - 20
+          raise CorruptObject, "truncated index extension" if offset + 8 > bytes.bytesize - 20
+          signature = bytes[offset, 4]
+          size = bytes[offset + 4, 4].unpack1("N")
+          # Lowercase extensions change index interpretation (e.g. split index).
+          raise CorruptObject, "unsupported mandatory index extension #{signature}" if signature[0].match?(/[a-z]/)
+          offset += 8 + size
+          raise CorruptObject, "truncated index extension payload" if offset > bytes.bytesize - 20
+        end
+      end
+    end
+  end
+end

data/lib/canopus/git/object_database.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require_relative "pack"
+
+module Canopus
+  module Git
+    class ObjectDatabase
+      attr_reader :directory
+
+      def initialize(directory)
+        @directory = File.expand_path(directory)
+        @packs = nil
+      end
+
+      def self.hash(type, data) = Digest::SHA1.hexdigest("#{type} #{data.bytesize}\0".b + data.b)
+
+      def read(oid, seen = [])
+        raise ArgumentError, "expected a full SHA-1 object id" unless /\A[0-9a-f]{40}\z/.match?(oid.to_s)
+        raise CorruptObject, "cyclic object reference" if seen.include?(oid) || seen.length > 128
+        loose = File.join(directory, oid[0, 2], oid[2..])
+        object = if File.file?(loose)
+          inflated = Zlib::Inflate.inflate(File.binread(loose))
+          header, data = inflated.split("\0", 2)
+          type, size = header.split(" ", 2)
+          raise CorruptObject, "invalid loose object header" unless %w[commit tree blob tag].include?(type) && size&.match?(/\A\d+\z/) && data && data.bytesize == size.to_i
+          [type, data]
+        else
+          pack = packs.find { |entry| entry.include?(oid) }
+          unless pack
+            @packs = nil # New packs may appear during background GC.
+            pack = packs.find { |entry| entry.include?(oid) }
+          end
+          raise KeyError, "Git object not found: #{oid}" unless pack
+          pack.read(oid) { |base| read(base, seen + [oid]) }
+        end
+        raise CorruptObject, "object SHA-1 mismatch: #{oid}" unless self.class.hash(*object) == oid
+        object
+      rescue Zlib::Error => error
+        raise CorruptObject, error.message
+      end
+
+      def packs = @packs ||= Dir[File.join(directory, "pack", "*.idx")].sort.map { |path| Pack.new(path) }
+    end
+  end
+end

data/lib/canopus/git/pack.rb

@@ -0,0 +1,186 @@
+# frozen_string_literal: true
+
+require "zlib"
+require "digest/sha1"
+require_relative "corrupt_object"
+
+module Canopus
+  module Git
+    class Pack
+      TYPES = {1 => "commit", 2 => "tree", 3 => "blob", 4 => "tag"}.freeze
+      MAX_OBJECT_SIZE = 512 * 1024 * 1024
+      attr_reader :path, :offsets
+
+      def initialize(index_path)
+        @path = index_path.sub(/\.idx\z/, ".pack")
+        @offsets = read_index(File.binread(index_path))
+        File.open(path, "rb") do |file|
+          header = file.read(12)
+          raise CorruptObject, "invalid pack header" unless header&.bytesize == 12 && header[0, 4] == "PACK" && [2, 3].include?(header[4, 4].unpack1("N"))
+          raise CorruptObject, "pack/index object count mismatch" unless header[8, 4].unpack1("N") == offsets.length
+          file.seek(-20, IO::SEEK_END)
+          raise CorruptObject, "pack/index checksum mismatch" unless file.read(20) == @pack_checksum
+        end
+        @cache = {}
+      end
+
+      def include?(oid) = offsets.key?(oid)
+
+      def read(oid, &resolve)
+        offset = offsets[oid]
+        raise KeyError, "object not in pack: #{oid}" unless offset
+        read_at(offset, [], &resolve)
+      end
+
+      def self.apply_delta(base, delta)
+        cursor = 0
+        read_size = lambda do
+          size = shift = 0
+          loop do
+            byte = delta.getbyte(cursor)
+            raise CorruptObject, "truncated delta header" unless byte && shift <= 63
+            cursor += 1
+            size |= (byte & 0x7f) << shift
+            break if (byte & 0x80).zero?
+            shift += 7
+          end
+          size
+        end
+        source_size = read_size.call
+        target_size = read_size.call
+        raise CorruptObject, "delta base size mismatch" unless base.bytesize == source_size
+        raise CorruptObject, "delta too large" if target_size > MAX_OBJECT_SIZE
+        output = +"".b
+        while cursor < delta.bytesize
+          opcode = delta.getbyte(cursor)
+          cursor += 1
+          if (opcode & 0x80).positive?
+            offset = length = 0
+            7.times do |bit|
+              next if (opcode & (1 << bit)).zero?
+              byte = delta.getbyte(cursor)
+              raise CorruptObject, "truncated delta copy" unless byte
+              cursor += 1
+              bit < 4 ? offset |= byte << (bit * 8) : length |= byte << ((bit - 4) * 8)
+            end
+            length = 0x10000 if length.zero?
+            raise CorruptObject, "delta copy outside base" if offset + length > base.bytesize
+            output << base.byteslice(offset, length)
+          elsif opcode.positive?
+            raise CorruptObject, "truncated delta insert" if cursor + opcode > delta.bytesize
+            output << delta.byteslice(cursor, opcode)
+            cursor += opcode
+          else
+            raise CorruptObject, "invalid delta opcode"
+          end
+          raise CorruptObject, "delta exceeds target size" if output.bytesize > target_size
+        end
+        raise CorruptObject, "delta target size mismatch" unless output.bytesize == target_size
+        output
+      end
+
+      private
+
+      def read_index(bytes)
+        raise CorruptObject, "truncated pack index" if bytes.bytesize < 1064
+        raise CorruptObject, "pack index checksum mismatch" unless Digest::SHA1.digest(bytes[0...-20]) == bytes[-20, 20]
+        @pack_checksum = bytes[-40, 20]
+        version = bytes.start_with?("\xfftOc".b) ? bytes[4, 4].unpack1("N") : 1
+        raise CorruptObject, "unsupported pack index version #{version}" unless [1, 2].include?(version)
+        start = version == 1 ? 0 : 8
+        fanout = bytes[start, 1024].unpack("N*")
+        raise CorruptObject, "invalid pack index fanout" unless fanout.each_cons(2).all? { |a, b| a <= b }
+        count = fanout.last
+        cursor = start + 1024
+        minimum = cursor + count * (version == 1 ? 24 : 28) + 40
+        raise CorruptObject, "truncated pack index entries" if minimum > bytes.bytesize
+        if version == 1
+          return count.times.to_h do |index|
+            position = cursor + index * 24
+            [bytes[position + 4, 20].unpack1("H*"), bytes[position, 4].unpack1("N")]
+          end
+        end
+        names = cursor
+        positions = cursor + count * 24
+        large_positions = positions + count * 4
+        count.times.to_h do |index|
+          offset = bytes[positions + index * 4, 4].unpack1("N")
+          if offset >= 0x80000000
+            location = large_positions + (offset & 0x7fffffff) * 8
+            raise CorruptObject, "truncated 64-bit pack offset" if location + 8 > bytes.bytesize - 40
+            offset = bytes[location, 8].unpack1("Q>")
+          end
+          [bytes[names + index * 20, 20].unpack1("H*"), offset]
+        end
+      end
+
+      def read_at(offset, stack, &resolve)
+        return @cache[offset] if @cache.key?(offset)
+        raise CorruptObject, "cyclic or excessive pack delta chain" if stack.include?(offset) || stack.length > 128
+        stack = stack + [offset]
+        type = data = base_offset = base_oid = nil
+        File.open(path, "rb") do |file|
+          raise CorruptObject, "object offset outside pack" unless offset >= 12 && offset < file.size - 20
+          file.seek(offset)
+          byte = file.getbyte
+          type = (byte >> 4) & 7
+          size = byte & 15
+          shift = 4
+          while (byte & 0x80).positive?
+            byte = file.getbyte
+            raise CorruptObject, "truncated pack object size" unless byte && shift <= 63
+            size |= (byte & 0x7f) << shift
+            shift += 7
+          end
+          raise CorruptObject, "pack object too large" if size > MAX_OBJECT_SIZE
+          if type == 6
+            byte = file.getbyte
+            raise CorruptObject, "truncated delta offset" unless byte
+            distance = byte & 0x7f
+            count = 0
+            while (byte & 0x80).positive?
+              byte = file.getbyte
+              count += 1
+              raise CorruptObject, "invalid delta offset" unless byte && count <= 9
+              distance = ((distance + 1) << 7) | (byte & 0x7f)
+            end
+            base_offset = offset - distance
+            raise CorruptObject, "invalid delta base offset" unless base_offset >= 12 && base_offset < offset
+          elsif type == 7
+            raw = file.read(20)
+            raise CorruptObject, "truncated delta reference" unless raw&.bytesize == 20
+            base_oid = raw.unpack1("H*")
+          elsif !TYPES.key?(type)
+            raise CorruptObject, "invalid packed object type #{type}"
+          end
+          inflater = Zlib::Inflate.new
+          begin
+            data = +"".b
+            until inflater.finished?
+              chunk = file.read(16_384)
+              raise CorruptObject, "truncated compressed object" unless chunk
+              inflater.inflate(chunk) do |part|
+                data << part
+                raise CorruptObject, "packed object exceeds declared size" if data.bytesize > size
+              end
+            end
+          rescue Zlib::Error => error
+            raise CorruptObject, error.message
+          ensure
+            inflater.close
+          end
+          raise CorruptObject, "packed object size mismatch" unless data.bytesize == size
+        end
+        object = if base_offset || base_oid
+          base_type, base = base_offset ? read_at(base_offset, stack, &resolve) : resolve.call(base_oid)
+          [base_type, self.class.apply_delta(base, data)]
+        else
+          [TYPES.fetch(type), data]
+        end
+        # ponytail: bound object cache by count; byte budgeting if large blobs dominate.
+        @cache.shift if @cache.length >= 128
+        @cache[offset] = object.map(&:freeze).freeze
+      end
+    end
+  end
+end