thuban 0.3.0 → 0.4.0

data/lib/thuban/remote/push.rb

@@ -0,0 +1,250 @@
+# frozen_string_literal: true
+
+require "stringio"
+
+module Thuban
+  module Remote
+    class Connection
+      MAX_PUSH_UPDATES = 1024
+      MAX_PUSH_RESPONSE_SIZE = 8 * 1024 * 1024
+      ZERO_OID = "0" * 40
+
+      def push(repository, updates, atomic: false)
+        ensure_open
+        raise TypeError, "expected Thuban::Repository" unless repository.is_a?(Repository)
+        raise ArgumentError, "atomic must be true or false" unless [true, false].include?(atomic)
+
+        updates = normalize_push_updates(repository, updates)
+        advertised = receive_refs
+        updates.each do |update|
+          current = advertised.find { |ref| ref.name == update[:ref] }&.oid || ZERO_OID
+          expected = update[:old] || current
+          raise TransportError, "remote reference changed: #{update[:ref]}" unless current == expected
+
+          update[:old] = expected
+        end
+        capabilities = push_capabilities(atomic, updates)
+        objects = push_objects(repository, updates.map { |update| update[:new] }, advertised)
+        output = StringIO.new(Protocol.commands(updates, capabilities))
+        output.seek(0, IO::SEEK_END)
+        Pack.write(output, objects) do |current, total|
+          ensure_open
+          yield Progress.new(phase: :pack, current: current, total: total, bytes: output.pos) if block_given?
+        end if updates.any? { |update| update[:new] != ZERO_OID }
+
+        response = request(:post, "/git-receive-pack", body: output.string,
+          headers: {"Content-Type" => "application/x-git-receive-pack-request",
+                    "Accept" => "application/x-git-receive-pack-result"},
+          content_type: "application/x-git-receive-pack-result", limit: MAX_PUSH_RESPONSE_SIZE)
+        parse_push_response(response, updates, capabilities) do |progress|
+          yield progress if block_given?
+        end
+        updates.map { |update| Ref.new(name: update[:ref], oid: update[:new] == ZERO_OID ? nil : update[:new]) }
+      ensure
+        @receive_refs = nil
+      end
+
+      private
+
+      def receive_refs
+        return @receive_refs if @receive_refs
+
+        fetch_capabilities = @capabilities
+        begin
+          body = request(:get, "/info/refs", query: "service=git-receive-pack",
+            headers: {"Accept" => "application/x-git-receive-pack-advertisement"},
+            content_type: "application/x-git-receive-pack-advertisement", limit: MAX_ADVERTISEMENT_SIZE)
+          reader = Protocol::Reader.new(StringIO.new(body), max_bytes: MAX_ADVERTISEMENT_SIZE)
+          first = reader.read
+          if first == "# service=git-receive-pack\n"
+            raise TransportError, "invalid receive-pack advertisement" unless reader.read == Protocol::FLUSH
+            first = reader.read
+          end
+          refs = read_v0_refs(reader, first)
+          @receive_capabilities = @capabilities
+          @receive_refs = refs
+        ensure
+          @capabilities = fetch_capabilities
+        end
+      end
+
+      def normalize_push_updates(repository, updates)
+        raise TypeError, "updates must be enumerable" unless updates.respond_to?(:each)
+
+        result = []
+        updates.each do |entry|
+          raise ArgumentError, "updates must be [ref, old_oid, new_oid]" unless entry.is_a?(Array) && entry.length == 3
+          raise ArgumentError, "too many push updates" if result.length >= MAX_PUSH_UPDATES
+
+          ref, old_oid, new_oid = entry
+          ref = Protocol.validate_ref(ref)
+          raise TransportError, "HEAD cannot be updated directly" if ref == "HEAD"
+          old_oid = Protocol.validate_oid(old_oid) unless old_oid.nil?
+          new_oid = Protocol.validate_oid(new_oid)
+          raise ArgumentError, "object to push was not found: #{new_oid}" if new_oid != ZERO_OID && !repository.odb.exist?(new_oid)
+          raise ArgumentError, "duplicate push destination: #{ref}" if result.any? { |update| update[:ref] == ref }
+
+          result << {ref: ref, old: old_oid, new: new_oid}
+        end
+        raise ArgumentError, "at least one push update is required" if result.empty?
+
+        result
+      end
+
+      def push_capabilities(atomic, updates)
+        advertised = @receive_capabilities
+        status = %w[report-status-v2 report-status].find { |capability| advertised.include?(capability) }
+        raise TransportError, "server does not report push status" unless status
+        raise TransportError, "server does not support reference deletion" if updates.any? { |update| update[:new] == ZERO_OID } && !advertised.include?("delete-refs")
+        raise TransportError, "server does not support atomic push" if atomic && !advertised.include?("atomic")
+
+        [status, ("side-band-64k" if advertised.include?("side-band-64k")), ("atomic" if atomic)].compact
+      end
+
+      def push_objects(repository, new_oids, advertised)
+        excluded = advertised.flat_map { |ref| [ref.oid, ref.peeled] }.compact.to_h { |oid| [oid, true] }
+        seen = {}
+        objects = []
+        pending = new_oids.reject { |oid| oid == ZERO_OID }
+        until pending.empty?
+          ensure_open
+          oid = pending.pop
+          next if seen[oid] || excluded[oid]
+
+          seen[oid] = true
+          type, data = repository.odb.read(oid)
+          objects << [type, data]
+          pending.concat(referenced_oids(type, data))
+        end
+        objects
+      end
+
+      def referenced_oids(type, data)
+        case type
+        when "commit"
+          data.split("\n\n", 2).first.lines.filter_map do |line|
+            oid = line[/\A(?:tree|parent) ([0-9a-f]{40})\n?\z/, 1]
+            oid && Protocol.validate_oid(oid)
+          end
+        when "tag"
+          oid = data[/\Aobject ([0-9a-f]{40})\n/, 1]
+          oid ? [Protocol.validate_oid(oid)] : []
+        when "tree"
+          tree_oids(data)
+        else
+          []
+        end
+      end
+
+      def tree_oids(data)
+        result = []
+        offset = 0
+        while offset < data.bytesize
+          ending = data.index("\0", offset)
+          raise CorruptObject, "invalid tree object" unless ending && ending + 21 <= data.bytesize
+
+          result << data.byteslice(ending + 1, 20).unpack1("H*")
+          offset = ending + 21
+        end
+        result
+      end
+
+      def parse_push_response(body, updates, capabilities)
+        packets = read_push_packets(StringIO.new(body), capabilities.include?("side-band-64k")) do |text|
+          yield Progress.new(phase: :remote, current: nil, total: nil, bytes: text.bytesize)
+        end
+        unpack = packets.shift
+        raise TransportError, "push response omitted unpack status" unless unpack&.start_with?("unpack ")
+        raise TransportError, "remote unpack failed: #{safe_message(unpack.delete_prefix("unpack "))}" unless unpack == "unpack ok\n"
+
+        statuses = {}
+        report_v2 = capabilities.include?("report-status-v2")
+        current_ref = nil
+        options = {}
+        packets.each do |packet|
+          state, ref, reason = packet.chomp.split(" ", 3)
+          if state == "option"
+            raise TransportError, "invalid push option" unless report_v2 && current_ref && ref && !options[ref]
+
+            validate_push_option(ref, reason)
+            options[ref] = true
+            next
+          end
+          valid = state == "ok" ? ref && reason.nil? : state == "ng" && ref && reason && !reason.empty?
+          raise TransportError, "invalid push status" unless valid
+
+          ref = Protocol.validate_ref(ref)
+          raise TransportError, "duplicate push status" if statuses.key?(ref)
+
+          statuses[ref] = [state, reason]
+          current_ref = state == "ok" ? ref : nil
+          options = {}
+        end
+        expected_refs = updates.map { |update| update[:ref] }
+        raise TransportError, "push status included an unknown reference" unless (statuses.keys - expected_refs).empty?
+
+        updates.each_with_index do |update, index|
+          state, reason = statuses.fetch(update[:ref]) { raise TransportError, "push status missing for #{update[:ref]}" }
+          raise TransportError, "push rejected for #{update[:ref]}: #{safe_message(reason.to_s)}" unless state == "ok"
+          yield Progress.new(phase: :push, current: index + 1, total: updates.length, bytes: nil)
+        end
+      end
+
+      def validate_push_option(name, value)
+        case name
+        when "refname" then Protocol.validate_ref(value)
+        when "old-oid", "new-oid" then Protocol.validate_oid(value)
+        when "forced-update" then raise TransportError, "invalid push option" unless value.nil?
+        else raise TransportError, "invalid push option"
+        end
+      end
+
+      def read_push_packets(input, sideband)
+        reader = Protocol::Reader.new(input, max_bytes: MAX_PUSH_RESPONSE_SIZE)
+        status = +"".b
+        result = []
+        loop do
+          packet = reader.read
+          raise TransportError, "truncated push response" if packet.nil?
+          break if packet == Protocol::FLUSH
+          raise TransportError, "invalid push response" unless packet.is_a?(String)
+
+          unless sideband
+            result << packet
+            next
+          end
+          band = packet.getbyte(0)
+          data = packet.byteslice(1..).to_s
+          case band
+          when 1 then status << data
+          when 2 then yield safe_message(data)
+          when 3 then raise TransportError, "remote error: #{safe_message(data)}"
+          else raise TransportError, "invalid sideband channel"
+          end
+        end
+        return result unless sideband
+
+        nested = Protocol::Reader.new(StringIO.new(status), max_bytes: MAX_PUSH_RESPONSE_SIZE)
+        loop do
+          packet = nested.read
+          raise TransportError, "truncated push status" if packet.nil?
+          break if packet == Protocol::FLUSH
+          raise TransportError, "invalid push status" unless packet.is_a?(String)
+          result << packet
+        end
+        raise TransportError, "invalid push status" if nested.read
+
+        result
+      end
+    end
+
+    module Protocol
+      def self.commands(updates, capabilities)
+        updates.each_with_index.map do |update, index|
+          suffix = index.zero? && !capabilities.empty? ? "\0#{capabilities.join(" ")}" : ""
+          packet("#{update.fetch(:old)} #{update.fetch(:new)} #{update.fetch(:ref)}#{suffix}\n")
+        end.join + flush
+      end
+    end
+  end
+end

data/lib/thuban/remote/repository.rb

@@ -0,0 +1,223 @@
+# frozen_string_literal: true
+
+module Thuban
+  class Repository
+    def remotes
+      remote_settings.transform_values { |settings| settings[:url] }.compact
+    end
+
+    def fetch(remote = "origin", refspecs: nil, depth: nil, filter: nil)
+      depth = Remote::Protocol.validate_depth(depth)
+      filter = Remote::Protocol.validate_filter(filter)
+      settings = remote_settings[remote]
+      direct = direct_remote?(remote)
+      url = settings&.fetch(:url, nil) || (remote if direct)
+      raise ArgumentError, "unknown remote: #{remote}" unless url
+      url = normalize_remote_url(url)
+
+      specs = refspecs
+      if specs.nil? && settings
+        specs = settings[:fetch]
+        specs = ["+refs/heads/*:refs/remotes/#{remote}/*"] if specs.empty?
+      end
+      specs = normalize_fetch_refspecs(specs)
+      connection = Remote.open(url)
+      advertised = connection.refs
+      updates = select_fetch_refs(advertised, specs)
+      wants = updates.map { |ref, _| ref.oid }.compact.uniq
+      previous = updates.filter_map { |_, destination| [destination, resolve(destination)] if destination }.to_h
+      unless wants.empty?
+        haves = refs.values.compact.uniq.select { |oid| odb.exist?(oid) }
+        connection.fetch(self, wants: wants, haves: haves, depth: depth, filter: filter) do |progress|
+          yield progress if block_given?
+        end
+        record_promisor(remote, filter) if filter && settings
+        updates.each do |ref, destination|
+          update_ref(destination, ref.oid, old_oid: previous.fetch(destination), message: "fetch #{remote}: #{ref.name}") if destination && ref.oid
+        end
+      end
+      advertised
+    ensure
+      connection&.close
+    end
+
+    def push(remote = "origin", refspecs:, force: false, lease: nil, atomic: false)
+      settings = remote_settings[remote]
+      direct = direct_remote?(remote)
+      url = settings&.fetch(:pushurl, nil) || settings&.fetch(:url, nil) || (remote if direct)
+      raise ArgumentError, "unknown remote: #{remote}" unless url
+      url = normalize_remote_url(url)
+      raise ArgumentError, "force must be true or false" unless [true, false].include?(force)
+      raise ArgumentError, "atomic must be true or false" unless [true, false].include?(atomic)
+
+      connection = Remote.open(url)
+      advertised = connection.send(:receive_refs)
+      updates = push_updates(refspecs, force: force, lease: lease, advertised: advertised)
+      connection.push(self, updates, atomic: atomic) { |progress| yield progress if block_given? }
+    ensure
+      connection&.close
+    end
+
+    private
+
+    def direct_remote?(remote)
+      value = remote.to_s
+      value.match?(/\A(?:https?|ssh|file):\/\//i) || (!value.include?("://") && value.include?(":")) ||
+        value.start_with?("/", "./", "../", "~") || File.exist?(File.expand_path(value, root || common_dir))
+    end
+
+    def normalize_remote_url(url)
+      local = !url.include?("://") && (url.start_with?("/", "./", "../", "~") ||
+        !url.include?(":") || url.match?(/\A[A-Za-z]:[\\\/]/))
+      local ? File.expand_path(url, root || common_dir) : url
+    end
+
+    def remote_settings
+      result = {}
+      current = nil
+      File.foreach(File.join(common_dir, "config"), encoding: "UTF-8") do |line|
+        stripped = line.strip
+        if (match = stripped.match(/\A\[remote\s+"([^"\r\n]+)"\]\z/i))
+          current = match[1]
+          result[current] ||= {fetch: [], push: []}
+        elsif stripped.start_with?("[")
+          current = nil
+        elsif current && (match = stripped.match(/\A(url|pushurl|fetch|push)\s*=\s*(.*)\z/i))
+          key = match[1].downcase.to_sym
+          value = IgnoreMatcher.send(:parse_config_value, match[2])
+          %i[fetch push].include?(key) ? result[current][key] << value : result[current][key] = value
+        end
+      end
+      result
+    rescue Errno::ENOENT, Errno::EACCES
+      {}
+    end
+
+    def record_promisor(remote, filter)
+      path = File.join(common_dir, "config")
+      raise ArgumentError, "unsafe Git config" if File.symlink?(path) || File.symlink?(path + ".lock")
+      lock = File.open(path + ".lock", File::WRONLY | File::CREAT | File::EXCL | File::BINARY, 0o644)
+      lines = File.readlines(path, mode: "rb")
+      start = lines.index { |line| line.strip.match?(/\A\[remote\s+"#{Regexp.escape(remote)}"\]\z/i) }
+      raise RefLockError, "remote configuration changed during fetch" unless start
+
+      finish = ((start + 1)...lines.length).find { |index| lines[index].lstrip.start_with?("[") } || lines.length
+      body = lines[(start + 1)...finish].reject { |line| line.strip.match?(/\A(?:promisor|partialclonefilter)\s*=/i) }
+      body[-1] = body[-1] + "\n" if body.any? && !body[-1].end_with?("\n")
+      body << "\tpromisor = true\n" << "\tpartialclonefilter = #{filter}\n"
+      lock.chmod(File.stat(path).mode & 0o777)
+      lock.write((lines[0..start] + body + lines[finish..].to_a).join)
+      lock.flush
+      lock.fsync
+      lock.close
+      File.rename(lock.path, path)
+    rescue Errno::EEXIST
+      raise RefLockError, "Git config is locked"
+    ensure
+      lock&.close unless lock&.closed?
+      File.unlink(lock.path) if lock && File.exist?(lock.path)
+    end
+
+    def normalize_fetch_refspecs(refspecs)
+      return if refspecs.nil?
+
+      refspecs = [refspecs] if refspecs.is_a?(String)
+      raise TypeError, "refspecs must be enumerable" unless refspecs.respond_to?(:each)
+
+      refspecs.map do |source|
+        raise ArgumentError, "invalid fetch refspec" unless source.is_a?(String)
+
+        source = source.delete_prefix("+")
+        from, to = source.split(":", 2)
+        valid = valid_refspec_name?(from) && (!to || valid_refspec_name?(to)) && from.count("*") <= 1 && to.to_s.count("*") <= 1 &&
+          from.include?("*") == to.to_s.include?("*")
+        raise ArgumentError, "invalid fetch refspec" unless valid
+
+        [/\A#{Regexp.escape(from).sub("\\*", "(.*)")}\z/, to]
+      end
+    end
+
+    def valid_refspec_name?(name)
+      name.is_a?(String) && name.start_with?("refs/") && !name.end_with?("/", ".") &&
+        !name.include?("..") && !name.include?("@{") && !name.match?(/[\x00-\x20\x7f~^:?\[\\]/) &&
+        name.split("/").none? { |part| part.empty? || part.start_with?(".") || part.downcase.end_with?(".lock") }
+    end
+
+    def select_fetch_refs(advertised, refspecs)
+      return advertised.map { |ref| [ref, nil] } if refspecs.nil?
+
+      refspecs.flat_map do |pattern, destination|
+        advertised.filter_map do |ref|
+          match = pattern.match(ref.name)
+          [ref, destination&.sub("*", match[1].to_s)] if match
+        end
+      end
+    end
+
+    def push_updates(refspecs, force:, lease:, advertised:)
+      specs = refspecs.is_a?(String) ? [refspecs] : refspecs
+      raise TypeError, "refspecs must be enumerable" unless specs.respond_to?(:each)
+
+      entries = specs.flat_map { |spec| expand_push_refspec(spec, force: force) }
+      raise ArgumentError, "at least one push refspec is required" if entries.empty?
+      expected = push_leases(lease, entries)
+      entries.map do |entry|
+        current = advertised.find { |ref| ref.name == entry[:destination] }&.oid || Remote::Connection::ZERO_OID
+        old_oid = expected.fetch(entry[:destination], current)
+        raise TransportError, "remote reference changed: #{entry[:destination]}" unless old_oid == current
+        unless entry[:forced] || expected.key?(entry[:destination]) || entry[:new_oid] == Remote::Connection::ZERO_OID
+          if current != Remote::Connection::ZERO_OID && current != entry[:new_oid] && (!entry[:destination].start_with?("refs/heads/") ||
+              !odb.exist?(current) || merge_base(current, entry[:new_oid]) != current)
+            raise TransportError, "non-fast-forward push requires force or lease: #{entry[:destination]}"
+          end
+        end
+        [entry[:destination], old_oid, entry[:new_oid]]
+      end
+    end
+
+    def expand_push_refspec(spec, force:)
+      raise ArgumentError, "invalid push refspec" unless spec.is_a?(String)
+
+      forced = force || spec.start_with?("+")
+      source, destination = spec.delete_prefix("+").split(":", 2)
+      valid = destination && valid_refspec_name?(destination) && source.count("*") <= 1 && destination.count("*") <= 1 &&
+        source.include?("*") == destination.include?("*")
+      raise ArgumentError, "invalid push refspec" unless valid
+
+      if source.empty?
+        raise ArgumentError, "wildcard deletion is not supported" if destination.include?("*")
+        return [{destination: destination, new_oid: Remote::Connection::ZERO_OID, forced: forced}]
+      end
+      if source.include?("*")
+        raise ArgumentError, "invalid push refspec" unless valid_refspec_name?(source)
+        pattern = /\A#{Regexp.escape(source).sub("\\*", "(.*)")}\z/
+        return refs.filter_map do |name, oid|
+          match = pattern.match(name)
+          {destination: destination.sub("*", match[1]), new_oid: oid, forced: forced} if match && oid
+        end
+      end
+
+      new_oid = resolve(source)
+      raise ArgumentError, "unknown push source: #{source}" unless new_oid && odb.exist?(new_oid)
+
+      [{destination: destination, new_oid: new_oid, forced: forced}]
+    end
+
+    def push_leases(lease, entries)
+      return {} if lease.nil?
+      if lease.is_a?(String)
+        raise ArgumentError, "a scalar lease requires one refspec" unless entries.length == 1
+        return {entries.first[:destination] => Remote::Protocol.validate_oid(lease)}
+      end
+      raise TypeError, "lease must be an object ID or a ref-to-object-ID Hash" unless lease.is_a?(Hash)
+
+      lease.to_h do |name, oid|
+        raise ArgumentError, "lease contains an invalid reference" unless valid_refspec_name?(name)
+        [name, Remote::Protocol.validate_oid(oid)]
+      end.tap do |leases|
+        missing = entries.map { |entry| entry[:destination] } - leases.keys
+        raise ArgumentError, "lease is missing a push destination" unless missing.empty?
+      end
+    end
+  end
+end