omnibus 3.2.2 → 4.0.0.beta.1

data/lib/omnibus/fetcher.rb

@@ -14,174 +14,133 @@
 # limitations under the License.
 #
 
-require 'pp'
-
 module Omnibus
-  # Base class for classes that fetch project sources from the internet.
-  #
-  # @abstract Subclass and override the {#clean}, {#description},
-  #   {#fetch}, {#fetch_required?}, and {#version_guid} methods
-  #
-  # @todo Is this class supposed to be abstract or not?  Pretty sure
-  #   it's supposed to be abstract
   class Fetcher
-    # Given an error and a fetcher that generated the error, print a
-    # formatted report of the error and stacktrace, along with fetcher
-    # details to stderr.
-    #
-    # @note Does not rethrow the error; that must currently be done manually.
-    #
-    # @todo Since this is always called from within a fetcher, and
-    #   since the fetcher always passes itself in in the {#initialize}
-    #   method, this really ought to be encapsulated in a method call on
-    #   {Omnibus::Fetcher}.
-    # @todo Also, since we always 'raise' after calling {#explain}, we
-    #   should just go ahead and exit from here.  No need to raise,
-    #   since we're already outputting the real error and stacktrace
-    #   here.
-    class ErrorReporter
-      def initialize(error, fetcher)
-        @error, @fetcher = error, fetcher
-      end
-
-      # @todo Why not just make an attribute for error?
-      #
-      # @todo And for that matter, why not make an attribute for the
-      #   fetcher as well?  Or why not just use `@error` like we use
-      #   `@fetcher`?
-      def e
-        @error
-      end
-
-      # @todo If {Omnibus::Fetcher#description} is meant to show
-      #   parameters (presumably the kind of fetcher and the software it
-      #   is fetching?),
-      # @todo make this use the logger
-      def explain(why)
-        $stderr.puts '* ' * 40
-        $stderr.puts why
-        $stderr.puts 'Fetcher params:'
-        $stderr.puts indent(@fetcher.description, 2)
-        $stderr.puts 'Exception:'
-        $stderr.puts indent("#{e.class}: #{e.message.strip}", 2)
-        Array(e.backtrace).each { |l| $stderr.puts indent(l, 4) }
-        $stderr.puts '* ' * 40
-      end
-
-      private
-
-      # Indent each line of a string with `n` spaces.
-      #
-      # Splits the string at `\n` characters and then pads the left
-      # side with `n` spaces.  Rejoins them all again with `\n`.
-      #
-      # @param string [String] the string to indent
-      # @param n [Fixnum] the number of " " characters to indent each line.
-      # @return [String]
-      def indent(string, n)
-        string.split("\n").map { |l| ' '.rjust(n) << l }.join("\n")
-      end
-    end
-
-    class UnsupportedSourceLocation < ArgumentError
-    end
-
-    NULL_ARG = Object.new
-
-    # Returns an implementation of {Fetcher} that can retrieve the
-    # given software.
-    #
-    # @param software [Omnibus::Software] the software the Fetcher should fetch
-    # @return [Omnibus::Fetcher]
-    def self.for(software)
-      if software.source
-        if software.source[:url] && Config.use_s3_caching
-          S3CacheFetcher.new(software)
-        else
-          without_caching_for(software)
-        end
-      else
-        Fetcher.new(software)
-      end
-    end
-
-    # @param software [Omnibus::Software] the software to fetch
-    # @raise [UnsupportedSourceLocation] if the software's source is not
-    #   one of `:url`, `:git`, or `:path`
-    # @see Omnibus::Software#source
-    # @todo Define an enumeration of the acceptable software types
-    # @todo This probably ought to be folded into {#for} method above.
-    #   It looks like this is called explicitly in
-    #   {Omnibus::S3Cache#fetch}, but that could be handled by having a
-    #   second optional parameter that always disables caching.
-    # @todo Since the software determines what fetcher must be used to
-    #   fetch it, perhaps this should be a method on {Omnibus::Software}
-    #   instead.
-    def self.without_caching_for(software)
-      if software.source[:url]
-        NetFetcher.new(software)
-      elsif software.source[:git]
-        GitFetcher.new(software)
-      elsif software.source[:path]
-        PathFetcher.new(software)
-      else
-        raise UnsupportedSourceLocation, "Don't know how to fetch software project #{software}"
-      end
-    end
-
+    include Digestable
     include Logging
     include Util
 
-    attr_reader :name
-
-    # @todo What is this?  It doesn't appear to be used anywhere
-    attr_reader :source_timefile
+    #
+    # The software for this fetcher.
+    #
+    # @return [Software]
+    #
+    attr_reader :software
 
+    #
+    # Create a new Fetcher object from the given software.
+    #
+    # @param [Software] software
+    #   the software to create this fetcher
+    #
     def initialize(software)
       @software = software
     end
 
-    # @!group Methods for Subclasses to Implement
-
-    # @todo All extenders of this class override this method.  Since
-    #   this class appears to be intended as an abstract one, this
-    #   should raise a NotImplementedError
-    def description
-      # Not as pretty as we'd like, but it's a sane default:
-      inspect
-    end
+    #
+    # @!group Abstract methods
+    #
+    # The following methods are all abstract and should be overriden in child
+    # classes.
+    # --------------------------------------------------
 
-    # @todo All extenders of this class override this method.  Since
-    #   this class appears to be intended as an abstract one, this
-    #   should raise a NotImplementedError
+    #
+    # @abstract
+    #
     def fetch_required?
-      false
+      raise NotImplementedError
     end
 
-    # @todo Empty method is very suspicious; raise NotImplementedError instead.
+    #
+    # @abstract
+    #
     def clean
+      raise NotImplementedError
     end
 
-    # @todo Empty method is very suspicious; raise NotImplementedError instead.
+    #
+    # @abstract
+    #
     def fetch
+      raise NotImplementedError
     end
 
-    # @todo Empty method is very suspicious; raise NotImplementedError instead.
+    #
+    # @abstract
+    #
     def version_guid
+      raise NotImplementedError
     end
 
-    # Returns the version to be used during build caching.
-    # By default fetchers don't override the versions.
+    #
+    # @abstract
+    #
     def version_for_cache
-      nil
+      raise NotImplementedError
     end
 
+    #
+    # @!endgroup
+    # --------------------------------------------------
+
     private
 
+    #
+    # The "source" for this software, with applied overrides.
+    #
+    # @return [Hash]
+    #
+    def source
+      software.source
+    end
+
+    #
+    # The path where extracted software should live.
+    #
+    # @see Software#project_dir
+    #
+    # @return [String]
+    #
+    def project_dir
+      software.project_dir
+    end
+
+    #
+    # The version for this sfotware, with applied overrides.
+    #
+    # @return [String]
+    #
+    def version
+      software.version
+    end
+
+    #
+    # Override the +log_key+ for this fetcher to include the name of the
+    # software during the fetch.
+    #
+    # @return [String]
+    #
     def log_key
-      @log_key ||= "#{super}: #{@software.name}"
+      @log_key ||= "#{super}: #{software.name}"
     end
 
-    # !@endgroup
+
+    #
+    # Idempotently create the required directories for building/downloading.
+    # Fetchers should call this method before performing any operations that
+    # manipulate the filesystem.
+    #
+    # @return [void]
+    #
+    def create_required_directories
+      [
+        Config.cache_dir,
+        Config.source_dir,
+        software.build_dir,
+        software.project_dir,
+      ].each do |directory|
+        FileUtils.mkdir_p(directory) unless File.directory?(directory)
+      end
+    end
   end
 end

data/lib/omnibus/fetchers/git_fetcher.rb

@@ -15,202 +15,167 @@
 #
 
 module Omnibus
-  # Fetcher implementation for projects in git.
   class GitFetcher < Fetcher
-    attr_reader :source
-    attr_reader :project_dir
-    attr_reader :version
-
-    def initialize(software)
-      @name         = software.name
-      @source       = software.source
-      @version      = software.version
-      @project_dir  = software.project_dir
-      super
-    end
-
-    def description
-      <<-EOH.gsub(/^ {8}/, '').strip
-        repo URI:       #{@source[:git]}
-        local location: #{@project_dir}
-      EOH
+    #
+    # A fetch is required if the git repository is not cloned or if the local
+    # revision does not match the desired revision.
+    #
+    # @return [true, false]
+    #
+    def fetch_required?
+      !(cloned? && same_revision?)
     end
 
+    #
+    # The version identifier for this git location. This is computed using the
+    # current revision on disk.
+    #
+    # @return [String]
+    #
     def version_guid
-      "git:#{current_revision}".chomp
-    rescue
+      "git:#{current_revision}"
     end
 
+    #
+    # Clean the project directory by removing the contents from disk.
+    #
+    # @return [true, false]
+    #   true if the project directory was removed, false otherwise
+    #
     def clean
-      if existing_git_clone?
-        log.info(log_key) { 'Cleaning existing build' }
-        shellout!('git clean -fdx', cwd: project_dir)
+      if cloned?
+        log.info(log_key) { 'Cleaning existing clone' }
+        git('clean -fdx')
+        true
+      else
+        false
       end
-    rescue Exception => e
-      ErrorReporter.new(e, self).explain("Failed to clean git repository '#{@source[:git]}'")
-      raise
-    end
-
-    def fetch_required?
-      !existing_git_clone? || !current_rev_matches_target_rev?
     end
 
+    #
+    # Fetch (clone) or update (fetch) the remote git repository.
+    #
+    # @return [void]
+    #
     def fetch
-      retries ||= 0
-      if existing_git_clone?
-        fetch_updates unless current_rev_matches_target_rev?
-      else
-        clone
-        checkout
-      end
-    rescue Exception => e
-      if retries >= 3
-        ErrorReporter.new(e, self).explain("Failed to fetch git repository '#{@source[:git]}'")
-        raise
+      log.info(log_key) { "Fetching from `#{source_url}'" }
+
+      create_required_directories
+
+      if cloned?
+        git_fetch unless same_revision?
       else
-        # Deal with github failing all the time :(
-        time_to_sleep = 5 * (2**retries)
-        retries += 1
-        log.warn(log_key) do
-          "git clone/fetch failed for #{@source} #{retries} time(s). " \
-          "Retrying in #{time_to_sleep}s..."
-        end
-        sleep(time_to_sleep)
-        retry
+        git_clone
+        git_checkout
       end
     end
 
-    # Return the target sha to be used during build caching
-    # This overrides the cases where software.version is similar to
-    # master, 11-stable etc..
+    #
+    # The version for this item in the cache. The is the parsed revision of the
+    # item on disk.
+    #
+    # @return [String]
+    #
     def version_for_cache
-      target_revision
+      "revision:#{current_revision}"
     end
 
     private
 
-    def clone
-      log.info(log_key) { 'Cloning the source from git' }
-      shellout!("git clone #{@source[:git]} #{project_dir}")
-    end
-
-    def checkout
-      sha_ref = target_revision
-      shellout!("git checkout #{sha_ref}", cwd: project_dir)
-    end
-
-    def fetch_updates
-      log.info(log_key) do
-        "Fetching updates and resetting to revision '#{target_revision}'"
-      end
-
-      fetch_cmd = "git fetch origin && " \
-                  "git fetch origin --tags && " \
-                  "git reset --hard #{target_revision}"
-      shellout!(fetch_cmd, cwd: project_dir)
-    end
-
-    def existing_git_clone?
+    #
+    # The URL where the git source should be downloaded from.
+    #
+    # @return [String]
+    #
+    def source_url
+      source[:git]
+    end
+
+    #
+    # Determine if the clone exists.
+    #
+    # @return [true, false]
+    #
+    def cloned?
       File.exist?("#{project_dir}/.git")
     end
 
-    def current_rev_matches_target_rev?
-      current_revision && current_revision.strip.to_i(16) == target_revision.strip.to_i(16)
-    end
-
+    #
+    # Clone the +source_url+ into the +project_dir+.
+    #
+    # @return [void]
+    #
+    def git_clone
+      git("clone #{source_url} .")
+    end
+
+    #
+    # Checkout the +target_revision+.
+    #
+    # @return [void]
+    #
+    def git_checkout
+      git("fetch --all")
+      git("checkout #{target_revision}")
+    end
+
+    #
+    # Fetch the remote tags and refs, and reset to +target_revision+.
+    #
+    # @return [void]
+    #
+    def git_fetch
+      git("fetch --all")
+      git("reset --hard #{target_revision}")
+    end
+
+    #
+    # The current revision for the cloned checkout.
+    #
+    # @return [String]
+    #
     def current_revision
-      return @current_rev if @current_rev
-
-      cmd = shellout!('git rev-parse HEAD', cwd: project_dir)
-      stdout = cmd.stdout
-
-      @current_rev = sha_hash?(stdout) ? stdout : nil
-      @current_rev
+      @current_revision ||= git('rev-parse HEAD').stdout.strip
+    rescue CommandFailed
+      nil
     end
 
+    #
+    # The target revision from the user.
+    #
+    # @return [String]
+    #
     def target_revision
-      @target_rev ||= if sha_hash?(version)
-                        version
-                      else
-                        revision_from_remote_reference(version)
-                      end
-    end
-
-    def sha_hash?(rev)
-      rev =~ /^[0-9a-f]{40}$/
-    end
-
-    # Return the SHA corresponding to ref. If ref is an annotated tag,
-    # return the SHA that was tagged not the SHA of the tag itself.
-    def revision_from_remote_reference(ref)
-      retries ||= 0
-      # execute `git ls-remote` the trailing '*' does globbing. This
-      # allows us to return the SHA of the tagged commit for annotated
-      # tags. We take care to only return exact matches in
-      # process_remote_list.
-      cmd = shellout!("git ls-remote origin #{ref}*", cwd: project_dir)
-      commit_ref = process_remote_list(cmd.stdout, ref)
-
-      unless commit_ref
-        raise UnresolvableGitReference.new("Could not resolve `#{ref}' to a SHA.")
-      end
-      commit_ref
-    rescue UnresolvableGitReference => e # skip retries
-      ErrorReporter.new(e, self).explain(<<-E)
-Command `#{cmd}' did not find a commit for reference `#{ref}'.
-The tag or branch you're looking for doesn't exist on the remote repo.
-If your project uses version tags like v1.2.3, include the 'v' in your
-software's version.
-E
-      raise
-    rescue Exception => e
-      if retries >= 3
-        ErrorReporter.new(e, self).explain("Failed to find any commits for the ref '#{ref}'")
-        raise
+      @target_revision ||= git("rev-parse #{version}").stdout.strip
+    rescue CommandFailed => e
+      if e.message.include?('ambiguous argument')
+        @target_revision = git("rev-parse origin/#{version}").stdout.strip
+        @target_revision
       else
-        # Deal with github failing all the time :(
-        time_to_sleep = 5 * (2**retries)
-        retries += 1
-        log.warn(log_key) do
-          "git ls-remote failed for #{@source} #{retries} time(s). " \
-          "Retrying in #{time_to_sleep}s..."
-        end
-        sleep(time_to_sleep)
-        retry
+        log.warn { 'Could not determine target revision!' }
+        nil
       end
     end
 
-    def process_remote_list(stdout, ref)
-      # Dereference annotated tags.
-      #
-      # Output will look like this:
-      #
-      # a2ed66c01f42514bcab77fd628149eccb4ecee28        refs/tags/rel-0.11.0
-      # f915286abdbc1907878376cce9222ac0b08b12b8        refs/tags/rel-0.11.0^{}
-      #
-      # The SHA with ^{} is the commit pointed to by an annotated
-      # tag. If ref isn't an annotated tag, there will not be a line
-      # with trailing ^{}.
-      #
-      # We'll return the SHA corresponding to the ^{} which is the
-      # commit pointed to by an annotated tag. If no such commit
-      # exists (not an annotated tag) then we return the SHA of the
-      # ref.  If nothing matches, return "".
-      lines = stdout.split("\n")
-      matches = lines.map { |line| line.split("\t") }
-      # first try for ^{} indicating the commit pointed to by an
-      # annotated tag
-      tagged_commit = matches.find { |m| m[1].end_with?("#{ref}^{}") }
-      if tagged_commit
-        tagged_commit[0]
-      else
-        found = matches.find { |m| m[1].end_with?("#{ref}") }
-        if found
-          found[0]
-        else
-          nil
-        end
-      end
+    #
+    # Determine if the given revision matches the current revision.
+    #
+    # @return [true, false]
+    #
+    def same_revision?
+      current_revision == target_revision
+    end
+
+    #
+    # Execute the given git command, inside the +project_dir+.
+    #
+    # @see Util#shellout!
+    #
+    # @return [Mixlib::ShellOut]
+    #   the shellout object
+    #
+    def git(command)
+      shellout!("git #{command}", cwd: project_dir)
     end
   end
 end