omnibus 1.0.0

data/lib/omnibus/fetcher.rb

@@ -0,0 +1,184 @@
+#
+# Copyright:: Copyright (c) 2012 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'pp'
+
+module Omnibus
+
+  # Base class for classes that fetch project sources from the internet.
+  #
+  # @abstract Subclass and override the {#clean}, {#description},
+  #   {#fetch}, {#fetch_required?}, and {#version_guid} methods
+  #
+  # @todo Is this class supposed to be abstract or not?  Pretty sure
+  #   it's supposed to be abstract
+  class Fetcher
+
+    # Given an error and a fetcher that generated the error, print a
+    # formatted report of the error and stacktrace, along with fetcher
+    # details to stderr.
+    #
+    # @note Does not rethrow the error; that must currently be done manually.
+    #
+    # @todo Since this is always called from within a fetcher, and
+    #   since the fetcher always passes itself in in the {#initialize}
+    #   method, this really ought to be encapsulated in a method call on
+    #   {Omnibus::Fetcher}.
+    # @todo Also, since we always 'raise' after calling {#explain}, we
+    #   should just go ahead and exit from here.  No need to raise,
+    #   since we're already outputting the real error and stacktrace
+    #   here.
+    class ErrorReporter
+
+      def initialize(error, fetcher)
+        @error, @fetcher = error, fetcher
+      end
+
+      # @todo Why not just make an attribute for error?
+      #
+      # @todo And for that matter, why not make an attribute for the
+      #   fetcher as well?  Or why not just use `@error` like we use
+      #   `@fetcher`?
+      def e
+        @error
+      end
+
+      # @todo If {Omnibus::Fetcher#description} is meant to show
+      #   parameters (presumably the kind of fetcher and the software it
+      #   is fetching?),
+      def explain(why)
+        $stderr.puts "* " * 40
+        $stderr.puts why
+        $stderr.puts "Fetcher params:"
+        $stderr.puts indent(@fetcher.description, 2)
+        $stderr.puts "Exception:"
+        $stderr.puts indent("#{e.class}: #{e.message.strip}", 2)
+        Array(e.backtrace).each {|l| $stderr.puts indent(l, 4) }
+        $stderr.puts "* " * 40
+      end
+
+      private
+
+      # Indent each line of a string with `n` spaces.
+      #
+      # Splits the string at `\n` characters and then pads the left
+      # side with `n` spaces.  Rejoins them all again with `\n`.
+      #
+      # @param string [String] the string to indent
+      # @param n [Fixnum] the number of " " characters to indent each line.
+      # @return [String]
+      def indent(string, n)
+        string.split("\n").map {|l| " ".rjust(n) << l }.join("\n")
+      end
+
+    end
+
+    class UnsupportedSourceLocation < ArgumentError
+    end
+
+    NULL_ARG = Object.new
+
+    # Returns an implementation of {Fetcher} that can retrieve the
+    # given software.
+    #
+    # @param software [Omnibus::Software] the software the Fetcher should fetch
+    # @return [Omnibus::Fetcher]
+    def self.for(software)
+      if software.source
+        if software.source[:url] && Omnibus.config.use_s3_caching
+          S3CacheFetcher.new(software)
+        else
+          without_caching_for(software)
+        end
+      else
+        Fetcher.new(software)
+      end
+    end
+
+    # @param software [Omnibus::Software] the software to fetch
+    # @raise [UnsupportedSourceLocation] if the software's source is not
+    #   one of `:url`, `:git`, or `:path`
+    # @see Omnibus::Software#source
+    # @todo Define an enumeration of the acceptable software types
+    # @todo This probably ought to be folded into {#for} method above.
+    #   It looks like this is called explicitly in
+    #   {Omnibus::S3Cache#fetch}, but that could be handled by having a
+    #   second optional parameter that always disables caching.
+    # @todo Since the software determines what fetcher must be used to
+    #   fetch it, perhaps this should be a method on {Omnibus::Software}
+    #   instead.
+    def self.without_caching_for(software)
+      if software.source[:url]
+        NetFetcher.new(software)
+      elsif software.source[:git]
+        GitFetcher.new(software)
+      elsif software.source[:path]
+        PathFetcher.new(software)
+      else
+        raise UnsupportedSourceLocation, "Don't know how to fetch software project #{software}"
+      end
+    end
+
+    def self.name(name=NULL_ARG)
+      @name = name unless name.equal?(NULL_ARG)
+      @name
+    end
+
+    attr_reader :name
+
+    # @todo What is this?  It doesn't appear to be used anywhere
+    attr_reader :source_timefile
+
+    def initialize(software)
+    end
+
+    def log(message)
+      puts "[fetcher:#{self.class.name}::#{name}] #{message}"
+    end
+
+    # @!group Methods for Subclasses to Implement
+
+    # @todo All extenders of this class override this method.  Since
+    #   this class appears to be intended as an abstract one, this
+    #   should raise a NotImplementedError
+    def description
+      # Not as pretty as we'd like, but it's a sane default:
+      inspect
+    end
+
+    # @todo All extenders of this class override this method.  Since
+    #   this class appears to be intended as an abstract one, this
+    #   should raise a NotImplementedError
+    def fetch_required?
+      false
+    end
+
+    # @todo Empty method is very suspicious; raise NotImplementedError instead.
+    def clean
+    end
+
+    # @todo Empty method is very suspicious; raise NotImplementedError instead.
+    def fetch
+    end
+
+    # @todo Empty method is very suspicious; raise NotImplementedError instead.
+    def version_guid
+    end
+
+    # !@endgroup
+  end
+end

data/lib/omnibus/fetchers.rb

@@ -0,0 +1,22 @@
+#
+# Copyright:: Copyright (c) 2012 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'omnibus/fetcher'
+require 'omnibus/fetchers/net_fetcher'
+require 'omnibus/fetchers/git_fetcher'
+require 'omnibus/fetchers/path_fetcher'
+require 'omnibus/fetchers/s3_cache_fetcher'

data/lib/omnibus/fetchers/git_fetcher.rb

@@ -0,0 +1,212 @@
+#
+# Copyright:: Copyright (c) 2012 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module Omnibus
+
+  # Fetcher implementation for projects in git.
+  class GitFetcher < Fetcher
+
+    name :git
+
+    attr_reader :source
+    attr_reader :project_dir
+    attr_reader :version
+
+    def initialize(software)
+      @name         = software.name
+      @source       = software.source
+      @version      = software.version
+      @project_dir  = software.project_dir
+    end
+
+    def description
+      s=<<-E
+repo URI:       #{@source[:git]}
+local location: #{@project_dir}
+E
+    end
+
+    def version_guid
+      "git:#{current_revision}".chomp
+    rescue
+    end
+
+    def clean
+      if existing_git_clone?
+        log "cleaning existing build"
+        clean_cmd = "git clean -fdx"
+        shell = Mixlib::ShellOut.new(clean_cmd, :live_stream => STDOUT, :cwd => project_dir)
+        shell.run_command
+        shell.error!
+      end
+    rescue Exception => e
+      ErrorReporter.new(e, self).explain("Failed to clean git repository '#{@source[:git]}'")
+      raise
+    end
+
+    def fetch_required?
+      !existing_git_clone? || !current_rev_matches_target_rev?
+    end
+
+    def fetch
+      retries ||= 0
+      if existing_git_clone?
+        fetch_updates unless current_rev_matches_target_rev?
+      else
+        clone
+        checkout
+      end
+    rescue Exception => e
+      if retries >= 3
+        ErrorReporter.new(e, self).explain("Failed to fetch git repository '#{@source[:git]}'")
+        raise
+      else
+        # Deal with github failing all the time :(
+        time_to_sleep = 5 * (2 ** retries)
+        retries += 1
+        log "git clone/fetch failed for #{@source} #{retries} time(s), retrying in #{time_to_sleep}s"
+        sleep(time_to_sleep)
+        retry
+      end
+    end
+
+    private
+
+    def clone
+      puts "cloning the source from git"
+      clone_cmd = "git clone #{@source[:git]} #{project_dir}"
+      shell = Mixlib::ShellOut.new(clone_cmd, :live_stream => STDOUT)
+      shell.run_command
+      shell.error!
+    end
+
+    def checkout
+      sha_ref = target_revision
+
+      checkout_cmd = "git checkout #{sha_ref}"
+      shell = Mixlib::ShellOut.new(checkout_cmd, :live_stream => STDOUT, :cwd => project_dir)
+      shell.run_command
+      shell.error!
+    end
+
+    def fetch_updates
+      puts "fetching updates and resetting to revision #{target_revision}"
+      fetch_cmd = "git fetch origin && git fetch origin --tags && git reset --hard #{target_revision}"
+      shell = Mixlib::ShellOut.new(fetch_cmd, :live_stream => STDOUT, :cwd => project_dir)
+      shell.run_command
+      shell.error!
+    end
+
+    def existing_git_clone?
+      File.exist?("#{project_dir}/.git")
+    end
+
+    def current_rev_matches_target_rev?
+      current_revision && current_revision.strip.to_i(16) == target_revision.strip.to_i(16)
+    end
+
+    def current_revision
+      @current_rev ||= begin
+                         rev_cmd = "git rev-parse HEAD"
+                         shell = Mixlib::ShellOut.new(rev_cmd, :live_stream => STDOUT, :cwd => project_dir)
+                         shell.run_command
+                         shell.error!
+                         output = shell.stdout
+
+                         sha_hash?(output) ? output : nil
+                       end
+    end
+
+    def target_revision
+      @target_rev ||= begin
+                        if sha_hash?(version)
+                          version
+                        else
+                          revision_from_remote_reference(version)
+                        end
+                      end
+    end
+
+    def sha_hash?(rev)
+      rev =~ /^[0-9a-f]{40}$/
+    end
+
+    # Return the SHA corresponding to ref. If ref is an annotated tag,
+    # return the SHA that was tagged not the SHA of the tag itself.
+    def revision_from_remote_reference(ref)
+      retries ||= 0
+      # execute `git ls-remote` the trailing '*' does globbing. This
+      # allows us to return the SHA of the tagged commit for annotated
+      # tags. We take care to only return exact matches in
+      # process_remote_list.
+      cmd = "git ls-remote origin #{ref}*"
+      shell = Mixlib::ShellOut.new(cmd, :live_stream => STDOUT, :cwd => project_dir)
+      shell.run_command
+      shell.error!
+      commit_ref = process_remote_list(shell.stdout, ref)
+      if !commit_ref
+        raise "Could not parse SHA reference"
+      end
+      commit_ref
+    rescue Exception => e
+      if retries >= 3
+        ErrorReporter.new(e, self).explain("Failed to fetch git repository '#{@source[:git]}'")
+        raise
+      else
+        # Deal with github failing all the time :(
+        time_to_sleep = 5 * (2 ** retries)
+        retries += 1
+        log "git ls-remote failed for #{@source} #{retries} time(s), retrying in #{time_to_sleep}s"
+        sleep(time_to_sleep)
+        retry
+      end
+    end
+
+    def process_remote_list(stdout, ref)
+      # Dereference annotated tags.
+      #
+      # Output will look like this:
+      #
+      # a2ed66c01f42514bcab77fd628149eccb4ecee28        refs/tags/rel-0.11.0
+      # f915286abdbc1907878376cce9222ac0b08b12b8        refs/tags/rel-0.11.0^{}
+      #
+      # The SHA with ^{} is the commit pointed to by an annotated
+      # tag. If ref isn't an annotated tag, there will not be a line
+      # with trailing ^{}.
+      #
+      # We'll return the SHA corresponding to the ^{} which is the
+      # commit pointed to by an annotated tag. If no such commit
+      # exists (not an annotated tag) then we return the SHA of the
+      # ref.  If nothing matches, return "".
+      lines = stdout.split("\n")
+      matches = lines.map { |line| line.split("\t") }
+      # first try for ^{} indicating the commit pointed to by an
+      # annotated tag
+      tagged_commit = matches.find { |m| m[1].end_with?("#{ref}^{}") }
+      if tagged_commit
+        tagged_commit[0]
+      else
+        found = matches.find { |m| m[1].end_with?("#{ref}") }
+        if found
+          found[0]
+        else
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/omnibus/fetchers/net_fetcher.rb

@@ -0,0 +1,191 @@
+#
+# Copyright:: Copyright (c) 2012 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module Omnibus
+
+  class UnsupportedURIScheme < ArgumentError
+  end
+
+  class InvalidSourceFile < RuntimeError
+  end
+
+  # Fetcher Implementation for HTTP and FTP hosted tarballs
+  class NetFetcher < Fetcher
+
+    name :net
+
+    attr_reader :name
+    attr_reader :project_file
+    attr_reader :source
+    attr_reader :source_uri
+    attr_reader :source_dir
+    attr_reader :project_dir
+
+    def initialize(software)
+      @name         = software.name
+      @checksum     = software.checksum
+      @source       = software.source
+      @project_file = software.project_file
+      @source_uri   = software.source_uri
+      @source_dir   = software.source_dir
+      @project_dir  = software.project_dir
+    end
+
+    def description
+      s=<<-E
+source URI:     #{source_uri}
+checksum:       #{@checksum}
+local location: #@project_file
+E
+    end
+
+    def version_guid
+      "md5:#{@checksum}"
+    end
+
+    def fetch_required?
+      !File.exists?(project_file) || Digest::MD5.file(project_file) != @checksum
+    end
+
+
+    def clean
+      if File.exists?(project_dir)
+        log "cleaning existing build from #{project_dir}"
+        FileUtils.rm_rf(project_dir)
+      end
+      extract
+    end
+
+    def fetch
+      if fetch_required?
+        download
+        verify_checksum!
+      else
+        log "Cached copy of source tarball up to date"
+      end
+    end
+
+    def get_with_redirect(url, headers, limit = 10)
+      raise ArgumentError, 'HTTP redirect too deep' if limit == 0
+      log "getting from #{url} with #{limit} redirects left"
+
+      if !url.kind_of?(URI)
+        url = URI.parse(url)
+      end
+
+      req = Net::HTTP::Get.new(url.request_uri, headers)
+      http_client = Net::HTTP.new(url.host, url.port)
+      http_client.use_ssl = (url.scheme == "https")
+
+      response = http_client.start { |http| http.request(req) }
+      case response
+      when Net::HTTPSuccess
+        open(project_file, "wb") do |f|
+          f.write(response.body)
+        end
+      when Net::HTTPRedirection
+        get_with_redirect(response['location'], headers, limit - 1)
+      else
+        response.error!
+      end
+    end
+
+    def download
+      tries = 5
+      begin
+        log "\033[1;31m#{source[:warning]}\033[0m" if source.has_key?(:warning)
+        log "fetching #{project_file} from #{source_uri}"
+
+        case source_uri.scheme
+        when /https?/
+          headers = {
+            'accept-encoding' => '',
+          }
+          if source.has_key?(:cookie)
+            headers['Cookie'] = source[:cookie]
+          end
+          get_with_redirect(source_uri, headers)
+        when "ftp"
+          Net::FTP.open(source_uri.host) do |ftp|
+            ftp.passive = true
+            ftp.login
+            ftp.getbinaryfile(source_uri.path, project_file)
+            ftp.close
+          end
+        else
+          raise UnsupportedURIScheme, "Don't know how to download from #{source_uri}"
+        end
+      rescue Exception => e
+        if ( tries -= 1 ) != 0
+          log "retrying failed download..."
+          retry
+        else
+          raise
+        end
+      end
+    rescue Exception => e
+      ErrorReporter.new(e, self).explain("Failed to fetch source from #source_uri (#{e.class}: #{e.message.strip})")
+      raise
+    end
+
+    def verify_checksum!
+      actual_md5 = Digest::MD5.file(project_file)
+      unless actual_md5 == @checksum
+        log "Invalid MD5 for #@name"
+        log "Expected: #{@checksum}"
+        log "Actual:   #{actual_md5}"
+        raise InvalidSourceFile, "Checksum of downloaded file #{project_file} doesn't match expected"
+      end
+    end
+
+    def extract
+      log "extracting the source in #{project_file} to #{source_dir}"
+      cmd = extract_cmd
+      case cmd
+      when Proc
+        cmd.call
+      when String
+        shell = Mixlib::ShellOut.new(cmd, :live_stream => STDOUT)
+        shell.run_command
+        shell.error!
+      else
+        raise "Don't know how to extract command for #{cmd.class} class"
+      end
+    rescue Exception => e
+      ErrorReporter.new(e, self).explain("Failed to unpack archive at #{project_file} (#{e.class}: #{e.message.strip})")
+      raise
+    end
+
+    def extract_cmd
+      if project_file.end_with?(".gz") || project_file.end_with?(".tgz")
+        "gzip -dc  #{project_file} | ( cd #{source_dir} && tar -xf - )"
+      elsif project_file.end_with?(".bz2")
+        "bzip2 -dc  #{project_file} | ( cd #{source_dir} && tar -xf - )"
+      elsif project_file.end_with?(".7z")
+        "7z.exe x #{project_file} -o#{source_dir} -r -y"
+      else
+        #if we don't recognize the extension, simply copy over the file
+        Proc.new do
+          log "#{project_file} not an archive. Copying to #{project_dir}"
+          # hack hack hack, no project dir yet
+          FileUtils.mkdir_p(project_dir)
+          FileUtils.cp(project_file, project_dir)
+        end
+      end
+    end
+  end
+end