omnibus-sonian 1.2.0.1

data/lib/omnibus/fetchers.rb

@@ -0,0 +1,22 @@
+#
+# Copyright:: Copyright (c) 2012 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'omnibus/fetcher'
+require 'omnibus/fetchers/net_fetcher'
+require 'omnibus/fetchers/git_fetcher'
+require 'omnibus/fetchers/path_fetcher'
+require 'omnibus/fetchers/s3_cache_fetcher'

data/lib/omnibus/fetchers/git_fetcher.rb

@@ -0,0 +1,212 @@
+#
+# Copyright:: Copyright (c) 2012 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module Omnibus
+
+  # Fetcher implementation for projects in git.
+  class GitFetcher < Fetcher
+
+    name :git
+
+    attr_reader :source
+    attr_reader :project_dir
+    attr_reader :version
+
+    def initialize(software)
+      @name         = software.name
+      @source       = software.source
+      @version      = software.version
+      @project_dir  = software.project_dir
+    end
+
+    def description
+      s=<<-E
+repo URI:       #{@source[:git]}
+local location: #{@project_dir}
+E
+    end
+
+    def version_guid
+      "git:#{current_revision}".chomp
+    rescue
+    end
+
+    def clean
+      if existing_git_clone?
+        log "cleaning existing build"
+        clean_cmd = "git clean -fdx"
+        shell = Mixlib::ShellOut.new(clean_cmd, :live_stream => STDOUT, :cwd => project_dir)
+        shell.run_command
+        shell.error!
+      end
+    rescue Exception => e
+      ErrorReporter.new(e, self).explain("Failed to clean git repository '#{@source[:git]}'")
+      raise
+    end
+
+    def fetch_required?
+      !existing_git_clone? || !current_rev_matches_target_rev?
+    end
+
+    def fetch
+      retries ||= 0
+      if existing_git_clone?
+        fetch_updates unless current_rev_matches_target_rev?
+      else
+        clone
+        checkout
+      end
+    rescue Exception => e
+      if retries >= 3
+        ErrorReporter.new(e, self).explain("Failed to fetch git repository '#{@source[:git]}'")
+        raise
+      else
+        # Deal with github failing all the time :(
+        time_to_sleep = 5 * (2 ** retries)
+        retries += 1
+        log "git clone/fetch failed for #{@source} #{retries} time(s), retrying in #{time_to_sleep}s"
+        sleep(time_to_sleep)
+        retry
+      end
+    end
+
+    private
+
+    def clone
+      puts "cloning the source from git"
+      clone_cmd = "git clone #{@source[:git]} #{project_dir}"
+      shell = Mixlib::ShellOut.new(clone_cmd, :live_stream => STDOUT)
+      shell.run_command
+      shell.error!
+    end
+
+    def checkout
+      sha_ref = target_revision
+
+      checkout_cmd = "git checkout #{sha_ref}"
+      shell = Mixlib::ShellOut.new(checkout_cmd, :live_stream => STDOUT, :cwd => project_dir)
+      shell.run_command
+      shell.error!
+    end
+
+    def fetch_updates
+      puts "fetching updates and resetting to revision #{target_revision}"
+      fetch_cmd = "git fetch origin && git fetch origin --tags && git reset --hard #{target_revision}"
+      shell = Mixlib::ShellOut.new(fetch_cmd, :live_stream => STDOUT, :cwd => project_dir)
+      shell.run_command
+      shell.error!
+    end
+
+    def existing_git_clone?
+      File.exist?("#{project_dir}/.git")
+    end
+
+    def current_rev_matches_target_rev?
+      current_revision && current_revision.strip.to_i(16) == target_revision.strip.to_i(16)
+    end
+
+    def current_revision
+      @current_rev ||= begin
+                         rev_cmd = "git rev-parse HEAD"
+                         shell = Mixlib::ShellOut.new(rev_cmd, :live_stream => STDOUT, :cwd => project_dir)
+                         shell.run_command
+                         shell.error!
+                         output = shell.stdout
+
+                         sha_hash?(output) ? output : nil
+                       end
+    end
+
+    def target_revision
+      @target_rev ||= begin
+                        if sha_hash?(version)
+                          version
+                        else
+                          revision_from_remote_reference(version)
+                        end
+                      end
+    end
+
+    def sha_hash?(rev)
+      rev =~ /^[0-9a-f]{40}$/
+    end
+
+    # Return the SHA corresponding to ref. If ref is an annotated tag,
+    # return the SHA that was tagged not the SHA of the tag itself.
+    def revision_from_remote_reference(ref)
+      retries ||= 0
+      # execute `git ls-remote` the trailing '*' does globbing. This
+      # allows us to return the SHA of the tagged commit for annotated
+      # tags. We take care to only return exact matches in
+      # process_remote_list.
+      cmd = "git ls-remote origin #{ref}*"
+      shell = Mixlib::ShellOut.new(cmd, :live_stream => STDOUT, :cwd => project_dir)
+      shell.run_command
+      shell.error!
+      commit_ref = process_remote_list(shell.stdout, ref)
+      if !commit_ref
+        raise "Could not parse SHA reference"
+      end
+      commit_ref
+    rescue Exception => e
+      if retries >= 3
+        ErrorReporter.new(e, self).explain("Failed to fetch git repository '#{@source[:git]}'")
+        raise
+      else
+        # Deal with github failing all the time :(
+        time_to_sleep = 5 * (2 ** retries)
+        retries += 1
+        log "git ls-remote failed for #{@source} #{retries} time(s), retrying in #{time_to_sleep}s"
+        sleep(time_to_sleep)
+        retry
+      end
+    end
+
+    def process_remote_list(stdout, ref)
+      # Dereference annotated tags.
+      #
+      # Output will look like this:
+      #
+      # a2ed66c01f42514bcab77fd628149eccb4ecee28        refs/tags/rel-0.11.0
+      # f915286abdbc1907878376cce9222ac0b08b12b8        refs/tags/rel-0.11.0^{}
+      #
+      # The SHA with ^{} is the commit pointed to by an annotated
+      # tag. If ref isn't an annotated tag, there will not be a line
+      # with trailing ^{}.
+      #
+      # We'll return the SHA corresponding to the ^{} which is the
+      # commit pointed to by an annotated tag. If no such commit
+      # exists (not an annotated tag) then we return the SHA of the
+      # ref.  If nothing matches, return "".
+      lines = stdout.split("\n")
+      matches = lines.map { |line| line.split("\t") }
+      # first try for ^{} indicating the commit pointed to by an
+      # annotated tag
+      tagged_commit = matches.find { |m| m[1].end_with?("#{ref}^{}") }
+      if tagged_commit
+        tagged_commit[0]
+      else
+        found = matches.find { |m| m[1].end_with?("#{ref}") }
+        if found
+          found[0]
+        else
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/omnibus/fetchers/net_fetcher.rb

@@ -0,0 +1,193 @@
+#
+# Copyright:: Copyright (c) 2012 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module Omnibus
+
+  class UnsupportedURIScheme < ArgumentError
+  end
+
+  class InvalidSourceFile < RuntimeError
+  end
+
+  # Fetcher Implementation for HTTP and FTP hosted tarballs
+  class NetFetcher < Fetcher
+
+    name :net
+
+    attr_reader :name
+    attr_reader :project_file
+    attr_reader :source
+    attr_reader :source_uri
+    attr_reader :source_dir
+    attr_reader :project_dir
+
+    def initialize(software)
+      @name         = software.name
+      @checksum     = software.checksum
+      @source       = software.source
+      @project_file = software.project_file
+      @source_uri   = software.source_uri
+      @source_dir   = software.source_dir
+      @project_dir  = software.project_dir
+    end
+
+    def description
+      s=<<-E
+source URI:     #{source_uri}
+checksum:       #{@checksum}
+local location: #@project_file
+E
+    end
+
+    def version_guid
+      "md5:#{@checksum}"
+    end
+
+    def fetch_required?
+      !File.exists?(project_file) || Digest::MD5.file(project_file) != @checksum
+    end
+
+
+    def clean
+      if File.exists?(project_dir)
+        log "cleaning existing build from #{project_dir}"
+        FileUtils.rm_rf(project_dir)
+      end
+      extract
+    end
+
+    def fetch
+      if fetch_required?
+        download
+        verify_checksum!
+      else
+        log "Cached copy of source tarball up to date"
+      end
+    end
+
+    def get_with_redirect(url, headers, limit = 10)
+      raise ArgumentError, 'HTTP redirect too deep' if limit == 0
+      log "getting from #{url} with #{limit} redirects left"
+
+      if !url.kind_of?(URI)
+        url = URI.parse(url)
+      end
+
+      req = Net::HTTP::Get.new(url.request_uri, headers)
+      http_client = Net::HTTP.new(url.host, url.port)
+      http_client.use_ssl = (url.scheme == "https")
+
+      response = http_client.start { |http| http.request(req) }
+      case response
+      when Net::HTTPSuccess
+        open(project_file, "wb") do |f|
+          f.write(response.body)
+        end
+      when Net::HTTPRedirection
+        get_with_redirect(response['location'], headers, limit - 1)
+      else
+        response.error!
+      end
+    end
+
+    def download
+      tries = 5
+      begin
+        log "\033[1;31m#{source[:warning]}\033[0m" if source.has_key?(:warning)
+        log "fetching #{project_file} from #{source_uri}"
+
+        case source_uri.scheme
+        when /https?/
+          headers = {
+            'accept-encoding' => '',
+          }
+          if source.has_key?(:cookie)
+            headers['Cookie'] = source[:cookie]
+          end
+          get_with_redirect(source_uri, headers)
+        when "ftp"
+          Net::FTP.open(source_uri.host) do |ftp|
+            ftp.passive = true
+            ftp.login
+            ftp.getbinaryfile(source_uri.path, project_file)
+            ftp.close
+          end
+        else
+          raise UnsupportedURIScheme, "Don't know how to download from #{source_uri}"
+        end
+      rescue Exception => e
+        if ( tries -= 1 ) != 0
+          log "retrying failed download..."
+          retry
+        else
+          raise
+        end
+      end
+    rescue Exception => e
+      ErrorReporter.new(e, self).explain("Failed to fetch source from #source_uri (#{e.class}: #{e.message.strip})")
+      raise
+    end
+
+    def verify_checksum!
+      actual_md5 = Digest::MD5.file(project_file)
+      unless actual_md5 == @checksum
+        log "Invalid MD5 for #@name"
+        log "Expected: #{@checksum}"
+        log "Actual:   #{actual_md5}"
+        raise InvalidSourceFile, "Checksum of downloaded file #{project_file} doesn't match expected"
+      end
+    end
+
+    def extract
+      log "extracting the source in #{project_file} to #{source_dir}"
+      cmd = extract_cmd
+      case cmd
+      when Proc
+        cmd.call
+      when String
+        shell = Mixlib::ShellOut.new(cmd, :live_stream => STDOUT)
+        shell.run_command
+        shell.error!
+      else
+        raise "Don't know how to extract command for #{cmd.class} class"
+      end
+    rescue Exception => e
+      ErrorReporter.new(e, self).explain("Failed to unpack archive at #{project_file} (#{e.class}: #{e.message.strip})")
+      raise
+    end
+
+    def extract_cmd
+      if project_file.end_with?(".gz") || project_file.end_with?(".tgz")
+        "gzip -dc  #{project_file} | ( cd #{source_dir} && tar -xf - )"
+      elsif project_file.end_with?(".bz2")
+        "bzip2 -dc  #{project_file} | ( cd #{source_dir} && tar -xf - )"
+      elsif project_file.end_with?(".7z")
+        "7z.exe x #{project_file} -o#{source_dir} -r -y"
+      elsif project_file.end_with?(".zip")
+        "unzip #{project_file} -d #{source_dir}"
+      else
+        #if we don't recognize the extension, simply copy over the file
+        Proc.new do
+          log "#{project_file} not an archive. Copying to #{project_dir}"
+          # hack hack hack, no project dir yet
+          FileUtils.mkdir_p(project_dir)
+          FileUtils.cp(project_file, project_dir)
+        end
+      end
+    end
+  end
+end

data/lib/omnibus/fetchers/path_fetcher.rb

@@ -0,0 +1,65 @@
+#
+# Copyright:: Copyright (c) 2012 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module Omnibus
+
+  # Fetcher implementation for projects on the filesystem
+  class PathFetcher < Fetcher
+
+    name :path
+
+    def initialize(software)
+      @name = software.name
+      @source = software.source
+      @project_dir = software.project_dir
+      @version = software.version
+    end
+
+    def description
+      s=<<-E
+source path:    #{@source[:path]}
+local location: #{@project_dir}
+E
+    end
+    
+    def rsync
+      if OHAI.platform == "windows"
+        # Robocopy's return code is 1 if it succesfully copies over the
+        # files and 0 if the files are already existing at the destination
+        sync_cmd = "robocopy #{@source[:path]}\\ #{@project_dir}\\ /MIR /S"
+        shell = Mixlib::ShellOut.new(sync_cmd, :returns => [0, 1])
+      else
+        sync_cmd = "rsync --delete -a #{@source[:path]}/ #{@project_dir}/"
+        shell = Mixlib::ShellOut.new(sync_cmd)
+      end
+      shell.run_command
+    end
+
+    def clean
+      # Here, clean will do the same as fetch: reset source to pristine state
+      rsync
+    end
+
+    def fetch
+      rsync
+    end
+
+    def fetch_required?
+      true
+    end
+  end
+end