vanagon 0.7.1 → 0.8.0

data/lib/vanagon/component/source/http.rb

@@ -1,16 +1,41 @@
 require 'vanagon/utilities'
+require 'vanagon/component/source/local'
 require 'net/http'
 require 'uri'
 
 class Vanagon
   class Component
     class Source
-      class Http
+      class Http < Vanagon::Component::Source::Local
         include Vanagon::Utilities
-        attr_accessor :url, :sum, :file, :extension, :workdir, :cleanup
 
-        # Extensions for files we intend to unpack during the build
-        ARCHIVE_EXTENSIONS = ['.tar.gz', '.tgz', '.zip'].freeze
+        # Accessors :url, :file, :extension, :workdir, :cleanup are inherited from Local
+        attr_accessor :sum
+
+        class << self
+          def valid_url?(target_url) # rubocop:disable Metrics/AbcSize
+            uri = URI.parse(target_url.to_s)
+            return false unless ['http', 'https'].include? uri.scheme
+
+            Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == 'https') do |http|
+              http.request(Net::HTTP::Head.new(uri)) do |response|
+                case response
+                when Net::HTTPRedirection
+                  # By parsing the location header, we get either an absolute
+                  # URI or a URI with a relative `path`. Adding it to `uri`
+                  # should correctly update the relative `path` or overwrite
+                  # the entire URI if it's absolute.
+                  location = URI.parse(response.header['location'])
+                  valid_url?(uri + location)
+                when Net::HTTPSuccess
+                  return true
+                else
+                  false
+                end
+              end
+            end
+          end
+        end
 
         # Constructor for the Http source type
         #
@@ -18,7 +43,7 @@ class Vanagon
         # @param sum [String] sum to verify the download against
         # @param workdir [String] working directory to download into
         # @raise [RuntimeError] an exception is raised is sum is nil
-        def initialize(url, sum, workdir)
+        def initialize(url, sum:, workdir:, **options)
           unless sum
             fail "sum is required to validate the http source"
           end
@@ -31,31 +56,34 @@ class Vanagon
         # file as @file and the @extension for the file as a side effect.
         def fetch
           @file = download(@url)
-          @extension = get_extension
+        end
+
+        def file
+          @file ||= fetch
         end
 
         # Verify the downloaded file matches the provided sum
         #
         # @raise [RuntimeError] an exception is raised if the sum does not match the sum of the file
         def verify
-          puts "Verifying file: #{@file} against sum: '#{@sum}'"
-          actual = get_md5sum(File.join(@workdir, @file))
-          unless @sum == actual
-            fail "Unable to verify '#{@file}'. Expected: '#{@sum}', got: '#{actual}'"
-          end
+          puts "Verifying file: #{file} against sum: '#{sum}'"
+          actual = get_md5sum(File.join(workdir, file))
+          return true if sum == actual
+
+          fail "Unable to verify '#{File.join(workdir, file)}': md5sum mismatch (expected '#{sum}', got '#{actual}'')"
         end
 
         # Downloads the file from @url into the @workdir
         # @param target_url [String, URI, Addressable::URI] url of an http source to retrieve with GET
         # @raise [RuntimeError, Vanagon::Error] an exception is raised if the URI scheme cannot be handled
-        def download(target_url, target_file = nil) # rubocop:disable Metrics/AbcSize
+        def download(target_url, target_file = nil, headers = { "Accept-Encoding" => "identity" }) # rubocop:disable Metrics/AbcSize
           uri = URI.parse(target_url.to_s)
           target_file ||= File.basename(uri.path)
 
           puts "Downloading file '#{target_file}' from url '#{target_url}'"
 
           Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == 'https') do |http|
-            http.request(Net::HTTP::Get.new(uri)) do |response|
+            http.request(Net::HTTP::Get.new(uri, headers)) do |response|
               case response
               when Net::HTTPRedirection
                 # By parsing the location header, we get either an absolute
@@ -81,66 +109,6 @@ class Vanagon
           Net::HTTPHeaderSyntaxError, Net::ProtocolError => e
           raise Vanagon::Error.wrap(e, "Problem downloading #{target_file} from '#{@url}'. Please verify you have the correct uri specified.")
         end
-
-        # Gets the command to extract the archive given if needed (uses @extension)
-        #
-        # @param tar [String] the tar command to use
-        # @return [String, nil] command to extract the source
-        # @raise [RuntimeError] an exception is raised if there is no known extraction method for @extension
-        def extract(tar)
-          if ARCHIVE_EXTENSIONS.include?(@extension)
-            case @extension
-            when ".tar.gz", ".tgz"
-              return "gunzip -c '#{@file}' | '#{tar}' xf -"
-            when ".zip"
-              return "unzip '#{@file}' || 7za x -r -tzip -o'#{File.basename(@file, '.zip')}' '#{@file}'"
-            end
-          else
-            # Extension does not appear to be an archive
-            return ':'
-          end
-        end
-
-        # Return the correct incantation to cleanup the source archive and source directory for a given source
-        #
-        # @return [String] command to cleanup the source
-        # @raise [RuntimeError] an exception is raised if there is no known extraction method for @extension
-        def cleanup
-          if ARCHIVE_EXTENSIONS.include?(@extension)
-            return "rm #{@file}; rm -r #{dirname}"
-          else
-            # Because dirname will be ./ here, we don't want to try to nuke it
-            return "rm #{@file}"
-          end
-        end
-
-        # Returns the extension for @file
-        #
-        # @return [String] the extension of @file
-        def get_extension
-          extension_match = @file.match(/.*(#{Regexp.union(ARCHIVE_EXTENSIONS)})/)
-          unless extension_match
-            if @file.split('.').last.include?('.')
-              return '.' +  @file.split('.').last
-            else
-              # This is the case where the file has no extension
-              return @file
-            end
-          end
-          extension_match[1]
-        end
-
-        # The dirname to reference when building from the source
-        #
-        # @return [String] the directory that should be traversed into to build this source
-        # @raise [RuntimeError] if the @extension for the @file isn't currently handled by the method
-        def dirname
-          if ARCHIVE_EXTENSIONS.include?(@extension)
-            return @file.chomp(@extension)
-          else
-            return './'
-          end
-        end
       end
     end
   end

data/lib/vanagon/component/source/local.rb

@@ -1,31 +1,52 @@
 require 'vanagon/utilities'
-require 'net/http'
-require 'uri'
 
 class Vanagon
   class Component
     class Source
       class Local
-        include Vanagon::Utilities
         attr_accessor :url, :file, :extension, :workdir, :cleanup
 
         # Extensions for files we intend to unpack during the build
-        ARCHIVE_EXTENSIONS = ['.tar.gz', '.tgz', '.zip'].freeze
+        ARCHIVE_EXTENSIONS = {
+          "7z" => %w(.7z),
+          "bzip2" => %w(.bz2 .bz),
+          "cpio" => %w(.cpio),
+          "gzip" => %w(.gz .z),
+          "rar" => %w(.rar),
+          "tar" => %w(.tar),
+          "tbz2" => %w(.tar.bz2 .tbz2 .tbz),
+          "tgz" => %w(.tar.gz .tgz),
+          "txz" => %w(.tar.xz .txz),
+          "xz" => %w(.xz),
+          "zip" => %w(.zip),
+        }.freeze
 
-        # Constructor for the File source type
-        #
-        # @param url [String] url of the http source to fetch
-        # @param workdir [String] working directory to download into
-        def initialize(url, workdir)
-          @url = url
-          @workdir = workdir
+        class << self
+          def valid_file?(target_file)
+            File.exist?(mangle(target_file.to_s))
+          end
+
+          # If a scheme is specified as "file://", this will return
+          # strip off the scheme and delimiters -- we need to do this because
+          # once upon a time we allowed specifying files with no strong
+          # specifications for where they should be located.
+          def mangle(path)
+            path.gsub(%r{^file://}, '')
+          end
+
+          def archive_extensions
+            ARCHIVE_EXTENSIONS.values.flatten
+          end
         end
 
-        # Download the source from the url specified. Sets the full path to the
-        # file as @file and the @extension for the file as a side effect.
-        def fetch
-          @file = download
-          @extension = get_extension
+
+        # Constructor for the File source type
+        #
+        # @param path [String] path of the local file to copy
+        # @param workdir [String] working directory to copy <path> to
+        def initialize(path, workdir:, **options)
+          @url = ::Pathname.new(mangle(path))
+          @workdir = ::Pathname.new(workdir)
         end
 
         # Local files need no checksum so this is a noop
@@ -36,21 +57,19 @@ class Vanagon
         # Moves file from source to workdir
         #
         # @raise [RuntimeError, Vanagon::Error] an exception is raised if the URI scheme cannot be handled
-        def download
-          uri = URI.parse(@url)
-          target_file = File.basename(uri.path)
-          puts "Moving file '#{target_file}' to workdir"
-
-          uri = @url.match(/^file:\/\/(.*)$/)
-          if uri
-            source_file = uri[1]
-            target_file = File.basename(source_file)
-            FileUtils.cp(source_file, File.join(@workdir, target_file))
-          else
-            raise Vanagon::Error, "Unable to parse '#{@url}' for local file path."
-          end
+        def copy
+          puts "Copying file '#{url.basename}' to workdir"
+
+          FileUtils.cp(url, file)
+        end
+        alias_method :fetch, :copy
+
+        def file
+          @file ||= workdir + File.basename(url)
+        end
 
-          target_file
+        def extension
+          @extension ||= extname
         end
 
         # Gets the command to extract the archive given if needed (uses @extension)
@@ -58,17 +77,40 @@ class Vanagon
         # @param tar [String] the tar command to use
         # @return [String, nil] command to extract the source
         # @raise [RuntimeError] an exception is raised if there is no known extraction method for @extension
-        def extract(tar)
-          if ARCHIVE_EXTENSIONS.include?(@extension)
-            case @extension
-            when ".tar.gz", ".tgz"
-              return "gunzip -c '#{@file}' | '#{tar}' xf -"
-            when ".zip"
-              return "unzip '#{@file}' || 7za x -r -tzip -o'#{File.basename(@file, '.zip')}' '#{@file}'"
-            end
+        def extract(tar = "tar") # rubocop:disable Metrics/AbcSize
+          # Extension does not appear to be an archive, so "extract" is a no-op
+          return ':' unless archive_extensions.include?(extension)
+
+          case decompressor
+          when "7z"
+            %(7z x "#{file}")
+          when "bzip2"
+            %(bunzip2 "#{file}")
+          when "cpio"
+            %(
+              mkdir "#{file.basename}" &&
+              pushd "#{file.basename}" 2>&1 > /dev/null &&
+              cpio -idv < "#{file}" &&
+              popd 2>&1 > /dev/null
+            ).undent
+          when "gzip"
+            %(gunzip "#{file}")
+          when "rar"
+            %(unrar x "#{file}")
+          when "tar"
+            %(#{tar} xf "#{file}")
+          when "tbz2"
+            %(bunzip2 -c "#{file}" | #{tar} xf -)
+          when "tgz"
+            %(gunzip -c "#{file}" | #{tar} xf -)
+          when "txz"
+            %(unxz -d "#{file}" | #{tar} xvf -)
+          when "xz"
+            %(unxz "#{file}")
+          when "zip"
+            "unzip '#{file}' || 7za x -r -tzip -o'#{File.basename(file, '.zip')}' '#{file}'"
           else
-            # Extension does not appear to be an archive
-            return ':'
+            raise Vanagon::Error, "Don't know how to decompress #{extension} archives"
           end
         end
 
@@ -77,28 +119,28 @@ class Vanagon
         # @return [String] command to cleanup the source
         # @raise [RuntimeError] an exception is raised if there is no known extraction method for @extension
         def cleanup
-          if ARCHIVE_EXTENSIONS.include?(@extension)
-            return "rm #{@file}; rm -r #{dirname}"
-          else
-            # Because dirname will be ./ here, we don't want to try to nuke it
-            return "rm #{@file}"
-          end
+          archive? ? "rm #{file}; rm -r #{dirname}" : "rm #{file}"
         end
 
         # Returns the extension for @file
         #
         # @return [String] the extension of @file
-        def get_extension
-          extension_match = @file.match(/.*(#{Regexp.union(ARCHIVE_EXTENSIONS)})/)
-          unless extension_match
-            if @file.split('.').last.include?('.')
-              return '.' +  @file.split('.').last
-            else
-              # This is the case where the file has no extension
-              return @file
-            end
-          end
-          extension_match[1]
+        def extname
+          extension_match = file.to_s.match %r{#{Regexp.union(archive_extensions)}\Z}
+          return extension_match.to_s if extension_match
+          File.extname(file)
+        end
+
+        def archive_extensions
+          self.class.archive_extensions
+        end
+
+        def archive?
+          archive_extensions.include?(extension)
+        end
+
+        def decompressor
+          @decompressor ||= ARCHIVE_EXTENSIONS.select { |k, v| v.include? extension }.keys.first
         end
 
         # The dirname to reference when building from the source
@@ -106,11 +148,12 @@ class Vanagon
         # @return [String] the directory that should be traversed into to build this source
         # @raise [RuntimeError] if the @extension for the @file isn't currently handled by the method
         def dirname
-          if ARCHIVE_EXTENSIONS.include?(@extension)
-            return @file.chomp(@extension)
-          else
-            return './'
-          end
+          archive? ? File.basename(file, extension) : './'
+        end
+
+        # Wrapper around the class method '.mangle'
+        def mangle(path)
+          self.class.mangle(path)
         end
       end
     end

data/lib/vanagon/platform/deb.rb

@@ -47,6 +47,14 @@ class Vanagon
         "#{project.name}_#{project.version}-#{project.release}#{@codename}_#{project.noarch ? 'all' : @architecture}.deb"
       end
 
+      # Get the expected output dir for the debian packages. This allows us to
+      # use some standard tools to ship internally.
+      #
+      # @return [String] relative path to where debian packages should be staged
+      def output_dir(target_repo = "")
+        @output_dir ||= File.join("deb", @codename, target_repo)
+      end
+
       # Returns the string to add a target repo to the platforms' provisioning
       #
       # @param definition [URI] A URI to a deb or list file

data/lib/vanagon/platform/rpm.rb

@@ -36,6 +36,10 @@ class Vanagon
         "#{project.name}-#{project.version}-#{project.release}.#{project.noarch ? 'noarch' : @architecture}.rpm"
       end
 
+      def output_dir(target_repo = "products")
+        super
+      end
+
       def rpm_defines
         defines =  %(--define '_topdir $(tempdir)/rpmbuild' )
         # RPM doesn't allow dashes in the os_name. This was added to

data/lib/vanagon/project/dsl.rb

@@ -144,8 +144,10 @@ class Vanagon
       # and reachable from the current commit in that repository.
       #
       def version_from_git
-        version = Vanagon::Utilities.git_version(File.expand_path("..", Vanagon::Driver.configdir))
+        version = Git.open(File.expand_path("..", Vanagon::Driver.configdir)).describe('HEAD', tags: true)
         @project.version = version.split('-').reject(&:empty?).join('.')
+      rescue Git::GitExecuteError
+        warn "Directory '#{dirname}' cannot be versioned by git. Maybe it hasn't been tagged yet?"
       end
 
       # Sets the vendor for the project. Used in packaging artifacts.

data/lib/vanagon/utilities.rb

@@ -5,10 +5,15 @@ require 'json'
 require 'digest'
 require 'erb'
 require 'timeout'
+# This stupid library requires a capital 'E' in its name
+# but it provides a wealth of useful constants
+require 'English'
 require 'vanagon/extensions/string'
 
 class Vanagon
   module Utilities
+    extend self
+
     # Utility to get the md5 sum of a file
     #
     # @param file [String] file to md5sum
@@ -93,7 +98,7 @@ class Vanagon
     # @raise [Vanagon::Error] If the command fails an exception is raised
     def ex(command)
       ret = %x(#{command})
-      unless $?.success?
+      unless $CHILD_STATUS.success?
         raise Vanagon::Error, "'#{command}' did not succeed"
       end
       ret
@@ -119,6 +124,7 @@ class Vanagon
         return false
       end
     end
+    alias_method :which, :find_program_on_path
 
     # Method to retry a ruby block and fail if the command does not succeed
     # within the number of tries and timeout.
@@ -148,55 +154,6 @@ class Vanagon
       raise Vanagon::Error, "Block failed maximum number of #{tries} tries"
     end
 
-    # Simple wrapper around git command line executes the given commands and
-    # returns the results.
-    #
-    # @param command_string [String] The commands to be run
-    # @param raise_error [boolean] if this function should raise an error
-    #                              on a git failure
-    # @return [String] The output of the command
-    def git(command_string, raise_error = false)
-      git_bin = find_program_on_path('git')
-      output = %x(#{git_bin} #{command_string})
-      if raise_error
-        unless $?.success?
-          raise %(git #{command_string} failed)
-        end
-      end
-      return output
-    end
-
-    # Determines if the given directory is a git repo or not
-    #
-    # @param directory [String] The directory to check
-    # @return [true, false] True if the directory is a git repo, false otherwise
-    def is_git_repo?(directory = Dir.pwd)
-      Dir.chdir(directory) do
-        git('rev-parse --git-dir > /dev/null 2>&1')
-        $?.success?
-      end
-    end
-
-    # Determines a version for the given directory based on the git describe
-    # for the repository
-    #
-    # @param directory [String] The directory to use in versioning
-    # @return [String] The version of the directory accoring to git describe
-    # @raise [RuntimeError] If the given directory is not a git repo
-    def git_version(directory = Dir.pwd)
-      if is_git_repo?(directory)
-        Dir.chdir(directory) do
-          version = git('describe --tags 2> /dev/null').chomp
-          if version.empty?
-            warn "Directory '#{directory}' cannot be versioned by git. Maybe it hasn't been tagged yet?"
-          end
-          return version
-        end
-      else
-        fail "Directory '#{directory}' is not a git repo, cannot get a version"
-      end
-    end
-
     # Sends the desired file/directory to the destination using rsync
     #
     # @param source [String] file or directory to send
@@ -258,14 +215,14 @@ class Vanagon
       puts "Executing '#{command}' on '#{target}'"
       if return_command_output
         ret = %x(#{ssh_command(port)} -T #{target} '#{command.gsub("'", "'\\\\''")}').chomp
-        if $?.success?
+        if $CHILD_STATUS.success?
           return ret
         else
           raise "Remote ssh command (#{command}) failed on '#{target}'."
         end
       else
         Kernel.system("#{ssh_command(port)} -T #{target} '#{command.gsub("'", "'\\\\''")}'")
-        $?.success? or raise "Remote ssh command (#{command}) failed on '#{target}'."
+        $CHILD_STATUS.success? or raise "Remote ssh command (#{command}) failed on '#{target}'."
       end
     end
 
@@ -281,14 +238,14 @@ class Vanagon
         puts "Executing '#{command}' locally"
         if return_command_output
           ret = %x(#{command}).chomp
-          if $?.success?
+          if $CHILD_STATUS.success?
             return ret
           else
             raise "Local command (#{command}) failed."
           end
         else
           Kernel.system(command)
-          $?.success? or raise "Local command (#{command}) failed."
+          $CHILD_STATUS.success? or raise "Local command (#{command}) failed."
         end
       end
     end