packaging 0.99.0

data/lib/packaging/util.rb

@@ -0,0 +1,146 @@
+# Utility methods used by the various rake tasks
+module Pkg::Util
+  require 'erb'
+  require 'benchmark'
+  require 'base64'
+  require 'io/console'
+  require 'packaging/util/date'
+  require 'packaging/util/execution'
+  require 'packaging/util/file'
+  require 'packaging/util/git'
+  require 'packaging/util/gpg'
+  require 'packaging/util/jenkins'
+  require 'packaging/util/misc'
+  require 'packaging/util/net'
+  require 'packaging/util/os'
+  require 'packaging/util/platform'
+  require 'packaging/util/serialization'
+  require 'packaging/util/ship'
+  require 'packaging/util/tool'
+  require 'packaging/util/rake_utils'
+  require 'packaging/util/version'
+  require 'packaging/util/git_tags'
+
+  def self.boolean_value(var)
+    return TRUE if var == TRUE || ( var.is_a?(String) && ( var.downcase == 'true' || var.downcase =~ /^y$|^yes$/))
+    FALSE
+  end
+
+  def self.in_project_root(&blk)
+    result = nil
+    fail "Cannot execute in project root if Pkg::Config.project_root is not set" unless Pkg::Config.project_root
+
+    Dir.chdir Pkg::Config.project_root do
+      result = blk.call
+    end
+    result
+  end
+
+  # Utility to get the contents of an Environment variable
+  #
+  # @param var [String] The name of the environment variable to return
+  # @return [String, Boolean, Hash, Array, nil] The contents of ENV[var]
+  def self.get_var(var)
+    self.check_var(var, ENV[var])
+    ENV[var]
+  end
+
+  # Utility to check if a variable is set
+  #
+  # @param varname [String] the name of the variable to be checked
+  # @param var [String, Boolean, Hash, Array, nil] the contents of the variable to be checked
+  # @return [String, Boolean, Hash, Array, nil] the contents of var
+  # @raise [RuntimeError] raises an exception if the variable is not set and is required
+  def self.check_var(varname, var)
+    fail "Requires #{varname} be set!" if var.nil?
+    var
+  end
+
+  def self.require_library_or_fail(library, library_name = nil)
+    library_name ||= library
+    begin
+      require library
+    rescue LoadError
+      fail "Could not load #{library_name}. #{library_name} is required by the packaging repo for this task"
+    end
+  end
+
+  def self.base64_encode(string)
+    Base64.encode64(string).strip
+  end
+
+  # Utility to retrieve command line input
+  # @param noecho [Boolean, nil] if we are retrieving command line input with or without privacy. This is mainly
+  #   for sensitive information like passwords.
+  def self.get_input(echo = true)
+    fail "Cannot get input on a noninteractive terminal" unless $stdin.tty?
+
+    system 'stty -echo' unless echo
+    $stdin.gets.chomp!
+  ensure
+    system 'stty echo'
+  end
+
+  def self.rand_string
+    rand.to_s.split('.')[1]
+  end
+
+  def self.ask_yes_or_no(force = false)
+    unless force
+      return Pkg::Util.boolean_value(Pkg::Config.answer_override) unless Pkg::Config.answer_override.nil?
+    end
+
+    answer = Pkg::Util.get_input
+    return true if answer =~ /^y$|^yes$/
+    return false if answer =~ /^n$|^no$/
+    puts "Nope, try something like yes or no or y or n, etc:"
+    Pkg::Util.ask_yes_or_no
+  end
+
+  def self.confirm_ship(files)
+    $stdout.puts "Artifacts will be shipped to the following hosts:"
+    Pkg::Util.filter_configs('host').each { |key, value| puts "#{key}: #{value}" }
+    $stdout.puts "Does this look right?? [y,n]"
+    Pkg::Util.ask_yes_or_no(true)
+    $stdout.puts "The following files have been built and are ready to ship:"
+    files.each { |file| puts "\t#{file}\n" unless File.directory?(file) }
+    $stdout.puts "Ship these files?? [y,n]"
+    Pkg::Util.ask_yes_or_no(true)
+  end
+
+  def self.filter_configs(filter = nil)
+    return Pkg::Config.instance_values.select { |key, _| key.match(/#{filter}/) } if filter
+    Pkg::Config.instance_values
+  end
+
+
+  # Construct a probably-correct (or correct-enough) URI for
+  # tools like ssh or rsync. Currently lacking support for intuitive
+  # joins, ports, protocols, fragments, or 75% of what Addressable::URI
+  # or URI would provide out of the box. The "win" here is that
+  # the returned String should "just work".
+  # @private pseudo_uri
+  # @return [String, nil] a string representing either a hostname:/path pair,
+  #   a hostname without a path, or a path without a hostname. Returns nil
+  #   if it is unable to construct a useful URI-like string.
+  # @param [Hash] opts fragments used to build the pseudo URI
+  # @option opts [String] :path URI-ish path component
+  # @option opts [String] :host URI-ish host component
+  def self.pseudo_uri(opts = {})
+    options = { path: nil, host: nil }.merge(opts)
+
+    # Prune empty values to determine what is returned
+    options.delete_if { |_, v| v.to_s.empty? }
+    return nil if options.empty?
+
+    [options[:host], options[:path]].compact.join(':')
+  end
+
+  def self.deprecate(old_cmd, new_cmd = nil)
+    msg = "!! #{old_cmd} is deprecated."
+    if new_cmd
+      msg << " Please use #{new_cmd} instead."
+    end
+    $stdout.puts("\n#{msg}\n")
+  end
+end

data/lib/packaging/util/date.rb

@@ -0,0 +1,15 @@
+# Utilities for managing/querying date/time
+
+module Pkg::Util::Date
+
+  class << self
+    def timestamp(separator = nil)
+      if s = separator
+        format = "%Y#{s}%m#{s}%d#{s}%H#{s}%M#{s}%S"
+      else
+        format = "%Y-%m-%d %H:%M:%S"
+      end
+      Time.now.strftime(format)
+    end
+  end
+end

data/lib/packaging/util/execution.rb

@@ -0,0 +1,85 @@
+# Utility methods for handling system calls and interactions
+
+module Pkg::Util::Execution
+
+  class << self
+
+    # Alias to $?.success? that makes success? slightly easier to test and stub
+    # If immediately run, $? will not be instanciated, so only call success? if
+    # $? exists, otherwise return nil
+    def success?(statusobject = $?)
+      return statusobject.success?
+    end
+
+    # ex combines the behavior of `%x{cmd}` and rake's `sh "cmd"`. `%x{cmd}` has
+    # the benefit of returning the standard out of an executed command, enabling us
+    # to query the file system, e.g. `contents = %x{ls}`. The drawback to `%x{cmd}`
+    # is that on failure of a command (something returned non-zero) the return of
+    # `%x{cmd}` is just an empty string. As such, we can't know if we succeeded.
+    # Rake's `sh "cmd"`, on the other hand, will raise a RuntimeError if a command
+    # does not return 0, but doesn't return any of the stdout from the command -
+    # only true or false depending on its success or failure. With `ex(cmd)` we
+    # purport to both return the results of the command execution (ala `%x{cmd}`)
+    # while also raising an exception if a command does not succeed (ala `sh "cmd"`).
+    def ex(command, debug = false)
+      puts "Executing '#{command}'..." if debug
+      ret = `#{command}`
+      unless Pkg::Util::Execution.success?
+        raise RuntimeError
+      end
+
+      if debug
+        puts "Command '#{command}' returned:"
+        puts ret
+      end
+
+      ret
+    end
+
+    # Turns out trying to change ex to use Open3 is SUPER DANGEROUS and destructive
+    # in ways I hadn't imagined. I'm going to add a new method here instead and start
+    # converting code to use that so I don't break more than I plan to.
+    def capture3(command, debug = false)
+      require 'open3'
+      puts "Executing '#{command}'..." if debug
+      stdout, stderr, ret = Open3.capture3(command)
+      unless Pkg::Util::Execution.success?(ret)
+        raise "#{stdout}#{stderr}"
+      end
+
+      if debug
+        puts "Command '#{command}' returned:"
+        puts stdout
+      end
+
+      return stdout, stderr, ret
+    end
+
+    # Loop a block up to the number of attempts given, exiting when we receive success
+    # or max attempts is reached. Raise an exception unless we've succeeded.
+    def retry_on_fail(args, &blk)
+      success = FALSE
+      exception = ''
+
+      if args[:times].respond_to?(:times) and block_given?
+        args[:times].times do |i|
+          if args[:delay]
+            sleep args[:delay]
+          end
+
+          begin
+            blk.call
+            success = TRUE
+            break
+          rescue => err
+            puts "An error was encountered evaluating block. Retrying.."
+            exception = err.to_s + "\n" + err.backtrace.join("\n")
+          end
+        end
+      else
+        fail "retry_on_fail requires and arg (:times => x) where x is an Integer/Fixnum, and a block to execute"
+      end
+      fail "Block failed maximum of #{args[:times]} tries. Exiting..\nLast failure was: #{exception}" unless success
+    end
+  end
+end

data/lib/packaging/util/file.rb

@@ -0,0 +1,125 @@
+# Utility methods for handling files and directories
+require 'fileutils'
+
+module Pkg::Util::File
+
+  class << self
+    def exist?(file)
+      ::File.exist?(file)
+    end
+    alias_method :exists?, :exist?
+
+    def directory?(file)
+      ::File.directory?(file)
+    end
+
+    def mktemp
+      mktemp = Pkg::Util::Tool.find_tool('mktemp', :required => true)
+      stdout, _, _ = Pkg::Util::Execution.capture3("#{mktemp} -d -t pkgXXXXXX")
+      stdout.strip
+    end
+
+    def empty_dir?(dir)
+      File.exist?(dir) and File.directory?(dir) and Dir["#{dir}/**/*"].empty?
+    end
+
+    # Returns an array of all the directories at the top level of #{dir}
+    #
+    def directories(dir)
+      if File.directory?(dir)
+        Dir.chdir(dir) do
+          Dir.glob("*").select { |entry| File.directory?(entry) }
+        end
+      end
+    end
+
+    # Returns an array of all files with #{ext} inside #{dir}
+    def files_with_ext(dir, ext)
+      Dir.glob("#{dir}/**/*#{ext}")
+    end
+
+
+    def file_exists?(file, args = { :required => false })
+      exists = File.exist? file
+      if !exists and args[:required]
+        fail "Required file #{file} could not be found"
+      end
+      exists
+    end
+
+    def file_writable?(file, args = { :required => false })
+      writable = File.writable? file
+      if !writable and args[:required]
+        fail "File #{file} is not writable"
+      end
+      writable
+    end
+
+    alias :get_temp :mktemp
+
+    def erb_string(erbfile, b = binding)
+      template = File.read(erbfile)
+      message  = ERB.new(template, nil, "-")
+      message.result(b)
+    end
+
+    def erb_file(erbfile, outfile = nil, remove_orig = false, opts = { :binding => binding })
+      outfile ||= File.join(mktemp, File.basename(erbfile).sub(File.extname(erbfile), ""))
+      output = erb_string(erbfile, opts[:binding])
+      File.open(outfile, 'w') { |f| f.write output }
+      puts "Generated: #{outfile}"
+      FileUtils.rm_rf erbfile if remove_orig
+      outfile
+    end
+
+    def untar_into(source, target = nil, options = "")
+      tar = Pkg::Util::Tool.find_tool('tar', :required => true)
+      # We only accept a writable directory as a target
+      if target and !target.empty? and file_writable?(target) and File.directory?(target)
+        target_opts = "-C #{target}"
+      end
+      if file_exists?(source, :required => true)
+        stdout, _, _ = Pkg::Util::Execution.capture3(%Q(#{tar} #{options} #{target_opts} -xf #{source}))
+        stdout
+      end
+    end
+
+    def install_files_into_dir(file_patterns, workdir)
+      install = []
+      # We need to add our list of file patterns from the configuration; this
+      # used to be a list of "things to copy recursively", which would install
+      # editor backup files and other nasty things.
+      #
+      # This handles that case correctly, with a deprecation warning, to augment
+      # our FileList with the right things to put in place.
+      #
+      # Eventually, when all our projects are migrated to the new standard, we
+      # can drop this in favour of just pushing the patterns directly into the
+      # FileList and eliminate many lines of code and comment.
+      Dir.chdir(Pkg::Config.project_root) do
+        file_patterns.each do |pattern|
+          if File.directory?(pattern) and !Pkg::Util::File.empty_dir?(pattern)
+            install << Dir[pattern + "/**/*"]
+          else
+            install << Dir[pattern]
+          end
+        end
+        install.flatten!
+
+        # Transfer all the files and symlinks into the working directory...
+        install = install.select { |x| File.file?(x) or File.symlink?(x) or Pkg::Util::File.empty_dir?(x) }
+
+        install.each do |file|
+          if Pkg::Util::File.empty_dir?(file)
+            FileUtils.mkpath(File.join(workdir, file), :verbose => false)
+          else
+            FileUtils.mkpath(File.dirname(File.join(workdir, file)), :verbose => false)
+            FileUtils.cp(file, File.join(workdir, file), :verbose => false, :preserve => true)
+          end
+        end
+      end
+      Pkg::Util::Version.versionbump(workdir) if Pkg::Config.update_version_file
+    end
+  end
+end
+

data/lib/packaging/util/git.rb

@@ -0,0 +1,174 @@
+require 'fileutils'
+
+# Utility methods for handling git
+module Pkg::Util::Git
+  class << self
+    # Git utility to create a new git commit
+    def commit_file(file, message = 'changes')
+      fail_unless_repo
+      puts 'Commiting changes:'
+      puts
+      diff, = Pkg::Util::Execution.capture3("#{Pkg::Util::Tool::GIT} diff HEAD #{file}")
+      puts diff
+      stdout, = Pkg::Util::Execution.capture3(%(#{Pkg::Util::Tool::GIT} commit #{file} -m "Commit #{message} in #{file}" &> #{Pkg::Util::OS::DEVNULL}))
+      stdout
+    end
+
+    # Git utility to create a new git tag
+    def tag(version)
+      fail_unless_repo
+      stdout, = Pkg::Util::Execution.capture3("#{Pkg::Util::Tool::GIT} tag -s -u #{Pkg::Util::Gpg.key} -m '#{version}' #{version}")
+      stdout
+    end
+
+    # Git utility to create a new git bundle
+    # rubocop:disable Metrics/AbcSize
+    def bundle(treeish, appendix = Pkg::Util.rand_string, temp = Pkg::Util::File.mktemp)
+      fail_unless_repo
+      Pkg::Util::Execution.capture3("#{Pkg::Util::Tool::GIT} bundle create #{temp}/#{Pkg::Config.project}-#{Pkg::Config.version}-#{appendix} #{treeish} --tags")
+      Dir.chdir(temp) do
+        Pkg::Util::Execution.capture3("#{Pkg::Util::Tool.find_tool('tar')} -czf #{Pkg::Config.project}-#{Pkg::Config.version}-#{appendix}.tar.gz #{Pkg::Config.project}-#{Pkg::Config.version}-#{appendix}")
+        FileUtils.rm_rf("#{Pkg::Config.project}-#{Pkg::Config.version}-#{appendix}")
+      end
+      "#{temp}/#{Pkg::Config.project}-#{Pkg::Config.version}-#{appendix}.tar.gz"
+    end
+
+    def pull(remote, branch)
+      fail_unless_repo
+      stdout, = Pkg::Util::Execution.capture3("#{Pkg::Util::Tool::GIT} pull #{remote} #{branch}")
+      stdout
+    end
+
+    # Check if we are currently working on a tagged commit.
+    def tagged?
+      ref_type == 'tag'
+    end
+
+    # Reports if a ref and its corresponding git repo points to
+    # a git tag.
+    #
+    # @param url [string] url of repo grabbed from json file
+    # @param ref [string] ref grabbed from json file
+    def remote_tagged?(url, ref)
+      reference = Pkg::Util::Git_tag.new(url, ref)
+      reference.tag?
+    end
+
+    # Checks out a specified ref. The ref must exist in the current repo.
+    # This also removes any uncommitted changes
+    def checkout(ref)
+      Pkg::Util.in_project_root do
+        _, _, ret = Pkg::Util::Execution.capture3("#{Pkg::Util::Tool::GIT} reset --hard ; #{Pkg::Util::Tool::GIT} checkout #{ref}")
+        Pkg::Util::Execution.success?(ret) || raise("Could not checkout #{ref} git branch to build package from...exiting")
+      end
+    end
+
+    # Returns the value of `git describe`. If this is not a git repo or
+    # `git describe` fails because there is no tag, this will return false
+    def describe(extra_opts = ['--tags', '--dirty'])
+      Pkg::Util.in_project_root do
+        stdout, _, ret = Pkg::Util::Execution.capture3("#{Pkg::Util::Tool::GIT} describe #{Array(extra_opts).join(' ')}")
+        if Pkg::Util::Execution.success?(ret)
+          stdout.strip
+        else
+          false
+        end
+      end
+    end
+
+    # return the sha of HEAD on the current branch
+    # You can specify the length you want from the sha. Default is 40, the
+    # length for sha1. If you specify anything higher, it will still return 40
+    # characters. Ideally, you're not going to specify anything under 7 characters,
+    # but I'll leave that discretion up to you.
+    def sha(length = 40)
+      Pkg::Util.in_project_root do
+        stdout, = Pkg::Util::Execution.capture3("#{Pkg::Util::Tool::GIT} rev-parse --short=#{length} HEAD")
+        stdout.strip
+      end
+    end
+
+    # Return the ref type of HEAD on the current branch
+    def ref_type
+      Pkg::Util.in_project_root do
+        stdout, = Pkg::Util::Execution.capture3("#{Pkg::Util::Tool::GIT} cat-file -t #{describe('')}")
+        stdout.strip
+      end
+    end
+
+    # If HEAD is a tag, return the tag. Otherwise return the sha of HEAD.
+    def sha_or_tag(length = 40)
+      if ref_type == 'tag'
+        describe
+      else
+        sha(length)
+      end
+    end
+
+    # Return true if we're in a git repo, otherwise false
+    def repo?
+      Pkg::Util.in_project_root do
+        _, _, ret = Pkg::Util::Execution.capture3("#{Pkg::Util::Tool::GIT} rev-parse --git-dir")
+        Pkg::Util::Execution.success?(ret)
+      end
+    end
+
+    # rubocop:disable Style/GuardClause
+    def fail_unless_repo
+      unless repo?
+        raise "Pkg::Config.project_root (#{Pkg::Config.project_root}) is not \
+          a valid git repository"
+      end
+    end
+
+    # Return the basename of the project repo
+    def project_name
+      Pkg::Util.in_project_root do
+        stdout, = Pkg::Util::Execution.capture3("#{Pkg::Util::Tool::GIT} config --get remote.origin.url")
+        stdout.split('/')[-1].chomp('.git').chomp
+      end
+    end
+
+    # Return the name of the current branch
+    def branch_name
+      Pkg::Util.in_project_root do
+        stdout, = Pkg::Util::Execution.capture3("#{Pkg::Util::Tool::GIT} rev-parse --abbrev-ref HEAD")
+        stdout.strip
+      end
+    end
+
+    def source_dirty?
+      describe.include?('dirty')
+    end
+
+    def fail_on_dirty_source
+      if source_dirty?
+        raise "The source tree is dirty, e.g. there are uncommited changes. \
+         Please commit/discard changes and try again."
+      end
+    end
+
+    ##########################################################################
+    # DEPRECATED METHODS
+    #
+    def git_commit_file(file, message = "changes")
+      Pkg::Util.deprecate('Pkg::Util::Git.git_commit_file', 'Pkg::Util::Git.commit_file')
+      Pkg::Util::Git.commit_file(file, message)
+    end
+
+    def git_tag(version)
+      Pkg::Util.deprecate('Pkg::Util::Git.git_tag', 'Pkg::Util::Git.tag')
+      Pkg::Util::Git.tag(version)
+    end
+
+    def git_bundle(treeish, appendix = Pkg::Util.rand_string, temp = Pkg::Util::File.mktemp)
+      Pkg::Util.deprecate('Pkg::Util::Git.git_bundle', 'Pkg::Util::Git.bundle')
+      Pkg::Util::Git.bundle(treeish, appendix, temp)
+    end
+
+    def git_pull(remote, branch)
+      Pkg::Util.deprecate('Pkg::Util::Git.git_pull', 'Pkg::Util::Git.pull')
+      Pkg::Util::Git.pull(remote, branch)
+    end
+  end
+end