packaging 0.99.0

data/lib/packaging/repo.rb

@@ -0,0 +1,55 @@
+module Pkg::Repo
+
+  class << self
+    def create_signed_repo_archive(path_to_repo, name_of_archive, versioning)
+      tar = Pkg::Util::Tool.check_tool('tar')
+      Dir.chdir("pkg") do
+        if versioning == 'ref'
+          local_target = File.join(Pkg::Config.project, Pkg::Config.ref)
+        elsif versioning == 'version'
+          local_target = File.join(Pkg::Config.project, Pkg::Util::Version.dot_version)
+        end
+        Dir.chdir(local_target) do
+          if Pkg::Util::File.empty_dir?(path_to_repo)
+            if ENV['FAIL_ON_MISSING_TARGET'] == "true"
+              raise "ERROR: missing packages under #{path_to_repo}"
+            else
+              warn "Skipping #{name_of_archive} because #{path_to_repo} has no files"
+            end
+          else
+            puts "Archiving #{path_to_repo} as #{name_of_archive}"
+            stdout, _, _ = Pkg::Util::Execution.capture3("#{tar} --owner=0 --group=0 --create --gzip --file #{File.join('repos', "#{name_of_archive}.tar.gz")} #{path_to_repo}")
+            stdout
+          end
+        end
+      end
+    end
+
+    def create_all_repo_archives(project, versioning)
+      platforms = Pkg::Config.platform_repos
+      platforms.each do |platform|
+        archive_name = "#{project}-#{platform['name']}"
+        create_signed_repo_archive(platform['repo_location'], archive_name, versioning)
+      end
+    end
+
+    def directories_that_contain_packages(artifact_directory, pkg_ext)
+      cmd = "[ -d #{artifact_directory} ] || exit 1 ; "
+      cmd << "pushd #{artifact_directory} > /dev/null && "
+      cmd << "find . -name '*.#{pkg_ext}' -print0 | xargs --no-run-if-empty -0 -I {} dirname {} "
+      stdout, stderr = Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.distribution_server, cmd, true)
+      return stdout.split
+    rescue => e
+      fail "Could not retrieve directories that contain #{pkg_ext} packages in #{Pkg::Config.distribution_server}:#{artifact_directory}"
+    end
+
+    def populate_repo_directory(artifact_parent_directory)
+      cmd = "[ -d #{artifact_parent_directory}/artifacts ] || exit 1 ; "
+      cmd << "pushd #{artifact_parent_directory} > /dev/null && "
+      cmd << 'rsync --archive --verbose --one-file-system --ignore-existing artifacts/ repos/ '
+      Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.distribution_server, cmd)
+    rescue => e
+      fail "Could not populate repos directory in #{Pkg::Config.distribution_server}:#{artifact_parent_directory}"
+    end
+  end
+end

data/lib/packaging/retrieve.rb

@@ -0,0 +1,46 @@
+module Pkg::Retrieve
+  module_function
+
+  # --no-parent = Only descend when recursing, never ascend
+  # --no-host-directories = Discard http://#{Pkg::Config.builds_server} when saving to disk
+  # --level=0 = infinitely recurse, no limit
+  # --cut-dirs 3 = will cut off #{Pkg::Config.project}, #{Pkg::Config.ref}, and the first directory in #{remote_target} from the url when saving to disk
+  # --directory-prefix = where to save to disk (defaults to ./)
+  # --reject = Reject all hits that match the supplied regex
+
+  def default_wget(local_target, url)
+    wget = Pkg::Util::Tool.check_tool('wget')
+    wget_command = "#{wget} --quiet --recursive --no-parent --no-host-directories --level=0 --cut-dirs 3 --directory-prefix=#{local_target} --reject 'index*' #{url}"
+    puts "Executing #{wget_command} . . ."
+    %x(#{wget_command})
+  end
+
+  # This will always retrieve from under the 'artifacts' directory
+  def foss_only_retrieve(build_url, local_target)
+    unless Pkg::Config.foss_platforms
+      fail "FOSS_ONLY specified, but I don't know anything about FOSS_PLATFORMS. Retrieve cancelled."
+    end
+    default_wget(local_target, "#{build_url}/artifacts/#{Pkg::Config.ref}.yaml")
+    yaml_path = File.join(local_target, "#{Pkg::Config.ref}.yaml")
+    unless File.readable?(yaml_path)
+      fail "Couldn't read #{Pkg::Config.ref}.yaml, which is necessary for FOSS_ONLY. Retrieve cancelled."
+    end
+    platform_data = Pkg::Util::Serialization.load_yaml(yaml_path)[:platform_data]
+    platform_data.each do |platform, paths|
+      default_wget(local_target, "#{build_url}/artifacts/#{paths[:artifact]}") if Pkg::Config.foss_platforms.include?(platform)
+    end
+  end
+
+  def retrieve_all(build_url, rsync_path, remote_target, local_target)
+    if Pkg::Util::Tool.find_tool("wget")
+      default_wget(local_target, "#{build_url}/#{remote_target}/")
+    else
+      warn "Could not find `wget` tool. Falling back to rsyncing from #{Pkg::Config.distribution_server}."
+      begin
+        Pkg::Util::Net.rsync_from("#{rsync_path}/#{remote_target}/", Pkg::Config.distribution_server, "#{local_target}/")
+      rescue => e
+        fail "Couldn't rsync packages from distribution server.\n#{e}"
+      end
+    end
+  end
+end

data/lib/packaging/rpm.rb

@@ -0,0 +1,5 @@
+# RPM methods used by various libraries and rake tasks
+
+module Pkg::Rpm
+  require 'packaging/rpm/repo'
+end

data/lib/packaging/rpm/repo.rb

@@ -0,0 +1,257 @@
+# Utilities for working with rpm repos
+require 'fileutils'
+require 'find'
+
+module Pkg::Rpm::Repo
+  class << self
+    def base_url
+      "http://#{Pkg::Config.builds_server}/#{Pkg::Config.project}/#{Pkg::Config.ref}"
+    end
+
+    def ship_repo_configs(target = "repo_configs")
+      if Pkg::Util::File.empty_dir?("pkg/#{target}/rpm")
+        warn "No repo configs have been generated! Try pl:rpm_repo_configs."
+        return
+      end
+
+      invoke_task("pl:fetch")
+      repo_dir = "#{Pkg::Config.jenkins_repo_path}/#{Pkg::Config.project}/#{Pkg::Config.ref}/#{target}/rpm"
+      Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.distribution_server, "mkdir -p #{repo_dir}")
+      Pkg::Util::Execution.retry_on_fail(:times => 3) do
+        Pkg::Util::Net.rsync_to("pkg/#{target}/rpm/", Pkg::Config.distribution_server, repo_dir)
+      end
+    end
+
+    def repo_creation_command(repo_directory, artifact_paths = nil)
+      cmd = "[ -d #{repo_directory} ] || exit 1 ; "
+      cmd << "pushd #{repo_directory} > /dev/null && "
+      cmd << 'echo "Checking for running repo creation. Will wait if detected." && '
+      cmd << 'while [ -f .lock ] ; do sleep 1 ; echo -n "." ; done && '
+      cmd << 'echo "Setting lock" && '
+      cmd << 'touch .lock && '
+      cmd << 'createrepo=$(which createrepo) ; '
+
+      # Added for compatibility.
+      # The nightly repo ships operate differently and do not want to be calculating
+      # the correct paths based on which packages are available on the distribution
+      # host, we just want to be `createrepo`ing for what we've staged locally
+      #
+      # We should only assume repo_directory exists locally if we didn't pass
+      # artifact paths
+      if artifact_paths.nil?
+        # Since the command already has a `pushd #{repo_directory}` let's make sure
+        # we're calculating artifact paths relative to that.
+        Dir.chdir repo_directory do
+          artifact_paths = Dir.glob('**/*.rpm').map { |package| File.dirname(package) }
+        end
+      end
+
+      artifact_paths.each do |path|
+        next if path.include? 'aix'
+        cmd << "if [ -d #{path}  ]; then "
+        cmd << "pushd #{path} && "
+        cmd << '$createrepo --checksum=sha --checkts --update --delta-workers=0 --database . && '
+        cmd << 'popd ; '
+        cmd << 'fi ;'
+      end
+      cmd
+    end
+
+    # @deprecated this command will die a painful death when we are
+    #   able to sit down with Operations and refactor our distribution infra.
+    #   At a minimum, it should be refactored alongside its Debian counterpart
+    #   into something modestly more generic.
+    #   - Ryan McKern 11/2015
+    #
+    # @param origin_path [String] path for RPM repos on local filesystem
+    # @param destination_path [String] path for RPM repos on remote filesystem
+    # @param destination [String] remote host to send rsynced content to. If
+    #        nil will copy locally
+    # @param dryrun [Boolean] whether or not to use '--dry-run'
+    #
+    # @return [String] an rsync command that can be executed on a remote host
+    #   to copy local content from that host to a remote node.
+    def repo_deployment_command(origin_path, destination_path, destination, dryrun = false)
+      path = Pathname.new(origin_path)
+      dest_path = Pathname.new(destination_path)
+
+      # You may think "rsync doesn't actually remove the sticky bit, let's
+      # remove the Dugo-s from the chmod". However, that will make your rsyncs
+      # fail due to permission errors.
+      options = %w(
+        rsync
+        --recursive
+        --links
+        --hard-links
+        --update
+        --human-readable
+        --itemize-changes
+        --progress
+        --verbose
+        --super
+        --delay-updates
+        --omit-dir-times
+        --no-perms
+        --no-owner
+        --no-group
+      )
+
+      options << '--dry-run' if dryrun
+      options << path
+
+      if destination
+        options << "#{destination}:#{dest_path.parent}"
+      else
+        options << "#{dest_path.parent}"
+      end
+
+      options.join("\s")
+    end
+
+    def sign_repos(directory)
+      files_to_sign = Find.find(directory).select { |file| file.match(/repomd.xml$/) }
+      files_to_sign.each do |file|
+        Pkg::Util::Gpg.sign_file(file)
+      end
+    end
+
+    def retrieve_repo_configs(target = "repo_configs")
+      wget = Pkg::Util::Tool.check_tool("wget")
+      FileUtils.mkdir_p("pkg/#{target}")
+      config_url = "#{base_url}/#{target}/rpm/"
+      begin
+        stdout, _, _ = Pkg::Util::Execution.capture3("#{wget} -r -np -nH --cut-dirs 3 -P pkg/#{target} --reject 'index*' #{config_url}")
+        stdout
+      rescue => e
+        fail "Couldn't retrieve rpm yum repo configs.\n#{e}"
+      end
+    end
+
+    # Generate yum configuration files that point to the repositories created
+    # on the distribution server with packages created from the current source
+    # repo commit. There is one for each dist/version that is packaged (e.g.
+    # el5, el6, etc). Files are created in pkg/repo_configs/rpm and are named
+    # pl-$project-$sha.conf, and can be placed in /etc/yum.repos.d to enable
+    # clients to install these packages.
+    #
+    def generate_repo_configs(source = "repos", target = "repo_configs", signed = false)
+      # We have a hard requirement on wget because of all the download magicks
+      # we have to do
+      #
+      wget = Pkg::Util::Tool.check_tool("wget")
+
+      # This is the standard path to all build artifacts on the distribution
+      # server for this commit
+      #
+      repo_base = "#{base_url}/#{source}/"
+
+      # First check if the artifacts directory exists
+      #
+
+      # We have to do two checks here - first that there are directories with
+      # repodata folders in them, and second that those same directories also
+      # contain rpms
+      #
+      stdout, _, _ = Pkg::Util::Execution.capture3("#{wget} --spider -r -l 5 --no-parent #{repo_base} 2>&1")
+      stdout = stdout.split.uniq.reject { |x| x =~ /\?|index/ }.select { |x| x =~ /http:.*repodata\/$/ }
+
+      # RPMs will always exist at the same directory level as the repodata
+      # folder, which means if we go up a level we should find rpms
+      #
+      yum_repos = []
+      stdout.map { |x| x.chomp('repodata/') }.each do |url|
+        output, _, _ = Pkg::Util::Execution.capture3("#{wget} --spider -r -l 1 --no-parent #{url} 2>&1")
+        unless output.split.uniq.reject { |x| x =~ /\?|index/ }.select { |x| x =~ /http:.*\.rpm$/ }.empty?
+          yum_repos << url
+        end
+      end
+
+      if yum_repos.empty?
+        warn "No rpm repos were found to generate configs from!"
+        return
+      end
+
+      FileUtils.mkdir_p(File.join("pkg", target, "rpm"))
+
+      # Parse the rpm configs file to generate repository configs. Each line in
+      # the rpm_configs file corresponds with a repo directory on the
+      # distribution server.
+      #
+      yum_repos.each do |url|
+        # We ship a base 'srpm' that gets turned into a repo, but we want to
+        # ignore this one because its an extra
+        next if url == "#{repo_base}srpm/"
+
+        platform_tag = Pkg::Paths.tag_from_artifact_path(url)
+        platform, version, arch = Pkg::Platforms.parse_platform_tag(platform_tag)
+
+        # Create an array of lines that will become our yum config
+        #
+        config = ["[pl-#{Pkg::Config.project}-#{Pkg::Config.ref}]"]
+        config << ["name=PL Repo for #{Pkg::Config.project} at commit #{Pkg::Config.ref}"]
+        config << ["baseurl=#{url}"]
+        config << ["enabled=1"]
+        if signed
+          config << ["gpgcheck=1"]
+          config << ["gpgkey=http://#{Pkg::Config.builds_server}/#{Pkg::Util::Gpg.key}"]
+        else
+          config << ["gpgcheck=0"]
+        end
+
+        # Write the new config to a file under our repo configs dir
+        #
+        config_file = File.join("pkg", target, "rpm", "pl-#{Pkg::Config.project}-#{Pkg::Config.ref}-#{platform}-#{version}-#{arch}.repo")
+        File.open(config_file, 'w') { |f| f.puts config }
+      end
+      puts "Wrote yum configuration files for #{Pkg::Config.project} at #{Pkg::Config.ref} to pkg/#{target}/rpm"
+    end
+
+    def create_local_repos(directory = "repos")
+      stdout, _, _ = Pkg::Util::Execution.capture3("bash -c '#{repo_creation_command(directory)}'")
+      stdout
+    end
+
+    def create_remote_repos(directory = 'repos')
+      artifact_directory = File.join(Pkg::Config.jenkins_repo_path, Pkg::Config.project, Pkg::Config.ref)
+      artifact_paths = Pkg::Repo.directories_that_contain_packages(File.join(artifact_directory, 'artifacts'), 'rpm')
+      Pkg::Repo.populate_repo_directory(artifact_directory)
+      command = Pkg::Rpm::Repo.repo_creation_command(File.join(artifact_directory, directory), artifact_paths)
+
+      begin
+        Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.distribution_server, command)
+        # Now that we've created our package repositories, we can generate repo
+        # configurations for use with downstream jobs, acceptance clients, etc.
+        Pkg::Rpm::Repo.generate_repo_configs
+
+        # Now that we've created the repo configs, we can ship them
+        Pkg::Rpm::Repo.ship_repo_configs
+      ensure
+        # Always remove the lock file, even if we've failed
+        Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.distribution_server, "rm -f #{artifact_directory}/repos/.lock")
+      end
+    end
+
+    def create_repos_from_artifacts(directory = "repos")
+      Pkg::Util.deprecate('Pkg::Rpm::Repo.create_repos_from_artifacts', 'Pkg::Rpm::Repo.create_remote_repos')
+      create_remote_repos(directory)
+    end
+
+    def create_repos(directory = "repos")
+      Pkg::Util.deprecate('Pkg::Rpm::Repo.create_repos', 'Pkg::Rpm::Repo.create_local_repos')
+      create_local_repos(directory)
+    end
+
+    # @deprecated this command is exactly as awful as you think it is.
+    #   -- Ryan McKern 12/2015
+    #
+    # @param yum_path [String] path for rpm repos on local and remote filesystem
+    # @param origin_server [String] remote host to start the  rsync from
+    # @param destination_server [String] remote host to send rsynced content to
+    # @param dryrun [Boolean] whether or not to use '--dry-run'
+    def deploy_repos(yum_path, origin_server, destination_server, dryrun = false)
+      rsync_command = repo_deployment_command(yum_path, yum_path, destination_server, dryrun)
+
+      Pkg::Util::Net.remote_ssh_cmd(origin_server, rsync_command)
+    end
+  end
+end

data/lib/packaging/tar.rb

@@ -0,0 +1,154 @@
+module Pkg
+  class Tar
+    require 'fileutils'
+    require 'pathname'
+    include FileUtils
+
+    attr_accessor :files, :project, :version, :excludes, :target, :templates
+    attr_reader :tar
+
+    def initialize
+      @tar      = Pkg::Util::Tool.find_tool('tar', :required => true)
+      @project  = Pkg::Config.project
+      @version  = Pkg::Config.version
+      @files    = Pkg::Config.files
+      @target   = File.join(Pkg::Config.project_root, "pkg", "#{@project}-#{@version}.tar.gz")
+
+      # We require that the excludes list be a string (which is space
+      # separated, we hope)(deprecated) or an array.
+      #
+      if Pkg::Config.tar_excludes
+        if Pkg::Config.tar_excludes.is_a?(String)
+          warn "warning: `tar_excludes` should be an array, not a string"
+          @excludes = Pkg::Config.tar_excludes.split(' ')
+        elsif Pkg::Config.tar_excludes.is_a?(Array)
+          @excludes = Pkg::Config.tar_excludes
+        else
+          fail "Tarball excludes must either be an array or a string, not #{@excludes.class}"
+        end
+      else
+        @excludes = []
+      end
+
+      # On the other hand, support for explicit templates started with Arrays,
+      # so that's all we support.
+      #
+      if Pkg::Config.templates
+        @templates = Pkg::Config.templates.dup
+        fail "templates must be an array" unless @templates.is_a?(Array)
+        expand_templates
+      end
+    end
+
+    def install_files_to(workdir)
+      # It is nice to use arrays in YAML to represent array content, but we used
+      # to support a mode where a space-separated string was used.  Support both
+      # to allow a gentle migration to a modern style...
+      patterns =
+        case @files
+        when String
+          $stderr.puts "warning: `files` should be an array, not a string"
+          @files.split(' ')
+        when Array
+          @files
+        else
+          raise "`files` must be a string or an array!"
+        end
+
+      Pkg::Util::File.install_files_into_dir(patterns, workdir)
+    end
+
+    # The templates of a project can include globs, which may expand to an
+    # arbitrary number of files. This method expands all of the templates using
+    # Dir.glob and then filters out any templates that live in the packaging
+    # tools themselves. If the template is a source/target combination, it is
+    # returned to the array untouched.
+    def expand_templates
+      @templates.map! do |tempfile|
+        if tempfile.is_a?(String)
+          # Expand possible globs to all matching entries
+          Dir.glob(File.join(Pkg::Config::project_root, tempfile))
+        elsif tempfile.is_a?(Hash)
+          tempfile
+        end
+      end
+      @templates.flatten!
+
+      # Reject matches that are templates from packaging itself. These will contain the packaging root.
+      # These tend to come from the current tar.rake implementation.
+      @templates.reject! { |temp| temp.is_a?(String) && temp.match(/#{Pkg::Config::packaging_root}/) }
+    end
+
+    # Given the tar object's template files (assumed to be in Pkg::Config.project_root), transform
+    # them, removing the originals. If workdir is passed, assume Pkg::Config.project_root
+    # exists in workdir
+    def template(workdir = nil)
+      workdir ||= Pkg::Config.project_root
+      root = Pathname.new(Pkg::Config.project_root)
+
+      # Templates can be either a string or a hash of source and target. If it
+      # is a string, the target is assumed to be the same path as the
+      # source,with the extension removed. If it is a hash, we assume nothing
+      # and use the provided source and target.
+      @templates.each do |cur_template|
+        if cur_template.is_a?(String)
+          template_file = File.expand_path(cur_template)
+          target_file = template_file.sub(File.extname(template_file), "")
+        elsif cur_template.is_a?(Hash)
+          template_file = File.expand_path(cur_template["source"])
+          target_file = File.expand_path(cur_template["target"])
+        end
+
+        #   We construct paths to the erb template and its proposed target file
+        #   relative to the project root, *not* fully qualified. This allows us
+        #   to, given a temporary workdir containing a copy of the project,
+        #   construct the full path to the erb and target file inside the
+        #   temporary workdir.
+        #
+        rel_path_to_template = Pathname.new(template_file).relative_path_from(root).to_s
+        rel_path_to_target = Pathname.new(target_file).relative_path_from(root).to_s
+
+        #   What we pass to Pkg::util::File.erb_file are the paths to the erb
+        #   and target inside of a temporary project directory. We are, in
+        #   essence, templating "in place." This is why we remove the original
+        #   files - they're not the originals in the authoritative project
+        #   directory, but the originals in the temporary working copy.
+        if File.exist?(File.join(workdir, rel_path_to_template))
+          mkpath(File.dirname(File.join(workdir, rel_path_to_target)), :verbose => false)
+          Pkg::Util::File.erb_file(File.join(workdir, rel_path_to_template), File.join(workdir, rel_path_to_target), true, :binding => Pkg::Config.get_binding)
+        elsif File.exist?(File.join(root, rel_path_to_template))
+          mkpath(File.dirname(File.join(workdir, rel_path_to_target)), :verbose => false)
+          Pkg::Util::File.erb_file(File.join(root, rel_path_to_template), File.join(workdir, rel_path_to_target), false, :binding => Pkg::Config.get_binding)
+        else
+          fail "Expected to find #{template_file} in #{root} for templating. But it was not there. Maybe you deleted it?"
+        end
+      end
+    end
+
+    def tar(target, source)
+      mkpath File.dirname(target)
+      cd File.dirname(source) do
+        %x(#{@tar} #{@excludes.map { |x| (" --exclude #{x} ") }.join if @excludes} -zcf '#{File.basename(target)}' '#{File.basename(source)}')
+        unless $?.success?
+          fail "Failed to create .tar.gz archive with #{@tar}. Please ensure the tar command in your path accepts the flags '-c', '-z', and '-f'"
+        end
+        mv File.basename(target), target
+      end
+    end
+
+    def clean_up(workdir)
+      rm_rf workdir
+    end
+
+    def pkg!
+      workdir = File.join(Pkg::Util::File.mktemp, "#{@project}-#{@version}")
+      mkpath workdir
+      self.install_files_to workdir
+      self.template(workdir)
+      self.tar(@target, workdir)
+      self.clean_up workdir
+    end
+
+  end
+end
+