packaging 0.88.77

data/lib/packaging/repo.rb

@@ -0,0 +1,155 @@
+module Pkg::Repo
+
+  class << self
+
+    ##
+    ## Construct a local_target based upon the versioning style
+    ##
+    def construct_local_target_path(project, versioning)
+      case versioning
+      when 'ref'
+        return File.join(project, Pkg::Config.ref)
+      when 'version'
+        return File.join(project, Pkg::Util::Version.dot_version)
+      else
+        fail "Error: Unknown versioning argument: #{versioning}"
+      end
+    end
+
+    ##
+    ## Put a single signed repo into a tarball stored in
+    ## 'pkg/<local_target>/<archive_name>.tar.gz'
+    ##
+    def create_signed_repo_archive(repo_location, archive_name, versioning)
+      tar = Pkg::Util::Tool.check_tool('tar')
+
+      local_target = construct_local_target_path(Pkg::Config.project, versioning)
+
+      if Pkg::Util::File.empty_dir?(File.join('pkg', local_target, repo_location))
+        if ENV['FAIL_ON_MISSING_TARGET'] == "true"
+          raise "Error: missing packages under #{repo_location}"
+        end
+        warn "Warn: Skipping #{archive_name} because #{repo_location} has no files"
+        return
+      end
+
+      Dir.chdir(File.join('pkg', local_target)) do
+        puts "Info: Archiving #{repo_location} as #{archive_name}"
+        target_tarball = File.join('repos', "#{archive_name}.tar.gz")
+        tar_command = "#{tar} --owner=0 --group=0 --create --gzip --file #{target_tarball} #{repo_location}"
+        stdout, _, _ = Pkg::Util::Execution.capture3(tar_command)
+        return stdout
+      end
+    end
+
+    ##
+    ## Add a single repo tarball into the 'all' tarball located in
+    ## 'pkg/<local_target>/<project>-all.tar'
+    ## Create the 'all' tarball if needed.
+    ##
+    def update_tarball_of_all_repos(project, platform, versioning)
+      tar = Pkg::Util::Tool.check_tool('tar')
+
+      all_repos_tarball_name = "#{project}-all.tar"
+      archive_name = "#{project}-#{platform['name']}"
+      local_target = construct_local_target_path(project, versioning)
+      repo_tarball_name = "#{archive_name}.tar.gz"
+      repo_tarball_path = File.join('repos', repo_tarball_name)
+
+      Dir.chdir(File.join('pkg', local_target)) do
+        unless Pkg::Util::File.exist?(repo_tarball_path)
+          warn "Skipping #{archive_name} because it (#{repo_tarball_path}) contains no files"
+          next
+        end
+
+        tar_action = "--create"
+        if File.exist?(all_repos_tarball_name)
+          tar_action = "--update"
+        end
+
+        tar_command = "#{tar} --owner=0 --group=0 #{tar_action} --file #{all_repos_tarball_name} #{repo_tarball_path}"
+        stdout, _, _ = Pkg::Util::Execution.capture3(tar_command)
+        puts stdout
+      end
+    end
+
+    ##
+    ## Invoke gzip to compress the 'all' tarball located in
+    ## 'pkg/<local_target>/<project>-all.tar'
+    ##
+    def compress_tarball_of_all_repos(all_repos_tarball_name)
+      gzip = Pkg::Util::Tool.check_tool('gzip')
+
+      gzip_command = "#{gzip} --fast #{all_repos_tarball_name}"
+      stdout, _, _ = Pkg::Util::Execution.capture3(gzip_command)
+      puts stdout
+    end
+
+    ##
+    ## Generate each of the repos listed in <Config.platform_repos>.
+    ## Update the 'all repos' tarball as we do each one.
+    ## Compress the 'all repos' tarball when all the repos have been generated
+    ##
+    def create_all_repo_archives(project, versioning)
+      platforms = Pkg::Config.platform_repos
+      local_target = construct_local_target_path(project, versioning)
+      all_repos_tarball_name = "#{project}-all.tar"
+
+      platforms.each do |platform|
+        archive_name = "#{project}-#{platform['name']}"
+        create_signed_repo_archive(platform['repo_location'], archive_name, versioning)
+        update_tarball_of_all_repos(project, platform, versioning)
+      end
+
+      Dir.chdir(File.join('pkg', local_target)) do
+        compress_tarball_of_all_repos(all_repos_tarball_name)
+      end
+    end
+
+    def directories_that_contain_packages(artifact_directory, pkg_ext)
+      cmd = "[ -d #{artifact_directory} ] || exit 1 ; "
+      cmd << "pushd #{artifact_directory} > /dev/null && "
+      cmd << "find . -name '*.#{pkg_ext}' -print0 | xargs --no-run-if-empty -0 -I {} dirname {} "
+      stdout, _ = Pkg::Util::Net.remote_execute(
+                Pkg::Config.distribution_server,
+                cmd,
+                { capture_output: true }
+              )
+      return stdout.split
+    rescue => e
+      fail "Error: Could not retrieve directories that contain #{pkg_ext} packages in #{Pkg::Config.distribution_server}:#{artifact_directory}"
+    end
+
+    def populate_repo_directory(artifact_parent_directory)
+      cmd = "[ -d #{artifact_parent_directory}/artifacts ] || exit 1 ; "
+      cmd << "pushd #{artifact_parent_directory} > /dev/null && "
+      cmd << 'rsync --archive --verbose --one-file-system --ignore-existing artifacts/ repos/ '
+      Pkg::Util::Net.remote_execute(Pkg::Config.distribution_server, cmd)
+    rescue => e
+      fail "Error: Could not populate repos directory in #{Pkg::Config.distribution_server}:#{artifact_parent_directory}"
+    end
+
+    def argument_required?(argument_name, repo_command)
+      repo_command.include?("__#{argument_name.upcase}__")
+    end
+
+    def update_repo(remote_host, command, options = {})
+      fail_message = "Error: Missing required argument '%s', update your build_defaults?"
+      [:repo_name, :repo_path, :repo_host, :repo_url].each do |option|
+        fail fail_message % option.to_s if argument_required?(option.to_s, command) && !options[option]
+      end
+
+      whitelist = {
+        __REPO_NAME__: options[:repo_name],
+        __REPO_PATH__: options[:repo_path],
+        __REPO_HOST__: options[:repo_host],
+        __REPO_URL__: options[:repo_url],
+        __APT_PLATFORMS__: Pkg::Config.apt_releases.join(' '),
+        __GPG_KEY__: Pkg::Util::Gpg.key
+      }
+      Pkg::Util::Net.remote_execute(
+        remote_host,
+        Pkg::Util::Misc.search_and_replace(command, whitelist))
+    end
+  end
+end

data/lib/packaging/retrieve.rb

@@ -0,0 +1,75 @@
+module Pkg::Retrieve
+  module_function
+
+  # --no-parent = Only descend when recursing, never ascend
+  # --no-host-directories = Discard http://#{Pkg::Config.builds_server} when saving to disk
+  # --level=0 = infinitely recurse, no limit
+  # --cut-dirs 3 = will cut off #{Pkg::Config.project}, #{Pkg::Config.ref}, and the first directory in #{remote_target} from the url when saving to disk
+  # --directory-prefix = where to save to disk (defaults to ./)
+  # --reject = Reject all hits that match the supplied regex
+
+  def default_wget_command(local_target, url, additional_options = {})
+    default_options = {
+      'quiet' => true,
+      'recursive' => true,
+      'no-parent' => true,
+      'no-host-directories' => true,
+      'level' => 0,
+      'cut-dirs' => 3,
+      'directory-prefix' => local_target,
+      'reject' => "'index*'",
+    }
+    options = default_options.merge(additional_options)
+    wget = Pkg::Util::Tool.check_tool('wget')
+    wget_command = wget
+    options.each do |option, value|
+      next unless value
+      if value.is_a?(TrueClass)
+        wget_command << " --#{option}"
+      else
+        wget_command << " --#{option}=#{value}"
+      end
+    end
+    wget_command << " #{url}"
+    return wget_command
+  end
+
+  # NOTE: When supplying additional options, if you want your value to be
+  # quoted (e.g. --reject='index*'), you must include the quotes as part of
+  # your string (e.g. {'reject' => "'index*'"}).
+  def default_wget(local_target, url, additional_options = {})
+    wget_command = default_wget_command(local_target, url, additional_options)
+    puts "Executing #{wget_command} . . ."
+    %x(#{wget_command})
+  end
+
+  # This will always retrieve from under the 'artifacts' directory
+  def foss_only_retrieve(build_url, local_target)
+    unless Pkg::Config.foss_platforms
+      fail "FOSS_ONLY specified, but I don't know anything about FOSS_PLATFORMS. Retrieve cancelled."
+    end
+    default_wget(local_target, "#{build_url}/", { 'level' => 1 })
+    yaml_path = File.join(local_target, "#{Pkg::Config.ref}.yaml")
+    unless File.readable?(yaml_path)
+      fail "Couldn't read #{Pkg::Config.ref}.yaml, which is necessary for FOSS_ONLY. Retrieve cancelled."
+    end
+    platform_data = Pkg::Util::Serialization.load_yaml(yaml_path)[:platform_data]
+    platform_data.each do |platform, paths|
+      path_to_retrieve = File.dirname(paths[:artifact])
+      default_wget(local_target, "#{build_url}/#{path_to_retrieve}/") if Pkg::Config.foss_platforms.include?(platform)
+    end
+  end
+
+  def retrieve_all(build_url, rsync_path, local_target)
+    if Pkg::Util::Tool.find_tool("wget")
+      default_wget(local_target, "#{build_url}/")
+    else
+      warn "Could not find `wget` tool. Falling back to rsyncing from #{Pkg::Config.distribution_server}."
+      begin
+        Pkg::Util::Net.rsync_from("#{rsync_path}/", Pkg::Config.distribution_server, "#{local_target}/")
+      rescue => e
+        fail "Couldn't rsync packages from distribution server.\n#{e}"
+      end
+    end
+  end
+end

data/lib/packaging/rpm.rb

@@ -0,0 +1,5 @@
+# RPM methods used by various libraries and rake tasks
+
+module Pkg::Rpm
+  require 'packaging/rpm/repo'
+end

data/lib/packaging/rpm/repo.rb

@@ -0,0 +1,254 @@
+# Utilities for working with rpm repos
+require 'fileutils'
+require 'find'
+
+module Pkg::Rpm::Repo
+  class << self
+    def base_url
+      "http://#{Pkg::Config.builds_server}/#{Pkg::Config.project}/#{Pkg::Config.ref}"
+    end
+
+    def ship_repo_configs(target = "repo_configs")
+      if Pkg::Util::File.empty_dir?("pkg/#{target}/rpm")
+        warn "No repo configs have been generated! Try pl:rpm_repo_configs."
+        return
+      end
+
+      Pkg::Util::RakeUtils.invoke_task("pl:fetch")
+      repo_dir = "#{Pkg::Config.jenkins_repo_path}/#{Pkg::Config.project}/#{Pkg::Config.ref}/#{target}/rpm"
+      Pkg::Util::Net.remote_execute(Pkg::Config.distribution_server, "mkdir -p #{repo_dir}")
+      Pkg::Util::Execution.retry_on_fail(:times => 3) do
+        Pkg::Util::Net.rsync_to("pkg/#{target}/rpm/", Pkg::Config.distribution_server, repo_dir)
+      end
+    end
+
+    def repo_creation_command(repo_directory, artifact_paths = nil)
+      cmd = "[ -d #{repo_directory} ] || exit 1 ; "
+      cmd << "pushd #{repo_directory} > /dev/null && "
+      cmd << 'echo "Checking for running repo creation. Will wait if detected." && '
+      cmd << 'while [ -f .lock ] ; do sleep 1 ; echo -n "." ; done && '
+      cmd << 'echo "Setting lock" && '
+      cmd << 'touch .lock && '
+      cmd << 'createrepo=$(which createrepo) ; '
+
+      # Added for compatibility.
+      # The nightly repo ships operate differently and do not want to be calculating
+      # the correct paths based on which packages are available on the distribution
+      # host, we just want to be `createrepo`ing for what we've staged locally
+      #
+      # We should only assume repo_directory exists locally if we didn't pass
+      # artifact paths
+      if artifact_paths.nil?
+        # Since the command already has a `pushd #{repo_directory}` let's make sure
+        # we're calculating artifact paths relative to that.
+        Dir.chdir repo_directory do
+          artifact_paths = Dir.glob('**/*.rpm').map { |package| File.dirname(package) }
+        end
+      end
+
+      artifact_paths.each do |path|
+        next if path.include? 'aix'
+        cmd << "if [ -d #{path}  ]; then "
+        cmd << "pushd #{path} && "
+        cmd << '$createrepo --checksum=sha --checkts --update --delta-workers=0 --database . && '
+        cmd << 'popd ; '
+        cmd << 'fi ;'
+      end
+      cmd
+    end
+
+    # @deprecated this command will die a painful death when we are
+    #   able to sit down with Operations and refactor our distribution infra.
+    #   At a minimum, it should be refactored alongside its Debian counterpart
+    #   into something modestly more generic.
+    #   - Ryan McKern 11/2015
+    #
+    # @param origin_path [String] path for RPM repos on local filesystem
+    # @param destination_path [String] path for RPM repos on remote filesystem
+    # @param destination [String] remote host to send rsynced content to. If
+    #        nil will copy locally
+    # @param dryrun [Boolean] whether or not to use '--dry-run'
+    #
+    # @return [String] an rsync command that can be executed on a remote host
+    #   to copy local content from that host to a remote node.
+    def repo_deployment_command(origin_path, destination_path, destination, dryrun = false)
+      path = Pathname.new(origin_path)
+      dest_path = Pathname.new(destination_path)
+
+      options = %w(
+        rsync
+        --recursive
+        --links
+        --hard-links
+        --update
+        --human-readable
+        --itemize-changes
+        --progress
+        --verbose
+        --super
+        --delay-updates
+        --omit-dir-times
+        --no-perms
+        --no-owner
+        --no-group
+      )
+
+      options << '--dry-run' if dryrun
+      options << path
+
+      if destination
+        options << "#{destination}:#{dest_path.parent}"
+      else
+        options << "#{dest_path.parent}"
+      end
+
+      options.join("\s")
+    end
+
+    def sign_repos(directory)
+      files_to_sign = Find.find(directory).select { |file| file.match(/repomd.xml$/) }
+      files_to_sign.each do |file|
+        Pkg::Util::Gpg.sign_file(file)
+      end
+    end
+
+    def retrieve_repo_configs(target = "repo_configs")
+      wget = Pkg::Util::Tool.check_tool("wget")
+      FileUtils.mkdir_p("pkg/#{target}")
+      config_url = "#{base_url}/#{target}/rpm/"
+      begin
+        stdout, _, _ = Pkg::Util::Execution.capture3("#{wget} -r -np -nH --cut-dirs 3 -P pkg/#{target} --reject 'index*' #{config_url}")
+        stdout
+      rescue => e
+        fail "Couldn't retrieve rpm yum repo configs.\n#{e}"
+      end
+    end
+
+    # Generate yum configuration files that point to the repositories created
+    # on the distribution server with packages created from the current source
+    # repo commit. There is one for each dist/version that is packaged (e.g.
+    # el5, el6, etc). Files are created in pkg/repo_configs/rpm and are named
+    # pl-$project-$sha.conf, and can be placed in /etc/yum.repos.d to enable
+    # clients to install these packages.
+    #
+    def generate_repo_configs(source = "repos", target = "repo_configs", signed = false)
+      # We have a hard requirement on wget because of all the download magicks
+      # we have to do
+      #
+      wget = Pkg::Util::Tool.check_tool("wget")
+
+      # This is the standard path to all build artifacts on the distribution
+      # server for this commit
+      #
+      repo_base = "#{base_url}/#{source}/"
+
+      # First check if the artifacts directory exists
+      #
+
+      # We have to do two checks here - first that there are directories with
+      # repodata folders in them, and second that those same directories also
+      # contain rpms
+      #
+      stdout, _, _ = Pkg::Util::Execution.capture3("#{wget} --spider -r -l 5 --no-parent #{repo_base} 2>&1")
+      stdout = stdout.split.uniq.reject { |x| x =~ /\?|index/ }.select { |x| x =~ /http:.*repodata\/$/ }
+
+      # RPMs will always exist at the same directory level as the repodata
+      # folder, which means if we go up a level we should find rpms
+      #
+      yum_repos = []
+      stdout.map { |x| x.chomp('repodata/') }.each do |url|
+        output, _, _ = Pkg::Util::Execution.capture3("#{wget} --spider -r -l 1 --no-parent #{url} 2>&1")
+        unless output.split.uniq.reject { |x| x =~ /\?|index/ }.select { |x| x =~ /http:.*\.rpm$/ }.empty?
+          yum_repos << url
+        end
+      end
+
+      if yum_repos.empty?
+        warn "No rpm repos were found to generate configs from!"
+        return
+      end
+
+      FileUtils.mkdir_p(File.join("pkg", target, "rpm"))
+
+      # Parse the rpm configs file to generate repository configs. Each line in
+      # the rpm_configs file corresponds with a repo directory on the
+      # distribution server.
+      #
+      yum_repos.each do |url|
+        # We ship a base 'srpm' that gets turned into a repo, but we want to
+        # ignore this one because its an extra
+        next if url == "#{repo_base}srpm/"
+
+        platform_tag = Pkg::Paths.tag_from_artifact_path(url)
+        platform, version, arch = Pkg::Platforms.parse_platform_tag(platform_tag)
+
+        # Create an array of lines that will become our yum config
+        #
+        config = ["[pl-#{Pkg::Config.project}-#{Pkg::Config.ref}]"]
+        config << ["name=PL Repo for #{Pkg::Config.project} at commit #{Pkg::Config.ref}"]
+        config << ["baseurl=#{url}"]
+        config << ["enabled=1"]
+        if signed
+          config << ["gpgcheck=1"]
+          config << ["gpgkey=http://#{Pkg::Config.builds_server}/#{Pkg::Util::Gpg.key}"]
+        else
+          config << ["gpgcheck=0"]
+        end
+
+        # Write the new config to a file under our repo configs dir
+        #
+        config_file = File.join("pkg", target, "rpm", "pl-#{Pkg::Config.project}-#{Pkg::Config.ref}-#{platform}-#{version}-#{arch}.repo")
+        File.open(config_file, 'w') { |f| f.puts config }
+      end
+      puts "Wrote yum configuration files for #{Pkg::Config.project} at #{Pkg::Config.ref} to pkg/#{target}/rpm"
+    end
+
+    def create_local_repos(directory = "repos")
+      stdout, _, _ = Pkg::Util::Execution.capture3("bash -c '#{repo_creation_command(directory)}'")
+      stdout
+    end
+
+    def create_remote_repos(directory = 'repos')
+      artifact_directory = File.join(Pkg::Config.jenkins_repo_path, Pkg::Config.project, Pkg::Config.ref)
+      artifact_paths = Pkg::Repo.directories_that_contain_packages(File.join(artifact_directory, 'artifacts'), 'rpm')
+      Pkg::Repo.populate_repo_directory(artifact_directory)
+      command = Pkg::Rpm::Repo.repo_creation_command(File.join(artifact_directory, directory), artifact_paths)
+
+      begin
+        Pkg::Util::Net.remote_execute(Pkg::Config.distribution_server, command)
+        # Now that we've created our package repositories, we can generate repo
+        # configurations for use with downstream jobs, acceptance clients, etc.
+        Pkg::Rpm::Repo.generate_repo_configs
+
+        # Now that we've created the repo configs, we can ship them
+        Pkg::Rpm::Repo.ship_repo_configs
+      ensure
+        # Always remove the lock file, even if we've failed
+        Pkg::Util::Net.remote_execute(Pkg::Config.distribution_server, "rm -f #{artifact_directory}/repos/.lock")
+      end
+    end
+
+    def create_repos_from_artifacts(directory = "repos")
+      Pkg::Util.deprecate('Pkg::Rpm::Repo.create_repos_from_artifacts', 'Pkg::Rpm::Repo.create_remote_repos')
+      create_remote_repos(directory)
+    end
+
+    def create_repos(directory = "repos")
+      Pkg::Util.deprecate('Pkg::Rpm::Repo.create_repos', 'Pkg::Rpm::Repo.create_local_repos')
+      create_local_repos(directory)
+    end
+
+    # @deprecated this command is exactly as awful as you think it is.
+    #   -- Ryan McKern 12/2015
+    #
+    # @param yum_path [String] path for rpm repos on local and remote filesystem
+    # @param origin_server [String] remote host to start the  rsync from
+    # @param destination_server [String] remote host to send rsynced content to
+    # @param dryrun [Boolean] whether or not to use '--dry-run'
+    def deploy_repos(yum_path, origin_server, destination_server, dryrun = false)
+      rsync_command = repo_deployment_command(yum_path, yum_path, destination_server, dryrun)
+
+      Pkg::Util::Net.remote_execute(origin_server, rsync_command)
+    end
+  end
+end