puppet 2.7.26 → 3.0.0.rc4

data/ext/packaging/tasks/retrieve.rake

@@ -1,45 +0,0 @@
-##
-# This task is intended to retrieve packages from the distribution server that
-# have been built by jenkins and placed in a specific location,
-# /opt/jenkins-builds/$PROJECT/$SHA where $PROJECT is the build project as
-# established in project_data.yaml and $SHA is the git sha/tag of the project that
-# was built into packages. The current day is assumed, but an environment
-# variable override exists to retrieve packages from another day. The sha/tag is
-# assumed to be the current project's HEAD, e.g.  to retrieve packages for a
-# release of 3.1.0, checkout 3.1.0 locally before retrieving.
-#
-
-namespace :pl do
-  namespace :jenkins do
-    desc "Retrieve packages from the distribution server\. Check out commit to retrieve"
-    task :retrieve, :target do |t, args|
-      target = args.target || "artifacts"
-      invoke_task("pl:fetch")
-      mkdir_p 'pkg'
-      package_url = "http://#{Pkg::Config.builds_server}/#{Pkg::Config.project}/#{Pkg::Config.ref}/#{target}"
-      if wget=Pkg::Util::Tool.find_tool("wget")
-        sh "#{wget} -r -np -nH --cut-dirs 3 -P pkg --reject 'index*' #{package_url}/"
-      else
-        warn "Could not find `wget` tool. Falling back to rsyncing from #{Pkg::Config.distribution_server}"
-        begin
-          rsync_from("#{Pkg::Config.jenkins_repo_path}/#{Pkg::Config.project}/#{Pkg::Config.ref}/#{target}/", Pkg::Config.distribution_server, "pkg/")
-        rescue
-          fail "Couldn't download packages from distribution server. Try installing wget!"
-        end
-      end
-      puts "Packages staged in pkg"
-    end
-  end
-end
-
-if Pkg::Config.build_pe
-  namespace :pe do
-    namespace :jenkins do
-      desc "Retrieve packages from the distribution server\. Check out commit to retrieve"
-      task :retrieve, :target do |t, args|
-        target = args.target || "artifacts"
-        invoke_task("pl:jenkins:retrieve", target)
-      end
-    end
-  end
-end

data/ext/packaging/tasks/rpm.rake

@@ -1,71 +0,0 @@
-def prep_rpm_build_dir
-  temp = Pkg::Util::File.mktemp
-  tarball = "#{Pkg::Config.project}-#{Pkg::Config.version}.tar.gz"
-  FileUtils.mkdir_p([temp, "#{temp}/SOURCES", "#{temp}/SPECS"])
-  cp_pr FileList["pkg/#{tarball}*"], "#{temp}/SOURCES"
-  # If the file ext/redhat/<project>.spec exists in the tarball, we use it. If
-  # it doesn't we try to 'erb' the file from a predicted template in source,
-  # ext/redhat/<project>.spec.erb. If that doesn't exist, we fail. To do this,
-  # we have to open the tarball.
-  cp_p("pkg/#{tarball}", temp)
-
-  # Test for specfile in tarball
-  %x{tar -tzf #{File.join(temp, tarball)}}.split.grep(/ext\/redhat\/#{Pkg::Config.project}.spec$/)
-
-  if $?.success?
-    sh "tar -C #{temp} -xzf #{File.join(temp, tarball)} #{Pkg::Config.project}-#{Pkg::Config.version}/ext/redhat/#{Pkg::Config.project}.spec"
-    cp("#{temp}/#{Pkg::Config.project}-#{Pkg::Config.version}/ext/redhat/#{Pkg::Config.project}.spec", "#{temp}/SPECS/")
-  elsif File.exists?("ext/redhat/#{Pkg::Config.project}.spec.erb")
-    Pkg::Util::File.erb_file("ext/redhat/#{Pkg::Config.project}.spec.erb", "#{temp}/SPECS/#{Pkg::Config.project}.spec", nil, :binding => Pkg::Config.get_binding)
-  else
-    fail "Could not locate redhat spec ext/redhat/#{Pkg::Config.project}.spec or ext/redhat/#{Pkg::Config.project}.spec.erb"
-  end
-  temp
-end
-
-def build_rpm(buildarg = "-bs")
-  Pkg::Util::Tool.check_tool('rpmbuild')
-  workdir = prep_rpm_build_dir
-  if dist = Pkg::Util::Version.el_version
-    if dist.to_i < 6
-      dist_string = "--define \"%dist .el#{dist}"
-    end
-  end
-  rpm_define = "#{dist_string} --define \"%_topdir  #{workdir}\" "
-  rpm_old_version = '--define "_source_filedigest_algorithm 1" --define "_binary_filedigest_algorithm 1" \
-     --define "_binary_payload w9.gzdio" --define "_source_payload w9.gzdio" \
-     --define "_default_patch_fuzz 2"'
-  args = rpm_define + ' ' + rpm_old_version
-  FileUtils.mkdir_p('pkg/srpm')
-  if buildarg == '-ba'
-    FileUtils.mkdir_p('pkg/rpm')
-  end
-  if Pkg::Config.sign_tar
-    Rake::Task["pl:sign_tar"].invoke
-  end
-  sh "rpmbuild #{args} #{buildarg} --nodeps #{workdir}/SPECS/#{Pkg::Config.project}.spec"
-  mv FileList["#{workdir}/SRPMS/*.rpm"], "pkg/srpm"
-  if buildarg == '-ba'
-    mv FileList["#{workdir}/RPMS/*/*.rpm"], "pkg/rpm"
-  end
-  rm_rf workdir
-  puts
-  output = FileList['pkg/*/*.rpm']
-  puts "Wrote:"
-  output.each do | line |
-    puts line
-  end
-end
-
-namespace :package do
-  desc "Create srpm from this git repository (unsigned)"
-  task :srpm => :tar do
-    build_rpm("-bs")
-  end
-
-  desc "Create .rpm from this git repository (unsigned)"
-  task :rpm => :tar do
-    build_rpm("-ba")
-  end
-end
-

data/ext/packaging/tasks/rpm_repos.rake

@@ -1,139 +0,0 @@
-##
-#
-# A set of functionality for creating yum rpm repositories throughout the
-# standard pkg/ directory layout that the packaging repo creates. The standard
-# layout is:
-# pkg/{el,fedora}/{5,6,f16,f17,f18}/{products,devel,dependencies,extras}/{i386,x86_64,SRPMS}
-#
-# Because we'll likely be creating the repos on a server that is remote, e.g.
-# the distribution server, the logic here assumes we'll be doing everything via
-# ssh commands.
-#
-namespace :pl do
-  namespace :jenkins do
-    desc "Create yum repositories of built RPM packages for this SHA on the distribution server"
-    task :rpm_repos => "pl:fetch" do
-      begin
-        # Formulate our command string, which will just find directories with rpms
-        # and create and update repositories.
-        #
-        artifact_directory = File.join(Pkg::Config.jenkins_repo_path, Pkg::Config.project, Pkg::Config.ref)
-
-        ##
-        # Test that the artifacts directory exists on the distribution server.
-        # This will give us some more helpful output.
-        #
-        cmd = 'echo "Checking for build artifacts. Will exit if not found." ; '
-        cmd << "[ -d #{artifact_directory}/artifacts ] || exit 1 ; "
-
-        ##
-        # Enter the directory containing the build artifacts and create repos.
-        #
-        cmd << "pushd #{artifact_directory} ; "
-        cmd << 'echo "Checking for running repo creation. Will wait if detected." ; '
-        cmd << "while [ -f .lock ] ; do sleep 1 ; echo -n '.' ; done ; "
-        cmd << 'echo "Setting lock" ; '
-        cmd << "touch .lock ; "
-        cmd << "rsync -avxl artifacts/ repos/ ; pushd repos ; "
-        cmd << "createrepo=$(which createrepo) ; "
-        cmd << 'for repodir in $(find ./ -name "*.rpm" | xargs -I {} dirname {}) ; do '
-        cmd << "[ -d ${repodir} ] || continue; "
-        cmd << "pushd ${repodir} && ${createrepo} --checksum=sha --database --update . ; popd ; "
-        cmd << "done ; popd "
-
-        remote_ssh_cmd(Pkg::Config.distribution_server, cmd)
-        # Now that we've created our repositories, we can create the configs for
-        # them
-        Rake::Task["pl:jenkins:generate_rpm_repo_configs"].execute
-
-        # And once they're created, we can ship them
-        Rake::Task["pl:jenkins:ship_repo_configs"].execute
-      ensure
-        # Always remove the lock file, even if we've failed
-        remote_ssh_cmd(Pkg::Config.distribution_server, "rm -f #{artifact_directory}/.lock")
-      end
-    end
-
-    # Generate yum configuration files that point to the repositories created
-    # on the distribution server with packages created from the current source
-    # repo commit. There is one for each dist/version that is packaged (e.g.
-    # el5, el6, etc). Files are created in pkg/repo_configs/rpm and are named
-    # pl-$project-$sha.conf, and can be placed in /etc/yum.repos.d to enable
-    # clients to install these packages.
-    #
-    desc "Create yum repository configs for package repos for this sha/tag on the distribution server"
-    task :generate_rpm_repo_configs => "pl:fetch" do
-
-      # We have a hard requirement on wget because of all the download magicks
-      # we have to do
-      #
-      wget = Pkg::Util::Tool.find_tool("wget") or fail "Could not find `wget` tool. This is needed for composing the yum repo configurations. Install `wget` and try again."
-
-      # This is the standard path to all build artifacts on the distribution
-      # server for this commit
-      #
-      base_url = "http://#{Pkg::Config.builds_server}/#{Pkg::Config.project}/#{Pkg::Config.ref}/repos/"
-
-      # First check if the artifacts directory exists
-      #
-
-      # We have to do two checks here - first that there are directories with
-      # repodata folders in them, and second that those same directories also
-      # contain rpms
-      #
-      repo_urls = %x{#{wget} --spider -r -l 5 --no-parent #{base_url} 2>&1}.split.uniq.reject{ |x| x =~ /\?|index/ }.select{|x| x =~ /http:.*repodata\/$/}
-
-      # RPMs will always exist at the same directory level as the repodata
-      # folder, which means if we go up a level we should find rpms
-      #
-      yum_repos = []
-      repo_urls.map{|x| x.chomp('repodata/')}.each do |url|
-        unless %x{#{wget} --spider -r -l 1 --no-parent #{url} 2>&1}.split.uniq.reject{ |x| x =~ /\?|index/ }.select{|x| x =~ /http:.*\.rpm$/}.empty?
-          yum_repos << url
-        end
-      end
-
-      yum_repos.empty? and fail "No rpm repos were found to generate configs from!"
-
-      mkdir_p File.join("pkg","repo_configs","rpm")
-
-      # Parse the rpm configs file to generate repository configs. Each line in
-      # the rpm_configs file corresponds with a repo directory on the
-      # distribution server.
-      #
-      yum_repos.each do |url|
-        # We ship a base 'srpm' that gets turned into a repo, but we want to
-        # ignore this one because its an extra
-        next if url == "#{base_url}srpm/"
-
-        dist,version,_subdir,arch = url.split('/')[-4..-1]
-
-        # Create an array of lines that will become our yum config
-        #
-        config = ["[pl-#{Pkg::Config.project}-#{Pkg::Config.ref}]"]
-        config << ["name=PL Repo for #{Pkg::Config.project} at commit #{Pkg::Config.ref}"]
-        config << ["baseurl=#{url}"]
-        config << ["enabled=1"]
-        config << ["gpgcheck=0"]
-
-        # Write the new config to a file under our repo configs dir
-        #
-        config_file = File.join("pkg", "repo_configs", "rpm", "pl-#{Pkg::Config.project}-#{Pkg::Config.ref}-#{dist}-#{version}-#{arch}.repo")
-        File.open(config_file, 'w') { |f| f.puts config }
-      end
-      puts "Wrote yum configuration files for #{Pkg::Config.project} at #{Pkg::Config.ref} to pkg/repo_configs/rpm"
-    end
-
-    desc "Retrieve rpm yum repository configs from distribution server"
-    task :rpm_repo_configs => "pl:fetch" do
-      wget = Pkg::Util::Tool.find_tool("wget") or fail "Could not find `wget` tool! wget is required to download the repository configs."
-      mkdir_p "pkg/repo_configs"
-      config_url = "#{Pkg::Config.builds_server}/#{Pkg::Config.project}/#{Pkg::Config.ref}/repo_configs/rpm/"
-      begin
-        sh "#{wget} -r -np -nH --cut-dirs 3 -P pkg/repo_configs --reject 'index*' #{config_url}"
-      rescue
-        fail "Couldn't retrieve rpm yum repo configs. See preceding http response for more info."
-      end
-    end
-  end
-end

data/ext/packaging/tasks/ship.rake

@@ -1,192 +0,0 @@
-namespace :pl do
-  desc "Ship mocked rpms to #{Pkg::Config.yum_host}"
-  task :ship_rpms do
-    ["el", "fedora"].each do |dist|
-      retry_on_fail(:times => 3) do
-        pkgs = Dir["pkg/#{dist}/**/*.rpm"].map { |f| "'#{f.gsub("pkg/#{dist}/", "#{Pkg::Config.yum_repo_path}/#{dist}/")}'"}
-        unless pkgs.empty?
-          rsync_to("pkg/#{dist}", Pkg::Config.yum_host, Pkg::Config.yum_repo_path)
-          remote_set_immutable(Pkg::Config.yum_host, pkgs)
-        end
-      end if File.directory?("pkg/#{dist}")
-    end
-  end
-
-  namespace :remote do
-    # These hacky bits execute a pre-existing rake task on the Pkg::Config.apt_host
-    # The rake task takes packages in a specific directory and freights them
-    # to various target yum and apt repositories based on their specific type
-    # e.g., final vs devel vs PE vs FOSS packages
-
-    desc "Update remote rpm repodata on #{Pkg::Config.yum_host}"
-    task :update_yum_repo do
-      STDOUT.puts "Really run remote repo update on #{Pkg::Config.yum_host}? [y,n]"
-      if ask_yes_or_no
-        remote_ssh_cmd(Pkg::Config.yum_host, 'rake -f /opt/repository/Rakefile mk_repo')
-      end
-    end
-
-    desc "remote freight packages to repos on #{Pkg::Config.apt_host}"
-    task :freight do
-      STDOUT.puts "Really run remote freight command on #{Pkg::Config.apt_host}? [y,n]"
-      if ask_yes_or_no
-        override = "OVERRIDE=1" if ENV['OVERRIDE']
-        remote_ssh_cmd(Pkg::Config.apt_host, "rake -f /opt/repository/Rakefile freight #{override}")
-      end
-    end
-  end
-
-  desc "Ship cow-built debs to #{Pkg::Config.apt_host}"
-  task :ship_debs do
-    retry_on_fail(:times => 3) do
-      if File.directory?("pkg/deb")
-        rsync_to('pkg/deb/', Pkg::Config.apt_host, Pkg::Config.apt_repo_path)
-      end
-    end
-  end
-
-  namespace :remote do
-  end
-
-  desc "Update remote ips repository on #{Pkg::Config.ips_host}"
-  task :update_ips_repo do
-    rsync_to('pkg/ips/pkgs/', Pkg::Config.ips_host, Pkg::Config.ips_store)
-    remote_ssh_cmd(Pkg::Config.ips_host, "pkgrecv -s #{Pkg::Config.ips_store}/pkgs/#{Pkg::Config.project}Pkg::Config.#{Pkg::Config.ipsversion}.p5p -d #{Pkg::Config.ips_repo} \\*")
-    remote_ssh_cmd(Pkg::Config.ips_host, "pkgrepo refresh -s #{Pkg::Config.ips_repo}")
-    remote_ssh_cmd(Pkg::Config.ips_host, "/usr/sbin/svcadm restart svc:/application/pkg/server")
-  end if Pkg::Config.build_ips
-
-  desc "Upload ips p5p packages to downloads"
-  task :ship_ips => 'pl:fetch' do
-    if Dir['pkg/ips/pkgs/**/*'].empty?
-      STDOUT.puts "There aren't any p5p packages in pkg/ips/pkgs. Maybe something went wrong?"
-    else
-      rsync_to('pkg/ips/pkgs/', Pkg::Config.ips_package_host, Pkg::Config.ips_path)
-    end
-  end if Pkg::Config.build_ips
-
-  # We want to ship a gem only for projects that build gems
-  if Pkg::Config.build_gem
-    desc "Ship built gem to rubygems"
-    task :ship_gem do
-      # Even if a project builds a gem, if it uses the odd_even or zero-based
-      # strategies, we only want to ship final gems because otherwise a
-      # development gem would be preferred over the last final gem
-      if Pkg::Config.version_strategy !~ /odd_even|zero_based/ || Pkg::Util::Version.is_final?
-        FileList["pkg/#{Pkg::Config.gem_name}-#{Pkg::Config.gemversion}*.gem"].each do |f|
-          puts "Shipping gem #{f} to rubygems"
-          ship_gem(f)
-        end
-      else
-        STDERR.puts "Not shipping development gem using odd_even strategy for the sake of your users."
-      end
-    end
-  end
-
-  desc "ship apple dmg to #{Pkg::Config.yum_host}"
-  task :ship_dmg => 'pl:fetch' do
-    retry_on_fail(:times => 3) do
-      rsync_to('pkg/apple/*.dmg', Pkg::Config.yum_host, Pkg::Config.dmg_path)
-    end
-  end if Pkg::Config.build_dmg
-
-  desc "ship tarball and signature to #{Pkg::Config.tar_host}"
-  task :ship_tar => 'pl:fetch' do
-    retry_on_fail(:times => 3) do
-      rsync_to("pkg/#{Pkg::Config.project}-#{Pkg::Config.version}.tar.gz*", Pkg::Config.tar_host, Pkg::Config.tarball_path)
-    end
-  end
-
-  desc "UBER ship: ship all the things in pkg"
-  task :uber_ship => 'pl:fetch' do
-    if confirm_ship(FileList["pkg/**/*"])
-      ENV['ANSWER_OVERRIDE'] = 'yes'
-      Rake::Task["pl:ship_gem"].invoke if Pkg::Config.build_gem
-      Rake::Task["pl:ship_rpms"].invoke
-      Rake::Task["pl:ship_debs"].invoke
-      Rake::Task["pl:ship_dmg"].execute if Pkg::Config.build_dmg
-      Rake::Task["pl:ship_tar"].execute
-      Rake::Task["pl:jenkins:ship"].invoke("shipped")
-      add_shipped_metrics(:pe_version => ENV['PE_VER'], :is_rc => (! Pkg::Util::Version.is_final?)) if Pkg::Config.benchmark
-      post_shipped_metrics if Pkg::Config.benchmark
-    else
-      puts "Ship canceled"
-      exit
-    end
-  end
-
-  # It is odd to namespace this ship task under :jenkins, but this task is
-  # intended to be a component of the jenkins-based build workflow even if it
-  # doesn't interact with jenkins directly. The :target argument is so that we
-  # can invoke this task with a subdirectory of the standard distribution
-  # server path. That way we can separate out built artifacts from
-  # signed/actually shipped artifacts e.g. $path/shipped/ or $path/artifacts.
-  namespace :jenkins do
-    desc "Ship pkg directory contents to distribution server"
-    task :ship, :target do |t, args|
-      invoke_task("pl:fetch")
-      target = args.target || "artifacts"
-      artifact_dir = "#{Pkg::Config.jenkins_repo_path}/#{Pkg::Config.project}/#{Pkg::Config.ref}/#{target}"
-
-      # In order to get a snapshot of what this build looked like at the time
-      # of shipping, we also generate and ship the params file
-      #
-      Pkg::Config.config_to_yaml('pkg')
-
-
-      # Sadly, the packaging repo cannot yet act on its own, without living
-      # inside of a packaging-repo compatible project. This means in order to
-      # use the packaging repo for shipping and signing (things that really
-      # don't require build automation, specifically) we still need the project
-      # clone itself.
-      git_bundle('HEAD', 'signing_bundle', 'pkg')
-
-      # While we're bundling things, let's also make a git bundle of the
-      # packaging repo that we're using when we invoke pl:jenkins:ship. We can
-      # have a reasonable level of confidence, later on, that the git bundle on
-      # the distribution server was, in fact, the git bundle used to create the
-      # associated packages. This is because this ship task is automatically
-      # called upon completion each cell of the pl:jenkins:uber_build, and we
-      # have --ignore-existing set below. As such, the only git bundle that
-      # should possibly be on the distribution is the one used to create the
-      # packages.
-      # We're bundling the packaging repo because it allows us to keep an
-      # archive of the packaging source that was used to create the packages,
-      # so that later on if we need to rebuild an older package to audit it or
-      # for some other reason we're assured that the new package isn't
-      # different by virtue of the packaging automation.
-      if defined?(PACKAGING_ROOT)
-        packaging_bundle = ''
-        cd PACKAGING_ROOT do
-          packaging_bundle = git_bundle('HEAD', 'packaging-bundle')
-        end
-        mv(packaging_bundle, 'pkg')
-      end
-
-      retry_on_fail(:times => 3) do
-        remote_ssh_cmd(Pkg::Config.distribution_server, "mkdir -p #{artifact_dir}")
-      end
-      retry_on_fail(:times => 3) do
-        ignore_existing = "--ignore-existing"
-        rsync_to("pkg/", Pkg::Config.distribution_server, "#{artifact_dir}/ #{ignore_existing} --exclude repo_configs")
-      end
-      # If we just shipped a tagged version, we want to make it immutable
-      files = Dir.glob("pkg/**/*").select { |f| File.file?(f) }.map do |file|
-        "#{artifact_dir}/#{file.sub(/^pkg\//,'')}"
-      end
-      remote_set_immutable(Pkg::Config.distribution_server, files)
-    end
-
-    desc "Ship generated repository configs to the distribution server"
-    task :ship_repo_configs do
-      Pkg::Util::File.empty_dir?("pkg/repo_configs") and fail "No repo configs have been generated! Try pl:deb_repo_configs or pl:rpm_repo_configs"
-      invoke_task("pl:fetch")
-      repo_dir = "#{Pkg::Config.jenkins_repo_path}/#{Pkg::Config.project}/#{Pkg::Config.ref}/repo_configs"
-      remote_ssh_cmd(Pkg::Config.distribution_server, "mkdir -p #{repo_dir}")
-      retry_on_fail(:times => 3) do
-        rsync_to("pkg/repo_configs/", Pkg::Config.distribution_server, repo_dir)
-      end
-    end
-  end
-end
-