puppet 2.7.20 → 2.7.21

data/ext/packaging/tasks/build.rake

@@ -0,0 +1,183 @@
+module Build
+  ##
+  # This class is meant to encapsulate all of the data we know about a build invoked with
+  # `rake package:<build>` or `rake pl:<build>`. It can read in this data via a yaml file,
+  # have it set via accessors, and serialize it back to yaml for easy transport.
+  #
+  class BuildInstance
+    @@build_params = [:apt_host,
+                      :apt_repo_path,
+                      :apt_repo_url,
+                      :author,
+                      :benchmark,
+                      :build_defaults,
+                      :build_dmg,
+                      :build_doc,
+                      :build_gem,
+                      :build_ips,
+                      :build_pe,
+                      :builder_data_file,
+                      :builds_server,
+                      :certificate_pem,
+                      :cows,
+                      :db_table,
+                      :deb_build_host,
+                      :debversion,
+                      :debug,
+                      :default_cow,
+                      :default_mock,
+                      :description,
+                      :distribution_server,
+                      :dmg_path,
+                      :email,
+                      :files,
+                      :final_mocks,
+                      :freight_conf,
+                      :gem_default_executables,
+                      :gem_dependencies,
+                      :gem_description,
+                      :gem_devel_dependencies,
+                      :gem_excludes,
+                      :gem_executables,
+                      :gem_files,
+                      :gem_forge_project,
+                      :gem_name,
+                      :gem_rdoc_options,
+                      :gem_require_path,
+                      :gem_runtime_dependencies,
+                      :gem_summary,
+                      :gem_test_files,
+                      :gemversion,
+                      :gpg_key,
+                      :gpg_name,
+                      :homepage,
+                      :ips_build_host,
+                      :ips_host,
+                      :ips_inter_cert,
+                      :ips_package_host,
+                      :ips_path,
+                      :ips_repo,
+                      :ips_store,
+                      :ipsversion,
+                      :jenkins_build_host,
+                      :jenkins_packaging_job,
+                      :jenkins_repo_path,
+                      :metrics,
+                      :name,
+                      :project,
+                      :origversion,
+                      :osx_build_host,
+                      :packager,
+                      :packaging_repo,
+                      :packaging_url,
+                      :pbuild_conf,
+                      :pe_name,
+                      :pe_version,
+                      :pg_major_version,
+                      :privatekey_pem,
+                      :random_mockroot,
+                      :rc_mocks,
+                      :release,
+                      :rpm_build_host,
+                      :rpmrelease,
+                      :rpmversion,
+                      :ref,
+                      :sign_tar,
+                      :sles_build_host,
+                      :sles_repo_path,
+                      :sles_repo_host,
+                      :sles_arch_repos,
+                      :summary,
+                      :tar_excludes,
+                      :tarball_path,
+                      :task,
+                      :team,
+                      :version,
+                      :version_file,
+                      :yum_host,
+                      :yum_repo_path]
+
+    @@build_params.each do |v|
+      attr_accessor v
+    end
+
+    def initialize
+      @task = { :task => $*[0], :args => $*[1..-1] }
+      @ref = git_sha_or_tag
+    end
+
+    ##
+    # Take a hash of parameters, and iterate over them,
+    # setting each build param to the corresponding hash key,value.
+    #
+    def set_params_from_hash(data = {})
+      data.each do |param, value|
+        if @@build_params.include?(param.to_sym)
+          self.instance_variable_set("@#{param}", value)
+        else
+          warn "Warning - No build data parameter found for '#{param}'. Perhaps you have an erroneous entry in your yaml file?"
+        end
+      end
+    end
+
+    ##
+    # Load build parameters from a yaml file. Uses #data_from_yaml in
+    # 00_utils.rake
+    #
+    def set_params_from_file(file)
+      build_data = data_from_yaml(file)
+      set_params_from_hash(build_data)
+    end
+
+    ##
+    # Return a hash of all build parameters and their values, nil if unassigned.
+    #
+    def params
+      data = {}
+      @@build_params.each do |param|
+        data.store(param, self.instance_variable_get("@#{param}"))
+      end
+      data
+    end
+
+    ##
+    # Write all build parameters to a yaml file in a temporary location. Print
+    # the path to the file and return it as a string. Accept an argument for
+    # the write target directory. The name of the params file is the current
+    # git commit sha or tag.
+    #
+    def params_to_yaml(output_dir=nil)
+      dir = output_dir.nil? ? get_temp : output_dir
+      unless File.writable?(dir)
+        warn "#{dir} does not exist or is not writable, skipping build params write. Exiting.."
+        exit 1
+      end
+      params_file = File.join(dir, "#{self.ref}.yaml")
+      File.open(params_file, 'w') do |f|
+        f.puts params.to_yaml
+      end
+      puts params_file
+      params_file
+    end
+
+    ##
+    # Print the names and values of all the params known to the build object
+    #
+    def print_params
+      params.each { |k,v| puts "#{k}: #{v}" }
+    end
+  end
+end
+
+# Perform a build exclusively from a build params file. Requires that the build
+# params file include a setting for task, which is an array of the arguments
+# given to rake originally, including, first, the task name. The params file is
+# always loaded when passed, so these variables are accessible immediately.
+namespace :pl do
+  desc "Build from a build params file"
+  task :build_from_params do
+    check_var('PARAMS_FILE', ENV['PARAMS_FILE'])
+    git_co(@build.ref)
+    Rake::Task[@build.task[:task]].invoke(@build.task[:args])
+  end
+end

data/ext/packaging/tasks/deb.rake

@@ -5,7 +5,7 @@ def pdebuild args
   set_cow_envs(cow)
   update_cow(cow, devel_repo)
   begin
-    sh "pdebuild  --configfile #{@pbuild_conf} \
+    sh "pdebuild  --configfile #{@build.pbuild_conf} \
                   --buildresult #{results_dir} \
                   --pbuilder cowbuilder -- \
                   --basepath /var/cache/pbuilder/#{cow}/"
@@ -16,14 +16,11 @@ def pdebuild args
 end
 
 def update_cow(cow, is_rc = nil)
-  ENV['FOSS_DEVEL'] = is_rc
+  ENV['FOSS_DEVEL'] = is_rc.to_s
   ENV['PATH'] = "/usr/sbin:#{ENV['PATH']}"
   set_cow_envs(cow)
-  begin
-    sh "sudo -E /usr/sbin/cowbuilder --update --override-config --configfile #{@pbuild_conf} --basepath /var/cache/pbuilder/#{cow} --distribution #{ENV['DIST']} --architecture #{ENV['ARCH']}"
-  rescue
-    STDERR.puts "Couldn't update the cow #{cow}. Perhaps you don't have sudo?"
-    exit 1
+  retry_on_fail(:times => 3) do
+    sh "sudo -E /usr/sbin/cowbuilder --update --override-config --configfile #{@build.pbuild_conf} --basepath /var/cache/pbuilder/#{cow} --distribution #{ENV['DIST']} --architecture #{ENV['ARCH']}"
   end
 end
 
@@ -32,40 +29,44 @@ def debuild args
   begin
     sh "debuild --no-lintian -uc -us"
   rescue
-    STDERR.puts "Something went wrong. Hopefully the backscroll or #{results_dir}/#{@name}_#{@debversion}.build file has a clue."
+    STDERR.puts "Something went wrong. Hopefully the backscroll or #{results_dir}/#{@build.project}_#{@build.debversion}.build file has a clue."
     exit 1
   end
 end
 
 task :prep_deb_tars, :work_dir do |t,args|
   work_dir = args.work_dir
-  cp_p "pkg/#{@name}-#{@version}.tar.gz", work_dir
+  cp_p "pkg/#{@build.project}-#{@build.version}.tar.gz", work_dir
   cd work_dir do
-    sh "tar zxf #{@name}-#{@version}.tar.gz"
-    mv "#{@name}-#{@version}", "#{@name}-#{@debversion}"
-    mv "#{@name}-#{@version}.tar.gz", "#{@name}_#{@origversion}.orig.tar.gz"
+    sh "tar zxf #{@build.project}-#{@build.version}.tar.gz"
+    mv "#{@build.project}-#{@build.version}", "#{@build.project}-#{@build.debversion}"
+    mv "#{@build.project}-#{@build.version}.tar.gz", "#{@build.project}_#{@build.origversion}.orig.tar.gz"
   end
 end
 
 task :build_deb, :deb_command, :cow, :devel do |t,args|
-  deb_build = args.deb_command
-  cow       = args.cow
-  devel     = args.devel
-  work_dir  = get_temp
-  subdir    = 'pe/' if @build_pe
-  dest_dir  = "#{@build_root}/pkg/#{subdir}deb/#{cow.split('-')[1] unless cow.nil?}"
-  check_tool(deb_build)
-  mkdir_p dest_dir
-  deb_args  = { :work_dir => work_dir, :cow => cow, :devel => devel}
-  Rake::Task[:prep_deb_tars].reenable
-  Rake::Task[:prep_deb_tars].invoke(work_dir)
-  cd "#{work_dir}/#{@name}-#{@debversion}" do
-    mv 'ext/debian', '.'
-    send(deb_build, deb_args)
-    cp FileList["#{work_dir}/*.deb", "#{work_dir}/*.dsc", "#{work_dir}/*.changes", "#{work_dir}/*.debian.tar.gz", "#{work_dir}/*.orig.tar.gz"], dest_dir
-    rm_rf "#{work_dir}/#{@name}-#{@debversion}"
-    rm_rf work_dir
+  bench = Benchmark.realtime do
+    deb_build = args.deb_command
+    cow       = args.cow
+    devel     = args.devel
+    work_dir  = get_temp
+    subdir    = 'pe/' if @build.build_pe
+    dest_dir  = "#{@build_root}/pkg/#{subdir}deb/#{cow.split('-')[1] unless cow.nil?}"
+    check_tool(deb_build)
+    mkdir_p dest_dir
+    deb_args  = { :work_dir => work_dir, :cow => cow, :devel => devel}
+    Rake::Task[:prep_deb_tars].reenable
+    Rake::Task[:prep_deb_tars].invoke(work_dir)
+    cd "#{work_dir}/#{@build.project}-#{@build.debversion}" do
+      mv 'ext/debian', '.'
+      send(deb_build, deb_args)
+      cp FileList["#{work_dir}/*.deb", "#{work_dir}/*.dsc", "#{work_dir}/*.changes", "#{work_dir}/*.debian.tar.gz", "#{work_dir}/*.orig.tar.gz"], dest_dir
+      rm_rf "#{work_dir}/#{@build.project}-#{@build.debversion}"
+      rm_rf work_dir
+    end
   end
+  # See 30_metrics.rake to see what this is doing
+  add_metrics({ :dist => ENV['DIST'], :bench => bench }) if @build.benchmark
 end
 
 namespace :package do
@@ -76,33 +77,37 @@ namespace :package do
 end
 
 namespace :pl do
-  desc "Create a deb from this repo using the default cow #{@default_cow}."
+  desc "Create a deb from this repo using the default cow #{@build.default_cow}."
   task :deb => "package:tar"  do
-    check_var('PE_VER', ENV['PE_VER']) if @build_pe
-    Rake::Task[:build_deb].invoke('pdebuild', @default_cow)
+    check_var('PE_VER', @build.pe_version) if @build.build_pe
+    Rake::Task[:build_deb].invoke('pdebuild', @build.default_cow, is_rc?)
+    post_metrics if @build.benchmark
   end
 
-  desc "Create an RC deb from this repo using the default cow #{@default_cow}."
   task :deb_rc => "package:tar" do
-    Rake::Task[:build_deb].invoke('pdebuild', @default_cow, 'devel')
+    deprecate("pl:deb_rc", "pl:deb")
+    Rake::Task[:build_deb].invoke('pdebuild', @build.default_cow, 'true')
+    post_metrics if @build.benchmark
   end
 
   desc "Create debs from this git repository using all cows specified in build_defaults yaml"
   task :deb_all do
-    check_var('PE_VER', ENV['PE_VER']) if @build_pe
-    @cows.split(' ').each do |cow|
+    check_var('PE_VER', @build.pe_version) if @build.build_pe
+    @build.cows.split(' ').each do |cow|
       Rake::Task["package:tar"].invoke
       Rake::Task[:build_deb].reenable
-      Rake::Task[:build_deb].invoke('pdebuild', cow)
+      Rake::Task[:build_deb].invoke('pdebuild', cow, is_rc?)
     end
+    post_metrics if @build.benchmark
   end
 
-  desc "Create RC debs from this git repository using all cows specified in build_defaults yaml"
   task :deb_all_rc do
-    @cows.split(' ').each do |cow|
+    deprecate("pl:deb_all_rc", "pl:deb_all")
+    @build.cows.split(' ').each do |cow|
       Rake::Task["package:tar"].invoke
       Rake::Task[:build_deb].reenable
-      Rake::Task[:build_deb].invoke('pdebuild', cow, 'devel')
+      Rake::Task[:build_deb].invoke('pdebuild', cow, 'true')
     end
   end
+  post_metrics if @build.benchmark
 end

data/ext/packaging/tasks/deb_repos.rake

@@ -0,0 +1,103 @@
+##
+# Create a debian repository under the standard pkg/ directory layout that the
+# packaging repo creates. The standard layout is pkg/deb/$distribution/files.
+# The repository is created in the 'repos' directory under the jenkins build
+# directories on the distribution server, e.g.
+# /opt/jenkins-builds/$project/$sha/repos. Because we're creating deb
+# repositories on the fly, we have to generate the configuration files as well.
+# We assume every directory under the `deb` directory is named for a
+# distribution, and we use this in creating our configurations.
+#
+namespace :pl do
+  namespace :jenkins do
+    desc "Create apt repositories of build DEB packages for this SHA on the distributions erver"
+    task :deb_repos => "pl:fetch" do
+
+      # First, we test that artifacts exist and set up the repos directory
+      artifact_directory = File.join(@build.jenkins_repo_path, @build.project, @build.ref)
+
+      cmd = 'echo " Checking for deb build artifacts. Will exit if not found.." ; '
+      cmd << "[ -d #{artifact_directory}/artifacts/deb ] || exit 0 ; "
+      # Descend into the deb directory and obtain the list of distributions
+      # we'll be building repos for
+      cmd << "pushd #{artifact_directory}/artifacts/deb && dists=$(ls) && popd; "
+      # We do one more check here to make sure we actually have distributions
+      # to build for. If deb is empty we want to just exit.
+      #
+      cmd << '[ -n "$dists" ] || exit 0 ; '
+      cmd << "pushd #{artifact_directory} ; "
+
+      cmd << 'echo "Checking for running repo creation. Will wait if detected." ; '
+      cmd << "while [ -f .lock ] ; do sleep 1 ; echo -n '.' ; done ; "
+      cmd << 'echo "Setting lock" ; '
+      cmd << "touch .lock ; "
+      cmd << "rsync -avxl artifacts/ repos/ ; pushd repos ; "
+
+      # Make the conf directory and write out our configuration file
+      cmd << "rm -rf apt && mkdir -p apt ; pushd apt ; "
+      cmd << 'for dist in $dists ; do mkdir -p $dist/conf ; pushd $dist ;
+      echo "
+Origin: Puppet Labs
+Label: Puppet Labs
+Codename: $dist
+Architectures: i386 amd64
+Components: main
+Description: Apt repository for acceptance testing" >> conf/distributions ; '
+
+      # Create the repositories using reprepro. Since these are for acceptance
+      # testing only, we'll just add the debs and ignore source files for now.
+      #
+      cmd << "reprepro=$(which reprepro) ; "
+      cmd << "$reprepro includedeb $dist ../../deb/$dist/*.deb ; popd ; done ; "
+      cmd << "popd ; popd "
+
+      remote_ssh_cmd(@build.distribution_server, cmd)
+
+      # Always remove the lock file, even if we've failed
+      remote_ssh_cmd(@build.distribution_server, "rm -f #{artifact_directory}/.lock")
+
+      # Now that we've created our package repositories, we can generate repo
+      # configurations for use with downstream jobs, acceptance clients, etc.
+      Rake::Task["pl:jenkins:deb_repo_configs"].execute
+    end
+
+    # Generate apt configuration files that point to the repositories created
+    # on the distribution server with packages created from the current source
+    # repo commit. There is one for each dist that is packaged for (e.g. lucid,
+    # squeeze, etc). Files are created in pkg/repo_configs/deb and are named
+    # pl-$project-$sha.list, and can be placed in /etc/apt/sources.list.d to
+    # enable clients to install these packages.
+    #
+    desc "Create apt repository configs for package repos for this sha/tag on the distribution server"
+    task :deb_repo_configs => "pl:fetch" do
+
+      # This is the standard path to all build artifacts on the distribution
+      # server for this commit
+      #
+      artifact_directory = File.join(@build.jenkins_repo_path, @build.project, @build.ref)
+
+      # We obtain the list of distributions in the debian repository with some hackery.
+      #
+      %x{ssh -t #{@build.distribution_server} 'ls #{artifact_directory}/repos/apt'}
+      if $?.success?
+        dists = %x{ssh -t #{@build.distribution_server} 'ls #{artifact_directory}/repos/apt'}.split
+      else
+        warn "No repos were found to generate configs from. Exiting.."
+        exit 0
+      end
+
+      # Create apt sources.list files that can be added to hosts for installing
+      # these packages. We use the list of distributions to create a config
+      # file for every distribution.
+      #
+      mkdir_p File.join("pkg", "repo_configs", "deb")
+      dists.each do |dist|
+        repoconfig = ["# Packages for #{@build.project} built from ref #{@build.ref}",
+                      "deb http://#{@build.builds_server}/#{@build.project}/#{@build.ref}/repos/apt/#{dist} #{dist} main"]
+        config = File.join("pkg", "repo_configs", "deb", "pl-#{@build.project}-#{@build.ref}-#{dist}.list")
+        File.open(config, 'w') { |f| f.puts repoconfig }
+      end
+      puts "Wrote apt repo configs for #{@build.project} at #{@build.ref} to pkg/repo_configs/deb."
+    end
+  end
+end

data/ext/packaging/tasks/doc.rake

@@ -1,7 +1,7 @@
 # Most projects set rdoc options in the context of gem building. However, mcollective
-# generates its own rdoc package. We can reuse the @gem_rdoc_options here
+# generates its own rdoc package. We can reuse the @build.gem_rdoc_options here
 
-if @build_doc
+if @build.build_doc
   begin
     require 'rdoc/task'
   rescue LoadError
@@ -11,10 +11,10 @@ if @build_doc
   namespace :package do
     RDoc::Task.new(:doc) do |rdoc|
       rdoc.rdoc_dir = 'doc'
-      rdoc.title = "#{@name} version #{@version}"
-      @gem_rdoc_options.each do |option|
+      rdoc.title = "#{@build.project} version #{@build.version}"
+      @build.gem_rdoc_options.each do |option|
         rdoc.options << option
-      end unless @gem_rdoc_options.nil?
+      end unless @build.gem_rdoc_options.nil?
     end
   end
 end