puppet 2.7.25 → 2.7.26

data/ext/packaging/tasks/30_metrics.rake

@@ -2,9 +2,9 @@
 def add_shipped_metrics args
   @metrics << {
     :type         => 'shipped',
-    :package      => ( args[:package]             || @build.project    ),
-    :version      => ( args[:version]             || @build.version    ),
-    :pe_version   => ( args[:pe_version]          || @build.pe_version ),
+    :package      => ( args[:package]             || Pkg::Config.project    ),
+    :version      => ( args[:version]             || Pkg::Config.version    ),
+    :pe_version   => ( args[:pe_version]          || Pkg::Config.pe_version ),
     :is_rc        => ( args[:is_rc]               || false             ),
   }
 end
@@ -18,7 +18,7 @@ def post_shipped_metrics
     pe_version   = metric[:pe_version]
     is_rc        = metric[:is_rc]
 
-    uri = URI(@build.metrics_url)
+    uri = URI(Pkg::Config.metrics_url)
     res = Net::HTTP.post_form(
       uri,
       {

data/ext/packaging/tasks/apple.rake

@@ -18,16 +18,16 @@ PKGBUILD      = '/usr/bin/pkgbuild'
 task :setup do
   # Read the Apple file-mappings
   begin
-    @source_files        = data_from_yaml('ext/osx/file_mapping.yaml')
+    @source_files        = Pkg::Util::Serialization.load_yaml('ext/osx/file_mapping.yaml')
   rescue
     fail "Could not load Apple file mappings from 'ext/osx/file_mapping.yaml'"
   end
-  @package_name          = @build.project
-  @title                 = "#{@build.project}-#{@build.version}"
-  @reverse_domain        = "com.#{@build.packager}.#{@package_name}"
-  @package_major_version = @build.version.split('.')[0]
-  @package_minor_version = @build.version.split('.')[1] +
-                           @build.version.split('.')[2].split('-')[0].split('rc')[0]
+  @package_name          = Pkg::Config.project
+  @title                 = "#{Pkg::Config.project}-#{Pkg::Config.version}"
+  @reverse_domain        = "com.#{Pkg::Config.packager}.#{@package_name}"
+  @package_major_version = Pkg::Config.version.split('.')[0]
+  @package_minor_version = Pkg::Config.version.split('.')[1] +
+                           Pkg::Config.version.split('.')[2].split('-')[0].split('rc')[0]
   @pm_restart            = 'None'
   @build_date            = Time.new.strftime("%Y-%m-%dT%H:%M:%SZ")
   @apple_bindir          = '/usr/bin'
@@ -43,7 +43,7 @@ end
 #               package-specific options) is built from an ERB template located
 #               in the ext/osx directory.
 def make_directory_tree
-  project_tmp    = "#{get_temp}/#{@package_name}"
+  project_tmp    = "#{Pkg::Util::File.mktemp}/#{@package_name}"
   @scratch       = "#{project_tmp}/#{@title}"
   @working_tree  = {
      'scripts'   => "#{@scratch}/scripts",
@@ -58,15 +58,15 @@ def make_directory_tree
   end
 
   if File.exists?('ext/osx/postflight.erb')
-    erb 'ext/osx/postflight.erb', "#{@working_tree["scripts"]}/postinstall"
+    Pkg::Util::File.erb_file 'ext/osx/postflight.erb', "#{@working_tree["scripts"]}/postinstall", false, :binding => binding
   end
 
   if File.exists?('ext/osx/preflight.erb')
-    erb 'ext/osx/preflight.erb', "#{@working_tree["scripts"]}/preinstall"
+    Pkg::Util::File.erb_file 'ext/osx/preflight.erb', "#{@working_tree["scripts"]}/preinstall", false, :binding => binding
   end
 
   if File.exists?('ext/osx/prototype.plist.erb')
-    erb 'ext/osx/prototype.plist.erb', "#{@scratch}/prototype.plist"
+    Pkg::Util::File.erb_file 'ext/osx/prototype.plist.erb', "#{@scratch}/prototype.plist", false, :binding => binding
   end
 
   if File.exists?('ext/packaging/static_artifacts/PackageInfo.plist')
@@ -96,7 +96,7 @@ def build_dmg
   system("sudo #{PKGBUILD} --root #{@working_tree['working']} \
     --scripts #{@working_tree['scripts']} \
     --identifier #{@reverse_domain} \
-    --version #{@version} \
+    --version #{Pkg::Config.version} \
     --install-location / \
     --ownership preserve \
     --info #{@scratch}/PackageInfo.plist \
@@ -225,9 +225,24 @@ def pack_source
       end
     end
   end
+
+  # Hackery here. Our packages were using /usr/bin/env ruby and installing to
+  # system ruby loadpath, which breaks horribly in a multi-ruby (rbenv, etc)
+  # environment. This goes into the workdir and looks for any files dropped in
+  # bin, and "seds" the shebang to /usr/bin/ruby. I would love to be using a
+  # ruby approach to this instead of shelling out to sed, but the problem is
+  # we've already set ownership on these files, almost exclusively to root, and
+  # thus we need to sudo out.
+  if @source_files['directories'] and @source_files['directories']['bin']
+    if bindir = @source_files['directories']['bin']['path']
+      Dir[File.join(work, bindir, '*')].each do |binfile|
+        sh "sudo /usr/bin/sed -E -i '' '1 s,^#![[:space:]]*/usr/bin/env[[:space:]]+ruby$,#!/usr/bin/ruby,' #{binfile}"
+      end
+    end
+  end
 end
 
-if @build.build_dmg
+if Pkg::Config.build_dmg
   namespace :package do
     desc "Task for building an Apple Package"
     task :apple => [:setup] do

data/ext/packaging/tasks/build.rake

@@ -1,177 +1,3 @@
-module Build
-  ##
-  # This class is meant to encapsulate all of the data we know about a build invoked with
-  # `rake package:<build>` or `rake pl:<build>`. It can read in this data via a yaml file,
-  # have it set via accessors, and serialize it back to yaml for easy transport.
-  #
-  class BuildInstance
-    @@build_params = [:apt_host,
-                      :apt_repo_path,
-                      :apt_repo_url,
-                      :author,
-                      :benchmark,
-                      :build_date,
-                      :build_defaults,
-                      :build_dmg,
-                      :build_doc,
-                      :build_gem,
-                      :build_ips,
-                      :build_pe,
-                      :builder_data_file,
-                      :builds_server,
-                      :certificate_pem,
-                      :cows,
-                      :db_table,
-                      :deb_build_host,
-                      :debversion,
-                      :debug,
-                      :default_cow,
-                      :default_mock,
-                      :description,
-                      :distribution_server,
-                      :dmg_path,
-                      :email,
-                      :files,
-                      :final_mocks,
-                      :freight_conf,
-                      :gem_default_executables,
-                      :gem_dependencies,
-                      :gem_description,
-                      :gem_devel_dependencies,
-                      :gem_development_dependencies,
-                      :gem_excludes,
-                      :gem_executables,
-                      :gem_files,
-                      :gem_forge_project,
-                      :gem_name,
-                      :gem_platform_dependencies,
-                      :gem_rdoc_options,
-                      :gem_require_path,
-                      :gem_runtime_dependencies,
-                      :gem_summary,
-                      :gem_test_files,
-                      :gemversion,
-                      :gpg_key,
-                      :gpg_name,
-                      :homepage,
-                      :ips_build_host,
-                      :ips_host,
-                      :ips_inter_cert,
-                      :ips_package_host,
-                      :ips_path,
-                      :ips_repo,
-                      :ips_store,
-                      :ipsversion,
-                      :jenkins_build_host,
-                      :jenkins_packaging_job,
-                      :jenkins_repo_path,
-                      :metrics,
-                      :metrics_url,
-                      :name,
-                      :notify,
-                      :project,
-                      :origversion,
-                      :osx_build_host,
-                      :packager,
-                      :packaging_repo,
-                      :packaging_url,
-                      :pbuild_conf,
-                      :pe_name,
-                      :pe_version,
-                      :pg_major_version,
-                      :pre_tar_task,
-                      :privatekey_pem,
-                      :random_mockroot,
-                      :rc_mocks,
-                      :release,
-                      :rpm_build_host,
-                      :rpmrelease,
-                      :rpmversion,
-                      :ref,
-                      :sign_tar,
-                      :summary,
-                      :tar_excludes,
-                      :tar_host,
-                      :tarball_path,
-                      :task,
-                      :team,
-                      :templates,
-                      :update_version_file,
-                      :version,
-                      :version_file,
-                      :version_strategy,
-                      :yum_host,
-                      :yum_repo_path]
-
-    @@build_params.each do |v|
-      attr_accessor v
-    end
-
-    def initialize
-      @task = { :task => $*[0], :args => $*[1..-1] }
-      @ref = git_sha_or_tag
-    end
-
-    ##
-    # Take a hash of parameters, and iterate over them,
-    # setting each build param to the corresponding hash key,value.
-    #
-    def set_params_from_hash(data = {})
-      data.each do |param, value|
-        if @@build_params.include?(param.to_sym)
-          self.instance_variable_set("@#{param}", value)
-        else
-          warn "Warning - No build data parameter found for '#{param}'. Perhaps you have an erroneous entry in your yaml file?"
-        end
-      end
-    end
-
-    ##
-    # Load build parameters from a yaml file. Uses #data_from_yaml in
-    # 00_utils.rake
-    #
-    def set_params_from_file(file)
-      build_data = data_from_yaml(file)
-      set_params_from_hash(build_data)
-    end
-
-    ##
-    # Return a hash of all build parameters and their values, nil if unassigned.
-    #
-    def params
-      data = {}
-      @@build_params.each do |param|
-        data.store(param, self.instance_variable_get("@#{param}"))
-      end
-      data
-    end
-
-    ##
-    # Write all build parameters to a yaml file in a temporary location. Print
-    # the path to the file and return it as a string. Accept an argument for
-    # the write target directory. The name of the params file is the current
-    # git commit sha or tag.
-    #
-    def params_to_yaml(output_dir=nil)
-      dir = output_dir.nil? ? get_temp : output_dir
-      File.writable?(dir) or fail "#{dir} does not exist or is not writable, skipping build params write. Exiting.."
-      params_file = File.join(dir, "#{self.ref}.yaml")
-      File.open(params_file, 'w') do |f|
-        f.puts params.to_yaml
-      end
-      puts params_file
-      params_file
-    end
-
-    ##
-    # Print the names and values of all the params known to the build object
-    #
-    def print_params
-      params.each { |k,v| puts "#{k}: #{v}" }
-    end
-  end
-end
-
 # Perform a build exclusively from a build params file. Requires that the build
 # params file include a setting for task, which is an array of the arguments
 # given to rake originally, including, first, the task name. The params file is
@@ -180,7 +6,7 @@ namespace :pl do
   desc "Build from a build params file"
   task :build_from_params do
     check_var('PARAMS_FILE', ENV['PARAMS_FILE'])
-    git_co(@build.ref)
-    Rake::Task[@build.task[:task]].invoke(@build.task[:args])
+    Pkg::Util::Version.git_co(Pkg::Config.ref)
+    Rake::Task[Pkg::Config.task[:task]].invoke(Pkg::Config.task[:args])
   end
 end

data/ext/packaging/tasks/deb.rake

@@ -1,9 +1,11 @@
+require 'pathname'
+
 def pdebuild args
   results_dir = args[:work_dir]
   cow         = args[:cow]
   set_cow_envs(cow)
   update_cow(cow)
-  sh "pdebuild  --configfile #{@build.pbuild_conf} \
+  sh "pdebuild  --configfile #{Pkg::Config.pbuild_conf} \
                 --buildresult #{results_dir} \
                 --pbuilder cowbuilder -- \
                 --basepath /var/cache/pbuilder/#{cow}/"
@@ -14,7 +16,7 @@ def update_cow(cow)
   ENV['PATH'] = "/usr/sbin:#{ENV['PATH']}"
   set_cow_envs(cow)
   retry_on_fail(:times => 3) do
-    sh "sudo -E /usr/sbin/cowbuilder --update --override-config --configfile #{@build.pbuild_conf} --basepath /var/cache/pbuilder/#{cow} --distribution #{ENV['DIST']} --architecture #{ENV['ARCH']}"
+    sh "sudo -E /usr/sbin/cowbuilder --update --override-config --configfile #{Pkg::Config.pbuild_conf} --basepath /var/cache/pbuilder/#{cow} --distribution #{ENV['DIST']} --architecture #{ENV['ARCH']}"
   end
 end
 
@@ -23,17 +25,40 @@ def debuild args
   begin
     sh "debuild --no-lintian -uc -us"
   rescue
-    fail "Something went wrong. Hopefully the backscroll or #{results_dir}/#{@build.project}_#{@build.debversion}.build file has a clue."
+    fail "Something went wrong. Hopefully the backscroll or #{results_dir}/#{Pkg::Config.project}_#{Pkg::Config.debversion}.build file has a clue."
   end
 end
 
 task :prep_deb_tars, :work_dir do |t,args|
   work_dir = args.work_dir
-  cp_p "pkg/#{@build.project}-#{@build.version}.tar.gz", work_dir
+  cp_p "pkg/#{Pkg::Config.project}-#{Pkg::Config.version}.tar.gz", work_dir
   cd work_dir do
-    sh "tar zxf #{@build.project}-#{@build.version}.tar.gz"
-    mv "#{@build.project}-#{@build.version}", "#{@build.project}-#{@build.debversion}"
-    mv "#{@build.project}-#{@build.version}.tar.gz", "#{@build.project}_#{@build.origversion}.orig.tar.gz"
+    sh "tar zxf #{Pkg::Config.project}-#{Pkg::Config.version}.tar.gz"
+    mv "#{Pkg::Config.project}-#{Pkg::Config.version}", "#{Pkg::Config.project}-#{Pkg::Config.debversion}"
+    mv "#{Pkg::Config.project}-#{Pkg::Config.version}.tar.gz", "#{Pkg::Config.project}_#{Pkg::Config.origversion}.orig.tar.gz"
+  end
+
+
+  # This assumes that work_dir is absolute, which I hope is a safe assumption.
+  #
+  # Also, it turns out that invoking 'find' on a directory that doesn't exist
+  # will fail in nasty ways, so we only do this if the target exists...
+  if Pathname('ext/debian').directory?
+    pkg_dir = "#{work_dir}/#{Pkg::Config.project}-#{Pkg::Config.debversion}"
+    cd 'ext' do
+      Pathname('debian').find do |file|
+        case
+        when file.to_s =~ /~$/, file.to_s =~ /^#/
+          next
+        when file.directory?
+          mkdir_p "#{pkg_dir}/#{file}"
+        when file.extname == '.erb'
+          Pkg::Util::File.erb_file(file, "#{pkg_dir}/#{file.sub(/\.[^\.]*$/, '')}", false, :binding => Pkg::Config.get_binding)
+        else
+          cp file, "#{pkg_dir}/#{file}"
+        end
+      end
+    end
   end
 end
 
@@ -41,19 +66,35 @@ task :build_deb, :deb_command, :cow do |t,args|
   bench = Benchmark.realtime do
     deb_build = args.deb_command
     cow       = args.cow
-    work_dir  = get_temp
-    subdir    = 'pe/' if @build.build_pe
-    dest_dir  = "#{@build_root}/pkg/#{subdir}deb/#{cow.split('-')[1] unless cow.nil?}"
-    check_tool(deb_build)
+    work_dir  = Pkg::Util::File.mktemp
+    subdir    = 'pe/' if Pkg::Config.build_pe
+    dest_dir  = "#{Pkg::Config.project_root}/pkg/#{subdir}deb/#{cow.split('-')[1] unless cow.nil?}"
+    Pkg::Util::Tool.check_tool(deb_build)
     mkdir_p dest_dir
     deb_args  = { :work_dir => work_dir, :cow => cow}
     Rake::Task[:prep_deb_tars].reenable
     Rake::Task[:prep_deb_tars].invoke(work_dir)
-    cd "#{work_dir}/#{@build.project}-#{@build.debversion}" do
-      mv 'ext/debian', '.'
+    cd "#{work_dir}/#{Pkg::Config.project}-#{Pkg::Config.debversion}" do
+      if !File.directory?('debian') and File.directory?('ext/debian')
+        mv 'ext/debian', 'debian'
+      end
+
+      # So this is terrible. It is a hacky hacky bandaid for until this can be
+      # totally refactored into a library with templates drawn entirely from
+      # the tarball. The following two lines are needed because the deb.rake
+      # logic currently re-templates all of the templates in ext/debian for use
+      # in packaging. Then, before the package is built, if the debian
+      # directory doesn't exist (this is really only the case for puppetdb),
+      # the ext/debian directory from the tarball is moved into place. This
+      # breaks ezbake because ezbake maps templates to differently named files
+      # in the tarball templating, but those newly generated templates are
+      # completely ignored without the following two lines that unconditionally
+      # copy anything in ext/debian into the debian directory.
+      mkdir_p 'debian'
+      cp_pr(Dir.glob("ext/debian/*"), 'debian')
       send(deb_build, deb_args)
-      cp FileList["#{work_dir}/*.deb", "#{work_dir}/*.dsc", "#{work_dir}/*.changes", "#{work_dir}/*.debian.tar.gz", "#{work_dir}/*.orig.tar.gz"], dest_dir
-      rm_rf "#{work_dir}/#{@build.project}-#{@build.debversion}"
+      cp FileList["#{work_dir}/*.deb", "#{work_dir}/*.dsc", "#{work_dir}/*.changes", "#{work_dir}/*.debian.tar.gz", "#{work_dir}/*.orig.tar.gz", "${work_dir}/*.diff.gz"], dest_dir
+      rm_rf "#{work_dir}/#{Pkg::Config.project}-#{Pkg::Config.debversion}"
       rm_rf work_dir
     end
   end
@@ -68,16 +109,16 @@ namespace :package do
 end
 
 namespace :pl do
-  desc "Create a deb from this repo using the default cow #{@build.default_cow}."
+  desc "Create a deb from this repo using the default cow #{Pkg::Config.default_cow}."
   task :deb => "package:tar"  do
-    check_var('PE_VER', @build.pe_version) if @build.build_pe
-    Rake::Task[:build_deb].invoke('pdebuild', @build.default_cow)
+    check_var('PE_VER', Pkg::Config.pe_version) if Pkg::Config.build_pe
+    Rake::Task[:build_deb].invoke('pdebuild', Pkg::Config.default_cow)
   end
 
   desc "Create debs from this git repository using all cows specified in build_defaults yaml"
   task :deb_all do
-    check_var('PE_VER', @build.pe_version) if @build.build_pe
-    @build.cows.split(' ').each do |cow|
+    check_var('PE_VER', Pkg::Config.pe_version) if Pkg::Config.build_pe
+    Pkg::Config.cows.split(' ').each do |cow|
       Rake::Task["package:tar"].invoke
       Rake::Task[:build_deb].reenable
       Rake::Task[:build_deb].invoke('pdebuild', cow)

data/ext/packaging/tasks/deb_repos.rake

@@ -12,10 +12,10 @@ namespace :pl do
   namespace :jenkins do
     desc "Create apt repositories of build DEB packages for this SHA on the distributions erver"
     task :deb_repos => "pl:fetch" do
-      prefix = @build.build_pe ? "pe/" : ""
+      prefix = Pkg::Config.build_pe ? "pe/" : ""
 
       # First, we test that artifacts exist and set up the repos directory
-      artifact_directory = File.join(@build.jenkins_repo_path, @build.project, @build.ref)
+      artifact_directory = File.join(Pkg::Config.jenkins_repo_path, Pkg::Config.project, Pkg::Config.ref)
 
       cmd = 'echo " Checking for deb build artifacts. Will exit if not found.." ; '
       cmd << "[ -d #{artifact_directory}/artifacts/#{prefix}deb ] || exit 1 ; "
@@ -53,7 +53,7 @@ Description: Apt repository for acceptance testing" >> conf/distributions ; '
       cmd << "popd ; popd "
 
       begin
-        remote_ssh_cmd(@build.distribution_server, cmd)
+        remote_ssh_cmd(Pkg::Config.distribution_server, cmd)
         # Now that we've created our package repositories, we can generate repo
         # configurations for use with downstream jobs, acceptance clients, etc.
         Rake::Task["pl:jenkins:generate_deb_repo_configs"].execute
@@ -62,7 +62,7 @@ Description: Apt repository for acceptance testing" >> conf/distributions ; '
         Rake::Task["pl:jenkins:ship_repo_configs"].execute
       ensure
         # Always remove the lock file, even if we've failed
-        remote_ssh_cmd(@build.distribution_server, "rm -f #{artifact_directory}/.lock")
+        remote_ssh_cmd(Pkg::Config.distribution_server, "rm -f #{artifact_directory}/.lock")
       end
 
     end
@@ -80,12 +80,12 @@ Description: Apt repository for acceptance testing" >> conf/distributions ; '
       # This is the standard path to all debian build artifact repositories on
       # the distribution server for this commit
       #
-      base_url = "http://#{@build.builds_server}/#{@build.project}/#{@build.ref}/repos/apt/"
+      base_url = "http://#{Pkg::Config.builds_server}/#{Pkg::Config.project}/#{Pkg::Config.ref}/repos/apt/"
 
       # We use wget to obtain a directory listing of what are presumably our deb repos
       #
       repo_urls = []
-      wget = find_tool("wget") or fail "Could not find `wget` tool. This is needed for composing the debian repo configurations. Install `wget` and try again."
+      wget = Pkg::Util::Tool.find_tool("wget") or fail "Could not find `wget` tool. This is needed for composing the debian repo configurations. Install `wget` and try again."
       # First test if the directory even exists
       #
       wget_results = %x{#{wget} --spider -r -l 1 --no-parent #{base_url} 2>&1}
@@ -93,7 +93,7 @@ Description: Apt repository for acceptance testing" >> conf/distributions ; '
         # We want to exclude index and robots files and only include the http: prefixed elements
         repo_urls = wget_results.split.uniq.reject{|x| x=~ /\?|index|robots/}.select{|x| x =~ /http:/}.map{|x| x.chomp('/')}
       else
-        fail "No debian repos available for #{@build.project} at #{@build.ref}."
+        fail "No debian repos available for #{Pkg::Config.project} at #{Pkg::Config.ref}."
       end
 
       # Create apt sources.list files that can be added to hosts for installing
@@ -105,19 +105,19 @@ Description: Apt repository for acceptance testing" >> conf/distributions ; '
         # We want to skip the base_url, which wget returns as one of the results
         next if "#{url}/" == base_url
         dist = url.split('/').last
-        repoconfig = ["# Packages for #{@build.project} built from ref #{@build.ref}",
+        repoconfig = ["# Packages for #{Pkg::Config.project} built from ref #{Pkg::Config.ref}",
                       "deb #{url} #{dist} main"]
-        config = File.join("pkg", "repo_configs", "deb", "pl-#{@build.project}-#{@build.ref}-#{dist}.list")
+        config = File.join("pkg", "repo_configs", "deb", "pl-#{Pkg::Config.project}-#{Pkg::Config.ref}-#{dist}.list")
         File.open(config, 'w') { |f| f.puts repoconfig }
       end
-      puts "Wrote apt repo configs for #{@build.project} at #{@build.ref} to pkg/repo_configs/deb."
+      puts "Wrote apt repo configs for #{Pkg::Config.project} at #{Pkg::Config.ref} to pkg/repo_configs/deb."
     end
 
     desc "Retrieve debian apt repository configs for this sha"
     task :deb_repo_configs => "pl:fetch" do
-      wget = find_tool("wget") or fail "Could not find `wget` tool. This is needed for composing the debian repo configurations. Install `wget` and try again."
+      wget = Pkg::Util::Tool.find_tool("wget") or fail "Could not find `wget` tool. This is needed for composing the debian repo configurations. Install `wget` and try again."
       mkdir_p "pkg/repo_configs"
-      config_url = "#{@build.builds_server}/#{@build.project}/#{@build.ref}/repo_configs/deb/"
+      config_url = "#{Pkg::Config.builds_server}/#{Pkg::Config.project}/#{Pkg::Config.ref}/repo_configs/deb/"
       begin
         sh "#{wget} -r -np -nH --cut-dirs 3 -P pkg/repo_configs --reject 'index*' #{config_url}"
       rescue