puppet 2.7.20 → 2.7.21

data/ext/packaging/tasks/10_setupvars.rake

@@ -1,72 +1,101 @@
 require 'yaml'
 require 'erb'
+require 'benchmark'
+load File.expand_path('../build.rake', __FILE__)
 
-begin
-  @project_specs            ||= YAML.load_file('ext/project_data.yaml')
-  @name                     = @project_specs['project']
-  @author                   = @project_specs['author']
-  @email                    = @project_specs['email']
-  @homepage                 = @project_specs['homepage']
-  @summary                  = @project_specs['summary']
-  @description              = @project_specs['description']
-  @files                    = @project_specs['files']
-  @tar_excludes             = @project_specs['tar_excludes']
-  @version_file             = @project_specs['version_file']
-  @gem_files                = @project_specs['gem_files']
-  @gem_require_path         = @project_specs['gem_require_path']
-  @gem_test_files           = @project_specs['gem_test_files']
-  @gem_executables          = @project_specs['gem_executables']
-  @gem_runtime_dependencies = @project_specs['gem_runtime_dependencies']
-  @gem_devel_dependencies   = @project_specs['gem_devel_dependencies']
-  @gem_rdoc_options         = @project_specs['gem_rdoc_options']
-  @gem_forge_project        = @project_specs['gem_forge_project']
-  @gem_excludes             = @project_specs['gem_excludes']
-rescue => e
-  STDERR.puts "There was an error loading the project specifications from the 'ext/project_data.yaml' file.\n" + e
-  exit 1
+##
+# Where we get the data for our project depends on if a PARAMS_FILE environment
+# variable is passed with the rake call. PARAMS_FILE should be a path to a yaml
+# file containing all of the build parameters for a project, which are read
+# into our BuildInstance object. If no build parameters file is specified, we
+# assume input via the original methods of build_data.yaml and
+# project_data.yaml. This also applies to the pl:fetch and pl:load_extras
+# tasks, which are supplementary means of gathering data. These two are not
+# used if a PARAMS_FILE is passed.
+
+##
+# Create our BuildInstance object, which will contain all the data about our
+# proposed build
+#
+@build = Build::BuildInstance.new
+
+if ENV['PARAMS_FILE'] && ENV['PARAMS_FILE'] != ''
+  @build.set_params_from_file(ENV['PARAMS_FILE'])
+else
+  # Load information about the project from the default params files
+  #
+  @build.set_params_from_file('ext/project_data.yaml') if File.readable?('ext/project_data.yaml')
+  @build.set_params_from_file('ext/build_defaults.yaml') if File.readable?('ext/build_defaults.yaml')
+
+  # Allow environment variables to override the settings we just read in. These
+  # variables are called out specifically because they are likely to require
+  # overriding in at least some cases.
+  #
+  @build.sign_tar        = boolean_value(ENV['SIGN_TAR']) if ENV['SIGN_TAR']
+  @build.build_gem       = boolean_value(ENV['GEM'])      if ENV['GEM']
+  @build.build_dmg       = boolean_value(ENV['DMG'])      if ENV['DMG']
+  @build.build_ips       = boolean_value(ENV['IPS'])      if ENV['IPS']
+  @build.build_doc       = boolean_value(ENV['DOC'])      if ENV['DOC']
+  @build.build_pe        = boolean_value(ENV['PE_BUILD']) if ENV['PE_BUILD']
+  @build.debug           = boolean_value(ENV['DEBUG'])    if ENV['DEBUG']
+  @build.default_cow     = ENV['COW']                     if ENV['COW']
+  @build.cows            = ENV['COW']                     if ENV['COW']
+  @build.pbuild_conf     = ENV['PBUILDCONF']              if ENV['PBUILDCONF']
+  @build.packager        = ENV['PACKAGER']                if ENV['PACKAGER']
+  @build.default_mock    = ENV['MOCK']                    if ENV['MOCK']
+  @build.final_mocks     = ENV['MOCK']                    if ENV['MOCK']
+  @build.rc_mocks        = ENV['MOCK']                    if ENV['MOCK']
+  @build.gpg_name        = ENV['GPG_NAME']                if ENV['GPG_NAME']
+  @build.gpg_key         = ENV['GPG_KEY']                 if ENV['GPG_KEY']
+  @build.certificate_pem = ENV['CERT_PEM']                if ENV['CERT_PEM']
+  @build.privatekey_pem  = ENV['PRIVATE_PEM']             if ENV['PRIVATE_PEM']
+  @build.yum_host        = ENV['YUM_HOST']                if ENV['YUM_HOST']
+  @build.yum_repo_path   = ENV['YUM_REPO']                if ENV['YUM_REPO']
+  @build.apt_host        = ENV['APT_HOST']                if ENV['APT_HOST']
+  @build.apt_repo_path   = ENV['APT_REPO']                if ENV['APT_REPO']
+end
+
+##
+# These parameters are either generated dynamically by the project, or aren't
+# sufficiently generic/multi-purpose enough to justify being in
+# build_defaults.yaml or project_data.yaml.
+#
+@build.release           ||= get_release
+@build.version           ||= get_dash_version
+@build.gemversion        ||= get_dot_version
+@build.ipsversion        ||= get_ips_version
+@build.debversion        ||= get_debversion
+@build.origversion       ||= get_origversion
+@build.rpmversion        ||= get_rpmversion
+@build.rpmrelease        ||= get_rpmrelease
+@build.builder_data_file ||= 'builder_data.yaml'
+@build.team              = ENV['TEAM'] || 'dev'
+@build.random_mockroot   = ENV['RANDOM_MOCKROOT'] ? boolean_value(ENV['RANDOM_MOCKROOT']) : true
+@keychain_loaded         ||= FALSE
+@build_root              ||= Dir.pwd
+##
+# For backwards compatibilty, we set build:@name to build:@project. @name was
+# renamed to @project in an effort to align the variable names with what has
+# been supported for parameter names in the params files.
+#
+@build.name = @build.project
+
+if @build.debug
+  @build.print_params
 end
 
-begin
-  @pkg_defaults    ||= YAML.load_file('ext/build_defaults.yaml')
-  @sign_tar        = boolean_value( ENV['SIGN_TAR'] || @pkg_defaults['sign_tar']  )
-  @build_gem       = boolean_value( ENV['GEM']      || @pkg_defaults['build_gem'] )
-  @build_dmg       = boolean_value( ENV['DMG']      || @pkg_defaults['build_dmg'] )
-  @build_ips       = boolean_value( ENV['IPS']      || @pkg_defaults['build_ips'] )
-  @build_doc       = boolean_value( ENV['DOC']      || @pkg_defaults['build_doc'] )
-  @build_pe        = boolean_value( ENV['PE']       || @pkg_defaults['build_pe'] )
-  @default_cow     = ENV['COW']          || @pkg_defaults['default_cow']
-  @cows            = ENV['COW']          || @pkg_defaults['cows']
-  @pbuild_conf     = ENV['PBUILDCONF']   || @pkg_defaults['pbuild_conf']
-  @packager        = ENV['PACKAGER']     || @pkg_defaults['packager']
-  @default_mock    = ENV['MOCK']         || @pkg_defaults['default_mock']
-  @final_mocks     = ENV['MOCK']         || @pkg_defaults['final_mocks']
-  @rc_mocks        = ENV['MOCK']         || @pkg_defaults['rc_mocks']
-  @gpg_name        = ENV['GPG_NAME']     || @pkg_defaults['gpg_name']
-  @gpg_key         = ENV['GPG_KEY']      || @pkg_defaults['gpg_key']
-  @certificate_pem = ENV['CERT_PEM']     || @pkg_defaults['certificate_pem']
-  @privatekey_pem  = ENV['PRIVATE_PEM']  || @pkg_defaults['privatekey_pem']
-  @yum_host        = @pkg_defaults['yum_host']
-  @yum_repo_path   = @pkg_defaults['yum_repo_path']
-  @apt_host        = @pkg_defaults['apt_host']
-  @apt_repo_url    = @pkg_defaults['apt_repo_url']
-  @apt_repo_path   = @pkg_defaults['apt_repo_path']
-  @ips_repo        = @pkg_defaults['ips_repo']
-  @ips_store       = @pkg_defaults['ips_store']
-  @ips_host        = @pkg_defaults['ips_host']
-  @ips_inter_cert  = @pkg_defaults['ips_inter_cert']
-rescue => e
-  STDERR.puts "There was an error loading the packaging defaults from the 'ext/build_defaults.yaml' file.\n" + e
-  exit 1
+##
+# MM 1-22-2013
+# We have long made all of the variables available to erb templates in the
+# various projects. The problem is now that we've switched to encapsulating all
+# of this inside a build object, that information is no longer available. This
+# section is for backwards compatibility only. It sets an instance variable
+# for all of the parameters inside the build object. This is repeated in
+# 20_setupextrasvars.rake. Note: The intention is to eventually abolish this
+# behavior. We want to access information from the build object, not in what
+# are essentially globally available rake variables.
+#
+@build.params.each do |param, value|
+  self.instance_variable_set("@#{param}", value)
 end
 
-@build_root        ||= Dir.pwd
-@version           ||= @build_pe ? get_dot_version : get_dash_version
-@gemversion        ||= get_dot_version
-@ipsversion        ||= get_ips_version
-@debversion        ||= get_debversion
-@origversion       ||= get_origversion
-@rpmversion        ||= get_rpmversion
-@rpmrelease        ||= get_rpmrelease
-@keychain_loaded   ||= FALSE
-@builder_data_file ||= 'builder_data.yaml'
-@team              ||= ENV['TEAM'] || 'pe'

data/ext/packaging/tasks/20_setupextravars.rake

@@ -6,33 +6,52 @@
 # generic tasks with data not generally useful outside the
 # PL Release team
 namespace :pl do
-  task :load_extras do
-    begin
-      @build_data = YAML.load_file("#{ENV['HOME']}/.packaging/#{@builder_data_file}")
-      @rpm_build_host   = @build_data['rpm_build_host']   if @build_data['rpm_build_host']
-      @deb_build_host   = @build_data['deb_build_host']   if @build_data['deb_build_host']
-      @osx_build_host   = @build_data['osx_build_host']   if @build_data['osx_build_host']
-      @ips_build_host   = @build_data['ips_build_host']   if @build_data['ips_build_host']
-      @tarball_path     = @build_data['tarball_path']     if @build_data['tarball_path']
-      @dmg_path         = @build_data['dmg_path']         if @build_data['dmg_path']
-      @pe_version       = @build_data['pe_version']       if @build_data['pe_version']
-      @team             = @build_data['team']             if @build_data['team']
-      @yum_repo_path    = @build_data['yum_repo_path']    if @build_data['yum_repo_path']
-      @apt_repo_path    = @build_data['apt_repo_path']    if @build_data['apt_repo_path']
-      @freight_conf     = @build_data['freight_conf']     if @build_data['freight_conf']
-      @sles_build_host  = @build_data['sles_build_host']  if @build_data['sles_build_host']
-      @sles_arch_repos  = @build_data['sles_arch_repos']  if @build_data['sles_arch_repos']
-      @sles_repo_path   = @build_data['sles_repo_path']   if @build_data['sles_repo_path']
-      @sles_repo_host   = @build_data['sles_repo_host']   if @build_data['sles_repo_host']
-      @ips_path         = @build_data['ips_path']         if @build_data['ips_path']
-      @ips_package_host = @build_data['ips_package_host'] if @build_data['ips_package_host']
-      @certificate_pem  = @build_data['certificate_pem']  if @build_data['certificate_pem']
-      @privatekey_pem   = @build_data['privatekey_pem']   if @build_data['privatekey_pem']
-      @ips_inter_cert   = @build_data['ips_inter_cert']   if @build_data['ips_inter_cert']
-    rescue => e
-      STDERR.puts "There was an error loading the builder data from #{ENV['HOME']}/.packaging/#{@builder_data_file}. Try rake pl:fetch to download the current extras builder data.\n" + e
-      exit 1
+  task :load_extras, :tempdir do |t, args|
+    unless ENV['PARAMS_FILE'] && ENV['PARAMS_FILE'] != ''
+      tempdir = args.tempdir
+      raise "pl:load_extras requires a directory containing extras data" if tempdir.nil?
+      @build.set_params_from_file("#{tempdir}/team/#{@build.builder_data_file}")
+      @build.set_params_from_file("#{tempdir}/project/#{@build.builder_data_file}")
+      # Overrideable
+      @build.build_pe   = boolean_value(ENV['PE_BUILD']) if ENV['PE_BUILD']
+      # right now, puppetdb is the only one to override these, because it needs
+      # two sets of cows, one for PE and the other for FOSS
+      @build.cows             = ENV['COW']      if ENV['COW']
+      @build.final_mocks      = ENV['MOCK']     if ENV['MOCK']
+      @build.packager         = ENV['PACKAGER'] if ENV['PACKAGER']
+      @build.pe_version       = ENV['PE_VER']   if ENV['PE_VER']
+      @build.yum_repo_path    = ENV['YUM_REPO'] if ENV['YUM_REPO']
+      @build.yum_host         = ENV['YUM_HOST'] if ENV['YUM_HOST']
+      @build.apt_host         = ENV['APT_HOST'] if ENV['APT_HOST']
+      @build.apt_repo_path    = ENV['APT_REPO'] if ENV['APT_REPO']
     end
   end
 end
+if @build.team == 'release'
+  @build.benchmark = TRUE
+end
+
+##
+# Starting with puppetdb, we'll maintain two separate build-data files, one for
+# PE and the other for FOSS. This is the start to maintaining both PE and FOSS
+# packaging in one source repo. As is done in 10_setupvars.rake, the @name
+# variable is set to the value of @project, for backwards compatibility.
+#
+unless @build.pe_name.nil?
+  @build.project = @build.pe_name
+  @build.name    = @build.project
+end
 
+##
+# MM 1-22-2013
+# We have long made all of the variables available to erb templates in the
+# various projects. The problem is now that we've switched to encapsulating all
+# of this inside a build object, that information is no longer available. This
+# section is for backwards compatibility only. It sets an instance variable
+# for all of the parameters inside the build object. This is repeated in
+# 10_setupvars.rake. Note that the intention is to eventually abolish this
+# behavior, and access the parameters via the build object only.
+#
+@build.params.each do |param, value|
+  self.instance_variable_set("@#{param}", value)
+end

data/ext/packaging/tasks/30_metrics.rake

@@ -0,0 +1,41 @@
+if @build.benchmark
+  @metrics          = []
+  @pg_major_version = nil
+  @db_table         = 'metrics'
+
+  def add_metrics args
+    @metrics << {
+      :bench      => args[:bench],
+      :dist       => ( args[:dist]        || ENV['DIST']       ),
+      :pkg        => ( args[:pkg]         || @build.project    ),
+      :version    => ( args[:version]     || @build.version    ),
+      :pe_version => ( args[:pe_version]  || @build.pe_version ),
+      :date       => ( args[:date]        || timestamp         ),
+      :who        => ( args[:who]         || ENV['USER']       ),
+      :where      => ( args[:where]       || hostname          )
+    }
+  end
+
+  def post_metrics
+    if psql = find_tool('psql')
+      ENV["PGCONNECT_TIMEOUT"]="10"
+
+      @metrics.each do |metric|
+        date        = metric[:date]
+        pkg         = metric[:pkg]
+        dist        = metric[:dist]
+        bench       = metric[:bench]
+        who         = metric[:who]
+        where       = metric[:where]
+        version     = metric[:version]
+        pe_version  = metric[:pe_version]
+        @pg_major_version ||= %x{/usr/bin/psql --version}.match(/psql \(PostgreSQL\) (\d)\..*/)[1].to_i
+        no_pass_fail = "-w" if @pg_major_version > 8
+        %x{#{psql} #{no_pass_fail} -c "INSERT INTO #{@db_table} \
+        (date, package, dist, build_time, build_user, build_loc, version, pe_version) \
+        VALUES ('#{date}', '#{pkg}', '#{dist}', #{bench}, '#{who}', '#{where}', '#{version}', '#{pe_version}')"}
+      end
+      @metrics = []
+    end
+  end
+end

data/ext/packaging/tasks/apple.rake

@@ -1,4 +1,4 @@
-# Title:        Rake task to build Apple packages for #{@name}.
+# Title:        Rake task to build Apple packages for #{@project}.
 # Author:       Gary Larizza
 # Date:         05/18/2012
 # Description:  This task will create a DMG-encapsulated package that will
@@ -20,17 +20,17 @@ SED           = '/usr/bin/sed'
 task :setup do
   # Read the Apple file-mappings
   begin
-    @source_files        = YAML.load_file('ext/osx/file_mapping.yaml')
+    @source_files        = data_from_yaml('ext/osx/file_mapping.yaml')
   rescue
     STDERR.puts "Could not load Apple file mappings from 'ext/osx/file_mapping.yaml'"
     exit 1
   end
-  @package_name          = @name
-  @title                 = "#{@name}-#{@version}"
-  @reverse_domain        = "com.#{@packager}.#{@package_name}"
-  @package_major_version = @version.split('.')[0]
-  @package_minor_version = @version.split('.')[1] +
-                           @version.split('.')[2].split('-')[0].split('rc')[0]
+  @package_name          = @build.project
+  @title                 = "#{@build.project}-#{@build.version}"
+  @reverse_domain        = "com.#{@build.packager}.#{@package_name}"
+  @package_major_version = @build.version.split('.')[0]
+  @package_minor_version = @build.version.split('.')[1] +
+                           @build.version.split('.')[2].split('-')[0].split('rc')[0]
   @pm_restart            = 'None'
   @build_date            = Time.new.strftime("%Y-%m-%dT%H:%M:%SZ")
   @apple_bindir          = '/usr/bin'
@@ -59,17 +59,26 @@ def make_directory_tree
     mkdir_p(val)
   end
 
-  erb 'ext/osx/preflight.erb', "#{@working_tree["scripts"]}/preflight"
-  erb 'ext/osx/prototype.plist.erb', "#{@scratch}/prototype.plist"
+  if File.exists?('ext/osx/postflight.erb')
+    erb 'ext/osx/postflight.erb', "#{@working_tree["scripts"]}/postflight"
+  end
+
+  if File.exists?('ext/osx/preflight.erb')
+    erb 'ext/osx/preflight.erb', "#{@working_tree["scripts"]}/preflight"
+  end
+
+  if File.exists?('ext/osx/prototype.plist.erb')
+    erb 'ext/osx/prototype.plist.erb', "#{@scratch}/prototype.plist"
+  end
 
 end
 
 # method:        build_dmg
 # description:   This method builds a package from the directory structure in
-#                /tmp/#{@name} and puts it in the
-#                /tmp/#{@name}/#{@name}-#{version}/payload directory. A DMG is
+#                /tmp/#{@project} and puts it in the
+#                /tmp/#{@project}/#{@project}-#{version}/payload directory. A DMG is
 #                created, using hdiutil, based on the contents of the
-#                /tmp/#{@name}/#{@name}-#{version}/payload directory. The resultant
+#                /tmp/#{@project}/#{@project}-#{version}/payload directory. The resultant
 #                DMG is placed in the pkg/apple directory.
 #
 def build_dmg
@@ -105,20 +114,20 @@ def build_dmg
     #{dmg_file}")
 
   if File.directory?("#{pwd}/pkg/apple")
-    mv("#{pwd}/#{dmg_file}", "#{pwd}/pkg/apple/#{dmg_file}")
+    sh "sudo mv #{pwd}/#{dmg_file} #{pwd}/pkg/apple/#{dmg_file}"
     puts "moved:   #{dmg_file} has been moved to #{pwd}/pkg/apple/#{dmg_file}"
   else
     mkdir_p("#{pwd}/pkg/apple")
-    mv(dmg_file, "#{pwd}/pkg/apple/#{dmg_file}")
+    sh "sudo mv #{pwd}/#{dmg_file} #{pwd}/pkg/apple/#{dmg_file}"
     puts "moved:   #{dmg_file} has been moved to #{pwd}/pkg/apple/#{dmg_file}"
   end
 end
 
 # method:        pack_source
-# description:   This method copies the #{@name} source into a directory
-#                structure in /tmp/#{@name}/#{@name}-#{version}/root mirroring the
+# description:   This method copies the #{@project} source into a directory
+#                structure in /tmp/#{@project}/#{@project}-#{version}/root mirroring the
 #                structure on the target system for which the package will be
-#                installed. Anything installed into /tmp/#{@name}/root will be
+#                installed. Anything installed into /tmp/#{@project}/root will be
 #                installed as the package's payload.
 #
 def pack_source
@@ -145,8 +154,16 @@ def pack_source
 
   # Setup a preflight script and replace variables in the files with
   # the correct paths.
-  chown('root', 'wheel', "#{@working_tree['scripts']}/preflight")
-  chmod(0644, "#{@working_tree['scripts']}/preflight")
+  if File.exists?("#{@working_tree['scripts']}/preflight")
+    chmod(0644, "#{@working_tree['scripts']}/preflight")
+    sh "sudo chown root:wheel #{@working_tree['scripts']}/preflight"
+  end
+
+  # Setup a postflight from from the erb created earlier
+  if File.exists?("#{@working_tree['scripts']}/postflight")
+    chmod(0755, "#{@working_tree['scripts']}/postflight")
+    sh "sudo chown root:wheel #{@working_tree['scripts']}/postflight"
+  end
 
   # Do a run through first setting the specified permissions then
   # making sure 755 is set for all directories
@@ -156,12 +173,42 @@ def pack_source
       group = params['group']
       perms = params['perms']
       path  = params['path']
-      chmod_R(Integer(perms), "#{work}/#{path}")
-      chown_R(owner, group, "#{work}/#{path}")
-      chmod(0755, "#{work}/#{path}")
+      ##
+      # Before setting our default permissions for all subdirectories/files of
+      # each directory listed in directories, we have to get a list of the
+      # directories. Otherwise, when we set the default perms (most likely
+      # 0644) we'll lose traversal on subdirectories, and later when we want to
+      # ensure they're 755 we won't be able to find them.
+      #
+      directories = []
       Dir["#{work}/#{path}/**/*"].each do |file|
-        chmod(0755, file) if File.directory?(file)
+        directories << file if File.directory?(file)
       end
+
+      ##
+      # Here we're setting the default permissions for all files as described
+      # in file_mapping.yaml. Since we have a listing of directories, it
+      # doesn't matter if we remove executable permission on directories, we'll
+      # reset it later.
+      #
+      sh "sudo chmod -R #{perms} #{work}/#{path}"
+
+      ##
+      # We know at least one directory, the one listed in file_mapping.yaml, so
+      # we set it executable.
+      #
+      sh "sudo chmod 0755 #{work}/#{path}"
+
+      ##
+      # Now that default perms are set, we go in and reset executable perms on
+      # directories
+      #
+      directories.each { |d| sh "sudo chmod 0755 #{d}" }
+
+      ##
+      # Finally we set the owner/group as described in file_mapping.yaml
+      #
+      sh "sudo chown -R #{owner}:#{group} #{work}/#{path}"
     end
   end
 
@@ -173,8 +220,8 @@ def pack_source
       perms = params['perms']
       dest  = params['path']
       # Allow for regexs like [A-Z]*
-      FileList[file].each do |file|
-        cmd = "#{INSTALL} -o #{owner} -g #{group} -m #{perms} #{source}/#{file} #{work}/#{dest}"
+      FileList[file].each do |f|
+        cmd = "sudo #{INSTALL} -o #{owner} -g #{group} -m #{perms} #{source}/#{f} #{work}/#{dest}"
         puts cmd
         system(cmd)
       end
@@ -182,20 +229,29 @@ def pack_source
   end
 end
 
-if @build_dmg
+if @build.build_dmg
   namespace :package do
     desc "Task for building an Apple Package"
     task :apple => [:setup] do
-      # Test for Root and Packagemaker binary
-      raise "Please run rake as root to build Apple Packages" unless Process.uid == 0
-      raise "Packagemaker must be installed. Please install XCode Tools" unless \
-        File.exists?('/Developer/usr/bin/packagemaker')
-
-      make_directory_tree
-      pack_source
-      build_dmg
-      chmod_R(0775, "#{pwd}/pkg")
+      bench = Benchmark.realtime do
+        # Test for Packagemaker binary
+        raise "Packagemaker must be installed. Please install XCode Tools" unless \
+          File.exists?(PACKAGEMAKER)
+
+        make_directory_tree
+        pack_source
+        build_dmg
+      end
+      if @build.benchmark
+        add_metrics({ :dist => 'osx', :bench => bench })
+        post_metrics
+      end
     end
   end
+
+  # An alias task to simplify our remote logic in jenkins.rake
+  namespace :pl do
+    task :dmg => "package:apple"
+  end
 end