puppet 2.7.23 → 2.7.24

data/ext/packaging/tasks/sign.rake

@@ -1,14 +1,36 @@
-def sign_el5(rpm)
+def sign_rpm(rpm, sign_flags = nil)
+
+  # To enable support for wrappers around rpm and thus support for gpg-agent
+  # rpm signing, we have to be able to tell the packaging repo what binary to
+  # use as the rpm signing tool.
+  #
+  rpm_cmd = ENV['RPM'] || find_tool('rpm')
+
+  # If we're using the gpg agent for rpm signing, we don't want to specify the
+  # input for the passphrase, which is what '--passphrase-fd 3' does. However,
+  # if we're not using the gpg agent, this is required, and is part of the
+  # defaults on modern rpm. The fun part of gpg-agent signing of rpms is
+  # specifying that the gpg check command always return true
+  #
+  if boolean_value(ENV['RPM_GPG_AGENT'])
+    gpg_check_cmd = "--define '%__gpg_check_password_cmd /bin/true'"
+  else
+    input_flag = "--passphrase-fd 3"
+  end
+
   # Try this up to 5 times, to allow for incorrect passwords
   retry_on_fail(:times => 5) do
-    sh "rpm --define '%_gpg_name #{@build.gpg_name}' --define '%__gpg_sign_cmd %{__gpg} gpg --force-v3-sigs --digest-algo=sha1 --batch --no-verbose --no-armor --passphrase-fd 3 --no-secmem-warning -u %{_gpg_name} -sbo %{__signature_filename} %{__plaintext_filename}' --addsign #{rpm} > /dev/null"
+    # This definition of %__gpg_sign_cmd is the default on modern rpm. We
+    # accept extra flags to override certain signing behavior for older
+    # versions of rpm, e.g. specifying V3 signatures instead of V4.
+    #
+    sh "#{rpm_cmd} #{gpg_check_cmd} --define '%_gpg_name #{@build.gpg_name}' --define '%__gpg_sign_cmd %{__gpg} gpg #{sign_flags} #{input_flag} --batch --no-verbose --no-armor --no-secmem-warning -u %{_gpg_name} -sbo %{__signature_filename} %{__plaintext_filename}' --addsign #{rpm}"
   end
+
 end
 
-def sign_modern(rpm)
-  retry_on_fail(:times => 5) do
-    sh "rpm --define '%_gpg_name #{@build.gpg_name}' --addsign #{rpm} > /dev/null"
-  end
+def sign_legacy_rpm(rpm)
+  sign_rpm(rpm, "--force-v3-sigs --digest-algo=sha1")
 end
 
 def rpm_has_sig(rpm)
@@ -17,7 +39,9 @@ def rpm_has_sig(rpm)
 end
 
 def sign_deb_changes(file)
-  %x{debsign --re-sign -k#{@build.gpg_key} #{file}}
+  # Lazy lazy lazy lazy lazy
+  sign_program = "-p'gpg --use-agent --no-tty'" if ENV['RPM_GPG_AGENT']
+  sh "debsign #{sign_program} --re-sign -k#{@build.gpg_key} #{file}"
 end
 
 # requires atleast a self signed prvate key and certificate pair
@@ -46,17 +70,17 @@ namespace :pl do
     modern_rpms = (Dir["pkg/el/6/**/*.rpm"] + Dir["pkg/fedora/**/*.rpm"]).join(' ')
     unless el5_rpms.empty?
       puts "Signing el5 rpms..."
-      sign_el5(el5_rpms)
+      sign_legacy_rpm(el5_rpms)
     end
 
     unless modern_rpms.empty?
       puts "Signing el6 and fedora rpms..."
-      sign_modern(modern_rpms)
+      sign_rpm(modern_rpms)
     end
     # Now we hardlink them back in
     Dir["pkg/*/*/*/i386/*.noarch.rpm"].each do |rpm|
       cd File.dirname(rpm) do
-        ln rpm, File.join("..","x86_64")
+        ln File.basename(rpm), File.join("..","x86_64"), :force => true
       end
     end
   end
@@ -109,11 +133,17 @@ namespace :pl do
       # Because rpms and debs are laid out differently in PE under pkg/ they
       # have a different sign task to address this. Rather than create a whole
       # extra :jenkins task for signing PE, we determine which sign task to use
-      # based on if we're building PE
+      # based on if we're building PE.
+      # We also listen in on the environment variable SIGNING_BUNDLE. This is
+      # _NOT_ intended for public use, but rather with the internal promotion
+      # workflow for Puppet Enterprise. SIGNING_BUNDLE is the path to a tarball
+      # containing a git bundle to be used as the environment for the packaging
+      # repo in a signing operation.
+      signing_bundle = ENV['SIGNING_BUNDLE']
       rpm_sign_task = @build.build_pe ? "pe:sign_rpms" : "pl:sign_rpms"
       deb_sign_task = @build.build_pe ? "pe:sign_deb_changes" : "pl:sign_deb_changes"
       sign_tasks    = ["pl:sign_tar", rpm_sign_task, deb_sign_task]
-      remote_repo   = remote_bootstrap(@build.distribution_server, 'HEAD')
+      remote_repo   = remote_bootstrap(@build.distribution_server, 'HEAD', nil, signing_bundle)
       build_params  = remote_buildparams(@build.distribution_server, @build)
       rsync_to('pkg', @build.distribution_server, remote_repo)
       remote_ssh_cmd(@build.distribution_server, "cd #{remote_repo} ; rake #{sign_tasks.join(' ')} PARAMS_FILE=#{build_params}")

data/ext/packaging/tasks/tar.rake

@@ -72,7 +72,7 @@ namespace :package do
     #
     # If you set this the version will only be modified in the temporary copy,
     # with the intent that it never change the official source tree.
-    ENV['NEW_STYLE_PACKAGE'] and Rake::Task["package:versionbump"].invoke(workdir)
+    Rake::Task["package:versionbump"].invoke(workdir) if @build.update_version_file
 
     cd "pkg" do
       sh "#{tar} --exclude #{tar_excludes.join(" --exclude ")} -zcf '#{@build.project}-#{@build.version}.tar.gz' #{@build.project}-#{@build.version}"

data/ext/packaging/tasks/template.rake

@@ -3,10 +3,24 @@ namespace :package do
   task :template, :workdir do |t, args|
     workdir = args.workdir
 
-    FileList["#{workdir}/ext/**/*.erb"].exclude(/#{workdir}\/ext\/packaging/).each do |template|
+    if @build.templates
+      if @build.templates.is_a?(Array)
+        templates = FileList[@build.templates.map {|path| File.join(workdir, path)}]
+      else
+        STDERR.puts "templates must be an Array, not '#{@build.templates.class}'"
+      end
+    else
+      templates = FileList["#{workdir}/ext/**/*.erb"].exclude(/#{workdir}\/ext\/(packaging|osx)/)
+    end
+
+    templates.each do |template|
       # process the template, stripping off the ERB extension
-      erb(template, template[0..-5])
-      rm_f(template)
+      if File.extname(template) == ".erb"
+        erb(template, template.gsub(/\.erb$/,""))
+        rm_f(template)
+      else
+        STDERR.puts "Skipping #{template} because it doesn't look like an erb template"
+      end
     end
   end
 end

data/ext/packaging/tasks/vendor_gems.rake

@@ -1,6 +1,6 @@
 # This is an optional pre-tar-task, so we only want to present it if we're
 # using it
-if @build.pre_tar_task == "package:vendor_gems"
+if @build.pre_tar_task
   namespace :package do
     desc "vendor gems required by project"
     task :vendor_gems do

data/ext/packaging/templates/downstream.xml.erb

@@ -13,7 +13,8 @@ When DOWNSTREAM_JOB is passed to the packaging repo, this job is created to wrap
     </jenkins.plugins.hipchat.HipChatNotifier_-HipChatJobProperty>
   </properties>
   <scm class="hudson.scm.NullSCM"/>
-  <canRoam>true</canRoam>
+  <assignedNode>downstream</assignedNode>
+  <canRoam>false</canRoam>
   <disabled>false</disabled>
   <blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
   <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
@@ -22,9 +23,21 @@ When DOWNSTREAM_JOB is passed to the packaging repo, this job is created to wrap
   <builders>
     <hudson.tasks.Shell>
       <command>#!/bin/bash
+## We have to check if this has been triggered by a successful package and repo build
+curl -s &quot;http://<%= "#{@build.jenkins_build_host}" %>/job/<%= "#{@build.project}-repo-#{@build.build_date}-#{@build.ref}" %>/lastBuild/api/json&quot; | grep result\&quot;:\&quot;SUCCESS\&quot; &gt; /dev/null
+
+UPSTREAM_BUILD_STATUS=$?
+
 set -e
+
+if [ $UPSTREAM_BUILD_STATUS -eq 0 ] ; then
+  UPSTREAM_BUILD_STATUS=&quot;success&quot;
+else
+  UPSTREAM_BUILD_STATUS=&quot;failure&quot;
+fi
+
 # This URI was passed in as an argument to the original rake package call
-curl -i &apos;<%= escape_html(ENV['DOWNSTREAM_JOB']) %>&apos;</command>
+curl --fail -i &quot;<%= escape_html(add_param_to_uri(ENV['DOWNSTREAM_JOB'], "PACKAGE_BUILD_STATUS=$UPSTREAM_BUILD_STATUS")) %>&quot;</command>
     </hudson.tasks.Shell>
   </builders>
   <publishers/>

data/ext/packaging/templates/packaging.xml.erb

@@ -82,6 +82,11 @@ return labelMap.get(binding.getVariables().get(&quot;command&quot;));</groovyScr
           <description></description>
           <defaultValue></defaultValue>
         </hudson.model.StringParameterDefinition>
+        <hudson.model.StringParameterDefinition>
+          <name>METRICS</name>
+          <description></description>
+          <defaultValue></defaultValue>
+        </hudson.model.StringParameterDefinition>
       </parameterDefinitions>
     </hudson.model.ParametersDefinitionProperty>
   </properties>
@@ -140,6 +145,143 @@ popd</command>
     </hudson.tasks.Shell>
   </builders>
   <publishers>
+    <org.jvnet.hudson.plugins.groovypostbuild.GroovyPostbuildRecorder plugin="groovy-postbuild@1.8">
+      <groovyScript>
+import java.util.regex.Matcher
+import java.util.regex.Pattern
+import java.net.HttpURLConnection
+import java.util.Date;
+
+def get_jenkins_build_time() {
+    start_time  = manager.build.getStartTimeInMillis()
+    end_time    = new Date().getTime()
+    return String.valueOf((end_time - start_time)/1000)
+}
+
+// Assemble metrics to post to build metrics server
+
+app_server    = &quot;<%= @build.metrics_url %>&quot;
+task_metrics  = manager.build.getEnvironment(manager.listener)[&apos;METRICS&apos;]
+charset       = &quot;UTF-8&quot;
+
+// Maintain backwards compatibility
+if ( task_metrics == null) {
+    build_user  = &quot;N/A&quot;
+    version     = &quot;N/A&quot;
+    pe_version  = &quot;N/A&quot;
+    build_team  = &quot;N/A&quot;
+} else {
+    build_user  = task_metrics.split(&quot;~&quot;)[0]
+    version     = task_metrics.split(&quot;~&quot;)[1]
+    pe_version  = task_metrics.split(&quot;~&quot;)[2]
+    build_team  = task_metrics.split(&quot;~&quot;)[3]
+}
+
+matcher = manager.getLogMatcher(/(?:Finished building in:) ([\d]+\.?[\d]*)/)
+if (matcher != null) {
+  package_build_time = matcher[0][1]
+} else {
+  package_build_time = &quot;N/A&quot;
+}
+
+cmd_string  = manager.build.getEnvironment(manager.listener)[&apos;command&apos;]
+if(cmd_string =~ /deb/) {
+  package_type = 'deb'
+} else if(cmd_string =~ /rpm|mock/) {
+  package_type = 'rpm'
+} else if(cmd_string =~ /gem/) {
+  package_type = 'gem'
+} else if(cmd_string =~ /apple/) {
+  package_type = 'dmg'
+} else if(cmd_string =~ /tar/) {
+  package_type = 'tar'
+}  else {
+  package_type = 'N/A'
+}
+
+switch (package_type) {
+  case 'deb':
+    dist = cmd_string.split('-')[1]
+    break
+  case 'rpm':
+    if(pe_version != 'N/A') {
+      dist = cmd_string.split('-')[2]
+    } else {
+      dist = cmd_string.split('-')[1] + cmd_string.split('-')[2]
+    }
+    break
+  case 'gem':
+    dist = 'gem'
+    break
+  case 'dmg':
+    dist = 'apple'
+    break
+  case 'tar':
+    dist = 'tar'
+    break
+  default:
+    dist = 'N/A'
+}
+
+jenkins_build_time  = get_jenkins_build_time()
+package_name        = manager.build.getEnvironment(manager.listener)[&apos;PROJECT&apos;]
+build_loc           = manager.build.getEnvironment(manager.listener)[&apos;NODE_NAME&apos;]
+build_log           = &quot;${manager.build.getEnvironment(manager.listener)[&apos;BUILD_URL&apos;]}&quot; + &quot;consoleText&quot;
+success             = String.valueOf(manager.build.result)
+
+String query = String.format(&quot;package_name=%s&amp;dist=%s&amp;package_type=%s&amp;build_user=%s&amp;build_team=%s&amp;build_loc=%s&amp;version=%s&amp;pe_version=%s&amp;success=%s&amp;build_log=%s&amp;jenkins_build_time=%s&amp;package_build_time=%s&quot;,
+     URLEncoder.encode(package_name, charset),
+     URLEncoder.encode(dist, charset),
+     URLEncoder.encode(package_type, charset),
+     URLEncoder.encode(build_user, charset),
+     URLEncoder.encode(build_team, charset),
+     URLEncoder.encode(build_loc, charset),
+     URLEncoder.encode(version, charset),
+     URLEncoder.encode(pe_version, charset),
+     URLEncoder.encode(success, charset),
+     URLEncoder.encode(build_log, charset),
+     URLEncoder.encode(jenkins_build_time, charset),
+     URLEncoder.encode(package_build_time, charset))
+
+// Make sure the server is listening before attempting to post data
+
+URLConnection connection = null
+serverAlive = false
+try {
+    URL u       = new URL(app_server);
+    connection  = (HttpURLConnection) u.openConnection();
+    connection.setRequestMethod(&quot;GET&quot;);
+    int code    = connection.getResponseCode();
+    serverAlive = true
+    connection.disconnect();
+
+} catch (MalformedURLException e) {
+    serverAlive = false
+    e.printStackTrace()
+
+} catch (IOException e) {
+    serverAlive = false
+    e.printStackTrace()
+
+} finally {
+    if (serverAlive == true) {
+        connection = new URL(app_server).openConnection()
+        connection.setDoOutput(true) // Triggers POST.
+        connection.setRequestProperty(&quot;Accept-Charset&quot;, charset);
+        connection.setRequestProperty(&quot;Content-Type&quot;, &quot;application/x-www-form-urlencoded;charset=&quot; + charset);
+        OutputStream output = null;
+
+        try {
+             output = connection.getOutputStream()
+             output.write(query.getBytes(charset))
+             InputStream response = connection.getInputStream()
+        } finally {
+             if (output != null) try { output.close(); } catch (IOException logOrIgnore) {}
+        }
+    }
+}     </groovyScript>
+      <behavior>0</behavior>
+    </org.jvnet.hudson.plugins.groovypostbuild.GroovyPostbuildRecorder>
     <hudson.tasks.ArtifactArchiver>
       <artifacts>**/PROJECT_BUNDLE</artifacts>
       <latestOnly>false</latestOnly>
@@ -157,7 +299,7 @@ popd</command>
             <hudson.plugins.parameterizedtrigger.CurrentBuildParameters/>
           </configs>
           <projects><%= "#{@build.project}-repo-#{@build.build_date}-#{@build.ref}" %></projects>
-          <condition>UNSTABLE_OR_BETTER</condition>
+          <condition>ALWAYS</condition>
           <triggerWithNoParameters>false</triggerWithNoParameters>
         </hudson.plugins.parameterizedtrigger.BuildTriggerConfig>
       </configs>

data/ext/packaging/templates/repo.xml.erb

@@ -41,36 +41,47 @@ This job will trigger the downstream job supplied with DOWNSTREAM_JOB if passed
     </hudson.plugins.copyartifact.CopyArtifact>
     <hudson.tasks.Shell>
       <command>#!/bin/bash
+## We have to check if this has been triggered by a successful package build
+curl -s &quot;http://<%= "#{@build.jenkins_build_host}" %>/job/<%= "#{@build.project}-packaging-#{@build.build_date}-#{@build.ref}" %>/lastBuild/api/json&quot; | grep result\&quot;:\&quot;SUCCESS\&quot; > /dev/null
+
+PACKAGE_BUILD_RESULT=$?
+
 set -e
-### We've retrieved the git bundle from the tarball build, so now we clone it
-### and use it to trigger our repo creation
-#
-# PROJECT_BUNDLE is a tarball containing a bundle file and git_repo is the
-# directory that will contain the git repository. First we untar the tarball
-# and then clone the git_bundle
 
-[ -f &quot;PROJECT_BUNDLE&quot; ] || exit 1
-mkdir project &amp;&amp; tar -xzf PROJECT_BUNDLE -C project/
+if [ $PACKAGE_BUILD_RESULT -eq 0 ] ; then
+  echo &quot;Detected upstream package build success. Building repos.&quot;
+  ### We've retrieved the git bundle from the tarball build, so now we clone it
+  ### and use it to trigger our repo creation
+  #
+  # PROJECT_BUNDLE is a tarball containing a bundle file and git_repo is the
+  # directory that will contain the git repository. First we untar the tarball
+  # and then clone the git_bundle
+
+  [ -f &quot;PROJECT_BUNDLE&quot; ] || exit 1
+
+  [ -d project ] &amp;&amp; rm -rf project
+
+  mkdir project &amp;&amp; tar -xzf PROJECT_BUNDLE -C project/
 
-pushd project
-  git clone --recursive $(ls) git_repo
+  pushd project
+    git clone --recursive $(ls) git_repo
 
-  pushd git_repo
+    pushd git_repo
 
-    ### Clone the packaging repo
-    rake package:bootstrap --trace
+      ### Clone the packaging repo
+      rake package:bootstrap --trace
 
-    ### Run repo creation
-    rake pl:jenkins:rpm_repos --trace
-    rake pl:jenkins:deb_repos --trace
+      ### Run repo creation
+      rake pl:jenkins:rpm_repos --trace
+      rake pl:jenkins:deb_repos --trace
 
+    popd
   popd
-popd
+else
+  echo &quot;Detected upstream package build failure. Failing repo creation.&quot;
+  exit 1
+fi
 
-# Delete the upstream project now that repos have been created. This is largely
-# to avoid clutter and reclaim space
-#
-curl -i -X POST <%= "http://#{@build.jenkins_build_host}/job/#{@build.project}-packaging-#{@build.build_date}-#{@build.ref}" %>/doDelete
 </command>
     </hudson.tasks.Shell>
   </builders><% if ENV['DOWNSTREAM_JOB'] %>
@@ -78,9 +89,9 @@ curl -i -X POST <%= "http://#{@build.jenkins_build_host}/job/#{@build.project}-p
     <hudson.tasks.BuildTrigger>
       <childProjects><%= "#{@build.project}-downstream-#{@build.build_date}-#{@build.ref}" %></childProjects>
       <threshold>
-        <name>SUCCESS</name>
-        <ordinal>0</ordinal>
-        <color>BLUE</color>
+        <name>FAILURE</name>
+        <ordinal>2</ordinal>
+        <color>RED</color>
       </threshold>
     </hudson.tasks.BuildTrigger>
   </publishers><% else %>

data/lib/puppet/transaction.rb

@@ -151,7 +151,7 @@ class Puppet::Transaction
     begin
       made = resource.eval_generate.uniq
       return false if made.empty?
-      made = made.inject({}) {|a,v| a.merge(v.name => v) }
+      made = made.inject({}) {|a,v| a[v.name] = v; a}
     rescue => detail
       puts detail.backtrace if Puppet[:trace]
       resource.err "Failed to generate additional resources using 'eval_generate: #{detail}"

data/lib/puppet/type/file.rb

@@ -736,36 +736,25 @@ Puppet::Type.newtype(:file) do
   def write(property)
     remove_existing(:file)
 
-    use_temporary_file = write_temporary_file?
-    if use_temporary_file
-      path = "#{self[:path]}.puppettmp_#{rand(10000)}"
-      path = "#{self[:path]}.puppettmp_#{rand(10000)}" while ::File.exists?(path) or ::File.symlink?(path)
-    else
-      path = self[:path]
-    end
+    assumed_default_mode = 0644
 
     mode = self.should(:mode) # might be nil
-    umask = mode ? 000 : 022
-    mode_int = mode ? symbolic_mode_to_int(mode, 0644) : nil
-
-    content_checksum = Puppet::Util.withumask(umask) { ::File.open(path, 'wb', mode_int ) { |f| write_content(f) } }
-
-    # And put our new file in place
-    if use_temporary_file # This is only not true when our file is empty.
-      begin
-        fail_if_checksum_is_wrong(path, content_checksum) if validate_checksum?
-        ::File.rename(path, self[:path])
-      rescue => detail
-        fail "Could not rename temporary file #{path} to #{self[:path]}: #{detail}"
-      ensure
-        # Make sure the created file gets removed
-        ::File.unlink(path) if FileTest.exists?(path)
+    mode_int = mode ? symbolic_mode_to_int(mode, assumed_default_mode) : nil
+
+    if write_temporary_file?
+      Puppet::Util.replace_file(self[:path], mode_int) do |file|
+        file.binmode
+        content_checksum = write_content(file)
+        file.flush
+        fail_if_checksum_is_wrong(file.path, content_checksum) if validate_checksum?
       end
+    else
+      umask = mode ? 000 : 022
+      Puppet::Util.withumask(umask) { ::File.open(self[:path], 'wb', mode_int ) { |f| write_content(f) } }
     end
 
     # make sure all of the modes are actually correct
     property_fix
-
   end
 
   private