puppet 2.7.20 → 2.7.21

data/ext/packaging/spec/spec_helper.rb

@@ -5,8 +5,8 @@ require 'rake'
 
 SPECDIR = Pathname(__FILE__).dirname
 
-def require_task(name)
+def load_task(name)
   return false if (@loaded ||= {})[name]
-  load SPECDIR + '..' + 'tasks' + "#{name}.rake"
+  load File.join(SPECDIR, '..', 'tasks', name)
   @loaded[name] = true
 end

data/ext/packaging/spec/tasks/00_utils_spec.rb

@@ -1,6 +1,7 @@
 # -*- ruby -*-
 require 'spec_helper'
-require_task '00_utils'
+load_task '00_utils.rake'
+load_task 'build.rake'
 
 describe "00_utils" do
   TestVersions = {
@@ -8,58 +9,102 @@ describe "00_utils" do
       :git_describe_version       => %w{0.7.0},
       :get_dash_version           => '0.7.0',
       :get_ips_version            => '0.7.0,3.14159-0',
-      :get_dot_version            => '0.7.0'
+      :get_dot_version            => '0.7.0',
+      :get_debversion             => '0.7.0-1puppetlabs1',
+      :get_rpmversion             => '0.7.0',
+      :get_rpmrelease             => '1',
+      :is_rc?                     => false,
     },
     '0.7.0rc1'                    => {
       :git_describe_version       => %w{0.7.0rc1},
       :get_dash_version           => '0.7.0rc1',
       :get_ips_version            => '0.7.0rc1,3.14159-0',
-      :get_dot_version            => '0.7.0rc1'
+      :get_dot_version            => '0.7.0rc1',
+      :get_debversion             => '0.7.0-0.1rc1puppetlabs1',
+      :get_rpmversion             => '0.7.0',
+      :get_rpmrelease             => '0.1rc1',
+      :is_rc?                     => true,
     },
     '0.7.0-rc1'                   => {
       :git_describe_version       => %w{0.7.0 rc1},
       :get_dash_version           => '0.7.0-rc1',
       :get_ips_version            => '0.7.0,3.14159-0',
-      :get_dot_version            => '0.7.0.rc1'
+      :get_dot_version            => '0.7.0.rc1',
+      :get_debversion             => '0.7.0-0.1rc1puppetlabs1',
+      :get_rpmversion             => '0.7.0',
+      :get_rpmrelease             => '0.1rc1',
+      :is_rc?                     => true,
     },
     '0.7.0-rc1-63-ge391f55'       => {
       :git_describe_version       => %w{0.7.0 rc1 63},
       :get_dash_version           => '0.7.0-rc1-63',
       :get_ips_version            => '0.7.0,3.14159-63',
-      :get_dot_version            => '0.7.0.rc1.63'
+      :get_dot_version            => '0.7.0.rc1.63',
+      :get_debversion             => '0.7.0-0.1rc1.63puppetlabs1',
+      :get_rpmversion             => '0.7.0',
+      :get_rpmrelease             => '0.1rc1.63',
+      :is_rc?                     => true,
     },
     '0.7.0-rc1-63-ge391f55-dirty' => {
       :git_describe_version       => %w{0.7.0 rc1 63 dirty},
       :get_dash_version           => '0.7.0-rc1-63-dirty',
       :get_ips_version            => '0.7.0,3.14159-63-dirty',
-      :get_dot_version            => '0.7.0.rc1.63.dirty'
+      :get_dot_version            => '0.7.0.rc1.63.dirty',
+      :get_debversion             => '0.7.0-0.1rc1.63dirtypuppetlabs1',
+      :get_rpmversion             => '0.7.0',
+      :get_rpmrelease             => '0.1rc1.63dirty',
+      :is_rc?                     => true,
+
     },
     '0.7.0-63-ge391f55'           => {
       :git_describe_version       => %w{0.7.0 63},
       :get_dash_version           => '0.7.0-63',
       :get_ips_version            => '0.7.0,3.14159-63',
-      :get_dot_version            => '0.7.0.63'
+      :get_dot_version            => '0.7.0.63',
+      :get_debversion             => '0.7.0.63-1puppetlabs1',
+      :get_rpmversion             => '0.7.0.63',
+      :get_rpmrelease             => '1',
+      :is_rc?                     => false,
+
     },
     '0.7.0-63-ge391f55-dirty'     => {
       :git_describe_version       => %w{0.7.0 63 dirty},
       :get_dash_version           => '0.7.0-63-dirty',
       :get_ips_version            => '0.7.0,3.14159-63-dirty',
-      :get_dot_version            => '0.7.0.63.dirty'
+      :get_dot_version            => '0.7.0.63.dirty',
+      :get_debversion             => '0.7.0.63.dirty-1puppetlabs1',
+      :get_rpmversion             => '0.7.0.63.dirty',
+      :get_rpmrelease             => '1',
+      :is_rc?                     => false,
     },
   }
 
+  before :all do
+    @build = Build::BuildInstance.new
+  end
+
   TestVersions.keys.sort.each do |input|
-    results = TestVersions[input]
-    results.keys.sort_by(&:to_s).each do |method|
-      it "using #{method} #{input.inspect} becomes #{results[method].inspect}" do
-        # We have to call the `stub!` alias because we are trying to stub on
-        # `self`, and in the scope of an rspec block that is overridden to
-        # return a new double, not to stub a method!
-        self.stub!(:uname_r) { "3.14159" }
-        self.stub!(:is_git_repo) { true }
-        self.should_receive(:run_git_describe_internal).and_return(input)
+    describe "Versioning based on #{input}" do
+      results = TestVersions[input]
+      results.keys.sort_by(&:to_s).each do |method|
+        it "using #{method} #{input.inspect} becomes #{results[method].inspect}" do
+          # We have to call the `stub!` alias because we are trying to stub on
+          # `self`, and in the scope of an rspec block that is overridden to
+          # return a new double, not to stub a method!
+          @build.release = "1"
 
-        self.send(method).should == results[method]
+          if method.to_s.include?("deb")
+            self.should_receive(:run_git_describe_internal).and_return(input)
+            @build.packager = "puppetlabs"
+          elsif method.to_s.include?("rpm")
+            self.should_receive(:run_git_describe_internal).and_return(input)
+          else
+            self.stub!(:uname_r) { "3.14159" }
+            self.stub!(:is_git_repo) { true }
+            self.should_receive(:run_git_describe_internal).and_return(input)
+          end
+          self.send(method).should == results[method]
+        end
       end
     end
   end

data/ext/packaging/spec/tasks/build_object_spec.rb

@@ -0,0 +1,171 @@
+# -*- ruby -*-
+require 'spec_helper'
+load_task '00_utils.rake'
+load_task 'build.rake'
+
+describe Build::BuildInstance do
+  Build_Params = [:apt_host,
+                  :apt_repo_path,
+                  :apt_repo_url,
+                  :author,
+                  :benchmark,
+                  :build_defaults,
+                  :build_dmg,
+                  :build_doc,
+                  :build_gem,
+                  :build_ips,
+                  :build_pe,
+                  :builder_data_file,
+                  :certificate_pem,
+                  :cows,
+                  :db_table,
+                  :deb_build_host,
+                  :debversion,
+                  :debug,
+                  :default_cow,
+                  :default_mock,
+                  :description,
+                  :dmg_path,
+                  :email,
+                  :files,
+                  :final_mocks,
+                  :freight_conf,
+                  :gem_default_executables,
+                  :gem_dependencies,
+                  :gem_description,
+                  :gem_devel_dependencies,
+                  :gem_excludes,
+                  :gem_executables,
+                  :gem_files,
+                  :gem_forge_project,
+                  :gem_name,
+                  :gem_rdoc_options,
+                  :gem_require_path,
+                  :gem_runtime_dependencies,
+                  :gem_summary,
+                  :gem_test_files,
+                  :gemversion,
+                  :gpg_key,
+                  :gpg_name,
+                  :homepage,
+                  :ips_build_host,
+                  :ips_host,
+                  :ips_inter_cert,
+                  :ips_package_host,
+                  :ips_path,
+                  :ips_repo,
+                  :ips_store,
+                  :ipsversion,
+                  :jenkins_build_host,
+                  :jenkins_packaging_job,
+                  :jenkins_repo_path,
+                  :metrics,
+                  :name,
+                  :project,
+                  :origversion,
+                  :osx_build_host,
+                  :packager,
+                  :packaging_repo,
+                  :packaging_url,
+                  :pbuild_conf,
+                  :pe_name,
+                  :pe_version,
+                  :pg_major_version,
+                  :privatekey_pem,
+                  :random_mockroot,
+                  :rc_mocks,
+                  :release,
+                  :rpm_build_host,
+                  :rpmrelease,
+                  :rpmversion,
+                  :ref,
+                  :sign_tar,
+                  :sles_build_host,
+                  :sles_repo_path,
+                  :sles_repo_host,
+                  :sles_arch_repos,
+                  :summary,
+                  :tar_excludes,
+                  :tarball_path,
+                  :team,
+                  :version,
+                  :version_file,
+                  :yum_host,
+                  :yum_repo_path]
+
+  before :each do
+    @build = Build::BuildInstance.new
+  end
+
+  describe "#new" do
+    Build_Params.each do |param|
+      it "should have r/w accessors for #{param}" do
+        @build.should respond_to(param)
+        @build.should respond_to("#{param.to_s}=")
+      end
+    end
+  end
+
+  describe "#set_params_from_hash" do
+    good_params = { :yum_host => 'foo', :pe_name => 'bar' }
+    context "given a valid params hash #{good_params}" do
+      it "should set instance variable values for each param" do
+        good_params.each do |param, value|
+          @build.should_receive(:instance_variable_set).with("@#{param}", value)
+        end
+        @build.set_params_from_hash(good_params)
+      end
+    end
+
+    bad_params = { :foo => 'bar' }
+    context "given an invalid params hash #{bad_params}" do
+      bad_params.each do |param, value|
+        it "should print a warning that param '#{param}' is not valid" do
+          @build.should_receive(:warn).with(/No build data parameter found for '#{param}'/)
+          @build.set_params_from_hash(bad_params)
+        end
+
+        it "should not try to set instance variable @:#{param}" do
+          @build.should_not_receive(:instance_variable_set).with("@#{param}", value)
+          @build.set_params_from_hash(bad_params)
+        end
+      end
+    end
+
+    mixed_params = { :sign_tar => TRUE, :baz => 'qux' }
+    context "given a hash with both valid and invalid params" do
+      it "should set the valid param" do
+        @build.should_receive(:instance_variable_set).with("@sign_tar", TRUE)
+        @build.set_params_from_hash(mixed_params)
+      end
+
+      it "should issue a warning that the invalid param is not valid" do
+        @build.should_receive(:warn).with(/No build data parameter found for 'baz'/)
+        @build.set_params_from_hash(mixed_params)
+      end
+
+      it "should not try to set instance variable @:baz" do
+        @build.should_not_receive(:instance_variable_set).with("@baz", "qux")
+        @build.set_params_from_hash(mixed_params)
+      end
+    end
+  end
+
+  describe "#params" do
+    it "should return a hash containing keys for all build parameters" do
+      params = @build.params
+      Build_Params.each { |param| params.has_key?(param).should == TRUE }
+    end
+  end
+
+  describe "#params_to_yaml" do
+    it "should write a valid yaml file" do
+      file = mock('file')
+      File.should_receive(:open).with(anything(), 'w').and_yield(file)
+      file.should_receive(:puts).with(instance_of(String))
+      YAML.should_receive(:load_file).with(file)
+      expect { YAML.load_file(file) }.to_not raise_error
+      @build.params_to_yaml
+    end
+  end
+end

data/ext/packaging/tasks/00_utils.rake

@@ -60,14 +60,37 @@ def mv_f(src, dest, options={})
   mv(src, dest, options.merge(mandatory))
 end
 
-def git_co(dist)
-  %x{git reset --hard ; git checkout #{dist}}
+def git_co(ref)
+  %x{git reset --hard ; git checkout #{ref}}
   unless $?.success?
-    STDERR.puts 'Could not checkout #{dist} git branch to build package from...exiting'
+    STDERR.puts "Could not checkout #{ref} git branch to build package from...exiting"
     exit 1
   end
 end
 
+def git_describe
+  %x{git describe}.strip
+end
+
+# return the sha of HEAD on the current branch
+def git_sha
+  %x{git rev-parse HEAD}.strip
+end
+
+# Return the ref type of HEAD on the current branch
+def git_ref_type
+  %x{git cat-file -t #{git_describe}}.strip
+end
+
+# If HEAD is a tag, return the tag. Otherwise return the sha of HEAD.
+def git_sha_or_tag
+  if git_ref_type == "tag"
+    git_describe
+  else
+    git_sha
+  end
+end
+
 def get_temp
   temp = `mktemp -d -t tmpXXXXXX`.strip
 end
@@ -175,39 +198,37 @@ def get_pwd_version
   %x{pwd}.strip.split('.')[-1]
 end
 
+def get_base_pkg_version
+  dash = get_dash_version
+  if dash.include?("rc")
+    # Grab the rc number
+    rc_num = dash.match(/rc(\d)+/)[1]
+    ver = dash.sub(/-?rc[0-9]+/, "-0.#{@build.release}rc#{rc_num}").gsub(/(rc[0-9]+)-(\d+)?-?/, '\1.\2')
+  else
+    ver = dash.gsub('-','.') + "-#{@build.release}"
+  end
+
+  ver.split('-')
+end
+
 def get_debversion
-  (@version.include?("rc") ? @version.sub(/rc[0-9]+/, '0.1\0') : "#{@version.gsub('-','.')}-1") + "#{@packager}#{get_debrelease}"
+  get_base_pkg_version.join('-') << "#{@build.packager}1"
 end
 
 def get_origversion
-  @debversion.split('-')[0]
+  @build.debversion.split('-')[0]
 end
 
 def get_rpmversion
-  @version.match(/^([0-9.]+)/)[1]
+  get_base_pkg_version[0]
 end
 
-def get_version_file_version
-  # Match version files containing 'VERSION = "x.x.x"' and just x.x.x
-  contents = IO.read(@version_file)
-  if version_string = contents.match(/VERSION =.*/)
-    version_string.to_s.split()[-1]
-  else
-    contents
-  end
-end
-
-def get_debrelease
+def get_release
   ENV['RELEASE'] || '1'
 end
 
 def get_rpmrelease
-  ENV['RELEASE'] ||
-    if @version.include?("rc")
-      "0.1" + @version.gsub('-', '_').match(/rc[0-9]+.*/)[0]
-    else
-      "1"
-    end
+  get_base_pkg_version[1]
 end
 
 def load_keychain
@@ -218,12 +239,16 @@ def load_keychain
   end
 end
 
+def source_dirty?
+  git_describe_version.include?('dirty')
+end
+
 def kill_keychain
   %x{keychain -k mine}
 end
 
 def start_keychain
-  keychain = %x{/usr/bin/keychain -q --agents gpg --eval #{@gpg_key}}.chomp
+  keychain = %x{/usr/bin/keychain -q --agents gpg --eval #{@build.gpg_key}}.chomp
   new_env = keychain.match(/(GPG_AGENT_INFO)=([^;]*)/)
   ENV[new_env[1]] = new_env[2]
 end
@@ -232,7 +257,7 @@ def gpg_sign_file(file)
   gpg ||= find_tool('gpg')
 
   if gpg
-    sh "#{gpg} --armor --detach-sign -u #{@gpg_key} #{file}"
+    sh "#{gpg} --armor --detach-sign -u #{@build.gpg_key} #{file}"
   else
     STDERR.puts "No gpg available. Cannot sign #{file}. Exiting..."
     exit 1
@@ -259,7 +284,9 @@ def set_cow_envs(cow)
     end
     arch = arch.split('.')[0] if arch.include?('.')
   end
-
+  if @build.build_pe
+    ENV['PE_VER'] = @build.pe_version
+  end
   ENV['DIST'] = dist
   ENV['ARCH'] = arch
 end
@@ -268,6 +295,10 @@ def ln(target, name)
   FileUtils.ln(name, target, :force => true, :verbose => true)
 end
 
+def ln_sfT(src, dest)
+  sh "ln -sfT #{src} #{dest}"
+end
+
 def git_commit_file(file, message=nil)
   if has_tool('git') and File.exist?('.git')
     message ||= "changes"
@@ -322,7 +353,7 @@ end
 
 def git_tag(version)
   begin
-    sh "git tag -s -u #{@gpg_key} -m '#{version}' #{version}"
+    sh "git tag -s -u #{@build.gpg_key} -m '#{version}' #{version}"
   rescue Exception => e
     STDERR.puts e
     STDERR.puts "Unable to tag repo at #{version}"
@@ -337,12 +368,12 @@ end
 def git_bundle(treeish)
   temp = get_temp
   appendix = rand_string
-  sh "git bundle create #{temp}/#{@name}-#{@version}-#{appendix} #{treeish} --tags"
+  sh "git bundle create #{temp}/#{@build.project}-#{@build.version}-#{appendix} #{treeish} --tags"
   cd temp do
-    sh "tar -czf #{@name}-#{@version}-#{appendix}.tar.gz #{@name}-#{@version}-#{appendix}"
-    rm_rf "#{@name}-#{@version}-#{appendix}"
+    sh "tar -czf #{@build.project}-#{@build.version}-#{appendix}.tar.gz #{@build.project}-#{@build.version}-#{appendix}"
+    rm_rf "#{@build.project}-#{@build.version}-#{appendix}"
   end
-  "#{temp}/#{@name}-#{@version}-#{appendix}.tar.gz"
+  "#{temp}/#{@build.project}-#{@build.version}-#{appendix}.tar.gz"
 end
 
 # We take a tar argument for cases where `tar` isn't best, e.g. Solaris
@@ -354,8 +385,18 @@ def remote_bootstrap(host, treeish, tar_cmd=nil)
   tarball_name = File.basename(tarball).gsub('.tar.gz','')
   rsync_to(tarball, host, '/tmp')
   appendix = rand_string
-  sh "ssh -t #{host} '#{tar} -zxvf /tmp/#{tarball_name}.tar.gz -C /tmp/ ; git clone --recursive /tmp/#{tarball_name} /tmp/#{@name}-#{appendix} ; cd /tmp/#{@name}-#{appendix} ; rake package:bootstrap'"
-  "/tmp/#{@name}-#{appendix}"
+  sh "ssh -t #{host} '#{tar} -zxvf /tmp/#{tarball_name}.tar.gz -C /tmp/ ; git clone --recursive /tmp/#{tarball_name} /tmp/#{@build.project}-#{appendix} ; cd /tmp/#{@build.project}-#{appendix} ; rake package:bootstrap'"
+  "/tmp/#{@build.project}-#{appendix}"
+end
+
+# Given a BuildInstance object and a host, send its params to the host. Return
+# the remote path to the params.
+def remote_buildparams(host, build)
+  params_file = build.params_to_yaml
+  params_file_name = File.basename(params_file)
+  params_dir = rand_string
+  rsync_to(params_file, host, "/tmp/#{params_dir}/")
+  "/tmp/#{params_dir}/#{params_file_name}"
 end
 
 def is_git_repo
@@ -385,3 +426,115 @@ def empty_dir?(dir)
   File.exist?(dir) and File.directory?(dir) and Dir["#{dir}/**/*"].empty?
 end
 
+def hostname
+  require 'socket'
+  host = Socket.gethostname
+end
+
+# Loop a block up to the number of attempts given, exiting when we receive success
+# or max attempts is reached. Raise an exception unless we've succeeded.
+def retry_on_fail(args, &blk)
+  success = FALSE
+  if args[:times].respond_to?(:times) and block_given?
+    args[:times].times do |i|
+      begin
+        blk.call
+        success = TRUE
+        break
+      rescue
+        puts "An error was encountered evaluating block. Retrying.."
+      end
+    end
+  else
+    raise "retry_on_fail requires and arg (:times => x) where x is an Integer/Fixnum, and a block to execute"
+  end
+  raise "Block failed maximum of #{args[:times]} tries. Exiting.." unless success
+end
+
+def deprecate(old_cmd, new_cmd=nil)
+  msg = "!! #{old_cmd} is deprecated."
+  if new_cmd
+    msg << " Please use #{new_cmd} instead."
+  end
+  STDOUT.puts
+  STDOUT.puts(msg)
+  STDOUT.puts
+end
+
+# Determines if this package is an rc package via the version
+# returned by get_dash_version method.
+# Assumes version strings in the formats:
+# final:
+# '0.7.0'
+# '0.7.0-63'
+# '0.7.0-63-dirty'
+# rc:
+# '0.7.0rc1 (we don't actually use this format anymore, but once did)
+# '0.7.0-rc1'
+# '0.7.0-rc1-63'
+# '0.7.0-rc1-63-dirty'
+def is_rc?
+  return TRUE if get_dash_version =~ /^\d+\.\d+\.\d+-*rc\d+/
+  FALSE
+end
+
+# Utility method to return the dist method if this is a redhat box. We use this
+# in rpm packaging to define a dist macro, and we use it in the pl:fetch task
+# to disable ssl checking for redhat 5 because it has a certs bundle so old by
+# default that it's useless for our purposes.
+def el_version()
+  if File.exists?('/etc/fedora-release')
+    nil
+  elsif File.exists?('/etc/redhat-release')
+    return %x{rpm -q --qf \"%{VERSION}\" $(rpm -q --whatprovides /etc/redhat-release )}
+  end
+end
+
+# Given the path to a yaml file, load the yaml file into an object and return
+# the object.
+def data_from_yaml(file)
+  file = File.expand_path(file)
+  begin
+    input_data = YAML.load_file(file) || {}
+  rescue => e
+    puts "There was an error loading data from #{file}."
+    puts e.backtrace.join("\n")
+    exit 1
+  end
+  input_data
+end
+
+# This is fairly absurd. We're implementing curl by shelling out. What do I
+# wish we were doing? Using a sweet ruby wrapper around curl, such as Curb or
+# Curb-fu. However, because we're using clean build systems and trying to
+# make this portable with minimal system requirements, we can't very well
+# depend on libraries that aren't in the ruby standard libaries. We could
+# also do this using Net::HTTP but that set of libraries is a rabbit hole to
+# go down when what we're trying to accomplish is posting multi-part form
+# data that includes file uploads to jenkins. It gets hairy fairly quickly,
+# but, as they say, pull requests accepted.
+#
+# This method takes two arguments
+# 1) String - the URL to post to
+# 2) Array  - Ordered array of name=VALUE curl form parameters
+def curl_form_data(uri, form_data=[])
+  unless curl = find_tool("curl")
+    warn "Couldn't find curl. Curl is required for posting jenkins to trigger a build. Please install curl and try again."
+    exit 1
+  end
+  #
+  # Begin constructing the post string.
+  # First, assemble the form_data arguments
+  #
+  post_string = "-i "
+  form_data.each do |param|
+    post_string << "#{param} "
+  end
+
+  # Add the uri and we're off
+  post_string << "#{uri}"
+  sh "#{curl} #{post_string}"
+  return $?.success?
+end
+
+