packaging 0.99.0

data/tasks/build.rake

@@ -0,0 +1,12 @@
+# Perform a build exclusively from a build params file. Requires that the build
+# params file include a setting for task, which is an array of the arguments
+# given to rake originally, including, first, the task name. The params file is
+# always loaded when passed, so these variables are accessible immediately.
+namespace :pl do
+  desc "Build from a build params file"
+  task :build_from_params do
+    Pkg::Util.check_var('PARAMS_FILE', ENV['PARAMS_FILE'])
+    Pkg::Util::Git.checkout(Pkg::Config.ref)
+    Rake::Task[Pkg::Config.task[:task]].invoke(Pkg::Config.task[:args])
+  end
+end

data/tasks/clean.rake

@@ -0,0 +1,5 @@
+desc "Clean all built packages, eg rm -rf pkg"
+task :clean do
+  rm_rf 'pkg'
+end
+

data/tasks/config.rake

@@ -0,0 +1,30 @@
+namespace :config do
+  desc "print Pkg::Config values for this repo"
+  task :print => 'pl:fetch' do
+    Pkg::Util.filter_configs.each do |key, value|
+      puts "#{key}: #{value}"
+    end
+  end
+
+  task :print_hosts => 'pl:fetch' do
+    Pkg::Util.filter_configs('host').each do |key, value|
+      puts "#{key}: #{value}"
+    end
+  end
+
+  desc "print environment variables that can override build-data and build_defaults"
+  task :environment_variables do
+    Pkg::Params::ENV_VARS.each do |values|
+      type = case values[:type]
+      when :array
+        "expects one or more space, comma, or semicolon delimited value; treated as an array"
+      when :bool
+        "expects a boolean value"
+      end
+
+      msg = "#{values[:var]}: #{values[:envvar]}"
+      msg += " (#{type})" if type
+      puts msg
+    end
+  end
+end

data/tasks/deb.rake

@@ -0,0 +1,129 @@
+require 'pathname'
+
+def pdebuild(args)
+  results_dir = args[:work_dir]
+  cow         = args[:cow]
+  Pkg::Deb.set_cow_envs(cow)
+  update_cow(cow)
+  sh "pdebuild  --configfile #{Pkg::Config.pbuild_conf} \
+                --buildresult #{results_dir} \
+                --pbuilder cowbuilder -- \
+                --basepath /var/cache/pbuilder/#{cow}/"
+  $?.success? or fail "Failed to build deb with #{cow}!"
+end
+
+def update_cow(cow)
+  ENV['PATH'] = "/usr/sbin:#{ENV['PATH']}"
+  Pkg::Deb.set_cow_envs(cow)
+  Pkg::Util::Execution.retry_on_fail(:times => 3) do
+    sh "sudo -E /usr/sbin/cowbuilder --update --override-config --configfile #{Pkg::Config.pbuild_conf} --basepath /var/cache/pbuilder/#{cow} --distribution #{ENV['DIST']} --architecture #{ENV['ARCH']}"
+  end
+end
+
+def debuild(args)
+  results_dir = args[:work_dir]
+  begin
+    sh "debuild --no-lintian -uc -us"
+  rescue => e
+    fail "Something went wrong. Hopefully the backscroll or #{results_dir}/#{Pkg::Config.project}_#{Pkg::Config.debversion}.build file has a clue.\n#{e}"
+  end
+end
+
+task :prep_deb_tars, :work_dir do |t, args|
+  work_dir = args.work_dir
+  FileUtils.cp "pkg/#{Pkg::Config.project}-#{Pkg::Config.version}.tar.gz", work_dir, { :preserve => true }
+  cd work_dir do
+    sh "tar zxf #{Pkg::Config.project}-#{Pkg::Config.version}.tar.gz"
+    mv "#{Pkg::Config.project}-#{Pkg::Config.version}", "#{Pkg::Config.project}-#{Pkg::Config.debversion}"
+    mv "#{Pkg::Config.project}-#{Pkg::Config.version}.tar.gz", "#{Pkg::Config.project}_#{Pkg::Config.origversion}.orig.tar.gz"
+  end
+
+
+  # This assumes that work_dir is absolute, which I hope is a safe assumption.
+  #
+  # Also, it turns out that invoking 'find' on a directory that doesn't exist
+  # will fail in nasty ways, so we only do this if the target exists...
+  if Pathname('ext/debian').directory?
+    pkg_dir = "#{work_dir}/#{Pkg::Config.project}-#{Pkg::Config.debversion}"
+    cd 'ext' do
+      Pathname('debian').find do |file|
+        case
+        when file.to_s =~ /~$/, file.to_s =~ /^#/
+          next
+        when file.directory?
+          mkdir_p "#{pkg_dir}/#{file}"
+        when file.extname == '.erb'
+          Pkg::Util::File.erb_file(file, "#{pkg_dir}/#{file.sub(/\.[^\.]*$/, '')}", false, :binding => Pkg::Config.get_binding)
+        else
+          cp file, "#{pkg_dir}/#{file}"
+        end
+      end
+    end
+  end
+end
+
+task :build_deb, :deb_command, :cow do |t, args|
+  subrepo = Pkg::Config.apt_repo_name
+  bench = Benchmark.realtime do
+    deb_build = args.deb_command
+    cow       = args.cow
+    work_dir  = Pkg::Util::File.mktemp
+    subdir    = 'pe/' if Pkg::Config.build_pe
+    codename = /base-(.*)-(.*)\.cow/.match(cow)[1] unless cow.nil?
+    dest_dir  = File.join(Pkg::Config.project_root, "pkg", "#{subdir}deb", codename.to_s, subrepo.to_s)
+    Pkg::Util::Tool.check_tool(deb_build)
+    mkdir_p dest_dir
+    deb_args  = { :work_dir => work_dir, :cow => cow }
+    Rake::Task[:prep_deb_tars].reenable
+    Rake::Task[:prep_deb_tars].invoke(work_dir)
+    cd "#{work_dir}/#{Pkg::Config.project}-#{Pkg::Config.debversion}" do
+      if !File.directory?('debian') and File.directory?('ext/debian')
+        mv 'ext/debian', 'debian'
+      end
+
+      # So this is terrible. It is a hacky hacky bandaid for until this can be
+      # totally refactored into a library with templates drawn entirely from
+      # the tarball. The following two lines are needed because the deb.rake
+      # logic currently re-templates all of the templates in ext/debian for use
+      # in packaging. Then, before the package is built, if the debian
+      # directory doesn't exist (this is really only the case for puppetdb),
+      # the ext/debian directory from the tarball is moved into place. This
+      # breaks ezbake because ezbake maps templates to differently named files
+      # in the tarball templating, but those newly generated templates are
+      # completely ignored without the following two lines that unconditionally
+      # copy anything in ext/debian into the debian directory.
+      mkdir_p 'debian'
+      FileUtils.cp_r(Dir.glob("ext/debian/*"), 'debian', { :preserve => true })
+      send(deb_build, deb_args)
+      cp FileList["#{work_dir}/*.deb", "#{work_dir}/*.dsc", "#{work_dir}/*.changes", "#{work_dir}/*.debian.tar.gz", "#{work_dir}/*.orig.tar.gz", "${work_dir}/*.diff.gz"], dest_dir
+      rm_rf "#{work_dir}/#{Pkg::Config.project}-#{Pkg::Config.debversion}"
+      rm_rf work_dir
+    end
+  end
+  puts "Finished building in: #{bench}"
+end
+
+namespace :package do
+  desc "Create a deb from this repo, using debuild (all builddeps must be installed)"
+  task :deb => :tar do
+    Rake::Task[:build_deb].invoke('debuild')
+  end
+end
+
+namespace :pl do
+  desc "Create a deb from this repo using the default cow #{Pkg::Config.default_cow}."
+  task :deb => "package:tar"  do
+    Pkg::Util.check_var('PE_VER', Pkg::Config.pe_version) if Pkg::Config.build_pe
+    Rake::Task[:build_deb].invoke('pdebuild', Pkg::Config.default_cow)
+  end
+
+  desc "Create debs from this git repository using all cows specified in build_defaults yaml"
+  task :deb_all do
+    Pkg::Util.check_var('PE_VER', Pkg::Config.pe_version) if Pkg::Config.build_pe
+    Pkg::Config.cows.split(' ').each do |cow|
+      Rake::Task["package:tar"].invoke
+      Rake::Task[:build_deb].reenable
+      Rake::Task[:build_deb].invoke('pdebuild', cow)
+    end
+  end
+end

data/tasks/deb_repos.rake

@@ -0,0 +1,28 @@
+##
+# Create a debian repository under the standard pkg/ directory layout that the
+# packaging repo creates. The standard layout is pkg/deb/$distribution/files.
+# The repository is created in the 'repos' directory under the jenkins build
+# directories on the distribution server, e.g.
+# /opt/jenkins-builds/$project/$sha/repos. Because we're creating deb
+# repositories on the fly, we have to generate the configuration files as well.
+# We assume every directory under the `deb` directory is named for a
+# distribution, and we use this in creating our configurations.
+#
+namespace :pl do
+  namespace :jenkins do
+    desc "Create apt repositories of build DEB packages for this SHA on the distributions erver"
+    task :deb_repos => "pl:fetch" do
+      Pkg::Deb::Repo.create_repos
+    end
+
+    desc "Create apt repository configs for package repos for this sha/tag on the distribution server"
+    task :generate_deb_repo_configs => "pl:fetch" do
+      Pkg::Deb::Repo.generate_repo_configs
+    end
+
+    desc "Retrieve debian apt repository configs for this sha"
+    task :deb_repo_configs => "pl:fetch" do
+      Pkg::Deb::Repo::retrieve_repo_configs
+    end
+  end
+end

data/tasks/deprecated.rake

@@ -0,0 +1,130 @@
+# Emit useful messages for tasks that have been moved or removed.
+
+
+deprecation_message_winston = "The ticket generation tasks have moved. These now live in Winston: https://github.com/puppetlabs/winston"
+
+namespace :pl do
+  desc "The ticket generation tasks have moved. These now live in Winston: https://github.com/puppetlabs/winston"
+  task :new_server_platform_tickets do
+    warn deprecation_message_winston
+  end
+
+  desc "The ticket generation tasks have moved. These now live in Winston: https://github.com/puppetlabs/winston"
+  task :platform_addition do
+    warn deprecation_message_winston
+  end
+
+  desc "The ticket generation tasks have moved. These now live in Winston: https://github.com/puppetlabs/winston"
+  task :agent_tickets do
+    warn deprecation_message_winston
+  end
+
+  desc "The ticket generation tasks have moved. These now live in Winston: https://github.com/puppetlabs/winston"
+  task :platform_removal do
+    warn deprecation_message_winston
+  end
+
+  desc "The ticket generation tasks have moved. These now live in Winston: https://github.com/puppetlabs/winston"
+  task :puppet_agent_release_tickets do
+    warn deprecation_message_winston
+  end
+
+  desc "The ticket generation tasks have moved. These now live in Winston: https://github.com/puppetlabs/winston"
+  task :tickets do
+    warn deprecation_message_winston
+  end
+end
+
+if Pkg::Config.build_ips
+  ips_dep_warning = "The IPS build tasks have been removed from puppetlabs/packaging. Please port all Solaris projects to vanagon (https://github.com/puppetlabs/vanagon)"
+  namespace :package do
+    namespace :ips do
+      desc ips_dep_warning
+      task :clean do
+        warn ips_dep_warning
+      end
+
+      desc ips_dep_warning
+      task :clean_pkgs do
+        warn ips_dep_warning
+      end
+
+      desc ips_dep_warning
+      task :prepare do
+        warn ips_dep_warning
+      end
+
+      desc ips_dep_warning
+      task :prototmpl do
+        warn ips_dep_warning
+      end
+
+      desc ips_dep_warning
+      task :protogen do
+        warn ips_dep_warning
+      end
+
+      desc ips_dep_warning
+      task :protodeps do
+        warn ips_dep_warning
+      end
+
+      desc ips_dep_warning
+      task :protomogrify do
+        warn ips_dep_warning
+      end
+
+      desc ips_dep_warning
+      task :protomogrify do
+        warn ips_dep_warning
+      end
+
+      desc ips_dep_warning
+      task :lint do
+        warn ips_dep_warning
+      end
+
+      desc ips_dep_warning
+      task :package do
+        warn ips_dep_warning
+      end
+
+      desc ips_dep_warning
+      task :createrepo do
+        warn ips_dep_warning
+      end
+
+      desc ips_dep_warning
+      task :send do
+        warn ips_dep_warning
+      end
+
+      desc ips_dep_warning
+      task :receive do
+        warn ips_dep_warning
+      end
+
+      desc ips_dep_warning
+      task :dry_install do
+        warn ips_dep_warning
+      end
+
+      desc ips_dep_warning
+      task :p5p do
+        warn ips_dep_warning
+      end
+
+      desc ips_dep_warning
+      task :ips do
+        warn ips_dep_warning
+      end
+    end
+  end
+
+  namespace :pl do
+    desc ips_dep_warning
+    task :ips do
+      warn ips_dep_warning
+    end
+  end
+end

data/tasks/doc.rake

@@ -0,0 +1,20 @@
+# Most projects set rdoc options in the context of gem building. However, mcollective
+# generates its own rdoc package. We can reuse the Pkg::Config.gem_rdoc_options here
+
+if Pkg::Config.build_doc
+  begin
+    require 'rdoc/task'
+  rescue LoadError
+    require 'rake/rdoctask'
+  end
+
+  namespace :package do
+    RDoc::Task.new(:doc) do |rdoc|
+      rdoc.rdoc_dir = 'doc'
+      rdoc.title = "#{Pkg::Config.project} version #{Pkg::Config.version}"
+      Pkg::Config.gem_rdoc_options.each do |option|
+        rdoc.options << option
+      end unless Pkg::Config.gem_rdoc_options.nil?
+    end
+  end
+end

data/tasks/education.rake

@@ -0,0 +1,57 @@
+namespace :pl do
+  namespace :jenkins do
+    task :deploy_learning_vm, [:vm, :md5, :target_bucket, :target_directory] => "pl:fetch" do |t, args|
+
+      vm = args.vm or fail ":vm is a required argument for #{t}"
+      md5 = args.md5 or fail ":md5 is a required argument for #{t}"
+      target_bucket = args.target_bucket or fail ":target_bucket is a required argument for #{t}"
+      target_directory = args.target_directory or fail ":target_directory is a required argument for #{t}"
+
+      Pkg::Util::Net.s3sync_to(vm, target_bucket, target_directory, ["--acl-public"])
+      Pkg::Util::Net.s3sync_to(md5, target_bucket, target_directory, ["--acl-public"])
+
+      puts "'#{vm}' and '#{md5}' have been shipped via s3 to '#{target_bucket}/#{target_directory}'"
+    end
+
+    task :deploy_training_vm, [:vm, :md5, :target_host, :target_directory] => "pl:fetch" do |t, args|
+
+      vm = args.vm or fail ":vm is a required argument for #{t}"
+      md5 = args.md5 or fail ":md5 is a required argument for #{t}"
+      target_host = args.target_host or fail ":target_host is a required argument for #{t}"
+      target_directory = args.target_directory or fail ":target_directory is a required argument for #{t}"
+
+      # Determine VM we are trying to ship and set our link_target accordingly
+      case vm
+      when /training/
+        link_target = ["puppet-training.ova", "puppet-student.ova", "puppet-training.box", "puppet-student.box"]
+        # We want the -*- in this regex so we don't pick up the links
+        archive_cmd = "mv #{target_directory}/puppet-*-training* #{target_directory}/archive"
+      when /master/
+        link_target = ["puppet-master.ova", "puppet-master.box"]
+        # We want the -*- in this regex so we don't pick up the links
+        archive_cmd = "mv #{target_directory}/puppet-*-master* #{target_directory}/archive"
+      else
+        fail "We do not know the type of VM you are trying to ship. Cannot update symlinks"
+      end
+
+      # If the archive directory exists, move old VMs to it
+      Pkg::Util::Net.remote_ssh_cmd(target_host, "if [ -d #{target_directory}/archive ]; then #{archive_cmd}; fi")
+
+      # Ship VM and md5 to host
+      Pkg::Util::Net.rsync_to(vm, target_host, target_directory)
+      Pkg::Util::Net.rsync_to(md5, target_host, target_directory)
+
+      # Update symlink to point to the VM we just shipped
+      link_target.each do |link|
+        link_path = File.join(target_directory, link)
+        link_md5_path = "#{link_path}.md5"
+        Pkg::Util::Net.remote_ssh_cmd(target_host, "if [[ -L '#{link_path}' ]] && [[ ! -e '#{link_path}' ]] ; then echo '#{link_path} is a broken link, deleting' ; unlink '#{link_path}' ; fi")
+        Pkg::Util::Net.remote_ssh_cmd(target_host, "if [[ -L '#{link_md5_path}' ]] && [[ ! -e '#{link_md5_path}' ]] ; then echo '#{link_md5_path} is a broken link, deleting' ; unlink '#{link_md5_path}' ; fi")
+        Pkg::Util::Net.remote_ssh_cmd(target_host, "cd #{target_directory} ; ln -sf #{File.basename(vm)} #{link}")
+        Pkg::Util::Net.remote_ssh_cmd(target_host, "cd #{target_directory} ; ln -sf #{File.basename(md5)} #{link}.md5")
+      end
+
+      puts "'#{vm}' and '#{md5}' have been shipped via rsync to '#{target_host}/#{target_directory}'"
+    end
+  end
+end

data/tasks/fetch.rake

@@ -0,0 +1,57 @@
+# Each team has a build-defaults file that specifies local infrastructure targets
+# for things like builders, target locations for build artifacts, etc Since much
+# of these don't change, one file can be maintained for the team.  Each project
+# also has a data file for information specific to it. If the project builds
+# both PE and not PE, it has two files, one for PE, and the other for FOSS
+#
+data_repo = Pkg::Config.build_data_repo
+
+if Pkg::Config.dev_build
+  puts "NOTICE: This is a dev build!"
+  project_data_branch = "#{Pkg::Config.project}-dev"
+else
+  project_data_branch = Pkg::Config.project
+end
+team_data_branch = Pkg::Config.team
+
+if Pkg::Config.build_pe
+  project_data_branch = 'pe-' + project_data_branch unless project_data_branch =~ /^pe-/
+  team_data_branch = 'pe-' + team_data_branch unless team_data_branch =~ /^pe-/
+end
+
+# The pl:fetch task pulls down two files from the build-data repo that contain additional
+# data specific to Puppet Labs release infrastructure intended to augment/override any
+# defaults specified in the source project repo, e.g. in ext/build_defaults.yaml
+#
+# It uses curl to download the files, and places them in a temporary
+# directory, e.g. /tmp/somedirectory/{project,team}/Pkg::Config.builder_data_file
+namespace :pl do
+  desc "retrieve build-data configurations to override/extend local build_defaults"
+  task :fetch do
+    # Remove .packaging directory from old-style extras loading
+    rm_rf "#{ENV['HOME']}/.packaging" if File.directory?("#{ENV['HOME']}/.packaging")
+    # Touch the .packaging file which is allows packaging to present remote tasks
+    touch "#{ENV['HOME']}/.packaging"
+
+    begin
+      temp_build_data_dir = Pkg::Util::File.mktemp
+      %x(git clone #{data_repo} #{temp_build_data_dir})
+      if $?.success?
+        Dir.chdir(temp_build_data_dir) do
+          [team_data_branch, project_data_branch].each do |branch|
+            %x(git checkout #{branch})
+            if $?.success?
+              Pkg::Util::RakeUtils.invoke_task("pl:load_extras", temp_build_data_dir)
+            else
+              warn "Unable to load build_defaults from branch '#{branch}' in '#{data_repo}'. Skipping."
+            end
+          end
+        end
+      else
+        fail "There was an error attempting to fetch the build-data repo. Maybe you do not have the correct permissions?"
+      end
+    ensure
+      rm_rf temp_build_data_dir
+    end
+  end
+end