packaging 0.88.77

data/tasks/sign.rake

@@ -0,0 +1,154 @@
+namespace :pl do
+  desc "Sign the tarball, defaults to PL key, pass GPG_KEY to override or edit build_defaults"
+  task :sign_tar do
+    unless Pkg::Config.vanagon_project
+      tarballs_to_sign = Pkg::Util::Ship.collect_packages(['pkg/*.tar.gz'], ['signing_bundle', 'packaging-bundle'])
+      Pkg::Util::Gpg.load_keychain if Pkg::Util::Tool.find_tool('keychain')
+      tarballs_to_sign.each do |file|
+        Pkg::Util::Gpg.sign_file file
+      end
+    end
+  end
+
+  # If no directory to sign is specified assume "pkg"
+  $DEFAULT_DIRECTORY = "pkg"
+
+  desc "Sign the Arista EOS swix packages, defaults to PL key, pass GPG_KEY to override or edit build_defaults"
+  task :sign_swix, :root_dir do |_t, args|
+    swix_dir = args.root_dir || $DEFAULT_DIRECTORY
+    packages = Dir["#{swix_dir}/**/*.swix"]
+    unless packages.empty?
+      Pkg::Util::Gpg.load_keychain if Pkg::Util::Tool.find_tool('keychain')
+      packages.each do |swix_package|
+        Pkg::Util::Gpg.sign_file swix_package
+      end
+    end
+  end
+
+  desc "Detach sign any solaris svr4 packages"
+  task :sign_svr4, :root_dir do |_t, args|
+    svr4_dir = args.root_dir || $DEFAULT_DIRECTORY
+    unless Dir["#{svr4_dir}/**/*.pkg.gz"].empty?
+      Pkg::Util::Gpg.load_keychain if Pkg::Util::Tool.find_tool('keychain')
+      Dir["#{svr4_dir}/**/*.pkg.gz"].each do |pkg|
+        Pkg::Util::Gpg.sign_file pkg
+      end
+    end
+  end
+
+  desc "Sign mocked rpms, Defaults to PL Key, pass GPG_KEY to override"
+  task :sign_rpms, :root_dir do |t, args|
+    rpm_directory = args.root_dir || $DEFAULT_DIRECTORY
+    Pkg::Sign::Rpm.sign_all(rpm_directory)
+  end
+
+  desc "Sign ips package, uses PL certificates by default, update privatekey_pem, certificate_pem, and ips_inter_cert in build_defaults.yaml to override."
+  task :sign_ips, :root_dir do |_t, args|
+    ips_dir = args.root_dir || $DEFAULT_DIRECTORY
+    Pkg::Sign::Ips.sign(ips_dir) unless Dir["#{ips_dir}/**/*.p5p"].empty?
+  end
+
+  desc "Sign built gems, defaults to PL key, pass GPG_KEY to override or edit build_defaults"
+  task :sign_gem, :root_dir do |_t, args|
+    gems_dir = args.root_dir || $DEFAULT_DIRECTORY
+    gems = FileList["#{gems_dir}/*.gem"]
+    gems.each do |gem|
+      puts "signing gem #{gem}"
+      Pkg::Util::Gpg.sign_file(gem)
+    end
+  end
+
+  desc "Check if all rpms are signed"
+  task :check_rpm_sigs, :root_dir do |_t, args|
+    rpm_dir = args.root_dir || $DEFAULT_DIRECTORY
+    signed = true
+    rpms = Dir["#{rpm_dir}/**/*.rpm"]
+    print 'Checking rpm signatures'
+    rpms.each do |rpm|
+      if Pkg::Sign::Rpm.has_sig? rpm
+        print '.'
+      else
+        puts "#{rpm} is unsigned."
+        signed = false
+      end
+    end
+    fail unless signed
+    puts "All rpms signed"
+  end
+
+  desc "Sign generated debian changes files. Defaults to PL Key, pass GPG_KEY to override"
+  task :sign_deb_changes, :root_dir do |_t, args|
+    begin
+      deb_dir = args.root_dir || $DEFAULT_DIRECTORY
+      change_files = Dir["#{deb_dir}/**/*.changes"]
+      unless change_files.empty?
+        Pkg::Util::Gpg.load_keychain if Pkg::Util::Tool.find_tool('keychain')
+        change_files.each do |file|
+          Pkg::Sign::Deb.sign_changes(file)
+        end
+      end
+    ensure
+      Pkg::Util::Gpg.kill_keychain
+    end
+  end
+
+  desc "Sign OSX packages"
+  task :sign_osx, [:root_dir] => "pl:fetch" do |_t, args|
+    dmg_dir = args.root_dir || $DEFAULT_DIRECTORY
+    Pkg::Sign::Dmg.sign(dmg_dir) unless Dir["#{dmg_dir}/**/*.dmg"].empty?
+  end
+
+  desc "Sign MSI packages"
+  task :sign_msi, [:root_dir] => "pl:fetch" do |_t, args|
+    msi_dir = args.root_dir || $DEFAULT_DIRECTORY
+    Pkg::Sign::Msi.sign(msi_dir) unless Dir["#{msi_dir}/**/*.msi"].empty?
+  end
+
+  ##
+  # This crazy piece of work establishes a remote repo on the signing
+  # server, ships our packages out to it, signs them, and brings them back.
+  #
+  namespace :jenkins do
+    desc "Sign all locally staged packages on #{Pkg::Config.signing_server}"
+    task :sign_all, :root_dir do |_t, args|
+      Pkg::Util::RakeUtils.invoke_task('pl:fetch')
+      root_dir = args.root_dir || $DEFAULT_DIRECTORY
+      Dir["#{root_dir}/*"].empty? and fail "There were no files found in #{root_dir}. Maybe you wanted to build/retrieve something first?"
+
+      # Because rpms and debs are laid out differently in PE under pkg/ they
+      # have a different sign task to address this. Rather than create a whole
+      # extra :jenkins task for signing PE, we determine which sign task to use
+      # based on if we're building PE.
+      # We also listen in on the environment variable SIGNING_BUNDLE. This is
+      # _NOT_ intended for public use, but rather with the internal promotion
+      # workflow for Puppet Enterprise. SIGNING_BUNDLE is the path to a tarball
+      # containing a git bundle to be used as the environment for the packaging
+      # repo in a signing operation.
+      signing_bundle = ENV['SIGNING_BUNDLE']
+      rpm_sign_task = Pkg::Config.build_pe ? "pe:sign_rpms" : "pl:sign_rpms"
+      deb_sign_task = Pkg::Config.build_pe ? "pe:sign_deb_changes" : "pl:sign_deb_changes"
+      sign_tasks    = [rpm_sign_task]
+      sign_tasks    << deb_sign_task unless Dir["#{root_dir}/**/*.changes"].empty?
+      sign_tasks    << "pl:sign_tar" if Pkg::Config.build_tar
+      sign_tasks    << "pl:sign_gem" if Pkg::Config.build_gem
+      sign_tasks    << "pl:sign_osx" if Pkg::Config.build_dmg || Pkg::Config.vanagon_project
+      sign_tasks    << "pl:sign_swix" if Pkg::Config.vanagon_project
+      sign_tasks    << "pl:sign_svr4" if Pkg::Config.vanagon_project
+      sign_tasks    << "pl:sign_ips" if Pkg::Config.vanagon_project
+      sign_tasks    << "pl:sign_msi" if Pkg::Config.build_msi || Pkg::Config.vanagon_project
+      remote_repo   = Pkg::Util::Net.remote_unpack_git_bundle(Pkg::Config.signing_server, 'HEAD', nil, signing_bundle)
+      build_params  = Pkg::Util::Net.remote_buildparams(Pkg::Config.signing_server, Pkg::Config)
+      Pkg::Util::Net.rsync_to(root_dir, Pkg::Config.signing_server, remote_repo)
+      rake_command = <<-DOC
+cd #{remote_repo} ;
+#{Pkg::Util::Net.remote_bundle_install_command}
+bundle exec rake #{sign_tasks.map { |task| task + "[#{root_dir}]" }.join(" ")} PARAMS_FILE=#{build_params}
+DOC
+      Pkg::Util::Net.remote_execute(Pkg::Config.signing_server, rake_command)
+      Pkg::Util::Net.rsync_from("#{remote_repo}/#{root_dir}/", Pkg::Config.signing_server, "#{root_dir}/")
+      Pkg::Util::Net.remote_execute(Pkg::Config.signing_server, "rm -rf #{remote_repo}")
+      Pkg::Util::Net.remote_execute(Pkg::Config.signing_server, "rm #{build_params}")
+      puts "Signed packages staged in #{root_dir}/ directory"
+    end
+  end
+end

data/tasks/tag.rake

@@ -0,0 +1,8 @@
+namespace 'pl' do
+  desc "Tag this repository, requires a TAG, e.g. TAG=1.1.1"
+  task "tag" do
+    Pkg::Util.check_var('TAG', ENV['TAG'])
+    Pkg::Util::Git.tag(ENV['TAG'])
+  end
+end
+

data/tasks/tar.rake

@@ -0,0 +1,28 @@
+namespace :package do
+  desc "Create a source tar archive"
+  task :tar => [:clean] do
+
+    if Pkg::Config.pre_tar_task
+      Pkg::Util::RakeUtils.invoke_task(Pkg::Config.pre_tar_task)
+    end
+
+    Rake::Task["package:doc"].invoke if Pkg::Config.build_doc
+
+    tar = Pkg::Tar.new
+
+    # If the user has specified templates via config file, they will be ack'd
+    # by the tar class. Otherwise, we load what we consider to be the "default"
+    # set, which is default for historical purposes.
+    #
+    tar.templates ||= Dir[File.join(Pkg::Config.project_root, "ext", "**", "*.erb")].select { |i| i !~ /ext\/packaging|ext\/osx/ }
+
+    tar.pkg!
+
+    puts "Wrote #{`pwd`.strip}/pkg/#{Pkg::Config.project}-#{Pkg::Config.version}.tar.gz"
+  end
+end
+
+namespace :pl do
+  task :tar => ["package:tar"]
+end
+

data/tasks/update.rake

@@ -0,0 +1,16 @@
+namespace :package do
+  desc "Update your clone of the packaging repo with `git pull`"
+  task :update do
+    cd 'ext/packaging' do
+      remote = Pkg::Config.packaging_url.split(' ')[0]
+      branch = Pkg::Config.packaging_url.split(' ')[1].split('=')[1]
+      if branch.nil? or remote.nil?
+        $stderr.puts "Couldn't parse the packaging repo URL from 'ext/build_defaults.yaml'."
+        $stderr.puts "Normally this is a string in the format git@github.com:<User>/<packaging_repo> --branch=<branch>"
+      else
+        Pkg::Util::Git.pull(remote, branch)
+      end
+    end
+  end
+end
+

data/tasks/vanagon.rake

@@ -0,0 +1,35 @@
+namespace :vanagon do
+  # vanagon:check_tags is used to report the tagged status of vanagon components
+  # with "ref" fields indicated in their json files. This function does not create
+  # any barriers or stop execution, it's only a report
+  desc "Check for tagged components in vanagon projects"
+  task :check_tags do
+    # only available to vanagon projects
+    unless Pkg::Config.vanagon_project
+      puts "This is not a vanagon project"
+      return
+    end
+    puts "*" * (ENV["COLUMNS"] || 70).to_i
+    Pkg::Util.in_project_root do
+      Pkg::Util::File.files_with_ext("configs/components", ".json").each do |json_file|
+        json_data = Pkg::Util::Misc.load_from_json(json_file)
+        # Don't report on anything without a ref
+        if json_data["ref"].is_a?(String)
+          Pkg::Util::Version.report_json_tags(json_data)
+        end
+      end
+    end
+  end
+
+  # This is just a wrapper for uber_build that runs the check_tags report and
+  # pauses for input, mainly so we can manually check for tagged components
+  # before spending time on a build
+  desc "Run component tag reports, if the report is okay, run an uber_build"
+  task :build_all, [:poll_interval] do |t, args|
+    args.with_defaults(:poll_interval => 0)
+    Rake::Task["vanagon:check_tags"].invoke
+    puts "Does this look correct?"
+    return unless Pkg::Util.ask_yes_or_no
+    Rake::Task["pl:jenkins:uber_build"].invoke(args.poll_interval)
+  end
+end

data/tasks/vendor_gems.rake

@@ -0,0 +1,117 @@
+# This is an optional pre-tar-task, so we only want to present it if we're
+# using it
+if Pkg::Config.pre_tar_task
+  namespace :package do
+    desc "vendor gems required by project"
+    task :vendor_gems do
+      Pkg::Util::Tool.check_tool("bundle")
+      require 'bundler'
+
+      class UI
+        LEVELS = %w(silent error warn confirm info debug)
+
+        def warn(message, newline = nil)
+          puts message
+        end
+
+        def debug(message, newline = nil)
+          puts message
+        end
+
+        def trace(message, newline = nil)
+          puts message
+        end
+
+        def error(message, newline = nil)
+          puts message
+        end
+
+        def info(message, newline = nil)
+          puts message
+        end
+
+        def confirm(message, newline = nil)
+        end
+
+        def debug?
+          true
+        end
+
+        def ask(message)
+        end
+
+        def quiet?
+          false
+        end
+
+        def level=(level)
+          raise ArgumentError unless LEVELS.include?(level.to_s)
+          @level = level
+        end
+
+        def level(name = nil)
+          name ? LEVELS.index(name) <= LEVELS.index(@level) : @level
+        end
+
+        def silence
+          old_level, @level = @level, "silent"
+          yield
+        ensure
+          @level = old_level
+        end
+
+      end
+
+      class RGProxy < ::Gem::SilentUI
+        def initialize(ui)
+          @ui = ui
+          super()
+        end
+
+        def say(message)
+          if message =~ /native extensions/
+            @ui.info "with native extensions "
+          else
+            @ui.debug(message)
+          end
+        end
+      end
+
+
+      # Cache all the gems locally without using the shared GEM_PATH
+      Bundler.settings[:cache_all] = true
+      Bundler.settings[:local] = true
+      Bundler.settings[:disable_shared_gems] = true
+      # Make sure we cache all the gems, despite what the local config file says...
+      Bundler.settings.without = []
+
+      # Stupid bundler requires this because it's not abstracted out into a library that doesn't need IO
+      Bundler.ui = UI.new
+      Bundler.rubygems.ui = ::RGProxy.new(Bundler.ui)
+      Bundler.ui.level = "debug"
+
+      # Load the the Gemfile and resolve gems using RubyGems.org
+      definition = Bundler.definition
+      definition.validate_ruby!
+      definition.resolve_remotely!
+
+      mkdir_p Bundler.app_cache
+
+      # Cache the gems
+      definition.specs.each do |spec|
+        # Fetch Rubygem specs
+        if spec.source.respond_to?(:fetch_gem, true)
+          spec.source.send(:fetch_gem, spec) if spec.source.is_a?(Bundler::Source::Rubygems)
+        elsif Bundler::Fetcher.respond_to?(:fetch, true)
+          # Retain this call in case we are dealing with an older version of bundler.
+          Bundler::Fetcher.fetch(spec) if spec.source.is_a?(Bundler::Source::Rubygems)
+        else
+          # shrug
+          fail "Cannot find a way to download the gem for caching! Did the `fetch_gem` method move?"
+        end
+        # Cache everything but bundler itself...
+        spec.source.cache(spec) unless spec.name == "bundler"
+      end
+    end
+  end
+end

data/tasks/version.rake

@@ -0,0 +1,33 @@
+# This is something of a work in progress. Unfortunately,
+# many of the projects that use the packaging repo carry
+# version files with hard-coded versions, and many of these
+# are in completely disparate formats.
+#
+# This task attempts to automate the updating of this file
+# with the version to be packaged, but given the many version
+# file formats in use, doing so cleanly is difficult. With
+# any luck, going forward some of these projects will move
+# away from maintaining hard-coded versions in source.
+# However, if this effort loses momentum, we may end up
+# revisiting this task and improving it substantially,
+# and/or standardizing the expected version file format.
+namespace :package do
+  desc "Set and commit the version in #{Pkg::Config.version_file}, requires VERSION."
+  task :versionset do
+    Pkg::Util.check_var('VERSION', ENV['VERSION'])
+    Pkg::Util::Version.versionbump
+    Pkg::Util::Git.commit_file(Pkg::Config.version_file, "update to #{ENV['VERSION']}")
+  end
+
+  task :versionbump, :workdir do |t, args|
+    Pkg::Util::Version.versionbump(args.workdir)
+  end
+
+  # A set of tasks for printing the version
+  [:version, :rpmversion, :rpmrelease, :debversion, :release].each do |task|
+    task "#{task}" do
+      $stdout.puts Pkg::Config.instance_variable_get("@#{task}")
+    end
+  end
+end
+

data/tasks/z_data_dump.rake

@@ -0,0 +1,65 @@
+##
+# These tasks are just wrappers for the build objects capabilities, exposed
+# for our debugging purposes. This file is prepended with `z_` to ensure it is
+# loaded last, so that any variable manipulations that occur in the rake tasks
+# happen prior to printing (although ideally all variables have been set after
+# loading `20_setupextrasvars.rake`).
+#
+namespace :pl do
+  ##
+  # Utility rake task that will dump all current build parameters and variables
+  # to a yaml file to a temporary location and print the path. Given the
+  # environment variable 'OUTPUT_DIR', output file at 'OUTPUT_DIR'. The
+  # environment variable TASK sets the task instance variable of the build to the
+  # supplied args, allowing us to use this file for later builds.
+  #
+  desc "Write all package build parameters to a yaml file, pass OUTPUT_DIR to specify outut location"
+  task :write_build_params do
+    if ENV['TASK']
+      task_args = ENV['TASK'].split(' ')
+      Pkg::Config.task = { :task => task_args[0], :args => task_args[1..-1] }
+    end
+    Pkg::Config.config_to_yaml(ENV['OUTPUT_DIR'])
+  end
+
+  ##
+  # Print all build parameters to $stdout.
+  #
+  desc "Print all package build parameters"
+  task :print_build_params do
+    Pkg::Config.print_config
+  end
+
+  ##
+  # Print a parameter passed as an argument to $stdout.
+  desc "Print a build parameter"
+  task :print_build_param, :param do |t, args|
+    # We want a string that is the from "@<param name>"
+    if param = args.param
+      getter = param.dup
+      if param[0] == ':'
+        getter = param[1..-1]
+        param[0] = "@"
+      elsif param[0] == "@"
+        getter = param[1..-1]
+      else
+        param.insert(0, "@")
+      end
+
+      # We want to fail if the param passed is bogus, print 'nil' if its not
+      # set, and print the value if its set.
+      if Pkg::Config.respond_to?(getter)
+        if val = Pkg::Config.instance_variable_get(param)
+          puts val
+        else
+          puts 'nil'
+        end
+      else
+        fail "Could not locate a build parameter called #{param}. For a list of available parameters, do `rake pl:print_build_params`"
+      end
+    else
+      fail "To print a build parameter, pass the param name as a rake argument. Ex: rake pl:print_build_param[:version]"
+    end
+  end
+end
+