packaging 0.101.0 → 0.105.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4a7e26777ec04e6bbb1824c27ec325110f49d6b52f8ca0dd84e933d3937f8fc3
-  data.tar.gz: 4b1932eb5b21ae8c1c3bb5ea6f21b5fb57479b3fe3e46273e8d075ba4beb86eb
+  metadata.gz: 296bfdcecf936f80297894792d9b9a1bb3a83a422a61e4976aca3916dd6c9a2b
+  data.tar.gz: 0c76c4482d82367cef9d988fc1e31080f718322e4eab1fe9a5176f4544c8a3ce
 SHA512:
-  metadata.gz: e7aca6b96f6e8886d0cfd8b5f381ec1e0bb92337dae30ed30601391feb11d141df89262c19996e9e01751434f318f6668570151421f3eadfd860f6db17a75632
-  data.tar.gz: b74047c635a19fbb0ecb308d0b45297d01c81437f77fccc3205c6434934f8e7378372f218d96aa90325496496a5b19f83aa5589be0082ce1a90d96e090e417dc
+  metadata.gz: 071534bcce41eb8b11517d21592399a9f94c45b20a70259aa5a13c4bf3ec5bfd6cab5e517698e6f8884736a932a62d0212d945cb97ad549bdf8c80242f594f8e
+  data.tar.gz: 532cca35cd6eeea218c8a6cdcdf00f0802f0c0ae33fc009f6f42bd7c39056f45b147abaf971879e2f5532541e1d08c0c6267425d32aa69dad9c11f036697c183

data/lib/packaging/config.rb

@@ -80,7 +80,7 @@ module Pkg
 
         dir = "/opt/jenkins-builds/#{self.project}/#{self.ref}"
         cmd = "if [ -s \"#{dir}/artifacts\" ]; then cd #{dir};"\
-              "find ./artifacts/ -mindepth 2 -type f; fi"
+              "find ./artifacts -mindepth 2 -type f; fi"
         artifacts, _ = Pkg::Util::Net.remote_execute(
                      self.builds_server,
                      cmd,
@@ -95,6 +95,7 @@ module Pkg
           # the correct place. For 5.x and 6.x release streams the f prefix
           # has been removed and so tag will equal original_tag
           original_tag = Pkg::Paths.tag_from_artifact_path(artifact)
+          fail "Error: unrecognized artifact \"#{artifact}\"" if original_tag.nil?
 
           # Remove the f-prefix from the fedora platform tag keys so that
           # beaker can rely on consistent keys once we rip out the f for good
@@ -203,15 +204,18 @@ module Pkg
       # string. Accept an argument for the write target file. If not specified,
       # the name of the params file is the current git commit sha or tag.
       #
-      def config_to_yaml(target = nil)
-        file = "#{self.ref}.yaml"
-        target = target.nil? ? File.join(Pkg::Util::File.mktemp, "#{self.ref}.yaml") : File.join(target, file)
-        Pkg::Util::File.file_writable?(File.dirname(target), :required => true)
-        File.open(target, 'w') do |f|
+      def config_to_yaml(destination_directory = nil)
+        destination_directory = Pkg::Util::File.mktemp if destination_directory.nil?
+        config_yaml_file_name = "#{self.ref}.yaml"
+
+        config_yaml_path = File.join(destination_directory, config_yaml_file_name)
+
+        Pkg::Util::File.file_writable?(File.dirname(config_yaml_path), :required => true)
+        File.open(config_yaml_path, 'w') do |f|
           f.puts self.config_to_hash.to_yaml
         end
-        puts target
-        target
+        puts config_yaml_path
+        return config_yaml_path
       end
 
       ##

data/lib/packaging/platforms.rb

@@ -22,14 +22,6 @@ module Pkg
       },
 
       'debian' => {
-        '8' => {
-          codename: 'jessie',
-          architectures: ['amd64', 'i386', 'powerpc'],
-          source_architecture: 'source',
-          package_format: 'deb',
-          source_package_formats: DEBIAN_SOURCE_FORMATS,
-          repo: true,
-        },
         '9' => {
           codename: 'stretch',
           architectures: ['amd64', 'i386'],
@@ -57,14 +49,6 @@ module Pkg
       },
 
       'el' => {
-        '5' => {
-          architectures: ['x86_64', 'i386'],
-          source_architecture: 'SRPMS',
-          package_format: 'rpm',
-          source_package_formats: ['src.rpm'],
-          signature_format: 'v3',
-          repo: true,
-        },
         '6' => {
           architectures: ['x86_64', 'i386'],
           source_architecture: 'SRPMS',
@@ -88,26 +72,18 @@ module Pkg
           source_package_formats: ['src.rpm'],
           signature_format: 'v4',
           repo: true,
-        }
-      },
-
-      'fedora' => {
-        '30' => {
-          architectures: ['x86_64'],
-          source_architecture: 'SRPMS',
-          package_format: 'rpm',
-          source_package_formats: ['src.rpm'],
-          signature_format: 'v4',
-          repo: true,
         },
-        '31' => {
+        '9' => {
           architectures: ['x86_64'],
           source_architecture: 'SRPMS',
           package_format: 'rpm',
           source_package_formats: ['src.rpm'],
           signature_format: 'v4',
           repo: true,
-        },
+        }
+      },
+
+      'fedora' => {
         '32' => {
           architectures: ['x86_64'],
           source_architecture: 'SRPMS',
@@ -127,16 +103,6 @@ module Pkg
       },
 
       'osx' => {
-        '10.13' => {
-          architectures: ['x86_64'],
-          package_format: 'dmg',
-          repo: false,
-        },
-        '10.14' => {
-          architectures: ['x86_64'],
-          package_format: 'dmg',
-          repo: false,
-        },
         '10.15' => {
           architectures: ['x86_64'],
           package_format: 'dmg',
@@ -233,14 +199,6 @@ module Pkg
           source_package_formats: DEBIAN_SOURCE_FORMATS,
           repo: true,
         },
-        '18.10' => {
-          codename: 'cosmic',
-          architectures: ['amd64', 'ppc64el'],
-          source_architecture: 'source',
-          package_format: 'deb',
-          source_package_formats: DEBIAN_SOURCE_FORMATS,
-          repo: true,
-        },
         '20.04' => {
           codename: 'focal',
           architectures: ['amd64', 'aarch64'],

data/lib/packaging/repo.rb

@@ -36,7 +36,8 @@ module Pkg::Repo
       Dir.chdir(File.join('pkg', local_target)) do
         puts "Info: Archiving #{repo_location} as #{archive_name}"
         target_tarball = File.join('repos', "#{archive_name}.tar.gz")
-        tar_command = "#{tar} --owner=0 --group=0 --create --gzip --file #{target_tarball} #{repo_location}"
+        tar_command = %W[#{tar} --owner=0 --group=0 --create --gzip
+          --file #{target_tarball} #{repo_location}].join(' ')
         stdout, _, _ = Pkg::Util::Execution.capture3(tar_command)
         return stdout
       end
@@ -62,12 +63,12 @@ module Pkg::Repo
           next
         end
 
-        tar_action = "--create"
-        if File.exist?(all_repos_tarball_name)
-          tar_action = "--update"
-        end
+        tar_action = '--create'
+        tar_action = '--update' if File.exist?(all_repos_tarball_name)
+
+        tar_command = %W[#{tar} --owner=0 --group=0 #{tar_action}
+          --file #{all_repos_tarball_name} #{repo_tarball_path}].join(' ')
 
-        tar_command = "#{tar} --owner=0 --group=0 #{tar_action} --file #{all_repos_tarball_name} #{repo_tarball_path}"
         stdout, _, _ = Pkg::Util::Execution.capture3(tar_command)
         puts stdout
       end
@@ -117,7 +118,8 @@ module Pkg::Repo
               )
       return stdout.split
     rescue => e
-      fail "Error: Could not retrieve directories that contain #{pkg_ext} packages in #{Pkg::Config.distribution_server}:#{artifact_directory}"
+      fail "Error: Could not retrieve directories that contain #{pkg_ext} " \
+           "packages in #{Pkg::Config.distribution_server}:#{artifact_directory}: #{e}"
     end
 
     def populate_repo_directory(artifact_parent_directory)
@@ -126,7 +128,8 @@ module Pkg::Repo
       cmd << 'rsync --archive --verbose --one-file-system --ignore-existing artifacts/ repos/ '
       Pkg::Util::Net.remote_execute(Pkg::Config.distribution_server, cmd)
     rescue => e
-      fail "Error: Could not populate repos directory in #{Pkg::Config.distribution_server}:#{artifact_parent_directory}"
+      fail "Error: Could not populate repos directory in " \
+           "#{Pkg::Config.distribution_server}:#{artifact_parent_directory}: #{e}"
     end
 
     def argument_required?(argument_name, repo_command)
@@ -134,12 +137,12 @@ module Pkg::Repo
     end
 
     def update_repo(remote_host, command, options = {})
-      fail_message = "Error: Missing required argument '%s', update your build_defaults?"
+      fail_message = "Error: Missing required argument '%s', perhaps update build_defaults?"
       [:repo_name, :repo_path, :repo_host, :repo_url].each do |option|
         fail fail_message % option.to_s if argument_required?(option.to_s, command) && !options[option]
       end
 
-      whitelist = {
+      repo_configuration = {
         __REPO_NAME__: options[:repo_name],
         __REPO_PATH__: options[:repo_path],
         __REPO_HOST__: options[:repo_host],
@@ -149,7 +152,7 @@ module Pkg::Repo
       }
       Pkg::Util::Net.remote_execute(
         remote_host,
-        Pkg::Util::Misc.search_and_replace(command, whitelist))
+        Pkg::Util::Misc.search_and_replace(command, repo_configuration))
     end
   end
 end

data/lib/packaging/util/apt_staging_server.rb

@@ -0,0 +1,8 @@
+# Utility methods for handling Apt staging server.
+
+module Pkg::Util::AptStagingServer
+  def self.send_packages(pkg_directory, apt_component = 'stable')
+    %x(apt-stage-artifacts --component=#{apt_component} #{pkg_directory})
+    fail 'APT artifact staging failed.' unless $CHILD_STATUS.success?
+  end
+end

data/lib/packaging/util/build_metadata.rb

@@ -0,0 +1,17 @@
+# Utility methods for handling miscellaneous build metadata
+
+require 'fileutils'
+
+module Pkg::Util::BuildMetadata
+  class << self
+    def add_misc_json_files(target_directory)
+      misc_json_files = Dir.glob('ext/build_metadata*.json')
+      misc_json_files.each do |source_file|
+        target_file = File.join(
+          target_directory, "#{Pkg::Config.ref}.#{File.basename(source_file)}"
+        )
+        FileUtils.cp(source_file, target_file)
+      end
+    end
+  end
+end

data/lib/packaging/util/distribution_server.rb

@@ -0,0 +1,42 @@
+# Utility methods for the older distribution server
+
+require 'fileutils'
+
+module Pkg::Util::DistributionServer
+  class << self
+    def send_packages(local_source_directory, remote_target_directory)
+      Pkg::Util::Execution.retry_on_fail(times: 3) do
+        Pkg::Util::Net.remote_execute(
+          Pkg::Config.distribution_server,
+          "mkdir --mode=775 --parents #{remote_target_directory}"
+        )
+        Pkg::Util::Net.rsync_to(
+          "#{local_source_directory}/",
+          Pkg::Config.distribution_server, "#{remote_target_directory}/",
+          extra_flags: ['--ignore-existing', '--exclude repo_configs']
+        )
+      end
+
+      # In order to get a snapshot of what this build looked like at the time
+      # of shipping, we also generate and ship the params file
+      #
+      Pkg::Config.config_to_yaml(local_source_directory)
+      Pkg::Util::Execution.retry_on_fail(times: 3) do
+        Pkg::Util::Net.rsync_to(
+          "#{local_source_directory}/#{Pkg::Config.ref}.yaml",
+          Pkg::Config.distribution_server, "#{remote_target_directory}/",
+          extra_flags: ["--exclude repo_configs"]
+        )
+      end
+
+      # If we just shipped a tagged version, we want to make it immutable
+      files = Dir.glob("#{local_source_directory}/**/*")
+                 .select { |f| File.file?(f) and !f.include? "#{Pkg::Config.ref}.yaml" }
+                 .map { |f| "#{remote_target_directory}/#{f.sub(/^#{local_source_directory}\//, '')}" }
+
+      Pkg::Util::Net.remote_set_ownership(Pkg::Config.distribution_server, 'root', 'release', files)
+      Pkg::Util::Net.remote_set_permissions(Pkg::Config.distribution_server, '0664', files)
+      Pkg::Util::Net.remote_set_immutable(Pkg::Config.distribution_server, files)
+    end
+  end
+end

data/lib/packaging/util/ezbake.rb

@@ -0,0 +1,26 @@
+# Utility methods for handling ezbake
+
+require 'fileutils'
+
+module Pkg::Util::EZbake
+  class << self
+    def add_manifest(target_directory)
+      ezbake_manifest = File.join('ext', 'ezbake.manifest')
+      ezbake_yaml = File.join('ext', 'ezbake.manifest.yaml')
+
+      if File.exist?(ezbake_manifest)
+        FileUtils.cp(
+          ezbake_manifest,
+          File.join(target_directory, "#{Pkg::Config.ref}.ezbake.manifest")
+        )
+      end
+
+      if File.exists?(ezbake_yaml)
+        FileUtils.cp(
+          ezbake_yaml,
+          File.join(target_directory, "#{Pkg::Config.ref}.ezbake.manifest.yaml")
+        )
+      end
+    end
+  end
+end

data/lib/packaging/util/file.rb

@@ -120,6 +120,85 @@ module Pkg::Util::File
       end
       Pkg::Util::Version.versionbump(workdir) if Pkg::Config.update_version_file
     end
+
+    # The fetch method pulls down two files from the build-data repo that contain additional
+    # data specific to Puppet Labs release infrastructure intended to augment/override any
+    # defaults specified in the source project repo, e.g. in ext/build_defaults.yaml
+    #
+    # It uses curl to download the files, and places them in a temporary
+    # directory, e.g. /tmp/somedirectory/{project,team}/Pkg::Config.builder_data_file
+    #
+    # Retrieve build-data configurations to override/extend local build_defaults
+    def fetch
+      # Each team has a build-defaults file that specifies local infrastructure targets
+      # for things like builders, target locations for build artifacts, etc Since much
+      # of these don't change, one file can be maintained for the team.  Each project
+      # also has a data file for information specific to it. If the project builds
+      # both PE and not PE, it has two files, one for PE, and the other for FOSS
+      #
+      data_repo = Pkg::Config.build_data_repo
+
+      if Pkg::Config.dev_build
+        puts "NOTICE: This is a dev build!"
+        project_data_branch = "#{Pkg::Config.project}-dev"
+      else
+        project_data_branch = Pkg::Config.project
+      end
+      team_data_branch = Pkg::Config.team
+
+      if Pkg::Config.build_pe
+        project_data_branch = 'pe-' + project_data_branch unless project_data_branch =~ /^pe-/
+        team_data_branch = 'pe-' + team_data_branch unless team_data_branch =~ /^pe-/
+      end
+
+      # Remove .packaging directory from old-style extras loading
+      FileUtils.rm_rf("#{ENV['HOME']}/.packaging") if File.directory?("#{ENV['HOME']}/.packaging")
+
+      # Touch the .packaging file which is allows packaging to present remote tasks
+      FileUtils.touch("#{ENV['HOME']}/.packaging")
+
+      begin
+        build_data_directory = Pkg::Util::File.mktemp
+        %x(git clone #{data_repo} #{build_data_directory})
+        unless $?.success?
+          fail 'Error: could not fetch the build-data repo. Maybe you do not have the correct permissions?'
+        end
+
+        Dir.chdir(build_data_directory) do
+          [team_data_branch, project_data_branch].each do |branch|
+            %x(git checkout #{branch})
+            unless $?.success?
+              warn "Warning: no build_defaults found in branch '#{branch}' of '#{data_repo}'. Skipping."
+              next
+            end
+            load_extras(build_data_directory)
+          end
+        end
+      ensure
+        FileUtils.rm_rf(build_data_directory)
+      end
+
+      Pkg::Config.perform_validations
+    end
+
+    # The load_extras method is intended to load variables
+    # from the extra yaml file downloaded by the pl:fetch task.
+    # The goal is to be able to augment/override settings in the
+    # source project's build_data.yaml and project_data.yaml with
+    # Puppet Labs-specific data, rather than having to clutter the
+    # generic tasks with data not generally useful outside the
+    # PL Release team
+    def load_extras(temp_directory)
+      unless ENV['PARAMS_FILE'] && ENV['PARAMS_FILE'] != ''
+        temp_directory = temp_directory
+        raise "load_extras requires a directory containing extras data" if temp_directory.nil?
+        Pkg::Config.config_from_yaml("#{temp_directory}/#{Pkg::Config.builder_data_file}")
+
+        # Environment variables take precedence over those loaded from configs,
+        # so we make sure that any we clobbered are reset.
+        Pkg::Config.load_envvars
+      end
+    end
   end
 end
 

data/lib/packaging/util/net.rb

@@ -159,7 +159,7 @@ module Pkg::Util::Net
       raise(ArgumentError, "Cannot sync path '#{origin}' because both origin_host and target_host are nil. Perhaps you need to set TEAM=release ?") unless
         options[:origin_host] || options[:target_host]
 
-      cmd = %W(
+      cmd = %W[
         #{options[:bin]}
         --recursive
         --hard-links
@@ -169,7 +169,7 @@ module Pkg::Util::Net
         --no-perms
         --no-owner
         --no-group
-      ) + [*options[:extra_flags]]
+      ] + [*options[:extra_flags]]
 
       cmd << '--dry-run' if options[:dryrun]
       cmd << Pkg::Util.pseudo_uri(path: origin, host: options[:origin_host])
@@ -392,9 +392,8 @@ DOC
     end
 
     def remote_bundle_install_command
-      export_packaging_location = ''
       export_packaging_location = "export PACKAGING_LOCATION='#{ENV['PACKAGING_LOCATION']}';" if ENV['PACKAGING_LOCATION'] && !ENV['PACKAGING_LOCATION'].empty?
-      command = "source /usr/local/rvm/scripts/rvm; rvm use ruby-2.5.1; #{export_packaging_location} bundle install --path .bundle/gems ;"
+      "source /usr/local/rvm/scripts/rvm; rvm use ruby-2.5.1; #{export_packaging_location} bundle install --path .bundle/gems ;"
     end
 
     # Given a BuildInstance object and a host, send its params to the host. Return

data/lib/packaging/util/repo.rb

@@ -0,0 +1,18 @@
+# Module for signing all packages to places
+
+module Pkg::Util::Repo
+  class << self
+
+    # Create yum repositories of built RPM packages for this SHA on the distribution server
+    def rpm_repos
+      Pkg::Util::File.fetch
+      Pkg::Rpm::Repo.create_remote_repos
+    end
+
+    # Create apt repositories of build DEB packages for this SHA on the distributions server
+    def deb_repos
+      Pkg::Util::File.fetch
+      Pkg::Deb::Repo.create_repos
+    end
+  end
+end