packaging 0.99.24 → 0.99.25

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 42a454c3b78537b5fc58e2c3dee9027327647eb4
-  data.tar.gz: 20760d44dce80abc0ee5a4fc3346d9df0dfca29c
+SHA256:
+  metadata.gz: 882ec9fd259317c5faf675e1316bc009966c065c0016db603e04a3a92ffda3ee
+  data.tar.gz: c309fe43c243bf7c149a46707eb4ddcd5a4d6d9e270999a5ffa22757b6b31fff
 SHA512:
-  metadata.gz: a49aba088a2371721113f05c7a05e4b2e27e5782c0e4484eb5dd95450c6f80de4290529e9fb4eb524b31db52b08afdbaca103313b102d819a170f5e2302dd941
-  data.tar.gz: f1f3e6a0d9fa645cf92e3400905e8bf41da8413f04ae99d7fb040834dd6cbcc423a3eafa7295230fbbdd8adf6b49db60a3816fad9eedee6f5df1af759efd3db1
+  metadata.gz: 50444de7d9e7ad4f7c1a583bcb7b68f368b3cce463fe1dcea274bcbd312fb934be4e77f4b0bc6790970a481a7600ca168264e59a9fa89905e9e934a39e4624bd
+  data.tar.gz: a1bd3e9837ec9540ebebac3b75dbd84716c99e2e0b5522723afed882b2d17a4ebb77d0a9de73942cb7282966719e9e65b01f1d880085d3503f982fde035b45d6

data/lib/packaging/artifactory.rb

@@ -1,3 +1,6 @@
+require 'uri'
+require 'open-uri'
+
 module Pkg
 
   # The Artifactory class
@@ -203,16 +206,19 @@ module Pkg
       raise "Attempt to upload '#{package}' to #{File.join(@artifactory_uri, data[:full_artifactory_path])} failed"
     end
 
-    # @param directory [String] optional, The directory where the yaml file will
-    #   be downloaded
-    # @return [String] The path to the downloaded file
-    def retrieve_yaml_data_file(directory = nil)
-      directory ||= Dir.mktmpdir
-      retrieve_package(DEFAULT_REPO_TYPE, "#{@project_version}.yaml", directory)
-      File.join(directory, "#{@project_version}.yaml")
+    # @param pkg [String] The package to download YAML for
+    #   i.e. 'puppet-agent' or 'puppetdb'
+    # @param ref [String] The git ref (sha or tag) we want the YAML for
+    #
+    # @return [String] The contents of the YAML file
+    def retrieve_yaml_data(pkg, ref)
+      yaml_url = "#{@artifactory_uri}/#{DEFAULT_REPO_TYPE}/#{DEFAULT_REPO_BASE}/#{pkg}/#{ref}/#{ref}.yaml"
+      open(yaml_url) { |f| f.read }
+    rescue
+      raise "Failed to load YAML data for #{pkg} at #{ref} from #{yaml_url}!"
     end
 
-    # @param platform_data [Hash] The has of the platform data that needs to be
+    # @param platform_data [Hash] The hash of the platform data that needs to be
     #   parsed
     # @param platform_tag [String] The tag that the data we want belongs to
     # @return [String] The name of the package for the given project,
@@ -230,47 +236,80 @@ module Pkg
       raise fail_message
     end
 
-    # @param platform_tags [Array[String], String] optional, either a string, or
-    #   an array of strings. These are the platform or platforms that we will
-    #   download packages for.
-    # @param package [String] optional, the name of the package to be
-    #   retrieved. If the user does not know this information, we can derive it
-    #   from the yaml data. This ignores everything but the package name. Any
-    #   customization for where the user wants to fetch the package is via the
-    #   download_path parameter.
-    # @param download_path [String] Optional, an optional path set to where
-    #   the user wants the retrieved package to end up. If no path is specified
-    #   this defaults to the pkg directory.
-    def retrieve_package(platform_tags = nil, package = nil, download_path = nil)
-
-      if platform_tags.nil? && !package.nil?
-        platform_tags = Pkg::Paths.tag_from_artifact_path(package) || DEFAULT_REPO_TYPE
-      elsif platform_tags.nil? && package.nil?
-        yaml_file = retrieve_yaml_data_file(download_path)
-        yaml_data = Pkg::Config.config_from_yaml(yaml_file)
-        platform_data = yaml_data[:platform_data]
-        platform_tags = platform_data.keys
-      end
+    # @param platform_data [Hash] The hash of the platform data that needs to be
+    #   parsed
+    # @param platform_tag [String] The tag that the data we want belongs to
+    # @return [Array] An array containing all packages for the given project,
+    #   project_version, and platform_tag
+    def all_package_names(platform_data, platform_tag)
+      packages = [platform_data[platform_tag][:artifact]]
+      packages << platform_data[platform_tag][:additional_artifacts]
+      packages.flatten!
+      packages.reject! { |package| package.empty? || package.nil? }
+      packages.map { |package| File.basename(package) }
+    rescue
+      fail_message = <<-DOC
+  Package name could not be found from loaded yaml data. Either this package
+  does not exist, or '#{@platform_tag}' is not present in this dataset.
+
+  The following are available platform tags for '#{@project}' '#{@project_version}':
+    #{platform_data.keys.sort}
+      DOC
+      raise fail_message
+    end
+
+    # Promotes a build based on build SHA or tag (or SNAPSHOT version, for ezbake)
+    # Depending on if it's an RPM or Deb package promote accordingly
+    # 'promote' by copying the package(s) to the enterprise directory on artifactory
+    #
+    # @param pkg [String] the package name ex. puppet-agent
+    # @param ref [String] tag or SHA of package(s) to be promoted
+    # @param platform_tag [String] the platform tag of the artifact
+    #   ex. el-7-x86_64, ubuntu-18.04-amd64
+    # @param repositories [Array(String)] the repositories to promote
+    #   the artifact to. Will prepend 'rpm_' or 'debian_' to the repositories
+    #   depending on package type
+    def promote_package(pkg, ref, platform_tag, repositories)
+      # load package metadata
+      yaml_content = retrieve_yaml_data(pkg, ref)
+      yaml_data = YAML::load(yaml_content)
+
+      # get the artifact name
+      artifact_names = all_package_names(yaml_data[:platform_data], platform_tag)
+      artifact_names.each do |artifact_name|
+        artifact_to_promote = Artifactory::Resource::Artifact.search(name: artifact_name, :artifactory_uri => @artifactory_uri)
+
+        if artifact_to_promote.empty?
+          raise "Error: could not find PKG=#{pkg} at REF=#{git_ref} for #{platform_tag}"
+        end
 
-      Array(platform_tags).each do |platform_tag|
-        puts "fetching package for #{platform_tag}"
-        data = platform_specific_data(platform_tag)
-        if package.nil?
-          package_for_tag = package_name(platform_data, platform_tag)
-          puts "package name is #{package_for_tag}"
+        # This makes an assumption that we're using some consistent repo names
+        # but need to either prepend 'rpm_' or 'debian_' based on package type
+        if File.extname(artifact_name) == '.rpm'
+          promotion_paths = Array(repositories).compact.map { |repo| "rpm_#{repo}/#{platform_tag}/#{artifact_name}" }
+        elsif File.extname(artifact_name) == '.deb'
+          promotion_paths = Array(repositories).compact.map { |repo| "debian_#{repo}/#{platform_tag}/#{artifact_name}" }
         else
-          package_for_tag = package
+          raise "Error: Unknown promotion repository for #{artifact_name}! Only .rpm and .deb files are supported!"
         end
-        download_path_for_tag = download_path || data[:repo_subdirectories].sub(@repo_base, 'pkg')
 
-        check_authorization
-        artifact = Artifactory::Resource::Artifact.new(
-          download_uri: File.join(@artifactory_uri, data[:full_artifactory_path], File.basename(package_for_tag))
-        )
-        artifact.download(download_path_for_tag)
+        begin
+          promotion_paths.each do |path|
+            puts "promoting #{artifact_name} to #{path}"
+            artifact_to_promote[0].copy(path)
+          end
+        rescue Artifactory::Error::HTTPError => e
+          if e.message =~ /destination and source are the same/i
+            puts "Skipping promotion of #{artifact_name}; it has already been promoted"
+          else
+            puts "#{e.level}: #{e.message}"
+            raise e
+          end
+        rescue => e
+          puts "Something went wrong promoting #{artifact_name}!"
+          raise e
+        end
       end
-    rescue
-      raise "Attempt to download '#{File.basename(package)}' from #{File.join(@artifactory_uri, data[:full_artifactory_path])} failed."
     end
 
     private :check_authorization

data/lib/packaging/config.rb

@@ -81,6 +81,13 @@ module Pkg
           data = {}
           artifacts.each do |artifact|
             tag = Pkg::Paths.tag_from_artifact_path(artifact)
+
+            # Remove the f-prefix from the fedora platform tag keys so that
+            # beaker can rely on consistent keys once we rip out the f for good
+            tag = tag.sub(/fedora-f/, 'fedora-')
+
+            data[tag] ||= {}
+
             platform, version, arch = Pkg::Platforms.parse_platform_tag(tag)
             package_format = Pkg::Platforms.get_attribute(tag, :package_format)
 
@@ -89,6 +96,7 @@ module Pkg
             # information, but we should report the versioned artifact in
             # platform_data
             next if platform == 'windows' && File.basename(artifact) == "#{self.project}-#{arch}.#{package_format}"
+
             # Sometimes we have source or debug packages. We don't want to save
             # these paths in favor of the artifact paths.
             if platform == 'solaris'
@@ -97,7 +105,14 @@ module Pkg
             else
               next if File.extname(artifact) != ".#{package_format}"
             end
-            next if /#{self.project}-[a-z]+/.match(File.basename(artifact))
+
+            # Don't want to include debian debug packages
+            next if /-dbgsym/.match(File.basename(artifact))
+
+            if /#{self.project}-[a-z]+/.match(File.basename(artifact))
+              add_additional_artifact(data, tag, artifact.sub('artifacts/', ''))
+              next
+            end
 
             case package_format
             when 'deb'
@@ -109,12 +124,17 @@ module Pkg
             else
               fail "Not sure what to do with packages with a package format of '#{package_format}' - maybe update PLATFORM_INFO?"
             end
-            # Remove the f-prefix from the fedora platform tag keys so that
-            # beaker can rely on consistent keys once we rip out the f for good
-            tag = tag.sub(/fedora-f/, 'fedora-')
-            data[tag] = { :artifact => artifact.sub('artifacts/', ''),
-                          :repo_config => repo_config,
-                        }
+
+            # handle the case where there are multiple artifacts but the artifacts are not
+            # named based on project name (e.g. puppet-enterprise-vanagon).
+            # In this case, the first one will get set as the artifact, everything else
+            # will be in the additional artifacts
+            if data[tag][:artifact].nil?
+              data[tag][:artifact] = artifact.sub('artifacts/', '')
+              data[tag][:repo_config] = repo_config
+            else
+              add_additional_artifact(data, tag, artifact.sub('artifacts/', ''))
+            end
           end
           return data
         else
@@ -123,6 +143,32 @@ module Pkg
         end
       end
 
+      # Add artifact to the `additional_artifacts` array in platform data.
+      # This will not add noarch package paths for the same noarch package
+      # multiple times.
+      #
+      # @param platform_data The platform data hash to update
+      # @param tag the platform tag
+      # @param artifact the path of the additional artifact path to add
+      def add_additional_artifact(platform_data, tag, artifact)
+        # Don't add noarch packages to additional_artifacts if the same package
+        # is already the artifact
+        if File.basename(platform_data[tag][:artifact]) == File.basename(artifact)
+          return
+        end
+
+        platform_data[tag][:additional_artifacts] ||= []
+
+        if platform_data[tag][:additional_artifacts].select { |a| File.basename(a) == File.basename(artifact) }.empty?
+          platform_data[tag][:additional_artifacts] << artifact
+        end
+
+        # try to avoid empty entries in the yaml for more concise output
+        if platform_data[tag][:additional_artifacts].empty?
+          platform_data[tag][:additional_artifacts] = nil
+        end
+      end
+
       ##
       # Return a hash of all build parameters and their values, nil if unassigned.
       #

data/spec/fixtures/config/ext/build_defaults.yaml

@@ -1 +1 @@
-spec/fixtures/config/ext/../params.yaml
+../params.yaml

data/spec/fixtures/config/ext/project_data.yaml

@@ -1 +1 @@
-spec/fixtures/config/ext/../params.yaml
+../params.yaml

data/spec/lib/packaging/artifactory_spec.rb

@@ -14,26 +14,32 @@ describe 'artifactory.rb' do
       'el-6-x86_64' => {
         :artifact => "./el/6/PC1/x86_64/puppet-agent-5.3.1.34.gf65f9ef-1.el6.x86_64.rpm",
         :repo_config => "../repo_configs/rpm/pl-puppet-agent-f65f9efbb727c3d2d72d6799c0fc345a726f27b5-el-6-x86_64.repo",
+        :additional_artifacts => ["./el/6/PC1/x86_64/puppet-agent-extras-5.3.1.34.gf65f9ef-1.el6.x86_64.rpm"],
       },
       'ubuntu-16.04-amd64' => {
         :artifact => "./deb/xenial/PC1/puppet-agent_5.3.1.34.gf65f9ef-1xenial_amd64.deb",
         :repo_config => "../repo_configs/deb/pl-puppet-agent-f65f9efbb727c3d2d72d6799c0fc345a726f27b5-xenial.list",
+        :additional_artifacts => ["./deb/xenial/PC1/puppet-agent-extras_5.3.1.34.gf65f9ef-1xenial_amd64.deb"],
       },
       'windows-2012-x86' => {
         :artifact => "./windows/puppet-agent-5.3.1.34-x86.msi",
         :repo_config => '',
+        :additional_artifacts => ["./windows/puppet-agent-extras-5.3.1.34-x86.msi"],
       },
       'eos-4-i386' => {
         :artifact => "./eos/4/PC1/i386/puppet-agent-5.3.1.34.gf65f9ef-1.eos4.i386.swix",
         :repo_config => '',
+        :additional_artifacts => ["./eos/4/PC1/i386/puppet-agent-extras-5.3.1.34.gf65f9ef-1.eos4.i386.swix"],
       },
       'osx-10.12-x86_64' => {
         :artifact => "./apple/10.12/PC1/x86_64/puppet-agent-5.3.1.34.gf65f9ef-1.osx10.12.dmg",
         :repo_config => '',
+        :additional_artifacts => ["./apple/10.12/PC1/x86_64/puppet-agent-extras-5.3.1.34.gf65f9ef-1.osx10.12.dmg"],
       },
       'solaris-10-sparc' => {
         :artifact => "./solaris/10/PC1/puppet-agent-5.3.1.34.gf65f9ef-1.sparc.pkg.gz",
         :repo_config => '',
+        :additional_artifacts => ["./solaris/10/PC1/puppet-agent-extras-5.3.1.34.gf65f9ef-1.sparc.pkg.gz"],
       },
     }
   }
@@ -44,6 +50,7 @@ describe 'artifactory.rb' do
       :repo_subdirectories => "#{default_repo_name}/#{project}/#{project_version}/el-6-x86_64",
       :package_format => 'rpm',
       :package_name => 'path/to/a/el/6/package/puppet-agent-5.3.1.34.gf65f9ef-1.el6.x86_64.rpm',
+      :all_package_names => ['puppet-agent-5.3.1.34.gf65f9ef-1.el6.x86_64.rpm', 'puppet-agent-extras-5.3.1.34.gf65f9ef-1.el6.x86_64.rpm']
     },
     'ubuntu-16.04-amd64' => {
       :toplevel_repo => 'debian__local',
@@ -51,26 +58,31 @@ describe 'artifactory.rb' do
       :codename => 'xenial',
       :arch => 'amd64',
       :package_name => 'path/to/a/xenial/package/puppet-agent_5.3.1.34.gf65f9ef-1xenial_amd64.deb',
+      :all_package_names => ['puppet-agent_5.3.1.34.gf65f9ef-1xenial_amd64.deb', 'puppet-agent-extras_5.3.1.34.gf65f9ef-1xenial_amd64.deb']
     },
     'windows-2012-x86' => {
       :toplevel_repo => 'generic',
       :repo_subdirectories => "#{default_repo_name}/#{project}/#{project_version}/windows-x86",
       :package_name => 'path/to/a/windows/package/puppet-agent-5.3.1.34-x86.msi',
+      :all_package_names => ['puppet-agent-5.3.1.34-x86.msi','puppet-agent-extras-5.3.1.34-x86.msi']
     },
     'eos-4-i386' => {
       :toplevel_repo => 'generic',
       :repo_subdirectories => "#{default_repo_name}/#{project}/#{project_version}/eos-4-i386",
       :package_name => 'path/to/an/eos/4/package/puppet-agent-5.3.1.34.gf65f9ef-1.eos4.i386.swix',
+      :all_package_names => ['puppet-agent-5.3.1.34.gf65f9ef-1.eos4.i386.swix', 'puppet-agent-extras-5.3.1.34.gf65f9ef-1.eos4.i386.swix']
     },
     'osx-10.12-x86_64' => {
       :toplevel_repo => 'generic',
       :repo_subdirectories => "#{default_repo_name}/#{project}/#{project_version}/osx-10.12-x86_64",
       :package_name => 'path/to/an/osx/10.12/package/puppet-agent-5.3.1.34.gf65f9ef-1.osx10.12.dmg',
+      :all_package_names => ['puppet-agent-5.3.1.34.gf65f9ef-1.osx10.12.dmg', 'puppet-agent-extras-5.3.1.34.gf65f9ef-1.osx10.12.dmg']
     },
     'solaris-10-sparc' => {
       :toplevel_repo => 'generic',
       :repo_subdirectories => "#{default_repo_name}/#{project}/#{project_version}/solaris-10-sparc",
       :package_name => 'path/to/a/solaris/10/package/puppet-agent-5.3.1.34.gf65f9ef-1.sparc.pkg.gz',
+      :all_package_names => ['puppet-agent-5.3.1.34.gf65f9ef-1.sparc.pkg.gz','puppet-agent-extras-5.3.1.34.gf65f9ef-1.sparc.pkg.gz']
     },
   }
 
@@ -124,6 +136,23 @@ describe 'artifactory.rb' do
       end
     end
 
+    describe '#all_package_names' do
+      it 'parses the retrieved yaml file and returns the correct package name' do
+        all_package_names = artifact.all_package_names(platform_data, platform_tag)
+        all_package_names_data = [platform_tag_data[:additional_artifacts], platform_tag_data[:all_package_names]].flatten.compact
+        all_package_names.map! { |p| File.basename(p) }
+        all_package_names_data.map! { |p| File.basename(p) }
+        expect(all_package_names.size).to eq(all_package_names_data.size)
+        expect(all_package_names.sort).to eq(all_package_names_data.sort)
+      end
+
+      it 'fails if it cannot find a valid platform name' do
+        new_platform_data = platform_data
+        new_platform_data.delete_if { |k| k.match(platform_tag) }
+        expect{artifact.package_name(new_platform_data, platform_tag)}.to raise_error
+      end
+    end
+
     describe '#deb_list_contents' do
       it "returns the correct contents for the debian list file for #{platform_tag}" do
         if platform_tag_data[:codename]

data/spec/lib/packaging/config_spec.rb

@@ -239,6 +239,11 @@ describe "Pkg::Config" do
       "./artifacts/deb/stretch/PC1/puppet-agent_5.3.2.658.gc79ef9a-1stretch_amd64.deb\n" \
       "./artifacts/deb/stretch/PC1/puppet-agent-dbgsym_5.3.2.658.gc79ef9a-1stretch_amd64.deb"
 
+    artifacts_not_matching_project = \
+      "./artifacts/deb/xenial/pe-postgresql-contrib_2019.1.9.6.12-1xenial_amd64.deb\n" \
+      "./artifacts/deb/xenial/pe-postgresql-devel_2019.1.9.6.12-1xenial_amd64.deb\n" \
+      "./artifacts/deb/xenial/pe-postgresql-server_2019.1.9.6.12-1xenial_amd64.deb\n" \
+      "./artifacts/deb/xenial/pe-postgresql_2019.1.9.6.12-1xenial_amd64.deb"
     project = 'puppet-agent'
     ref = '5.3.2'
 
@@ -295,6 +300,16 @@ describe "Pkg::Config" do
       expect(data['debian-9-amd64']).to include(:artifact => './deb/stretch/PC1/puppet-agent_5.3.2.658.gc79ef9a-1stretch_amd64.deb')
     end
 
+    it "should collect packages that don't match the project name" do
+      allow(Pkg::Util::Net).to receive(:remote_ssh_cmd).and_return(artifacts_not_matching_project, nil)
+      data = Pkg::Config.platform_data
+      expect(data['ubuntu-16.04-amd64']).to include(:artifact => './deb/xenial/pe-postgresql-contrib_2019.1.9.6.12-1xenial_amd64.deb')
+      expect(data['ubuntu-16.04-amd64'][:additional_artifacts].size).to eq(3)
+      expect(data['ubuntu-16.04-amd64'][:additional_artifacts]).to include('./deb/xenial/pe-postgresql-devel_2019.1.9.6.12-1xenial_amd64.deb')
+      expect(data['ubuntu-16.04-amd64'][:additional_artifacts]).to include('./deb/xenial/pe-postgresql-server_2019.1.9.6.12-1xenial_amd64.deb')
+      expect(data['ubuntu-16.04-amd64'][:additional_artifacts]).to include('./deb/xenial/pe-postgresql_2019.1.9.6.12-1xenial_amd64.deb')
+    end
+
     it "should use 'ppc' instead of 'power' in aix paths" do
       allow(Pkg::Util::Net).to receive(:remote_ssh_cmd).and_return(aix_artifacts, nil)
       data = Pkg::Config.platform_data

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: packaging
 version: !ruby/object:Gem::Version
-  version: 0.99.24
+  version: 0.99.25
 platform: ruby
 authors:
 - Puppet Labs
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-03-12 00:00:00.000000000 Z
+date: 2019-04-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -223,8 +223,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.9
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Puppet Labs' packaging automation