packaging 0.99.58 → 0.99.59

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2bfce2acc825d7d873b03566196c2a55291d471fb19fc657a6049b983f6c38fa
-  data.tar.gz: 63b40774e1540cc1fa17e28d10406ef8122bd395ed8896359f72bd31a68f4b93
+  metadata.gz: 0f37ea2f3dd50cd5a9e64bf12c3046212317ec540c00570c2407c5280db76fd0
+  data.tar.gz: 2e9bb139d99cff830d21affc34c307edb786cb32688965e678c9593babb6c5ea
 SHA512:
-  metadata.gz: 0672e07b27b347bbc849fc0bb55eed9f4c1eb595038dc7f5bcdf2759c7a55ece048f0785f820eb5c5fa59d9fc384e9b84fcfc29f59a4ed2c3fa705451573eb57
-  data.tar.gz: e10e09461d51b8af7c010f110fe124ceac66589bf86942c1638ad330a9d145af8d4955b392b44a4baabd555e4319f7f0b7f006f6bf88723053a1518d7cc13f46
+  metadata.gz: 819a3182d71f837ca6ba1757e0f37d283fca7fc452437cf45de2f105c436170b7f8913ac98e9f12157ad9327776b9b2e287b2e3ddc65456b8d96cc0120830a73
+  data.tar.gz: 2973cc69e5ed2789059ef672edb76133c7bcadfa8a9c7503ffecb19589ab6c87fa8d33870a1a9b6c376ea9313c8cb1f248a07b1df0820318245fd6b88cc1e305

data/lib/packaging/artifactory.rb

@@ -408,15 +408,33 @@ module Pkg
     # Using the manifest provided by enterprise-dist, grab the appropropriate packages from artifactory based on md5sum
     # @param staging_directory [String] location to download packages to
     # @param manifest [File] JSON file containing information about what packages to download and the corresponding md5sums
-    def download_packages(staging_directory, manifest)
+    # @param remote_path [String] Optional partial path on the remote host containing packages
+    #        Used to specify which subdirectories packages will be downloaded from.
+    def download_packages(staging_directory, manifest, remote_path = '')
       check_authorization
       manifest.each do |dist, packages|
         puts "Grabbing the #{dist} packages from artifactory"
         packages.each do |name, info|
-          artifact_to_download = Artifactory::Resource::Artifact.checksum_search(md5: "#{info["md5"]}", repos: ["rpm_enterprise__local", "debian_enterprise__local"]).first
+          artifacts = Artifactory::Resource::Artifact.checksum_search(md5: "#{info["md5"]}", repos: ["rpm_enterprise__local", "debian_enterprise__local"])
+          artifact_to_download = artifacts.select { |artifact| artifact.download_uri.include? remote_path }.first
+          # If we found matching artifacts, but not in the correct path, copy the artifact to the correct path
+          # This should help us keep repos up to date with the packages we are expecting to be there
+          # while helping us avoid 'what the hell, could not find package' errors
+          if artifact_to_download.nil? && !artifacts.empty?
+            artifact_to_copy = artifacts.first
+            filename = info['filename'] || File.basename(artifact_to_copy.download_uri)
+            copy_artifact(artifact_to_copy, artifact_to_copy.repo, "#{remote_path}/#{dist}/#{filename}")
+            artifacts = Artifactory::Resource::Artifact.checksum_search(md5: "#{info["md5"]}", repos: ["rpm_enterprise__local", "debian_enterprise__local"])
+            artifact_to_download = artifacts.select { |artifact| artifact.download_uri.include? remote_path }.first
+          end
+
           if artifact_to_download.nil?
-            filename = info["filename"] || info["name"]
-            raise "Error: what the hell, could not find package #{filename} with md5sum #{info["md5"]}"
+            filename = info["filename"] || name
+            message = "Error: what the hell, could not find package #{filename} with md5sum #{info["md5"]}"
+            unless remote_path.empty?
+              message += " in #{remote_path}"
+            end
+            raise message
           else
             full_staging_path = "#{staging_directory}/#{dist}"
             filename = info['filename'] || File.basename(artifact_to_download.download_uri)
@@ -583,6 +601,33 @@ module Pkg
       end
     end
 
+    # Copy an artifact to a target repo/path
+    #
+    # @param artifact [Artifactory::Resource::Artifact] The artifact to be copied
+    # @param target_repo [String] The repository to copy the artifact to
+    # @param target_path [String] The path in the target repository to copy the artifact to
+    # @param target_debian_component [String] `deb.component` property to set on the copied artifact
+    #        defaults to `Pkg::Paths.debian_component_from_path(target_path)`
+    def copy_artifact(artifact, target_repo, target_path, target_debian_component = nil)
+      filename = File.basename(artifact.download_uri)
+      artifactory_target_path = "#{target_repo}/#{target_path}"
+      puts "Copying #{artifact.download_uri} to #{artifactory_target_path}"
+      begin
+        artifact.copy(artifactory_target_path)
+      rescue Artifactory::Error::HTTPError
+        STDERR.puts "Could not copy #{artifactory_target_path}. Source and destination are the same. Skipping..."
+      end
+
+      if File.extname(filename) == '.deb'
+        target_debian_component ||= Pkg::Paths.debian_component_from_path(target_path)
+        copied_artifact_search = search_with_path(filename, target_repo, target_path)
+        fail "Error: what the hell, could not find just-copied package #{filename} under #{target_repo}/#{target_path}" if copied_artifact_search.empty?
+        copied_artifact = copied_artifact_search.first
+        properties = { 'deb.component' => target_debian_component }
+        copied_artifact.properties(properties)
+      end
+     end
+
     # When we cut a new PE branch, we need to copy the pe components into <pe_version>/{repos,feature,release}/<platform>
     # @param manifest [File] JSON file containing information about what packages to download and the corresponding md5sums
     # @param target_path [String] path on artifactory to copy components to, e.g. <pe_version>/release
@@ -593,24 +638,11 @@ module Pkg
         packages.each do |name, info|
           artifact = Artifactory::Resource::Artifact.checksum_search(md5: "#{info["md5"]}", repos: ["rpm_enterprise__local", "debian_enterprise__local"]).first
           if artifact.nil?
-            filename = info["filename"] || info["name"]
+            filename = info["filename"] || name
             raise "Error: what the hell, could not find package #{filename} with md5sum #{info["md5"]}"
           end
-          begin
-            filename = info["filename"] || File.basename(artifact.download_uri)
-            artifact_target_path = "#{artifact.repo}/#{target_path}/#{dist}/#{filename}"
-            puts "Copying #{artifact.download_uri} to #{artifact_target_path}"
-            artifact.copy(artifact_target_path)
-          rescue Artifactory::Error::HTTPError
-            STDERR.puts "Could not copy #{artifact_target_path}. Source and destination are the same. Skipping..."
-          end
-          if File.extname(filename) == '.deb'
-            copied_artifact_search = Artifactory::Resource::Artifact.pattern_search(repo: 'debian_enterprise__local', pattern: "#{target_path}/*/#{filename}")
-            fail "Error: what the hell, could not find just-copied package #{filename} under debian_enterprise__local/#{target_path}" if copied_artifact_search.nil?
-            copied_artifact = copied_artifact_search.first
-            properties = { 'deb.component' => Pkg::Paths.debian_component_from_path(target_path) }
-            copied_artifact.properties(properties)
-          end
+          filename = info["filename"] || File.basename(artifact.download_uri)
+          copy_artifact(artifact, artifact.repo, "#{target_path}/#{dist}/#{filename}")
         end
       end
     end

data/lib/packaging/paths.rb

@@ -304,9 +304,18 @@ module Pkg::Paths
   end
 
   def debian_component_from_path(path)
-    matches = path.match(/(\d+\.\d+)\/(\w+)/)
+    # substitute '.' and '/' since those aren't valid characters for debian components
+    matches = path.match(/([\d+\.\d+|master])\/(\w+)/)
+    regex_for_substitution = /[\.\/]/
     fail "Error: Could not determine Debian Component from path #{path}" if matches.nil?
-    return matches[1] if matches[2] == 'repos'
-    return matches[0]
+    base_component = matches[1]
+    component_qualifier = matches[2]
+    full_component = "#{base_component}/#{component_qualifier}"
+    unless regex_for_substitution.nil?
+      base_component.gsub!(regex_for_substitution, '_')
+      full_component.gsub!(regex_for_substitution, '_')
+    end
+    return base_component if component_qualifier == 'repos'
+    return full_component
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: packaging
 version: !ruby/object:Gem::Version
-  version: 0.99.58
+  version: 0.99.59
 platform: ruby
 authors:
 - Puppet Labs
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-03-03 00:00:00.000000000 Z
+date: 2020-03-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec