packaging 0.99.80 → 0.99.81

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3c641cdc322d55fe7c595fb4802ababf87830fe6a63a852f3acf291b601deeed
-  data.tar.gz: 39d19f13de74f2736b3aa8efcd9519345a81ee056f33ae7f1960cecc40e7d896
+  metadata.gz: 5b6b5822d617cc1452e3bb9cdf7fc9c5788fc71cfe3e05557bf5a0efb0e7db07
+  data.tar.gz: 19266ebee48e9231a450a0a511e26583aeb161c8d0bf7d45fd1e4cdedd6a1673
 SHA512:
-  metadata.gz: 690143636f97cba9f3a97918c60dd1f875b2b0a84eada68ae456f3b7c71a0295fcf38fcafcaf1584237ce2dbec6b3c4c8350e4e0d4112ac60f5c1f057ddd27ed
-  data.tar.gz: 3cfd026361b7b485eaffc9e9bb1ec8c0a2a296300631d3399f9edb3ee8d2e59a4cf3866cd0f1f7243f70f9ff15b3dc2b372477426ebf55fe5657362f531a76c9
+  metadata.gz: 84a29597af4cd4c654f127eac2179be39721af304bf71d3c7e6f0d8a216930ab9bcb782c2575d92fd20a259572ff63684efe869bb2b08019ed80e1bbf1f65585
+  data.tar.gz: d7e30e71d5f197c958dc688d9904c7a431dee71ad3669dd6d3b8792e80f5cda5153fea85272b0be19e6701b645862e0be14d2ad452e7bad46235d5764e38d874

data/lib/packaging/artifactory.rb

@@ -342,40 +342,69 @@ module Pkg
       end
     end
 
-    # Using the manifest provided by enterprise-dist, grab the appropropriate packages from artifactory based on md5sum
+    # Using the manifest provided by enterprise-dist, grab the
+    # appropropriate packages from artifactory based on md5sum
+
     # @param staging_directory [String] location to download packages to
-    # @param manifest [File] JSON file containing information about what packages to download and the corresponding md5sums
+    # @param manifest [File] JSON file listing which packages to download and
+    #   the corresponding md5sums
     # @param remote_path [String] Optional partial path on the remote host containing packages
     #        Used to specify which subdirectories packages will be downloaded from.
     def download_packages(staging_directory, manifest, remote_path = '')
       check_authorization
+      download_repositories = %w(rpm_enterprise__local debian_enterprise__local)
       manifest.each do |dist, packages|
-        puts "Grabbing the #{dist} packages from artifactory"
         packages.each do |name, info|
-          filename = info['filename']
-          artifacts = Artifactory::Resource::Artifact.checksum_search(md5: "#{info["md5"]}", repos: ["rpm_enterprise__local", "debian_enterprise__local"], name: filename)
-          artifact_to_download = artifacts.select { |artifact| artifact.download_uri.include? remote_path }.first
-          # If we found matching artifacts, but not in the correct path, copy the artifact to the correct path
-          # This should help us keep repos up to date with the packages we are expecting to be there
-          # while helping us avoid 'what the hell, could not find package' errors
+          package_file_name = info['filename']
+          puts format(
+                 "Searching Artifactory [%s]%s for %s (md5: %s)",
+                 download_repositories.join(', '),
+                 remote_path.empty? ? '' : "/#{remote_path}",
+                 package_file_name,
+                 info['md5']
+               )
+          artifacts = Artifactory::Resource::Artifact.checksum_search(
+            md5: info['md5'],
+            repos: download_repositories,
+            name: package_file_name
+          )
+
+          artifact_to_download = artifacts.select do |artifact|
+            artifact.download_uri.include? remote_path
+          end.first
+
+          # If we found matching artifacts, but not in the correct
+          # path, copy the artifact to the correct path This should
+          # help us keep repos up to date with the packages we are
+          # expecting to be there while helping us avoid 'could not
+          # find package' errors
           if artifact_to_download.nil? && !artifacts.empty?
             artifact_to_copy = artifacts.first
-            copy_artifact(artifact_to_copy, artifact_to_copy.repo, "#{remote_path}/#{dist}/#{filename}")
-            artifacts = Artifactory::Resource::Artifact.checksum_search(md5: "#{info["md5"]}", repos: ["rpm_enterprise__local", "debian_enterprise__local"], name: filename)
-            artifact_to_download = artifacts.select { |artifact| artifact.download_uri.include? remote_path }.first
+            copy_artifact(artifact_to_copy, artifact_to_copy.repo,
+                          "#{remote_path}/#{dist}/#{package_file_name}")
+
+            # Now, search again to make sure we find them in the correct path.
+            artifacts = Artifactory::Resource::Artifact.checksum_search(
+              md5: info['md5'],
+              repos: download_repositories,
+              name: package_file_name
+            )
+            artifact_to_download = artifacts.select do |artifact|
+              artifact.download_uri.include? remote_path
+            end.first
           end
 
           if artifact_to_download.nil?
-            message = "Error: what the hell, could not find package #{filename} with md5sum #{info["md5"]}"
-            unless remote_path.empty?
-              message += " in #{remote_path}"
-            end
+            message = "Error: could not find package #{package_file_name} " \
+                      "with md5sum #{info['md5']}"
+            message += " in #{remote_path}" unless remote_path.empty?
             raise message
-          else
-            full_staging_path = "#{staging_directory}/#{dist}"
-            puts "downloading #{artifact_to_download.download_uri} to #{File.join(full_staging_path, filename)}"
-            artifact_to_download.download(full_staging_path, filename: filename)
           end
+
+          full_staging_path = "#{staging_directory}/#{dist}"
+          puts "Downloading #{artifact_to_download.download_uri} to " \
+               "#{File.join(full_staging_path, package_file_name)}"
+          artifact_to_download.download(full_staging_path, filename: package_file_name)
         end
       end
     end

data/lib/packaging/platforms.rb

@@ -157,6 +157,14 @@ module Pkg
           source_package_formats: ['src.rpm'],
           signature_format: 'v3',
           repo: true,
+        },
+        '8' => {
+          architectures: ['x86_64'],
+          source_architecture: 'SRPMS',
+          package_format: 'rpm',
+          source_package_formats: ['src.rpm'],
+          signature_format: 'v3',
+          repo: true,
         }
       },
 

data/tasks/nightly_repos.rake

@@ -93,7 +93,7 @@ DOC
         # The repo configs have Pkg::Config.builds_server used in them, but that
         # is internal, so we need to replace it with our public server. We also
         # want them only to see repos, and not signed repos, since the host is
-        # called nightlies.puppetlabs.com. Here we replace those values in each
+        # called nightlies.puppet.com. Here we replace those values in each
         # config with the desired value.
         Dir.glob("#{local_target}/repo_configs/**/*").select { |t_config| File.file?(t_config) }.each do |config|
           new_contents = File.read(config).gsub(Pkg::Config.builds_server, target_host).gsub(/#{target_prefix}_repos/, "repos")

data/tasks/ship.rake

@@ -172,20 +172,10 @@ namespace :pl do
       end
     end
 
-    desc "Copy signed deb repos from #{Pkg::Config.apt_signing_server} to AWS S3"
-    task :deploy_apt_repo_to_s3 => 'pl:fetch' do
-      puts "Really run S3 sync to deploy Debian repos from #{Pkg::Config.apt_signing_server} to AWS S3? [y,n]"
-      if Pkg::Util.ask_yes_or_no
-        Pkg::Util::Execution.retry_on_fail(:times => 3) do
-          command = 'sudo /usr/local/bin/s3_repo_sync.sh apt.puppetlabs.com'
-          Pkg::Util::Net.remote_execute(Pkg::Config.apt_signing_server, command)
-        end
-      end
-    end
-
     desc "Copy rpm repos from #{Pkg::Config.yum_staging_server} to #{Pkg::Config.yum_host}"
     task deploy_yum_repo: 'pl:fetch' do
-      puts "Really run remote rsync to deploy yum repos from #{Pkg::Config.yum_staging_server} to #{Pkg::Config.yum_host}? [y,n]"
+      puts "Really run remote rsync to deploy yum repos from #{Pkg::Config.yum_staging_server} " \
+           "to #{Pkg::Config.yum_host}? [y,n]"
       if Pkg::Util.ask_yes_or_no
         Pkg::Util::Execution.retry_on_fail(times: 3) do
           Pkg::Rpm::Repo.deploy_repos(
@@ -198,27 +188,67 @@ namespace :pl do
       end
     end
 
-    desc "Copy signed RPM repos from #{Pkg::Config.yum_staging_server} to AWS S3"
+    ##
+    ## S3 / GCP syncing
+    S3_REPO_SYNC = 'sudo /usr/local/bin/s3_repo_sync.sh'
+    GCP_REPO_SYNC = '/usr/local/bin/gcp_repo_sync'
+
+    desc "Sync signed apt repos from #{Pkg::Config.apt_signing_server} to AWS S3"
+    task :deploy_apt_repo_to_s3 => 'pl:fetch' do
+      sync_command = "#{S3_REPO_SYNC} apt.puppetlabs.com"
+      puts "Sync apt repos from #{Pkg::Config.apt_signing_server} to AWS S3? [y,n]"
+      next unless Pkg::Util.ask_yes_or_no
+
+      Pkg::Util::Execution.retry_on_fail(times: 3) do
+        Pkg::Util::Net.remote_execute(Pkg::Config.apt_signing_server, sync_command)
+      end
+    end
+
+    desc "Sync signed yum repos from #{Pkg::Config.yum_staging_server} to AWS S3"
     task :deploy_yum_repo_to_s3 => 'pl:fetch' do
-      puts "Really run S3 sync to deploy RPM repos from #{Pkg::Config.yum_staging_server} to AWS S3? [y,n]"
-      if Pkg::Util.ask_yes_or_no
-        Pkg::Util::Execution.retry_on_fail(:times => 3) do
-          command = 'sudo /usr/local/bin/s3_repo_sync.sh yum.puppetlabs.com'
-          Pkg::Util::Net.remote_execute(Pkg::Config.yum_staging_server, command)
-        end
+      sync_command = "#{S3_REPO_SYNC} yum.puppetlabs.com"
+      puts "Sync yum repos from #{Pkg::Config.yum_staging_server} to AWS S3? [y,n]"
+      next unless Pkg::Util.ask_yes_or_no
+      Pkg::Util::Execution.retry_on_fail(times: 3) do
+        Pkg::Util::Net.remote_execute(Pkg::Config.yum_staging_server, sync_command)
       end
     end
 
     desc "Sync downloads.puppetlabs.com from #{Pkg::Config.staging_server} to AWS S3"
     task :deploy_downloads_to_s3 => 'pl:fetch' do
-      puts "Really run S3 sync to sync downloads.puppetlabs.com from #{Pkg::Config.staging_server} to AWS S3? [y,n]"
-      if Pkg::Util.ask_yes_or_no
-        Pkg::Util::Execution.retry_on_fail(:times => 3) do
-          command = 'sudo /usr/local/bin/s3_repo_sync.sh downloads.puppetlabs.com'
-          Pkg::Util::Net.remote_execute(Pkg::Config.staging_server, command)
-        end
+      sync_command = "#{S3_REPO_SYNC} downloads.puppetlabs.com"
+      puts "Sync downloads.puppetlabs.com from #{Pkg::Config.staging_server} to AWS S3? [y,n]"
+      next unless Pkg::Util.ask_yes_or_no
+      Pkg::Util::Execution.retry_on_fail(times: 3) do
+        Pkg::Util::Net.remote_execute(Pkg::Config.staging_server, sync_command)
+      end
+    end
+
+    desc "Sync nightlies.puppet.com from #{Pkg::Config.staging_server} to AWS S3"
+    task :deploy_nightlies_to_s3 => 'pl:fetch' do
+      sync_command = "#{S3_REPO_SYNC} nightlies.puppet.com"
+      puts "Syncing nightly builds from #{Pkg::Config.staging_server} to AWS S3"
+      Pkg::Util::Execution.retry_on_fail(times: 3) do
+        Pkg::Util::Net.remote_execute(Pkg::Config.staging_server, sync_command)
+      end
+    end
+
+    desc "Sync signed apt repos from #{Pkg::Config.apt_signing_server} to Google Cloud Platform"
+    task :sync_apt_repo_to_gcp => 'pl:fetch' do
+      target_site = 'apt.repos.puppetlabs.com'
+      sync_command_puppet_6 = "#{GCP_REPO_SYNC} apt.repos.puppet.com puppet6"
+      sync_command_puppet_7 = "#{GCP_REPO_SYNC} apt.repos.puppet.com puppet7"
+      print "Sync apt repos from #{Pkg::Config.apt_signing_server} to #{target_site}? [y,n] "
+      next unless Pkg::Util.ask_yes_or_no
+      puts
+
+      Pkg::Util::Execution.retry_on_fail(times: 3) do
+        Pkg::Util::Net.remote_execute(Pkg::Config.apt_signing_server, sync_command_puppet_6)
+        Pkg::Util::Net.remote_execute(Pkg::Config.apt_signing_server, sync_command_puppet_7)
       end
     end
+    # Keep 'deploy' for backward compatibility
+    task :deploy_apt_repo_to_gcp => :sync_apt_repo_to_gcp
 
     desc "Sync apt, yum, and downloads.pl.com to AWS S3"
     task :deploy_final_builds_to_s3 => "pl:fetch" do
@@ -227,15 +257,6 @@ namespace :pl do
       Rake::Task['pl:remote:deploy_downloads_to_s3'].invoke
     end
 
-    desc "Sync nightlies.puppetlabs.com from #{Pkg::Config.staging_server} to AWS S3"
-    task :deploy_nightlies_to_s3 => 'pl:fetch' do
-      puts "Deploying nightly builds from #{Pkg::Config.staging_server} to AWS S3..."
-      Pkg::Util::Execution.retry_on_fail(:times => 3) do
-        command = 'sudo /usr/local/bin/s3_repo_sync.sh nightlies.puppet.com'
-        Pkg::Util::Net.remote_execute(Pkg::Config.staging_server, command)
-      end
-    end
-
     desc "Sync yum and apt from #{Pkg::Config.staging_server} to rsync servers"
     task :deploy_to_rsync_server => 'pl:fetch' do
       # This task must run after the S3 sync has run, or else /opt/repo-s3-stage won't be up-to-date

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: packaging
 version: !ruby/object:Gem::Version
-  version: 0.99.80
+  version: 0.99.81
 platform: ruby
 authors:
 - Puppet Labs
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-08-17 00:00:00.000000000 Z
+date: 2021-09-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -260,28 +260,28 @@ signing_key:
 specification_version: 4
 summary: Puppet Labs' packaging automation
 test_files:
-- spec/lib/packaging_spec.rb
-- spec/lib/packaging/repo_spec.rb
-- spec/lib/packaging/tar_spec.rb
-- spec/lib/packaging/retrieve_spec.rb
+- spec/lib/packaging/config_spec.rb
+- spec/lib/packaging/sign_spec.rb
+- spec/lib/packaging/platforms_spec.rb
 - spec/lib/packaging/util/rake_utils_spec.rb
-- spec/lib/packaging/util/misc_spec.rb
-- spec/lib/packaging/util/ship_spec.rb
-- spec/lib/packaging/util/os_spec.rb
+- spec/lib/packaging/util/jenkins_spec.rb
+- spec/lib/packaging/util/execution_spec.rb
 - spec/lib/packaging/util/net_spec.rb
-- spec/lib/packaging/util/version_spec.rb
+- spec/lib/packaging/util/file_spec.rb
+- spec/lib/packaging/util/misc_spec.rb
 - spec/lib/packaging/util/gpg_spec.rb
+- spec/lib/packaging/util/version_spec.rb
+- spec/lib/packaging/util/os_spec.rb
 - spec/lib/packaging/util/git_spec.rb
 - spec/lib/packaging/util/git_tag_spec.rb
-- spec/lib/packaging/util/file_spec.rb
-- spec/lib/packaging/util/jenkins_spec.rb
-- spec/lib/packaging/util/execution_spec.rb
+- spec/lib/packaging/util/ship_spec.rb
+- spec/lib/packaging/tar_spec.rb
+- spec/lib/packaging/repo_spec.rb
 - spec/lib/packaging/deb_spec.rb
-- spec/lib/packaging/deb/repo_spec.rb
-- spec/lib/packaging/artifactory_spec.rb
-- spec/lib/packaging/sign_spec.rb
-- spec/lib/packaging/config_spec.rb
-- spec/lib/packaging/gem_spec.rb
-- spec/lib/packaging/platforms_spec.rb
-- spec/lib/packaging/rpm/repo_spec.rb
 - spec/lib/packaging/paths_spec.rb
+- spec/lib/packaging/rpm/repo_spec.rb
+- spec/lib/packaging/gem_spec.rb
+- spec/lib/packaging/retrieve_spec.rb
+- spec/lib/packaging/artifactory_spec.rb
+- spec/lib/packaging/deb/repo_spec.rb
+- spec/lib/packaging_spec.rb