packaging 0.99.30 → 0.99.31

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7cfddfe342677b102df76026925a4f2d6c63d3ab462f11c663f370a66a54bcc0
-  data.tar.gz: 7b4b4bf61cd61f5b131ece5020bf6cb99ef6551118ddaeec5884036718c41078
+  metadata.gz: 46032268fefeac3b8976ae2cdd44b27bab028dd4154d6dae1b99c62832510e8c
+  data.tar.gz: 56f74df3abe80d40e39caeb2db1c2a4ba025e22ba2e90876191f29cb82a43f33
 SHA512:
-  metadata.gz: 779471e6fc2b8b379ac5a1e30c8f0d4002406b2ef267c8ff3fdd7384a97d61e29a9616b77b03851fd65ece2c95007a079b933eeeb0465d89e82260ca4cfb959d
-  data.tar.gz: 83d313b8e9342138e946e0d652c3b88880a7f16c7d2eda18f54bfb3cd793102201cd787be0f39eb0981757dd79bd63707f41b6ce36733d831f80f511c526dc9d
+  metadata.gz: da7b1a51187cf880d42109e753c9166b5f5147f2f01faf59c880b334ad9abaaacf8ee663a92674b852d2e70133079e649955a925b188df88c256969f49aca972
+  data.tar.gz: 108cfe68681a7c93f1e1b53f6a2a1b620ee5ec5930598c1ab5a01d332ed5460fad9b79fdfb98b082686dbeea3b0442712835783bc6d82384398676084c041536

data/lib/packaging.rb

@@ -18,6 +18,7 @@ module Pkg
   require 'packaging/artifactory'
   require 'packaging/retrieve'
   require 'packaging/sign'
+  require 'packaging/archive'
 
   # Load configuration defaults
   Pkg::Config.load_defaults

data/lib/packaging/archive.rb

@@ -0,0 +1,144 @@
+module Pkg::Archive
+  module_function
+
+  # Array of base paths for foss artifacts on weth
+  def base_paths
+    [Pkg::Config.yum_repo_path, Pkg::Config.apt_repo_staging_path, Pkg::Config.apt_repo_path, '/opt/downloads'].compact.freeze
+  end
+
+  # Array of paths for temporarily staging artifacts before syncing to release-archives on s3
+  def archive_paths
+    [Pkg::Config.yum_archive_path, Pkg::Config.apt_archive_path, Pkg::Config.freight_archive_path, Pkg::Config.downloads_archive_path].compact.freeze
+  end
+
+  # Move yum directories from repo path to archive staging path
+  def stage_yum_archives(directory)
+    # /opt/repository/yum/#{directory}
+    full_directory = File.join(Pkg::Config.yum_repo_path, directory)
+    archive_path = File.join(Pkg::Config.yum_archive_path, directory)
+    command = <<-CMD
+      if [ ! -d #{full_directory} ]; then
+        if [ -d #{archive_path} ]; then
+          echo "Directory #{full_directory} has already been staged, skipping . . ."
+          exit 0
+        else
+          echo "ERROR: Couldn't find directory #{full_directory}, exiting . . ."
+          exit 1
+        fi
+      fi
+      find #{full_directory} -type l -delete
+      sudo chattr -i -R #{full_directory}
+      sudo mkdir --parents #{File.dirname(archive_path)}
+      sudo chown root:release -R #{Pkg::Config.yum_archive_path}
+      sudo chmod g+w -R #{Pkg::Config.yum_archive_path}
+      mv #{full_directory} #{archive_path}
+    CMD
+    Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.staging_server, command)
+  end
+
+  # Move apt directories from repo path (and repo staging path aka freight path) to archive staging paths
+  def stage_apt_archives(directory)
+    # /opt/tools/freight/apt/$codename/#{directory}
+    # /opt/repository/apt/pool/$codename/#{directory}
+    pool_directory = File.join(Pkg::Config.apt_repo_path, 'pool')
+    command = <<-CMD
+      for full_directory in $(find #{pool_directory} -type d -name #{directory}); do
+        find $full_directory -type l -delete
+        sudo chattr -i -R $full_directory
+        subdirectory=${full_directory##{Pkg::Config.apt_repo_path}/}
+        codename=$(echo $subdirectory | cut -d'/' -f 2)
+        sudo mkdir --parents #{Pkg::Config.apt_archive_path}/$subdirectory
+        sudo chown root:release -R #{Pkg::Config.apt_archive_path}/$subdirectory
+        sudo chmod g+w -R #{Pkg::Config.apt_archive_path}/$subdirectory
+        mv $full_directory #{Pkg::Config.apt_archive_path}/$(dirname $subdirectory)
+        sudo mkdir --parents #{Pkg::Config.freight_archive_path}/$codename
+        sudo chown root:release -R #{Pkg::Config.freight_archive_path}/$codename
+        sudo chmod g+w -R #{Pkg::Config.freight_archive_path}/$codename
+        if [ #{directory} = 'main' ]; then
+          freight_directory=#{Pkg::Config.apt_repo_staging_path}/$codename
+          if [ ! -d $freight_directory ]; then
+            echo "ERROR: Couldn't find freight directory $freight_directory, exiting . . ."
+            exit 1
+          fi
+          for file in $(find $freight_directory -maxdepth 1 -type f); do
+            mv $file #{Pkg::Config.freight_archive_path}/$codename
+          done
+        else
+          freight_directory=#{Pkg::Config.apt_repo_staging_path}/$codename/#{directory}
+          if [ ! -d $freight_directory ]; then
+            if [ -d #{Pkg::Config.freight_archive_path}/$codename/#{directory} ]; then
+              echo "Directory $freight_directory has already been staged, skipping . . ."
+              exit 0
+            else
+              echo "ERROR: Couldn't find freight directory $freight_directory, exiting . . ."
+              exit 1
+            fi
+          fi
+          mv $freight_directory #{Pkg::Config.freight_archive_path}/$codename/#{directory}
+        fi
+      done
+    CMD
+    Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.staging_server, command)
+  end
+
+  # Move downloads directories to archive staging path
+  def stage_downloads_archives(directory)
+    # /opt/downloads/#{directory}
+    full_directory = File.join('/', 'opt', 'downloads', directory)
+    archive_path = File.join(Pkg::Config.downloads_archive_path, directory)
+    command = <<-CMD
+      if [ ! -d #{full_directory} ]; then
+        if [ -d #{archive_path} ]; then
+          echo "Directory #{full_directory} has already been staged, skipping . . ."
+          exit 0
+        else
+          echo "ERROR: Couldn't find directory #{full_directory}, exiting . . ."
+          exit 1
+        fi
+      fi
+      find #{full_directory} -type l -delete
+      sudo chattr -i -R #{full_directory}
+      sudo mkdir --parents #{File.dirname(archive_path)}
+      sudo chown root:release -R #{Pkg::Config.downloads_archive_path}
+      sudo chmod g+w -R #{Pkg::Config.downloads_archive_path}
+      mv #{full_directory} #{archive_path}
+    CMD
+    Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.staging_server, command)
+  end
+
+  # Delete empty directories from repo paths on weth
+  def remove_empty_directories
+    base_paths.each do |path|
+      command = <<-CMD
+        for directory in $(find #{path} -type d); do
+          if [ ! -d $directory ]; then
+            echo "Can't find directory $directory, it was probably already deleted, skipping . . ."
+            continue
+          fi
+          files=$(find $directory -type f)
+          if [ -z "$files" ]; then
+            echo "No files in directory $directory, deleting . . ."
+            sudo rm -rf $directory
+          fi
+        done
+      CMD
+      Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.staging_server, command)
+    end
+  end
+
+  # Delete broken symlinks from repo paths on weth
+  def remove_dead_symlinks
+    base_paths.each do |path|
+      command = "find #{path} -xtype l -delete"
+      Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.staging_server, command)
+    end
+  end
+
+  # Delete artifacts from archive staging paths (after they've been synced to s3)
+  def delete_staged_archives
+    archive_paths.each do |archive_path|
+      command = "sudo rm -rf #{File.join(archive_path, '*')}"
+      Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.staging_server, command)
+    end
+  end
+end

data/lib/packaging/config/params.rb

@@ -5,6 +5,7 @@ require "packaging/platforms"
 module Pkg::Params
   BUILD_PARAMS = [:allow_dirty_tree,
                   :answer_override,
+                  :apt_archive_path,
                   :apt_host,
                   :apt_releases,
                   :apt_repo_command,
@@ -45,11 +46,13 @@ module Pkg::Params
                   :dmg_host,
                   :dmg_path,
                   :dmg_staging_server,
+                  :downloads_archive_path,
                   :email,
                   :files,
                   :final_mocks,
                   :foss_only,
                   :foss_platforms,
+                  :freight_archive_path,
                   :freight_conf,
                   :gem_default_executables,
                   :gem_dependencies,
@@ -179,6 +182,7 @@ module Pkg::Params
                   :version,
                   :version_file,
                   :version_strategy,
+                  :yum_archive_path,
                   :yum_host,
                   :yum_repo_command,
                   :yum_repo_name,
@@ -195,6 +199,7 @@ module Pkg::Params
   ENV_VARS = [
               { :var => :allow_dirty_tree,        :envvar => :ALLOW_DIRTY_TREE, :type => :bool },
               { :var => :answer_override,         :envvar => :ANSWER_OVERRIDE },
+              { :var => :apt_archive_path,        :envvar => :APT_ARCHIVE_PATH },
               { :var => :apt_host,                :envvar => :APT_HOST },
               { :var => :apt_releases,            :envvar => :APT_RELEASES,    :type => :array },
               { :var => :apt_repo_path,           :envvar => :APT_REPO },
@@ -218,9 +223,11 @@ module Pkg::Params
               { :var => :dmg_host,                :envvar => :DMG_HOST },
               { :var => :dmg_path,                :envvar => :DMG_PATH },
               { :var => :dmg_staging_server,      :envvar => :DMG_STAGING_SERVER },
+              { :var => :downloads_archive_path,  :envvar => :DOWNLOADS_ARCHIVE_PATH },
               { :var => :final_mocks,             :envvar => :MOCK },
               { :var => :foss_only,               :envvar => :FOSS_ONLY,       :type => :bool },
               { :var => :foss_platforms,          :envvar => :FOSS_PLATFORMS,  :type => :array },
+              { :var => :freight_archive_path,    :envvar => :FREIGHT_ARCHIVE_PATH },
               { :var => :gem_host,                :envvar => :GEM_HOST },
               { :var => :gpg_key,                 :envvar => :GPG_KEY },
               { :var => :gpg_name,                :envvar => :GPG_NAME },
@@ -289,6 +296,7 @@ module Pkg::Params
               { :var => :team,                    :envvar => :TEAM },
               { :var => :update_version_file,     :envvar => :NEW_STYLE_PACKAGE },
               { :var => :vanagon_project,         :envvar => :VANAGON_PROJECT, :type => :bool },
+              { :var => :yum_archive_path,        :envvar => :YUM_ARCHIVE_PATH },
               { :var => :yum_host,                :envvar => :YUM_HOST },
               { :var => :yum_repo_path,           :envvar => :YUM_REPO },
               { :var => :yum_staging_server,      :envvar => :YUM_STAGING_SERVER },

data/lib/packaging/util/rake_utils.rb

@@ -67,6 +67,7 @@ module Pkg::Util::RakeUtils
         '00_utils.rake',
         '30_metrics.rake',
         'apple.rake',
+        'archive.rake',
         'build.rake',
         'clean.rake',
         'deb.rake',

data/spec/fixtures/config/ext/build_defaults.yaml

@@ -1 +1 @@
-spec/fixtures/config/ext/../params.yaml
+../params.yaml

data/spec/fixtures/config/ext/project_data.yaml

@@ -1 +1 @@
-spec/fixtures/config/ext/../params.yaml
+../params.yaml

data/spec/lib/packaging/config_spec.rb

@@ -4,7 +4,8 @@ require 'yaml'
 
 describe "Pkg::Config" do
 
-  Build_Params = [:apt_host,
+  Build_Params = [:apt_archive_path,
+                  :apt_host,
                   :apt_releases,
                   :apt_repo_path,
                   :apt_repo_url,
@@ -34,9 +35,11 @@ describe "Pkg::Config" do
                   :default_mock,
                   :description,
                   :dmg_path,
+                  :downloads_archive_path,
                   :email,
                   :files,
                   :final_mocks,
+                  :freight_archive_path,
                   :freight_conf,
                   :gem_default_executables,
                   :gem_dependencies,
@@ -128,6 +131,7 @@ describe "Pkg::Config" do
                   :version,
                   :version_file,
                   :version_strategy,
+                  :yum_archive_path,
                   :yum_host,
                   :yum_repo_path,
                   :yum_repo_name,

data/tasks/archive.rake

@@ -0,0 +1,37 @@
+namespace :pl do
+  namespace :remote do
+
+    desc "Move packages from repo paths to archive staging paths"
+    task :stage_archives, [:yum_directories, :apt_directories, :downloads_directories] => 'pl:fetch' do |_t, args|
+      yum_directories = args.yum_directories ? args.yum_directories.split(' ') : []
+      apt_directories = args.apt_directories ? args.apt_directories.split(' ') : []
+      downloads_directories = args.downloads_directories ? args.downloads_directories.split(' ') : []
+
+      yum_directories.each do |directory|
+        Pkg::Archive.stage_yum_archives(directory)
+      end
+      apt_directories.each do |directory|
+        Pkg::Archive.stage_apt_archives(directory)
+      end
+      downloads_directories.each do |directory|
+        Pkg::Archive.stage_downloads_archives(directory)
+      end
+    end
+
+    desc "Sync archived packages to s3"
+    task :deploy_staged_archives_to_s3 => 'pl:fetch' do
+      command = 'sudo /usr/local/bin/s3_repo_sync.sh release-archives.puppet.com'
+      Pkg::Util::Execution.retry_on_fail(:times => 3) do
+        Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.staging_server, command)
+      end
+    end
+
+    desc "Delete packages in archive staging directory"
+    task :archive_cleanup => 'pl:fetch' do
+      Pkg::Archive.remove_empty_directories
+      Pkg::Archive.remove_dead_symlinks
+      Pkg::Archive.delete_staged_archives
+    end
+  end
+end
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: packaging
 version: !ruby/object:Gem::Version
-  version: 0.99.30
+  version: 0.99.31
 platform: ruby
 authors:
 - Puppet Labs
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-04-25 00:00:00.000000000 Z
+date: 2019-05-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -91,6 +91,7 @@ files:
 - README-Solaris.md
 - README.md
 - lib/packaging.rb
+- lib/packaging/archive.rb
 - lib/packaging/artifactory.rb
 - lib/packaging/config.rb
 - lib/packaging/config/params.rb
@@ -167,6 +168,7 @@ files:
 - tasks/00_utils.rake
 - tasks/30_metrics.rake
 - tasks/apple.rake
+- tasks/archive.rake
 - tasks/build.rake
 - tasks/clean.rake
 - tasks/config.rake
@@ -223,8 +225,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Puppet Labs' packaging automation