packaging 0.99.28 → 0.99.29

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f6f5b755d756b480d14f212a1d984687482ee67f10cd7e4e4c2714b586a1392b
-  data.tar.gz: 82cd5cb8e8db8c430a1b9784759cad218bd49a425d6f04eb4ad564085a52c43a
+  metadata.gz: 9f423cc0b3db951ead3f977e29ce01bb1ef130b86ce40de7b21c45f8c7cb7952
+  data.tar.gz: e15f8d7c8bb1f03dea062857bc5e8669e175b11d11855affa24e653fa45b87f1
 SHA512:
-  metadata.gz: 3701863f1127c41ded7f2a9542867288f6562d224335bf9ad713a931f350f541595e3fb22e96a0ce78c61bd18c5619d0b70531fd13eb4796b49a635378872ffb
-  data.tar.gz: 9bc45ceff5bd337bee49c138347499d72599672f7076edc0c8aa993544fa0ece4dba7c7d9655e34dc2e73e8448e6b756831b69d81675811dd9f7b3014d508ded
+  metadata.gz: 40aa070d4b58734c6b16be7ceac959b860560016f407dfb124349cb51e5afd6d6cbe1f506c3aaecbce345e029ca861527d2d419167235114083b9555ec466f0c
+  data.tar.gz: aa7e8edc975af46c0b08370e94e2335381bdba55cc281f908e207b6b80db1c60a79f6319c88782500b8a2e5b24865a6aa90362be4e9a3fb36f4167053cf15382

data/lib/packaging/paths.rb

@@ -266,17 +266,35 @@ module Pkg::Paths
     end
   end
 
+  def remote_repo_base(platform_tag, nonfinal = false)
+    package_format = Pkg::Platforms.package_format_for_tag(platform_tag)
+    case package_format
+    when 'rpm'
+      nonfinal ? Pkg::Config.nonfinal_yum_repo_path : Pkg::Config.yum_repo_path
+    when 'deb'
+      nonfinal ? Pkg::Config.nonfinal_apt_repo_path : Pkg::Config.apt_repo_path
+    else
+      raise "Can't determine remote repo base path for package format '#{package_format}'."
+    end
+  end
+
+  # This is where deb packages end up after freight repo updates
+  def apt_package_base_path(platform_tag, repo_name, project, nonfinal = false)
+    fail "Can't determine path for non-debian platform #{platform_tag}." unless Pkg::Platforms.package_format_for_tag(platform_tag) == 'deb'
+    platform, version, _ = Pkg::Platforms.parse_platform_tag(platform_tag)
+    codename = Pkg::Platforms.codename_for_platform_version(platform, version)
+    return File.join(remote_repo_base(platform_tag, nonfinal), 'pool', codename, repo_name, project[0], project)
+  end
+
   def release_package_link_path(platform_tag, nonfinal = false)
     platform, version, arch = Pkg::Platforms.parse_platform_tag(platform_tag)
     package_format = Pkg::Platforms.package_format_for_tag(platform_tag)
     case package_format
     when 'rpm'
-      base_path = nonfinal ? Pkg::Config.nonfinal_yum_repo_path : Pkg::Config.yum_repo_path
-      return File.join(base_path, "#{repo_name(nonfinal)}-release-#{platform}-#{version}.noarch.rpm")
+      return File.join(remote_repo_base(platform_tag, nonfinal), "#{repo_name(nonfinal)}-release-#{platform}-#{version}.noarch.rpm")
     when 'deb'
-      base_path = nonfinal ? Pkg::Config.nonfinal_apt_repo_path : Pkg::Config.apt_repo_path
       codename = Pkg::Platforms.codename_for_platform_version(platform, version)
-      return File.join(base_path, "#{repo_name(nonfinal)}-release-#{codename}.deb")
+      return File.join(remote_repo_base(platform_tag, nonfinal), "#{repo_name(nonfinal)}-release-#{codename}.deb")
     else
       warn "No release packages for package format '#{package_format}', skipping . . ."
       return nil

data/lib/packaging/sign/msi.rb

@@ -66,23 +66,42 @@ module Pkg::Sign::Msi
       if "/cygdrive/c/tools/osslsigncode-fork/osslsigncode.exe" verify -in "C:/#{work_dir}/$msi" ; then
         echo "$msi is already signed, skipping . . ." ;
       else
-        "/cygdrive/c/tools/osslsigncode-fork/osslsigncode.exe" sign \
-          -n "Puppet" -i "http://www.puppet.com" \
-          -h sha1 \
-          -pkcs12 "#{Pkg::Config.msi_signing_cert}" \
-          -pass "#{Pkg::Config.msi_signing_cert_pw}" \
-          -t "http://timestamp.verisign.com/scripts/timstamp.dll" \
-          -in "C:/#{work_dir}/$msi" \
-          -out "C:/#{work_dir}/signed-$msi"
-        "/cygdrive/c/tools/osslsigncode-fork/osslsigncode.exe" sign \
-          -n "Puppet" -i "http://www.puppet.com" \
-          -nest -h sha256 \
-          -pkcs12 "#{Pkg::Config.msi_signing_cert}" \
-          -pass "#{Pkg::Config.msi_signing_cert_pw}" \
-          -ts "http://sha256timestamp.ws.symantec.com/sha256/timestamp" \
-          -in "C:/#{work_dir}/signed-$msi" \
-          -out "C:/#{work_dir}/$msi"
-        rm "C:/#{work_dir}/signed-$msi" ;
+          tries=5
+          sha1Servers=(http://timestamp.verisign.com/scripts/timstamp.dll
+            http://timestamp.globalsign.com/scripts/timstamp.dll
+            http://www.startssl.com/timestamp)
+              for timeserver in "${sha1Servers[@]}"; do
+                for ((try=1; try<=$tries; try++)) do
+                  ret=$(/cygdrive/c/tools/osslsigncode-fork/osslsigncode.exe sign \
+                            -n "Puppet" -i "http://www.puppet.com" \
+                            -h sha1 \
+                            -pkcs12 "#{Pkg::Config.msi_signing_cert}" \
+                            -pass "#{Pkg::Config.msi_signing_cert_pw}" \
+                            -t "$timeserver" \
+                            -in "C:/#{work_dir}/$msi" \
+                            -out "C:/#{work_dir}/signed-$msi")
+                    if [[ $ret == *"Succeeded"* ]]; then break; fi
+                  done;
+              if [[ $ret == *"Succeeded"* ]]; then break; fi
+            done;
+            echo $ret
+            sha256Servers=(http://sha256timestamp.ws.symantec.com/sha256/timestamp
+              http://timestamp.comodoca.com?td=sha256)
+                for timeserver in "${sha256Servers[@]}"; do
+                  for ((try=1; try<=$tries; try++)) do
+                    ret=$(/cygdrive/c/tools/osslsigncode-fork/osslsigncode.exe sign \
+                      -n "Puppet" -i "http://www.puppet.com" \
+                      -nest -h sha256 \
+                      -pkcs12 "#{Pkg::Config.msi_signing_cert}" \
+                      -pass "#{Pkg::Config.msi_signing_cert_pw}" \
+                      -ts "$timeserver" \
+                      -in "C:/#{work_dir}/signed-$msi" \
+                      -out "C:/#{work_dir}/$msi")
+                      if [[ $ret == *"Succeeded"* ]]; then break; fi
+                    done;
+                if [[ $ret == *"Succeeded"* ]]; then break; fi
+              done;
+              echo $ret
       fi
     done))
     msis.each do | msi |

data/lib/packaging/util/ship.rb

@@ -208,6 +208,40 @@ module Pkg::Util::Ship
     fail "Failed to create rolling repo link for '#{platform_tag}'.\n#{e}"
   end
 
+  def update_release_package_symlinks(local_staging_directory, nonfinal = false)
+    local_packages = collect_packages(["#{local_staging_directory}/**/*.rpm", "#{local_staging_directory}/**/*.deb"])
+    local_packages.each do |package|
+      platform_tag = Pkg::Paths.tag_from_artifact_path(package)
+      package_format = Pkg::Platforms.package_format_for_tag(platform_tag)
+      case package_format
+      when 'rpm'
+        remote_base = Pkg::Paths.artifacts_path(platform_tag, Pkg::Paths.remote_repo_base(platform_tag, nonfinal), nonfinal)
+      when 'deb'
+        remote_base = Pkg::Paths.apt_package_base_path(platform_tag, Pkg::Paths.repo_name(nonfinal), Pkg::Config.project, nonfinal)
+      else
+        fail "Unexpected package format #{package_format}, cannot create symlinks."
+      end
+      remote_path = File.join(remote_base, File.basename(package))
+      link_path = Pkg::Paths.release_package_link_path(platform_tag, nonfinal)
+      link_command = <<-CMD
+        if [ ! -e #{remote_path} ]; then
+          echo "Uh oh! #{remote_path} doesn't exist! Can't create symlink."
+          exit 1
+        fi
+        if [ -e #{link_path} ] && [ ! -L #{link_path} ]; then
+          echo "Uh oh! #{link_path} exists but isn't a link, I don't know what to do with this."
+          exit 1
+        fi
+        if [ -L #{link_path} ] && [ ! #{remote_path} -ef #{link_path} ]; then
+          echo "Removing old link from $(readlink #{link_path}) to #{link_path} . . ."
+          rm #{link_path}
+        fi
+        ln -sf #{remote_path} #{link_path}
+      CMD
+      Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.staging_server, link_command)
+    end
+  end
+
   def test_ship(vm, ship_task)
     command = 'getent group release || groupadd release'
     Pkg::Util::Net.remote_ssh_cmd(vm, command)

data/spec/fixtures/config/ext/build_defaults.yaml

@@ -1 +1 @@
-spec/fixtures/config/ext/../params.yaml
+../params.yaml

data/spec/fixtures/config/ext/project_data.yaml

@@ -1 +1 @@
-spec/fixtures/config/ext/../params.yaml
+../params.yaml

data/spec/lib/packaging/paths_spec.rb

@@ -236,6 +236,46 @@ describe 'Pkg::Paths' do
     end
   end
 
+  describe '#remote_repo_base' do
+    before :each do
+      allow(Pkg::Config).to receive(:yum_repo_path).and_return('foo')
+      allow(Pkg::Config).to receive(:apt_repo_path).and_return('bar')
+      allow(Pkg::Config).to receive(:nonfinal_yum_repo_path).and_return('foo-nightly')
+      allow(Pkg::Config).to receive(:nonfinal_apt_repo_path).and_return('bar-nightly')
+    end
+    it 'returns yum_repo_path for rpms' do
+      expect(Pkg::Paths.remote_repo_base('el-7-x86_64')).to eq('foo')
+    end
+    it 'returns apt_repo_path for debs' do
+      expect(Pkg::Paths.remote_repo_base('ubuntu-18.04-amd64')).to eq('bar')
+    end
+    it 'returns nonfinal_yum_repo_path for nonfinal rpms' do
+      expect(Pkg::Paths.remote_repo_base('fedora-29-x86_64', true)).to eq('foo-nightly')
+    end
+    it 'returns nonfinal_apt_repo_path for nonfinal debs' do
+      expect(Pkg::Paths.remote_repo_base('debian-9-amd64', true)).to eq('bar-nightly')
+    end
+    it 'fails for all other package formats' do
+      expect { Pkg::Paths.remote_repo_base('osx-10.14-x86_64') }.to raise_error(/Can't determine remote repo base path/)
+    end
+  end
+
+  describe '#apt_package_base_path' do
+    it 'fails for non-debian platforms' do
+      expect { Pkg::Paths.apt_package_base_path('el-7-x86_64', 'puppet6', 'puppet-agent') }.to raise_error(/Can't determine path for non-debian platform/)
+    end
+    it 'returns the approprate apt repo path' do
+      allow(Pkg::Paths).to receive(:remote_repo_base).and_return('/opt/repository/apt')
+      expect(Pkg::Paths.apt_package_base_path('ubuntu-18.04-amd64', 'puppet6', 'puppet-agent')).to eq('/opt/repository/apt/pool/bionic/puppet6/p/puppet-agent')
+      expect(Pkg::Paths.apt_package_base_path('debian-9-amd64', 'puppet6', 'razor-server')).to eq('/opt/repository/apt/pool/stretch/puppet6/r/razor-server')
+    end
+    it 'returns the appropriate nonfinal repo path' do
+      allow(Pkg::Paths).to receive(:remote_repo_base).and_return('/opt/repository-nightlies/apt')
+      expect(Pkg::Paths.apt_package_base_path('ubuntu-18.04-amd64', 'puppet6-nightly', 'puppet-agent', true)).to eq('/opt/repository-nightlies/apt/pool/bionic/puppet6-nightly/p/puppet-agent')
+      expect(Pkg::Paths.apt_package_base_path('debian-9-amd64', 'puppet6-nightly', 'razor-server', true)).to eq('/opt/repository-nightlies/apt/pool/stretch/puppet6-nightly/r/razor-server')
+    end
+  end
+
   describe '#release_package_link_path' do
     repo_name = 'puppet6'
     nonfinal_repo_name = 'puppet6-nightly'

data/tasks/jenkins.rake

@@ -298,6 +298,24 @@ namespace :pl do
       Rake::Task['pl:remote:deploy_to_rsync_server'].invoke
     end
 
+    task :stage_release_packages => "pl:fetch" do
+      Rake::Task['pl:jenkins:uber_ship_lite'].invoke
+      # Deb packages only appear in the freight directory until repo updates.
+      # We must run that before creating symlinks so we can link from packages
+      # in the apt repository.
+      Rake::Task['pl:remote:update_apt_repo'].invoke
+      Pkg::Util::Ship.update_release_package_symlinks('pkg')
+    end
+
+    task :stage_nightly_release_packages => "pl:fetch" do
+      Rake::Task['pl:jenkins:stage_nightlies'].invoke
+      # Deb packages only appear in the freight directory until repo updates.
+      # We must run that before creating symlinks so we can link from packages
+      # in the apt repository.
+      Rake::Task['pl:remote:update_nightlies_apt_repo'].invoke
+      Pkg::Util::Ship.update_release_package_symlinks('pkg', true)
+    end
+
     desc "Retrieve packages built by jenkins, sign, and ship all!"
     task :uber_ship => "pl:fetch" do
       uber_tasks = %w(

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: packaging
 version: !ruby/object:Gem::Version
-  version: 0.99.28
+  version: 0.99.29
 platform: ruby
 authors:
 - Puppet Labs
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-04-11 00:00:00.000000000 Z
+date: 2019-04-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -223,8 +223,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Puppet Labs' packaging automation