packaging 0.104.0 → 0.106.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (74) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +4 -4
  3. data/lib/packaging/archive.rb +2 -2
  4. data/lib/packaging/artifactory/extensions.rb +1 -0
  5. data/lib/packaging/artifactory.rb +27 -23
  6. data/lib/packaging/config/params.rb +191 -193
  7. data/lib/packaging/config/validations.rb +0 -2
  8. data/lib/packaging/config.rb +8 -8
  9. data/lib/packaging/deb/repo.rb +11 -14
  10. data/lib/packaging/gem.rb +2 -2
  11. data/lib/packaging/metrics.rb +7 -7
  12. data/lib/packaging/nuget.rb +0 -1
  13. data/lib/packaging/paths.rb +11 -13
  14. data/lib/packaging/platforms.rb +14 -52
  15. data/lib/packaging/repo.rb +11 -12
  16. data/lib/packaging/retrieve.rb +1 -1
  17. data/lib/packaging/rpm/repo.rb +8 -8
  18. data/lib/packaging/sign/dmg.rb +8 -7
  19. data/lib/packaging/sign/ips.rb +64 -32
  20. data/lib/packaging/sign/msi.rb +48 -48
  21. data/lib/packaging/sign/rpm.rb +1 -1
  22. data/lib/packaging/sign.rb +0 -1
  23. data/lib/packaging/tar.rb +2 -4
  24. data/lib/packaging/util/date.rb +0 -1
  25. data/lib/packaging/util/distribution_server.rb +2 -2
  26. data/lib/packaging/util/execution.rb +2 -4
  27. data/lib/packaging/util/file.rb +81 -3
  28. data/lib/packaging/util/git.rb +1 -3
  29. data/lib/packaging/util/git_tags.rb +3 -3
  30. data/lib/packaging/util/gpg.rb +3 -4
  31. data/lib/packaging/util/jenkins.rb +0 -3
  32. data/lib/packaging/util/misc.rb +1 -1
  33. data/lib/packaging/util/net.rb +25 -23
  34. data/lib/packaging/util/repo.rb +17 -0
  35. data/lib/packaging/util/serialization.rb +1 -2
  36. data/lib/packaging/util/ship.rb +150 -3
  37. data/lib/packaging/util/sign.rb +47 -0
  38. data/lib/packaging/util/tool.rb +1 -4
  39. data/lib/packaging/util/version.rb +1 -5
  40. data/lib/packaging/util.rb +3 -1
  41. data/lib/packaging.rb +1 -2
  42. data/spec/lib/packaging/config_spec.rb +3 -3
  43. data/spec/lib/packaging/deb/repo_spec.rb +1 -1
  44. data/spec/lib/packaging/paths_spec.rb +2 -3
  45. data/spec/lib/packaging/platforms_spec.rb +6 -6
  46. data/spec/lib/packaging/sign_spec.rb +1 -13
  47. data/spec/lib/packaging/util/git_spec.rb +2 -2
  48. data/spec/lib/packaging/util/git_tag_spec.rb +5 -5
  49. data/spec/lib/packaging/util/ship_spec.rb +0 -2
  50. data/tasks/30_metrics.rake +2 -2
  51. data/tasks/apple.rake +8 -14
  52. data/tasks/archive.rake +1 -2
  53. data/tasks/deb.rake +7 -8
  54. data/tasks/deb_repos.rake +1 -0
  55. data/tasks/doc.rake +5 -3
  56. data/tasks/education.rake +2 -4
  57. data/tasks/fetch.rake +1 -0
  58. data/tasks/gem.rake +20 -12
  59. data/tasks/jenkins.rake +27 -15
  60. data/tasks/jenkins_dynamic.rake +10 -10
  61. data/tasks/load_extras.rake +1 -0
  62. data/tasks/mock.rake +8 -9
  63. data/tasks/nightly_repos.rake +14 -14
  64. data/tasks/pe_ship.rake +10 -17
  65. data/tasks/retrieve.rake +2 -2
  66. data/tasks/rpm.rake +1 -1
  67. data/tasks/rpm_repos.rake +1 -0
  68. data/tasks/ship.rake +24 -12
  69. data/tasks/sign.rake +6 -6
  70. data/tasks/tar.rake +2 -3
  71. data/tasks/update.rake +2 -2
  72. data/tasks/vendor_gems.rake +5 -7
  73. data/tasks/version.rake +2 -2
  74. metadata +42 -40
@@ -1,15 +1,13 @@
1
1
  # Utility methods for handling network calls and interactions
2
2
 
3
3
  module Pkg::Util::Net
4
-
5
4
  class << self
6
-
7
5
  # This simple method does an HTTP get of a URI and writes it to a file
8
6
  # in a slightly more platform agnostic way than curl/wget
9
7
  def fetch_uri(uri, target)
10
8
  require 'open-uri'
11
9
  if Pkg::Util::File.file_writable?(File.dirname(target))
12
- File.open(target, 'w') { |f| f.puts(open(uri).read) }
10
+ File.open(target, 'w') { |f| f.puts(URI.open(uri).read) }
13
11
  end
14
12
  end
15
13
 
@@ -37,7 +35,7 @@ module Pkg::Util::Net
37
35
  Array(hosts).flatten.each do |host|
38
36
  begin
39
37
  remote_execute(host, 'exit', { extra_options: '-oBatchMode=yes' })
40
- rescue
38
+ rescue StandardError
41
39
  errs << host
42
40
  end
43
41
  end
@@ -56,7 +54,7 @@ module Pkg::Util::Net
56
54
  begin
57
55
  remote_execute(host, "gpg --list-secret-keys #{gpg} > /dev/null 2&>1",
58
56
  { extra_options: '-oBatchMode=yes' })
59
- rescue
57
+ rescue StandardError
60
58
  errs << host
61
59
  end
62
60
  end
@@ -112,13 +110,14 @@ module Pkg::Util::Net
112
110
  ###
113
111
  ### Deprecated method implemented as a shim to the new `remote_execute` method
114
112
  ###
115
- def remote_ssh_cmd(target, command, capture_output = false, extra_options = '', fail_fast = true, trace = false) # rubocop:disable Style/ParameterLists
113
+ def remote_ssh_cmd(target, command, capture_output = false, extra_options = '', fail_fast = true, trace = false) # rubocop:disable Metrics/ParameterLists
116
114
  puts "Warn: \"remote_ssh_cmd\" call in packaging is deprecated. Use \"remote_execute\" instead."
117
115
  remote_execute(target, command, {
118
116
  capture_output: capture_output,
119
117
  extra_options: extra_options,
120
118
  fail_fast: fail_fast,
121
- trace: trace })
119
+ trace: trace
120
+ })
122
121
  end
123
122
 
124
123
  # Construct a valid rsync command
@@ -149,7 +148,8 @@ module Pkg::Util::Net
149
148
  target_path: nil,
150
149
  target_host: nil,
151
150
  extra_flags: nil,
152
- dryrun: false }.merge(opts)
151
+ dryrun: false
152
+ }.merge(opts)
153
153
  origin = Pathname.new(origin_path)
154
154
  target = options[:target_path] || origin.parent
155
155
 
@@ -187,9 +187,10 @@ module Pkg::Util::Net
187
187
  target_path: nil,
188
188
  target_host: nil,
189
189
  extra_flags: nil,
190
- dryrun: ENV['DRYRUN'] }.merge(opts.delete_if { |_, value| value.nil? })
190
+ dryrun: ENV['DRYRUN']
191
+ }.merge(opts.delete_if { |_, value| value.nil? })
191
192
 
192
- stdout, _, _ = Pkg::Util::Execution.capture3(rsync_cmd(source, options), true)
193
+ stdout, = Pkg::Util::Execution.capture3(rsync_cmd(source, options), true)
193
194
  stdout
194
195
  end
195
196
 
@@ -223,7 +224,7 @@ module Pkg::Util::Net
223
224
  s3cmd = Pkg::Util::Tool.check_tool('s3cmd')
224
225
 
225
226
  if Pkg::Util::File.file_exists?(File.join(ENV['HOME'], '.s3cfg'))
226
- stdout, _, _ = Pkg::Util::Execution.capture3("#{s3cmd} sync #{flags.join(' ')} '#{source}' s3://#{target_bucket}/#{target_directory}/")
227
+ stdout, = Pkg::Util::Execution.capture3("#{s3cmd} sync #{flags.join(' ')} '#{source}' s3://#{target_bucket}/#{target_directory}/")
227
228
  stdout
228
229
  else
229
230
  fail "#{File.join(ENV['HOME'], '.s3cfg')} does not exist. It is required to ship files using s3cmd."
@@ -279,7 +280,7 @@ module Pkg::Util::Net
279
280
  '--write-out "%{http_code}"',
280
281
  '--output /dev/null'
281
282
  ]
282
- stdout, _ = Pkg::Util::Net.curl_form_data(uri, data)
283
+ stdout, = Pkg::Util::Net.curl_form_data(uri, data)
283
284
  stdout
284
285
  end
285
286
 
@@ -292,18 +293,18 @@ module Pkg::Util::Net
292
293
  end
293
294
 
294
295
  def remote_set_ownership(host, owner, group, files)
295
- remote_cmd = "for file in #{files.join(" ")}; do if [[ -d $file ]] || ! `lsattr $file | grep -q '\\-i\\-'`; then sudo chown #{owner}:#{group} $file; else echo \"$file is immutable\"; fi; done"
296
+ remote_cmd = "for file in #{files.join(' ')}; do if [[ -d $file ]] || ! `lsattr $file | grep -q '\\-i\\-'`; then sudo chown #{owner}:#{group} $file; else echo \"$file is immutable\"; fi; done"
296
297
  Pkg::Util::Net.remote_execute(host, remote_cmd)
297
298
  end
298
299
 
299
300
  def remote_set_permissions(host, permissions, files)
300
- remote_cmd = "for file in #{files.join(" ")}; do if [[ -d $file ]] || ! `lsattr $file | grep -q '\\-i\\-'`; then sudo chmod #{permissions} $file; else echo \"$file is immutable\"; fi; done"
301
+ remote_cmd = "for file in #{files.join(' ')}; do if [[ -d $file ]] || ! `lsattr $file | grep -q '\\-i\\-'`; then sudo chmod #{permissions} $file; else echo \"$file is immutable\"; fi; done"
301
302
  Pkg::Util::Net.remote_execute(host, remote_cmd)
302
303
  end
303
304
 
304
305
  # Remotely set the immutable bit on a list of files
305
306
  def remote_set_immutable(host, files)
306
- Pkg::Util::Net.remote_execute(host, "sudo chattr +i #{files.join(" ")}")
307
+ Pkg::Util::Net.remote_execute(host, "sudo chattr +i #{files.join(' ')}")
307
308
  end
308
309
 
309
310
  # Create a symlink indicating the latest version of a package
@@ -350,8 +351,9 @@ module Pkg::Util::Net
350
351
  CMD
351
352
 
352
353
  _, err = Pkg::Util::Net.remote_execute(
353
- Pkg::Config.staging_server, cmd, { capture_output: true })
354
- $stderr.puts err
354
+ Pkg::Config.staging_server, cmd, { capture_output: true }
355
+ )
356
+ warn err
355
357
  end
356
358
 
357
359
  def escape_html(uri)
@@ -383,18 +385,18 @@ module Pkg::Util::Net
383
385
  Pkg::Util::Net.rsync_to(tarball, host, '/tmp')
384
386
  appendix = Pkg::Util.rand_string
385
387
  git_bundle_directory = File.join('/tmp', "#{Pkg::Config.project}-#{appendix}")
386
- command = <<-DOC
387
- #{tar} -zxvf /tmp/#{tarball_name}.tar.gz -C /tmp/ ;
388
- git clone --recursive /tmp/#{tarball_name} #{git_bundle_directory} ;
389
- DOC
388
+ command = <<~DOC
389
+ #{tar} -zxvf /tmp/#{tarball_name}.tar.gz -C /tmp/ ;
390
+ git clone --recursive /tmp/#{tarball_name} #{git_bundle_directory} ;
391
+ DOC
390
392
  Pkg::Util::Net.remote_execute(host, command)
391
393
  return git_bundle_directory
392
394
  end
393
395
 
394
396
  def remote_bundle_install_command
395
- export_packaging_location = ''
396
397
  export_packaging_location = "export PACKAGING_LOCATION='#{ENV['PACKAGING_LOCATION']}';" if ENV['PACKAGING_LOCATION'] && !ENV['PACKAGING_LOCATION'].empty?
397
- command = "source /usr/local/rvm/scripts/rvm; rvm use ruby-2.5.1; #{export_packaging_location} bundle install --path .bundle/gems ;"
398
+ export_vanagon_location = "export VANAGON_LOCATION='#{ENV['VANAGON_LOCATION']}';" if ENV['VANAGON_LOCATION'] && !ENV['VANAGON_LOCATION'].empty?
399
+ "source /usr/local/rvm/scripts/rvm; rvm use ruby-2.5.1; #{export_packaging_location} #{export_vanagon_location} bundle install --path .bundle/gems ;"
398
400
  end
399
401
 
400
402
  # Given a BuildInstance object and a host, send its params to the host. Return
@@ -0,0 +1,17 @@
1
+ # Module for signing all packages to places
2
+
3
+ module Pkg::Util::Repo
4
+ class << self
5
+ # Create yum repositories of built RPM packages for this SHA on the distribution server
6
+ def rpm_repos
7
+ Pkg::Util::File.fetch
8
+ Pkg::Rpm::Repo.create_remote_repos
9
+ end
10
+
11
+ # Create apt repositories of build DEB packages for this SHA on the distributions server
12
+ def deb_repos
13
+ Pkg::Util::File.fetch
14
+ Pkg::Deb::Repo.create_repos
15
+ end
16
+ end
17
+ end
@@ -2,14 +2,13 @@
2
2
 
3
3
  module Pkg::Util::Serialization
4
4
  class << self
5
-
6
5
  # Given the path to a yaml file, load the yaml file into an object and return the object.
7
6
  def load_yaml(file)
8
7
  require 'yaml'
9
8
  file = File.expand_path(file)
10
9
  begin
11
10
  input_data = YAML.load_file(file) || {}
12
- rescue => e
11
+ rescue StandardError => e
13
12
  fail "There was an error loading data from #{file}.\n#{e}"
14
13
  end
15
14
  input_data
@@ -87,7 +87,7 @@ module Pkg::Util::Ship
87
87
  puts "Do you want to ship the above files to (#{staging_server})?"
88
88
  return false unless Pkg::Util.ask_yes_or_no
89
89
 
90
- extra_flags = %w(--ignore-existing --delay-updates)
90
+ extra_flags = %w[--ignore-existing --delay-updates]
91
91
  extra_flags << '--dry-run' if ENV['DRYRUN']
92
92
 
93
93
  staged_pkgs.each do |pkg|
@@ -330,7 +330,7 @@ module Pkg::Util::Ship
330
330
  def test_ship(vm, ship_task)
331
331
  command = 'getent group release || groupadd release'
332
332
  Pkg::Util::Net.remote_execute(vm, command)
333
- hosts_to_override = %w(
333
+ hosts_to_override = %w[
334
334
  APT_HOST
335
335
  DMG_HOST
336
336
  GEM_HOST
@@ -349,10 +349,157 @@ module Pkg::Util::Ship
349
349
  TAR_STAGING_SERVER
350
350
  YUM_STAGING_SERVER
351
351
  STAGING_SERVER
352
- )
352
+ ]
353
353
  hosts_to_override.each do |host|
354
354
  ENV[host] = vm
355
355
  end
356
356
  Rake::Task[ship_task].invoke
357
357
  end
358
+
359
+ # Ship pkg directory contents to distribution server
360
+ def ship(target = 'artifacts', local_directory = 'pkg')
361
+ Pkg::Util::File.fetch
362
+
363
+ unless Pkg::Config.project
364
+ fail "You must set the 'project' in build_defaults.yaml or with the 'PROJECT_OVERRIDE' environment variable."
365
+ end
366
+
367
+ project_basedir = "#{Pkg::Config.jenkins_repo_path}/#{Pkg::Config.project}/#{Pkg::Config.ref}"
368
+ artifact_directory = "#{project_basedir}/#{target}"
369
+
370
+ # For EZBake builds, we also want to include the ezbake.manifest file to
371
+ # get a snapshot of this build and all dependencies. We eventually will
372
+ # create a yaml version of this file, but until that point we want to
373
+ # make the original ezbake.manifest available
374
+ #
375
+ ezbake_manifest = File.join('ext', 'ezbake.manifest')
376
+ if File.exist?(ezbake_manifest)
377
+ FileUtils.cp(ezbake_manifest, File.join(local_directory, "#{Pkg::Config.ref}.ezbake.manifest"))
378
+ end
379
+ ezbake_yaml = File.join("ext", "ezbake.manifest.yaml")
380
+ if File.exists?(ezbake_yaml)
381
+ FileUtils.cp(ezbake_yaml, File.join(local_directory, "#{Pkg::Config.ref}.ezbake.manifest.yaml"))
382
+ end
383
+
384
+ # Inside build_metadata*.json files there is additional metadata containing
385
+ # information such as git ref and dependencies that are needed at build
386
+ # time. If these files exist, copy them downstream.
387
+ # Typically these files are named 'ext/build_metadata.<project>.<platform>.json'
388
+ build_metadata_json_files = Dir.glob('ext/build_metadata*.json')
389
+ build_metadata_json_files.each do |source_file|
390
+ target_file = File.join(local_directory, "#{Pkg::Config.ref}.#{File.basename(source_file)}")
391
+ FileUtils.cp(source_file, target_file)
392
+ end
393
+
394
+ # Sadly, the packaging repo cannot yet act on its own, without living
395
+ # inside of a packaging-repo compatible project. This means in order to
396
+ # use the packaging repo for shipping and signing (things that really
397
+ # don't require build automation, specifically) we still need the project
398
+ # clone itself.
399
+ Pkg::Util::Git.bundle('HEAD', 'signing_bundle', local_directory)
400
+
401
+ # While we're bundling things, let's also make a git bundle of the
402
+ # packaging repo that we're using when we invoke pl:jenkins:ship. We can
403
+ # have a reasonable level of confidence, later on, that the git bundle on
404
+ # the distribution server was, in fact, the git bundle used to create the
405
+ # associated packages. This is because this ship task is automatically
406
+ # called upon completion each cell of the pl:jenkins:uber_build, and we
407
+ # have --ignore-existing set below. As such, the only git bundle that
408
+ # should possibly be on the distribution is the one used to create the
409
+ # packages.
410
+ # We're bundling the packaging repo because it allows us to keep an
411
+ # archive of the packaging source that was used to create the packages,
412
+ # so that later on if we need to rebuild an older package to audit it or
413
+ # for some other reason we're assured that the new package isn't
414
+ # different by virtue of the packaging automation.
415
+ if defined?(PACKAGING_ROOT)
416
+ packaging_bundle = ''
417
+ Dir.chdir(PACKAGING_ROOT) do
418
+ packaging_bundle = Pkg::Util::Git.bundle('HEAD', 'packaging-bundle')
419
+ end
420
+ FileUtils.mv(packaging_bundle, local_directory)
421
+ end
422
+
423
+ # This is functionality to add the project-arch.msi links that have no
424
+ # version. The code itself looks for the link (if it's there already)
425
+ # and if the source package exists before linking. Searching for the
426
+ # packages has been restricted specifically to just the pkg/windows dir
427
+ # on purpose, as this is where we currently have all windows packages
428
+ # building to. Once we move the Metadata about the output location in
429
+ # to one source of truth we can refactor this to use that to search
430
+ # -Sean P. M. 08/12/16
431
+
432
+ {
433
+ 'windows' => ['x86', 'x64'],
434
+ 'windowsfips' => ['x64']
435
+ }.each_pair do |platform, archs|
436
+ packages = Dir["#{local_directory}/#{platform}/*"]
437
+
438
+ archs.each do |arch|
439
+ package_version = Pkg::Util::Git.describe.tr('-', '.')
440
+ package_filename = File.join(local_directory, platform, "#{Pkg::Config.project}-#{package_version}-#{arch}.msi")
441
+ link_filename = File.join(local_directory, platform, "#{Pkg::Config.project}-#{arch}.msi")
442
+
443
+ next unless !packages.include?(link_filename) && packages.include?(package_filename)
444
+ # Dear future code spelunkers:
445
+ # Using symlinks instead of hard links causes failures when we try
446
+ # to set these files to be immutable. Also be wary of whether the
447
+ # linking utility you're using expects the source path to be relative
448
+ # to the link target or pwd.
449
+ #
450
+ FileUtils.ln(package_filename, link_filename)
451
+ end
452
+ end
453
+
454
+ Pkg::Util::Execution.retry_on_fail(times: 3) do
455
+ Pkg::Util::Net.remote_execute(Pkg::Config.distribution_server, "mkdir --mode=775 -p #{project_basedir}")
456
+ Pkg::Util::Net.remote_execute(Pkg::Config.distribution_server, "mkdir -p #{artifact_directory}")
457
+ Pkg::Util::Net.rsync_to("#{local_directory}/", Pkg::Config.distribution_server, "#{artifact_directory}/", extra_flags: ['--ignore-existing', '--exclude repo_configs'])
458
+ end
459
+
460
+ # In order to get a snapshot of what this build looked like at the time
461
+ # of shipping, we also generate and ship the params file
462
+ #
463
+ Pkg::Config.config_to_yaml(local_directory)
464
+ Pkg::Util::Execution.retry_on_fail(:times => 3) do
465
+ Pkg::Util::Net.rsync_to("#{local_directory}/#{Pkg::Config.ref}.yaml", Pkg::Config.distribution_server, "#{artifact_directory}/", extra_flags: ["--exclude repo_configs"])
466
+ end
467
+
468
+ # If we just shipped a tagged version, we want to make it immutable
469
+ files = Dir.glob("#{local_directory}/**/*").select { |f| File.file?(f) and !f.include? "#{Pkg::Config.ref}.yaml" }.map do |file|
470
+ "#{artifact_directory}/#{file.sub(/^#{local_directory}\//, '')}"
471
+ end
472
+
473
+ Pkg::Util::Net.remote_set_ownership(Pkg::Config.distribution_server, 'root', 'release', files)
474
+ Pkg::Util::Net.remote_set_permissions(Pkg::Config.distribution_server, '0664', files)
475
+ Pkg::Util::Net.remote_set_immutable(Pkg::Config.distribution_server, files)
476
+ end
477
+
478
+ def ship_to_artifactory(local_directory = 'pkg')
479
+ Pkg::Util::File.fetch
480
+ unless Pkg::Config.project
481
+ fail "You must set the 'project' in build_defaults.yaml or with the 'PROJECT_OVERRIDE' environment variable."
482
+ end
483
+ artifactory = Pkg::ManageArtifactory.new(Pkg::Config.project, Pkg::Config.ref)
484
+
485
+ artifacts = Dir.glob("#{local_directory}/**/*").reject { |e| File.directory? e }
486
+ artifacts.sort! do |a, b|
487
+ if File.extname(a) =~ /(md5|sha\d+)/ && File.extname(b) !~ /(md5|sha\d+)/
488
+ 1
489
+ elsif File.extname(b) =~ /(md5|sha\d+)/ && File.extname(a) !~ /(md5|sha\d+)/
490
+ -1
491
+ else
492
+ a <=> b
493
+ end
494
+ end
495
+ artifacts.each do |artifact|
496
+ if File.extname(artifact) == ".yaml" || File.extname(artifact) == ".json"
497
+ artifactory.deploy_package(artifact)
498
+ elsif artifactory.package_exists_on_artifactory?(artifact)
499
+ warn "Attempt to upload '#{artifact}' failed. Package already exists!"
500
+ else
501
+ artifactory.deploy_package(artifact)
502
+ end
503
+ end
504
+ end
358
505
  end
@@ -0,0 +1,47 @@
1
+ # Module for signing all packages to places
2
+
3
+
4
+ module Pkg::Util::Sign
5
+ class << self
6
+ # Sign all locally staged packages on signing server.
7
+ def sign_all(root_directory = nil)
8
+ Pkg::Util::File.fetch
9
+ root_directory ||= ENV['DEFAULT_DIRECTORY']
10
+ Dir["#{root_directory}/*"].empty? and fail "There were no files found in #{root_directory}. \
11
+ Maybe you wanted to build/retrieve something first?"
12
+
13
+ # Because rpms and debs are laid out differently in PE under pkg/ they
14
+ # have a different sign task to address this. Rather than create a whole
15
+ # extra :jenkins task for signing PE, we determine which sign task to use
16
+ # based on if we're building PE.
17
+ # We also listen in on the environment variable SIGNING_BUNDLE. This is
18
+ # _NOT_ intended for public use, but rather with the internal promotion
19
+ # workflow for Puppet Enterprise. SIGNING_BUNDLE is the path to a tarball
20
+ # containing a git bundle to be used as the environment for the packaging
21
+ # repo in a signing operation.
22
+ signing_bundle = ENV['SIGNING_BUNDLE']
23
+ sign_tasks = ["pl:sign_rpms"]
24
+ sign_tasks << "pl:sign_deb_changes" unless Dir["#{root_directory}/**/*.changes"].empty?
25
+ sign_tasks << "pl:sign_tar" if Pkg::Config.build_tar
26
+ sign_tasks << "pl:sign_gem" if Pkg::Config.build_gem
27
+ sign_tasks << "pl:sign_osx" if Pkg::Config.build_dmg || Pkg::Config.vanagon_project
28
+ sign_tasks << "pl:sign_swix" if Pkg::Config.vanagon_project
29
+ sign_tasks << "pl:sign_svr4" if Pkg::Config.vanagon_project
30
+ sign_tasks << "pl:sign_ips" if Pkg::Config.vanagon_project
31
+ sign_tasks << "pl:sign_msi" if Pkg::Config.build_msi || Pkg::Config.vanagon_project
32
+ remote_repo = Pkg::Util::Net.remote_unpack_git_bundle(Pkg::Config.signing_server, 'HEAD', nil, signing_bundle)
33
+ build_params = Pkg::Util::Net.remote_buildparams(Pkg::Config.signing_server, Pkg::Config)
34
+ Pkg::Util::Net.rsync_to(root_directory, Pkg::Config.signing_server, remote_repo)
35
+ rake_command = <<~DOC
36
+ cd #{remote_repo} ;
37
+ #{Pkg::Util::Net.remote_bundle_install_command}
38
+ bundle exec rake #{sign_tasks.map { |task| task + "[#{root_directory}]" }.join(' ')} PARAMS_FILE=#{build_params}
39
+ DOC
40
+ Pkg::Util::Net.remote_execute(Pkg::Config.signing_server, rake_command)
41
+ Pkg::Util::Net.rsync_from("#{remote_repo}/#{root_directory}/", Pkg::Config.signing_server, "#{root_directory}/")
42
+ Pkg::Util::Net.remote_execute(Pkg::Config.signing_server, "rm -rf #{remote_repo}")
43
+ Pkg::Util::Net.remote_execute(Pkg::Config.signing_server, "rm #{build_params}")
44
+ puts "Signed packages staged in #{root_directory}/ directory"
45
+ end
46
+ end
47
+ end
@@ -1,7 +1,6 @@
1
1
  # Utility methods for handling system binaries
2
2
 
3
3
  module Pkg::Util::Tool
4
-
5
4
  # Set up utility methods for handling system binaries
6
5
  #
7
6
  class << self
@@ -15,7 +14,7 @@ module Pkg::Util::Tool
15
14
 
16
15
  if Pkg::Util::OS.windows? && File.extname(location).empty?
17
16
  exts = ENV['PATHEXT']
18
- exts = exts ? exts.split(File::PATH_SEPARATOR) : %w(.EXE .BAT .CMD .COM)
17
+ exts = exts ? exts.split(File::PATH_SEPARATOR) : %w[.EXE .BAT .CMD .COM]
19
18
  exts.each do |ext|
20
19
  locationext = File.expand_path(location + ext)
21
20
 
@@ -30,12 +29,10 @@ module Pkg::Util::Tool
30
29
  end
31
30
 
32
31
  alias :has_tool :find_tool
33
-
34
32
  end
35
33
 
36
34
  # Set up paths to system tools we use in the packaging repo
37
35
  # no matter what distribution we're packaging for.
38
36
 
39
37
  GIT = Pkg::Util::Tool.check_tool('git')
40
-
41
38
  end
@@ -80,7 +80,6 @@ module Pkg::Util::Version
80
80
  # 5.3.0.rc4-1
81
81
  # 3.0.5.rc6.24.g431768c-1
82
82
  #
83
- # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
84
83
  def base_pkg_version(version = Pkg::Config.version)
85
84
  return "#{dot_version(version)}-#{Pkg::Config.release}".split('-') if final?(version) || Pkg::Config.vanagon_project
86
85
 
@@ -142,9 +141,6 @@ module Pkg::Util::Version
142
141
  # If you invoke this the version will only be modified in the temporary copy,
143
142
  # with the intent that it never change the official source tree.
144
143
  #
145
- # rubocop:disable Metrics/AbcSize
146
- # rubocop:disable Metrics/CyclomaticComplexity
147
- # rubocop:disable Metrics/PerceivedComplexity
148
144
  def versionbump(workdir = nil)
149
145
  version = ENV['VERSION'] || Pkg::Config.version.to_s.strip
150
146
  new_version = '"' + version + '"'
@@ -182,7 +178,7 @@ module Pkg::Util::Version
182
178
  # input json file and output if it "looks tagged" or not
183
179
  #
184
180
  # @param json_data [hash] json data hash containing the ref to check
185
- def report_json_tags(json_data) # rubocop:disable Metrics/AbcSize
181
+ def report_json_tags(json_data)
186
182
  puts 'component: ' + File.basename(json_data['url'])
187
183
  puts 'ref: ' + json_data['ref'].to_s
188
184
  if Pkg::Util::Git.remote_tagged?(json_data['url'], json_data['ref'].to_s)
@@ -25,9 +25,11 @@ module Pkg::Util
25
25
  require 'packaging/util/version'
26
26
  require 'packaging/util/windows'
27
27
  require 'packaging/util/git_tags'
28
+ require 'packaging/util/sign'
29
+ require 'packaging/util/repo'
28
30
 
29
31
  def self.boolean_value(var)
30
- return true if var == true || ( var.is_a?(String) && ( var.downcase == 'true' || var.downcase =~ /^y$|^yes$/))
32
+ return true if var == true || (var.is_a?(String) && (var.downcase == 'true' || var.downcase =~ /^y$|^yes$/))
31
33
  return false
32
34
  end
33
35
 
data/lib/packaging.rb CHANGED
@@ -1,6 +1,5 @@
1
1
  module Pkg
2
-
3
- LIBDIR = File.expand_path(File.dirname(__FILE__))
2
+ LIBDIR = __dir__
4
3
 
5
4
  $:.unshift(LIBDIR) unless
6
5
  $:.include?(File.dirname(__FILE__)) || $:.include?(LIBDIR)
@@ -223,7 +223,7 @@ describe "Pkg::Config" do
223
223
  "./artifacts/aix/7.1/PC1/ppc/puppet-agent-5.3.2-1.aix7.1.ppc.rpm"
224
224
 
225
225
  fedora_artifacts = \
226
- "./artifacts/fedora/31/PC1/x86_64/puppet-agent-5.3.2-1.fc31.x86_64.rpm"
226
+ "./artifacts/fedora/32/PC1/x86_64/puppet-agent-5.3.2-1.fc32.x86_64.rpm"
227
227
 
228
228
  windows_artifacts = \
229
229
  "./artifacts/windows/puppet-agent-x64.msi\n" \
@@ -280,8 +280,8 @@ describe "Pkg::Config" do
280
280
  it "should not use 'f' in fedora platform tags" do
281
281
  allow(Pkg::Util::Net).to receive(:remote_execute).and_return(fedora_artifacts, nil)
282
282
  data = Pkg::Config.platform_data
283
- expect(data).to include('fedora-31-x86_64')
284
- expect(data).not_to include('fedora-f31-x86_64')
283
+ expect(data).to include('fedora-32-x86_64')
284
+ expect(data).not_to include('fedora-f32-x86_64')
285
285
  end
286
286
 
287
287
  it "should collect packages whose extname differ from package_format" do
@@ -6,7 +6,7 @@ describe "Pkg::Deb::Repo" do
6
6
  let(:project) { "deb_repos" }
7
7
  let(:ref) { "1234abcd" }
8
8
  let(:base_url) { "http://#{builds_server}/#{project}/#{ref}" }
9
- let(:cows) { ["xenial", "jessie", "trusty", "stretch", ""] }
9
+ let(:cows) { ["xenial", "trusty", "stretch", ""] }
10
10
  let(:wget_results) { cows.map {|cow| "#{base_url}/repos/apt/#{cow}" }.join("\n") }
11
11
  let(:wget_garbage) { "\n and an index\nhttp://somethingelse.com/robots" }
12
12
  let(:repo_configs) { cows.reject {|cow| cow.empty?}.map {|dist| "pkg/repo_configs/deb/pl-#{project}-#{ref}-#{dist}.list" } }
@@ -5,12 +5,11 @@ describe 'Pkg::Paths' do
5
5
  arch_transformations = {
6
6
  ['pkg/el-8-x86_64/puppet-agent-6.9.0-1.el8.x86_64.rpm', 'el', '8'] => 'x86_64',
7
7
  ['pkg/el/8/puppet6/aarch64/puppet-agent-6.5.0.3094.g16b6fa6f-1.el8.aarch64.rpm', 'el', '8'] => 'aarch64',
8
- ['artifacts/fedora/32/puppet6/x86_64/puppet-agent-6.9.0-1.fc30.x86_64.rpm', 'fedora', '32'] => 'x86_64',
8
+ ['artifacts/fedora/32/puppet6/x86_64/puppet-agent-6.9.0-1.fc32.x86_64.rpm', 'fedora', '32'] => 'x86_64',
9
9
  ['pkg/ubuntu-16.04-amd64/puppet-agent_4.99.0-1xenial_amd64.deb', 'ubuntu', '16.04'] => 'amd64',
10
10
  ['artifacts/deb/focal/puppet6/puppet-agent_6.5.0.3094.g16b6fa6f-1focal_arm64.deb', 'ubuntu', '20.04'] => 'aarch64',
11
11
 
12
12
  ['artifacts/ubuntu-16.04-i386/puppetserver_5.0.1-0.1SNAPSHOT.2017.07.27T2346puppetlabs1.debian.tar.gz', 'ubuntu', '16.04'] => 'source',
13
- ['artifacts/deb/jessie/PC1/puppetserver_5.0.1.master.orig.tar.gz', 'debian', '8'] => 'source',
14
13
  ['artifacts/el/6/PC1/SRPMS/puppetserver-5.0.1.master-0.1SNAPSHOT.2017.08.18T0951.el6.src.rpm', 'el', '6'] => 'SRPMS'
15
14
  }
16
15
  arch_transformations.each do |path_array, arch|
@@ -273,7 +272,7 @@ describe 'Pkg::Paths' do
273
272
  .to eq(fake_apt_repo_path)
274
273
  end
275
274
  it 'returns nonfinal_yum_repo_path for nonfinal rpms' do
276
- expect(Pkg::Paths.remote_repo_base('fedora-31-x86_64', nonfinal: true))
275
+ expect(Pkg::Paths.remote_repo_base('fedora-34-x86_64', nonfinal: true))
277
276
  .to eq(fake_yum_nightly_repo_path)
278
277
  end
279
278
  it 'returns nonfinal_apt_repo_path for nonfinal debs' do
@@ -26,7 +26,7 @@ describe 'Pkg::Platforms' do
26
26
 
27
27
  describe '#versions_for_platform' do
28
28
  it 'should return all supported versions for a given platform' do
29
- expect(Pkg::Platforms.versions_for_platform('el')).to match_array(['5', '6', '7', '8'])
29
+ expect(Pkg::Platforms.versions_for_platform('el')).to match_array(['6', '7', '8', '9'])
30
30
  end
31
31
 
32
32
  it 'should raise an error if given a nonexistent platform' do
@@ -36,7 +36,7 @@ describe 'Pkg::Platforms' do
36
36
 
37
37
  describe '#codenames' do
38
38
  it 'should return all codenames for a given platform' do
39
- codenames = ['focal', 'bionic', 'bullseye', 'buster', 'cosmic', 'jessie', 'stretch', 'trusty', 'xenial']
39
+ codenames = ['focal', 'bionic', 'bullseye', 'buster', 'stretch', 'trusty', 'xenial']
40
40
  expect(Pkg::Platforms.codenames).to match_array(codenames)
41
41
  end
42
42
  end
@@ -97,16 +97,16 @@ describe 'Pkg::Platforms' do
97
97
  end
98
98
 
99
99
  describe '#platform_lookup' do
100
- ['osx-10.15-x86_64', 'osx-11-x86_64'].each do |platform|
100
+ ['osx-10.15-x86_64', 'osx-11-x86_64', 'osx-12-x86_64'].each do |platform|
101
101
  it 'should return a hash of platform info' do
102
102
  expect(Pkg::Platforms.platform_lookup(platform)).to be_instance_of(Hash)
103
103
  end
104
-
104
+
105
105
  it 'should include at least arch and package format keys' do
106
106
  expect(Pkg::Platforms.platform_lookup(platform).keys).to include(:architectures)
107
107
  expect(Pkg::Platforms.platform_lookup(platform).keys).to include(:package_format)
108
108
  end
109
- end
109
+ end
110
110
  end
111
111
 
112
112
  describe '#get_attribute' do
@@ -166,7 +166,7 @@ describe 'Pkg::Platforms' do
166
166
 
167
167
  describe '#generic_platform_tag' do
168
168
  it 'fails for unsupported platforms' do
169
- expect { Pkg::Platforms.generic_platform_tag('butts') }.to raise_error
169
+ expect { Pkg::Platforms.generic_platform_tag('noplatform') }.to raise_error
170
170
  end
171
171
 
172
172
  it 'returns a supported platform tag containing the supplied platform' do
@@ -15,13 +15,6 @@ Header V4 RSA/SHA256 Signature, key ID ef8d349f: NOKEY
15
15
  Header SHA1 digest: OK (3cb7e9861e8bc09783a1b6c8d88243a3c16daa81)
16
16
  V4 RSA/SHA256 Signature, key ID ef8d349f: NOKEY
17
17
  MD5 digest: OK (d5f06ba2a9053de532326d0659ec0d11)
18
- DOC
19
- }
20
- let(:el5_signed_response) { <<-DOC
21
- Header V3 RSA/SHA1 signature: NOKEY, key ID ef8d349f
22
- Header SHA1 digest: OK (12ea7bd578097a3aecc5deb8ada6aca6147d68e3)
23
- V3 RSA/SHA1 signature: NOKEY, key ID ef8d349f
24
- MD5 digest: OK (27353c6153068a3c9902fcb4ad5b8b92)
25
18
  DOC
26
19
  }
27
20
  let(:sles12_signed_response) { <<-DOC
@@ -40,10 +33,6 @@ DOC
40
33
  allow(Pkg::Sign::Rpm).to receive(:`).and_return(el7_signed_response)
41
34
  expect(Pkg::Sign::Rpm.has_sig?(rpm)).to be true
42
35
  end
43
- it 'returns true if rpm has been signed (el5)' do
44
- allow(Pkg::Sign::Rpm).to receive(:`).and_return(el5_signed_response)
45
- expect(Pkg::Sign::Rpm.has_sig?(rpm)).to be true
46
- end
47
36
  it 'returns true if rpm has been signed (sles12)' do
48
37
  allow(Pkg::Sign::Rpm).to receive(:`).and_return(sles12_signed_response)
49
38
  expect(Pkg::Sign::Rpm.has_sig?(rpm)).to be true
@@ -63,12 +52,11 @@ DOC
63
52
  end
64
53
 
65
54
  describe '#sign_all' do
66
- let(:rpm_directory) { 'foo' }
55
+ let(:rpm_directory) { Dir.mktmpdir }
67
56
  let(:rpms_not_to_sign) { [
68
57
  "#{rpm_directory}/aix/7.1/PC1/ppc/puppet-agent-5.5.3-1.aix7.1.ppc.rpm",
69
58
  ] }
70
59
  let(:v3_rpms) { [
71
- "#{rpm_directory}/el/5/PC1/i386/puppet-agent-5.5.3-1.el5.i386.rpm",
72
60
  "#{rpm_directory}/sles/11/PC1/x86_64/puppet-agent-5.5.3-1.sles11.x86_64.rpm",
73
61
  ] }
74
62
  let(:v4_rpms) { [
@@ -150,11 +150,11 @@ describe 'Pkg::Util::Git' do
150
150
 
151
151
  context '#remote_tagged?' do
152
152
  it 'reports Yes on tagged component' do
153
- expect(Pkg::Util::Git.remote_tagged?('git://github.com/puppetlabs/leatherman.git', 'refs/tags/0.6.2')).to be(true)
153
+ expect(Pkg::Util::Git.remote_tagged?('https://github.com/puppetlabs/leatherman.git', 'refs/tags/0.6.2')).to be(true)
154
154
  end
155
155
 
156
156
  it 'reports No on non-tagged component' do
157
- expect(Pkg::Util::Git.remote_tagged?('git://github.com/puppetlabs/leatherman.git', '4eef05389ebf418b62af17406c7f9f13fa51f975')).to be(false)
157
+ expect(Pkg::Util::Git.remote_tagged?('https://github.com/puppetlabs/leatherman.git', '4eef05389ebf418b62af17406c7f9f13fa51f975')).to be(false)
158
158
  end
159
159
  end
160
160
  end