packaging 0.88.77 → 0.99.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. checksums.yaml +5 -5
  2. data/README.md +128 -74
  3. data/lib/packaging/artifactory.rb +60 -433
  4. data/lib/packaging/config/params.rb +7 -28
  5. data/lib/packaging/config.rb +50 -150
  6. data/lib/packaging/deb/repo.rb +19 -20
  7. data/lib/packaging/gem.rb +83 -41
  8. data/lib/packaging/ips.rb +57 -0
  9. data/lib/packaging/msi.rb +89 -0
  10. data/lib/packaging/nuget.rb +1 -1
  11. data/lib/packaging/osx.rb +36 -0
  12. data/lib/packaging/paths.rb +87 -225
  13. data/lib/packaging/platforms.rb +416 -443
  14. data/lib/packaging/repo.rb +22 -122
  15. data/lib/packaging/retrieve.rb +7 -36
  16. data/lib/packaging/rpm/repo.rb +8 -5
  17. data/lib/packaging/tar.rb +0 -9
  18. data/lib/packaging/util/date.rb +0 -5
  19. data/lib/packaging/util/execution.rb +2 -2
  20. data/lib/packaging/util/git.rb +1 -1
  21. data/lib/packaging/util/gpg.rb +1 -5
  22. data/lib/packaging/util/net.rb +37 -79
  23. data/lib/packaging/util/rake_utils.rb +0 -1
  24. data/lib/packaging/util/ship.rb +13 -142
  25. data/lib/packaging/util/tool.rb +1 -1
  26. data/lib/packaging/util/version.rb +0 -8
  27. data/lib/packaging/util.rb +2 -2
  28. data/lib/packaging.rb +3 -3
  29. data/spec/fixtures/config/params.yaml +2 -0
  30. data/spec/lib/packaging/artifactory_spec.rb +16 -66
  31. data/spec/lib/packaging/config_spec.rb +29 -49
  32. data/spec/lib/packaging/deb/repo_spec.rb +7 -16
  33. data/spec/lib/packaging/paths_spec.rb +56 -321
  34. data/spec/lib/packaging/platforms_spec.rb +21 -46
  35. data/spec/lib/packaging/repo_spec.rb +40 -78
  36. data/spec/lib/packaging/retrieve_spec.rb +8 -47
  37. data/spec/lib/packaging/rpm/repo_spec.rb +4 -4
  38. data/spec/lib/packaging/tar_spec.rb +40 -34
  39. data/spec/lib/packaging/util/git_tag_spec.rb +1 -1
  40. data/spec/lib/packaging/util/gpg_spec.rb +1 -1
  41. data/spec/lib/packaging/util/net_spec.rb +15 -35
  42. data/spec/lib/packaging/util/ship_spec.rb +63 -145
  43. data/spec/spec_helper.rb +14 -0
  44. data/tasks/00_utils.rake +6 -4
  45. data/tasks/apple.rake +0 -2
  46. data/tasks/config.rake +0 -5
  47. data/tasks/education.rake +5 -5
  48. data/tasks/fetch.rake +14 -17
  49. data/tasks/gem.rake +121 -134
  50. data/tasks/jenkins.rake +7 -51
  51. data/tasks/nightly_repos.rake +69 -20
  52. data/tasks/pe_ship.rake +11 -16
  53. data/tasks/retrieve.rake +6 -13
  54. data/tasks/ship.rake +256 -196
  55. data/tasks/sign.rake +135 -63
  56. data/tasks/tar.rake +6 -0
  57. data/templates/packaging.xml.erb +7 -9
  58. data/templates/repo.xml.erb +3 -6
  59. metadata +27 -80
  60. data/lib/packaging/archive.rb +0 -126
  61. data/lib/packaging/artifactory/extensions.rb +0 -94
  62. data/lib/packaging/config/validations.rb +0 -13
  63. data/lib/packaging/metrics.rb +0 -15
  64. data/lib/packaging/sign/deb.rb +0 -9
  65. data/lib/packaging/sign/dmg.rb +0 -41
  66. data/lib/packaging/sign/ips.rb +0 -57
  67. data/lib/packaging/sign/msi.rb +0 -124
  68. data/lib/packaging/sign/rpm.rb +0 -115
  69. data/lib/packaging/sign.rb +0 -8
  70. data/spec/lib/packaging/gem_spec.rb +0 -86
  71. data/spec/lib/packaging/sign_spec.rb +0 -133
  72. data/tasks/archive.rake +0 -69
@@ -5,8 +5,6 @@ require "packaging/platforms"
5
5
  module Pkg::Params
6
6
  BUILD_PARAMS = [:allow_dirty_tree,
7
7
  :answer_override,
8
- :apt_archive_path,
9
- :apt_archive_repo_command,
10
8
  :apt_host,
11
9
  :apt_releases,
12
10
  :apt_repo_command,
@@ -47,13 +45,11 @@ module Pkg::Params
47
45
  :dmg_host,
48
46
  :dmg_path,
49
47
  :dmg_staging_server,
50
- :downloads_archive_path,
51
48
  :email,
52
49
  :files,
53
50
  :final_mocks,
54
51
  :foss_only,
55
52
  :foss_platforms,
56
- :freight_archive_path,
57
53
  :freight_conf,
58
54
  :gem_default_executables,
59
55
  :gem_dependencies,
@@ -80,6 +76,8 @@ module Pkg::Params
80
76
  :gpg_name,
81
77
  :homepage,
82
78
  :internal_gem_host,
79
+ :internal_nexus_host,
80
+ :internal_stickler_host,
83
81
  :ips_build_host,
84
82
  :ips_host,
85
83
  :ips_inter_cert,
@@ -110,7 +108,6 @@ module Pkg::Params
110
108
  :nonfinal_apt_repo_path,
111
109
  :nonfinal_apt_repo_staging_path,
112
110
  :nonfinal_dmg_path,
113
- :nonfinal_gem_path,
114
111
  :nonfinal_ips_path,
115
112
  :nonfinal_msi_path,
116
113
  :nonfinal_p5p_path,
@@ -137,7 +134,6 @@ module Pkg::Params
137
134
  :packaging_url,
138
135
  :pbuild_conf,
139
136
  :pe_feature_branch,
140
- :pe_release_branch,
141
137
  :pe_name,
142
138
  :pe_platforms,
143
139
  :pe_version,
@@ -183,7 +179,6 @@ module Pkg::Params
183
179
  :version,
184
180
  :version_file,
185
181
  :version_strategy,
186
- :yum_archive_path,
187
182
  :yum_host,
188
183
  :yum_repo_command,
189
184
  :yum_repo_name,
@@ -200,8 +195,6 @@ module Pkg::Params
200
195
  ENV_VARS = [
201
196
  { :var => :allow_dirty_tree, :envvar => :ALLOW_DIRTY_TREE, :type => :bool },
202
197
  { :var => :answer_override, :envvar => :ANSWER_OVERRIDE },
203
- { :var => :apt_archive_path, :envvar => :APT_ARCHIVE_PATH },
204
- { :var => :apt_archive_repo_command, :envvar => :APT_ARCHIVE_REPO_COMMAND },
205
198
  { :var => :apt_host, :envvar => :APT_HOST },
206
199
  { :var => :apt_releases, :envvar => :APT_RELEASES, :type => :array },
207
200
  { :var => :apt_repo_path, :envvar => :APT_REPO },
@@ -225,11 +218,9 @@ module Pkg::Params
225
218
  { :var => :dmg_host, :envvar => :DMG_HOST },
226
219
  { :var => :dmg_path, :envvar => :DMG_PATH },
227
220
  { :var => :dmg_staging_server, :envvar => :DMG_STAGING_SERVER },
228
- { :var => :downloads_archive_path, :envvar => :DOWNLOADS_ARCHIVE_PATH },
229
221
  { :var => :final_mocks, :envvar => :MOCK },
230
222
  { :var => :foss_only, :envvar => :FOSS_ONLY, :type => :bool },
231
223
  { :var => :foss_platforms, :envvar => :FOSS_PLATFORMS, :type => :array },
232
- { :var => :freight_archive_path, :envvar => :FREIGHT_ARCHIVE_PATH },
233
224
  { :var => :gem_host, :envvar => :GEM_HOST },
234
225
  { :var => :gpg_key, :envvar => :GPG_KEY },
235
226
  { :var => :gpg_name, :envvar => :GPG_NAME },
@@ -253,7 +244,6 @@ module Pkg::Params
253
244
  { :var => :nonfinal_apt_repo_path, :envvar => :NONFINAL_APT_REPO_PATH },
254
245
  { :var => :nonfinal_apt_repo_staging_path, :envvar => :NONFINAL_APT_REPO_STAGING_PATH },
255
246
  { :var => :nonfinal_dmg_path, :envvar => :NONFINAL_DMG_PATH },
256
- { :var => :nonfinal_gem_path, :envvar => :NONFINAL_GEM_PATH },
257
247
  { :var => :nonfinal_ips_path, :envvar => :NONFINAL_IPS_PATH },
258
248
  { :var => :nonfinal_msi_path, :envvar => :NONFINAL_MSI_PATH },
259
249
  { :var => :nonfinal_p5p_path, :envvar => :NONFINAL_P5P_PATH },
@@ -277,7 +267,6 @@ module Pkg::Params
277
267
  { :var => :pe_feature_branch, :envvar => :PE_FEATURE_BRANCH },
278
268
  { :var => :pe_version, :envvar => :PE_VER },
279
269
  { :var => :privatekey_pem, :envvar => :PRIVATE_PEM },
280
- { :var => :project, :envvar => :PROJECT_OVERRIDE },
281
270
  { :var => :project_root, :envvar => :PROJECT_ROOT },
282
271
  { :var => :random_mockroot, :envvar => :RANDOM_MOCKROOT, :type => :bool },
283
272
  { :var => :rc_mocks, :envvar => :MOCK },
@@ -298,12 +287,12 @@ module Pkg::Params
298
287
  { :var => :team, :envvar => :TEAM },
299
288
  { :var => :update_version_file, :envvar => :NEW_STYLE_PACKAGE },
300
289
  { :var => :vanagon_project, :envvar => :VANAGON_PROJECT, :type => :bool },
301
- { :var => :version, :envvar => :PACKAGING_PACKAGE_VERSION },
302
- { :var => :yum_archive_path, :envvar => :YUM_ARCHIVE_PATH },
303
290
  { :var => :yum_host, :envvar => :YUM_HOST },
304
291
  { :var => :yum_repo_path, :envvar => :YUM_REPO },
305
292
  { :var => :yum_staging_server, :envvar => :YUM_STAGING_SERVER },
306
293
  { :var => :internal_gem_host, :envvar => :INTERNAL_GEM_HOST },
294
+ { :var => :internal_nexus_host, :envvar => :INTERNAL_NEXUS_HOST },
295
+ { :var => :internal_stickler_host, :envvar => :INTERNAL_STICKLER_HOST },
307
296
  ]
308
297
  # Default values that are supplied if the user does not supply them
309
298
  #
@@ -318,7 +307,7 @@ module Pkg::Params
318
307
  { :var => :build_data_repo, :val => 'https://github.com/puppetlabs/build-data.git' },
319
308
  { :var => :build_date, :val => Pkg::Util::Date.timestamp('-') },
320
309
  { :var => :release, :val => '1' },
321
- { :var => :internal_gem_host, :val => 'https://artifactory.delivery.puppetlabs.net/artifactory/api/gems/rubygems' },
310
+ { :var => :internal_gem_host, :val => 'https://artifactory.delivery.puppetlabs.net/artifactory/api/gems/rubygems/' },
322
311
  { :var => :build_tar, :val => true },
323
312
  { :var => :dev_build, :val => false },
324
313
  { :var => :osx_signing_cert, :val => '$OSX_SIGNING_CERT' },
@@ -331,7 +320,6 @@ module Pkg::Params
331
320
  { :var => :msi_signing_cert, :val => '$MSI_SIGNING_CERT' },
332
321
  { :var => :msi_signing_cert_pw, :val => '$MSI_SIGNING_CERT_PW' },
333
322
  { :var => :pe_feature_branch, :val => false },
334
- { :var => :pe_release_branch, :val => false },
335
323
  { :var => :s3_ship, :val => false },
336
324
  { :var => :apt_releases, :val => Pkg::Platforms.codenames }]
337
325
 
@@ -342,6 +330,8 @@ module Pkg::Params
342
330
  # in case it is not set.
343
331
  #
344
332
  REASSIGNMENTS = [
333
+ { :oldvar => :internal_gem_host, :newvar => :internal_nexus_host },
334
+ { :oldvar => :internal_gem_host, :newvar => :internal_stickler_host },
345
335
  # These are fall-through values for shipping endpoints
346
336
  { :oldvar => :staging_server, :newvar => :apt_staging_server },
347
337
  { :oldvar => :staging_server, :newvar => :dmg_staging_server },
@@ -363,7 +353,6 @@ module Pkg::Params
363
353
  { :oldvar => :yum_host, :newvar => :tar_host },
364
354
  ]
365
355
 
366
-
367
356
  # These are variables that we have deprecated. If they are encountered in a
368
357
  # project's config, we issue deprecations for them.
369
358
  #
@@ -374,14 +363,4 @@ module Pkg::Params
374
363
  { :var => :gpg_name, :message => "
375
364
  DEPRECATED, 29-Jul-2014: 'gpg_name' has been replaced with 'gpg_key'.
376
365
  Please update this field in your build_defaults.yaml" }]
377
-
378
- # Provide an open-ended template for validating BUILD_PARAMS.
379
- #
380
- # Each validatation contains the variable name as ':var' and a list of validations it
381
- # must pass from the Pkg::Params::Validations class.
382
- #
383
- VALIDATIONS = [
384
- { :var => :project, :validations => [:not_empty?] }
385
- ]
386
-
387
366
  end
@@ -6,7 +6,6 @@ module Pkg
6
6
  #
7
7
  class Config
8
8
  require 'packaging/config/params.rb'
9
- require 'packaging/config/validations.rb'
10
9
  require 'yaml'
11
10
 
12
11
  class << self
@@ -74,114 +73,54 @@ module Pkg
74
73
  # This is to be consumed by beaker and later replaced with our metadata service.
75
74
  #
76
75
  def platform_data
77
- # Return nil if something is not right..
78
- return nil unless self.project && self.ref &&
79
- Pkg::Util::Net.check_host_ssh([self.builds_server]).empty?
80
-
81
- dir = "/opt/jenkins-builds/#{self.project}/#{self.ref}"
82
- cmd = "if [ -s \"#{dir}/artifacts\" ]; then cd #{dir};"\
83
- "find ./artifacts/ -mindepth 2 -type f; fi"
84
- artifacts, _ = Pkg::Util::Net.remote_execute(
85
- self.builds_server,
86
- cmd,
87
- { capture_output: true }
88
- )
89
-
90
- artifacts = artifacts.split("\n")
91
- data = {}
92
- artifacts.each do |artifact|
93
- # We need to preserve the original tag to make sure we look for
94
- # fedora repo configs in the 1.10.x branch of puppet-agent in
95
- # the correct place. For 5.x and 6.x release streams the f prefix
96
- # has been removed and so tag will equal original_tag
97
- original_tag = Pkg::Paths.tag_from_artifact_path(artifact)
98
-
99
- # Remove the f-prefix from the fedora platform tag keys so that
100
- # beaker can rely on consistent keys once we rip out the f for good
101
- tag = original_tag.sub(/fedora-f/, 'fedora-')
102
-
103
- data[tag] ||= {}
104
-
105
- platform, version, arch = Pkg::Platforms.parse_platform_tag(tag)
106
- package_format = Pkg::Platforms.get_attribute(tag, :package_format)
107
-
108
- # Skip this if it's an unversioned MSI. We create these to help
109
- # beaker install the msi without having to know any version
110
- # information, but we should report the versioned artifact in
111
- # platform_data
112
- next if platform =~ /^windows.*$/ &&
113
- File.basename(artifact) == "#{self.project}-#{arch}.#{package_format}"
114
-
115
- # Sometimes we have source or debug packages. We don't want to save
116
- # these paths in favor of the artifact paths.
117
- if platform == 'solaris'
118
- next if version == '10' && File.extname(artifact) != '.gz'
119
- next if version == '11' && File.extname(artifact) != '.p5p'
120
- else
121
- next if File.extname(artifact) != ".#{package_format}"
122
- end
123
-
124
- # Don't want to include debian debug packages
125
- next if /-dbgsym/.match(File.basename(artifact))
126
-
127
- if /#{self.project}-[a-z]+/.match(File.basename(artifact))
128
- add_additional_artifact(data, tag, artifact.sub('artifacts/', ''))
129
- next
130
- end
131
-
132
- case package_format
133
- when 'deb'
134
- repo_config = "../repo_configs/deb/pl-#{self.project}-#{self.ref}-"\
135
- "#{Pkg::Platforms.get_attribute(tag, :codename)}.list"
136
- when 'rpm'
137
- # Using original_tag here to not break legacy fedora repo targets
138
- unless tag.include? 'aix'
139
- repo_config = "../repo_configs/rpm/pl-#{self.project}-"\
140
- "#{self.ref}-#{original_tag}.repo"
76
+ if self.project && self.ref && Pkg::Util::Net.check_host_ssh([self.builds_server]).empty?
77
+ dir = "/opt/jenkins-builds/#{self.project}/#{self.ref}"
78
+ cmd = "if [ -s \"#{dir}/artifacts\" ]; then cd #{dir}; find ./artifacts/ -mindepth 2 -type f; fi"
79
+ artifacts, _ = Pkg::Util::Net.remote_ssh_cmd(self.builds_server, cmd, true)
80
+ artifacts = artifacts.split("\n")
81
+ data = {}
82
+ artifacts.each do |artifact|
83
+ tag = Pkg::Paths.tag_from_artifact_path(artifact)
84
+ platform, version, arch = Pkg::Platforms.parse_platform_tag(tag)
85
+ arch = 'ppc' if platform == 'aix'
86
+ package_format = Pkg::Platforms.get_attribute(tag, :package_format)
87
+
88
+ # Skip this if it's an unversioned MSI. We create these to help
89
+ # beaker install the msi without having to know any version
90
+ # information, but we should report the versioned artifact in
91
+ # platform_data
92
+ next if platform == 'windows' && File.basename(artifact) == "#{self.project}-#{arch}.#{package_format}"
93
+ # Sometimes we have source or debug packages. We don't want to save
94
+ # these paths in favor of the artifact paths.
95
+ if platform == 'solaris'
96
+ next if version == '10' && File.extname(artifact) != '.gz'
97
+ next if version == '11' && File.extname(artifact) != '.p5p'
98
+ else
99
+ next if File.extname(artifact) != ".#{package_format}"
141
100
  end
142
- when 'swix', 'svr4', 'ips', 'dmg', 'msi'
143
- # No repo_configs for these platforms, so do nothing.
144
- else
145
- fail "Error: Unknown package format: '#{package_format}'. Maybe update PLATFORM_INFO?"
146
- end
147
-
148
- # handle the case where there are multiple artifacts but the artifacts are not
149
- # named based on project name (e.g. puppet-enterprise-vanagon).
150
- # In this case, the first one will get set as the artifact, everything else
151
- # will be in the additional artifacts
152
- if data[tag][:artifact].nil?
153
- data[tag][:artifact] = artifact.sub('artifacts/', '')
154
- data[tag][:repo_config] = repo_config
155
- else
156
- add_additional_artifact(data, tag, artifact.sub('artifacts/', ''))
101
+ next if /#{self.project}-[a-z]+/.match(File.basename(artifact))
102
+
103
+ case package_format
104
+ when 'deb'
105
+ repo_config = "../repo_configs/deb/pl-#{self.project}-#{self.ref}-#{Pkg::Platforms.get_attribute(tag, :codename)}.list"
106
+ when 'rpm'
107
+ repo_config = "../repo_configs/rpm/pl-#{self.project}-#{self.ref}-#{tag}.repo" unless tag.include? 'aix'
108
+ when 'swix', 'svr4', 'ips', 'dmg', 'msi'
109
+ # No repo_configs for these platforms, so do nothing.
110
+ else
111
+ fail "Not sure what to do with packages with a package format of '#{package_format}' - maybe update PLATFORM_INFO?"
112
+ end
113
+ # Remove the f-prefix from the fedora platform tag keys so that
114
+ # beaker can rely on consistent keys once we rip out the f for good
115
+ tag = tag.sub(/fedora-f/, 'fedora-')
116
+ data[tag] = { :artifact => artifact.sub('artifacts/', ''),
117
+ :repo_config => repo_config,
118
+ }
157
119
  end
158
- end
159
- return data
160
- end
161
-
162
- # Add artifact to the `additional_artifacts` array in platform data.
163
- # This will not add noarch package paths for the same noarch package
164
- # multiple times.
165
- #
166
- # @param platform_data The platform data hash to update
167
- # @param tag the platform tag
168
- # @param artifact the path of the additional artifact path to add
169
- def add_additional_artifact(platform_data, tag, artifact)
170
- # Don't add noarch packages to additional_artifacts if the same package
171
- # is already the artifact
172
- if !platform_data[tag][:artifact].nil? && File.basename(platform_data[tag][:artifact]) == File.basename(artifact)
173
- return
174
- end
175
-
176
- platform_data[tag][:additional_artifacts] ||= []
177
-
178
- if platform_data[tag][:additional_artifacts].select { |a| File.basename(a) == File.basename(artifact) }.empty?
179
- platform_data[tag][:additional_artifacts] << artifact
180
- end
181
-
182
- # try to avoid empty entries in the yaml for more concise output
183
- if platform_data[tag][:additional_artifacts].empty?
184
- platform_data[tag][:additional_artifacts] = nil
120
+ return data
121
+ else
122
+ warn "Skipping platform_data collection, but don't worry about it."
123
+ return nil
185
124
  end
186
125
  end
187
126
 
@@ -394,31 +333,6 @@ module Pkg
394
333
  end
395
334
  end
396
335
 
397
- ##
398
- # Ask for validation of BUILD_PARAMS
399
- #
400
- # Issued as warnings initially but the intent is to turn this into
401
- # a failure.
402
- #
403
- def perform_validations
404
- error_count = 0
405
- Pkg::Params::VALIDATIONS.each do |v|
406
- variable_name = v[:var]
407
- variable_value = self.instance_variable_get("@#{v[:var]}")
408
- validations = v[:validations]
409
- validations.each do |validation|
410
- unless Pkg::ConfigValidations.send(validation, variable_value)
411
- warn "Warning: variable \"#{variable_name}\" failed validation \"#{validation}\""
412
- error_count += 1
413
- end
414
- end
415
- end
416
-
417
- if error_count != 0
418
- warn "Warning: #{error_count} validation failure(s)."
419
- end
420
- end
421
-
422
336
  def string_to_array(str)
423
337
  delimiters = /[,\s;]/
424
338
  return str if str.respond_to?('each')
@@ -440,10 +354,8 @@ module Pkg
440
354
 
441
355
  def deb_build_targets
442
356
  if self.vanagon_project
443
- fail "ERROR: Could not find any deb targets. Try adding `deb_targets` to your build_defaults.yaml. If you don't want to build any debs, set this to an empty string." unless self.deb_targets
444
357
  self.deb_targets.split(' ')
445
358
  else
446
- fail "ERROR: Could not find any deb targets. Try adding `cows` to your build_defaults.yaml. If you don't want to build any debs, set this to an empty string." unless self.cows
447
359
  self.cows.split(' ').map do |cow|
448
360
  codename, arch = self.cow_to_codename_arch(cow)
449
361
  "#{codename}-#{arch}"
@@ -453,10 +365,8 @@ module Pkg
453
365
 
454
366
  def rpm_build_targets
455
367
  if self.vanagon_project
456
- fail "ERROR: Could not find any rpm targets. Try adding `rpm_targets` to your build_defaults.yaml. If you don't want to build any rpms, set this to an empty string." unless self.rpm_targets
457
368
  self.rpm_targets.split(' ')
458
369
  else
459
- fail "ERROR: Could not find any rpm targets. Try adding `final_mocks` to your build_defaults.yaml. If you don't want to build any rpms, set this to an empty string." unless self.final_mocks
460
370
  self.final_mocks.split(' ').map do |mock|
461
371
  platform, version, arch = self.mock_to_dist_version_arch(mock)
462
372
  "#{platform}-#{version}-#{arch}"
@@ -465,27 +375,17 @@ module Pkg
465
375
  end
466
376
 
467
377
  def yum_target_path(feature_branch = false)
468
- target_path = "#{Pkg::Config.yum_repo_path}/#{Pkg::Config.pe_version}"
469
- # Target path is different for feature (PEZ) or release branches
470
378
  if feature_branch || Pkg::Config.pe_feature_branch
471
- return "#{target_path}/feature/repos/"
472
- elsif Pkg::Config.pe_release_branch
473
- return "#{target_path}/release/repos/"
474
- else
475
- return "#{target_path}/repos/"
379
+ return "#{Pkg::Config.yum_repo_path}/#{Pkg::Config.pe_version}/feature/repos/"
476
380
  end
381
+ "#{Pkg::Config.yum_repo_path}/#{Pkg::Config.pe_version}/repos/"
477
382
  end
478
383
 
479
384
  def apt_target_path(feature_branch = false)
480
- target_path = "#{Pkg::Config.apt_repo_path}/#{Pkg::Config.pe_version}"
481
- # Target path is different for feature (PEZ) or release branches
482
385
  if feature_branch || Pkg::Config.pe_feature_branch
483
- return "#{target_path}/feature/repos/"
484
- elsif Pkg::Config.pe_release_branch
485
- return "#{target_path}/release/repos/"
486
- else
487
- return "#{target_path}/repos/"
386
+ return "#{Pkg::Config.apt_repo_path}/#{Pkg::Config.pe_version}/feature/repos/"
488
387
  end
388
+ "#{Pkg::Config.apt_repo_path}/#{Pkg::Config.pe_version}/repos/"
489
389
  end
490
390
  end
491
391
  end
@@ -125,7 +125,7 @@ Description: Apt repository for acceptance testing" >> conf/distributions ; )
125
125
  command = repo_creation_command(File.join(artifact_directory, 'repos'), artifact_paths)
126
126
 
127
127
  begin
128
- Pkg::Util::Net.remote_execute(Pkg::Config.distribution_server, command)
128
+ Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.distribution_server, command)
129
129
  # Now that we've created our package repositories, we can generate repo
130
130
  # configurations for use with downstream jobs, acceptance clients, etc.
131
131
  Pkg::Deb::Repo.generate_repo_configs
@@ -134,7 +134,7 @@ Description: Apt repository for acceptance testing" >> conf/distributions ; )
134
134
  Pkg::Deb::Repo.ship_repo_configs
135
135
  ensure
136
136
  # Always remove the lock file, even if we've failed
137
- Pkg::Util::Net.remote_execute(Pkg::Config.distribution_server, "rm -f #{artifact_directory}/repos/.lock")
137
+ Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.distribution_server, "rm -f #{artifact_directory}/repos/.lock")
138
138
  end
139
139
  end
140
140
 
@@ -144,9 +144,9 @@ Description: Apt repository for acceptance testing" >> conf/distributions ; )
144
144
  return
145
145
  end
146
146
 
147
- Pkg::Util::RakeUtils.invoke_task("pl:fetch")
147
+ invoke_task("pl:fetch")
148
148
  repo_dir = "#{Pkg::Config.jenkins_repo_path}/#{Pkg::Config.project}/#{Pkg::Config.ref}/#{target}/deb"
149
- Pkg::Util::Net.remote_execute(Pkg::Config.distribution_server, "mkdir -p #{repo_dir}")
149
+ Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.distribution_server, "mkdir -p #{repo_dir}")
150
150
  Pkg::Util::Execution.retry_on_fail(:times => 3) do
151
151
  Pkg::Util::Net.rsync_to("pkg/#{target}/deb/", Pkg::Config.distribution_server, repo_dir)
152
152
  end
@@ -159,28 +159,27 @@ Description: Apt repository for acceptance testing" >> conf/distributions ; )
159
159
  dists = Pkg::Util::File.directories("#{target}/apt")
160
160
  supported_codenames = Pkg::Platforms.codenames
161
161
 
162
- unless dists
163
- warn "No repos found to sign. Maybe you didn't build any debs, or the repo creation failed?"
164
- return
165
- end
166
-
167
- dists.each do |dist|
168
- next unless supported_codenames.include?(dist)
169
- arches = Pkg::Platforms.arches_for_codename(dist)
170
- Dir.chdir("#{target}/apt/#{dist}") do
171
- File.open("conf/distributions", "w") do |f|
172
- f.puts "Origin: Puppet Labs
162
+ if dists
163
+ dists.each do |dist|
164
+ next unless supported_codenames.include?(dist)
165
+ arches = Pkg::Platforms.arches_for_codename(dist)
166
+ Dir.chdir("#{target}/apt/#{dist}") do
167
+ File.open("conf/distributions", "w") do |f|
168
+ f.puts "Origin: Puppet Labs
173
169
  Label: Puppet Labs
174
170
  Codename: #{dist}
175
171
  Architectures: #{(DEBIAN_PACKAGING_ARCHES + arches).uniq.join(' ')}
176
172
  Components: #{reprepro_repo_name}
177
173
  Description: #{message} for #{dist}
178
174
  SignWith: #{Pkg::Config.gpg_key}"
179
- end
175
+ end
180
176
 
181
- stdout, _, _ = Pkg::Util::Execution.capture3("#{reprepro} -vvv --confdir ./conf --dbdir ./db --basedir ./ export")
182
- stdout
177
+ stdout, _, _ = Pkg::Util::Execution.capture3("#{reprepro} -vvv --confdir ./conf --dbdir ./db --basedir ./ export")
178
+ stdout
179
+ end
183
180
  end
181
+ else
182
+ warn "No repos found to sign. Maybe you didn't build any debs, or the repo creation failed?"
184
183
  end
185
184
  end
186
185
 
@@ -252,11 +251,11 @@ SignWith: #{Pkg::Config.gpg_key}"
252
251
  rsync_command = repo_deployment_command(apt_path, destination_staging_path, destination_server, dryrun)
253
252
  cp_command = repo_deployment_command(destination_staging_path, apt_path, nil, dryrun)
254
253
 
255
- Pkg::Util::Net.remote_execute(origin_server, rsync_command)
254
+ Pkg::Util::Net.remote_ssh_cmd(origin_server, rsync_command)
256
255
  if dryrun
257
256
  puts "[DRYRUN] not executing #{cp_command} on #{destination_server}"
258
257
  else
259
- Pkg::Util::Net.remote_execute(destination_server, cp_command)
258
+ Pkg::Util::Net.remote_ssh_cmd(destination_server, cp_command)
260
259
  end
261
260
  end
262
261
 
data/lib/packaging/gem.rb CHANGED
@@ -1,54 +1,104 @@
1
- require 'json'
2
1
  module Pkg::Gem
2
+ @nexus_config = "#{ENV['HOME']}/.gem/nexus"
3
+
3
4
  class << self
5
+ # This is preserved because I don't want to update the deprecated code path
6
+ # yet; I'm not entirely sure I've fixed everything that might attempt
7
+ # to call this method so this is now a wrapper for a wrapper.
4
8
  def ship(file)
9
+ ship_to_stickler(file)
10
+ ship_to_nexus(file)
5
11
  rsync_to_downloads(file)
6
12
  ship_to_rubygems(file)
7
13
  end
8
14
 
9
- # Use rsync to deploy a file and any associated detached signatures,
10
- # checksums, or other glob-able artifacts to an external download server.
11
- def rsync_to_downloads(file)
12
- Pkg::Util.deprecate('Pkg::Gem.rsync_to_downloads', 'Pkg::Util::Ship.ship_pkgs')
13
- Pkg::Util::Ship.ship_pkgs(["#{file}*"], Pkg::Config.gem_host,
14
- Pkg::Config.gem_path, platform_independent: true)
15
+ def load_nexus_config
16
+ if Pkg::Util::File.file_exists?(@nexus_config)
17
+ config = YAML.load_file(@nexus_config)
18
+ end
19
+ config || {}
20
+ end
21
+
22
+ def write_nexus_config
23
+ hash = load_nexus_config
24
+ if hash["GEM_INTERNAL"].nil? || hash["GEM_INTERNAL"][:authorization].nil?
25
+ puts "Please enter nexus username:"
26
+ username = Pkg::Util.get_input
27
+ puts "Please enter nexus password:"
28
+ password = Pkg::Util.get_input(false)
29
+ hash["GEM_INTERNAL"] = { :authorization => "Basic #{Pkg::Util.base64_encode("#{username}:#{password}")}" }
30
+ end
31
+ if hash["GEM_INTERNAL"][:url].nil? || hash["GEM_INTERNAL"][:url] != Pkg::Config.internal_nexus_host
32
+ hash["GEM_INTERNAL"][:url] = Pkg::Config.internal_nexus_host
33
+ end
34
+ File.open(@nexus_config, "w") do |file|
35
+ file.write(hash.to_yaml)
36
+ end
15
37
  end
16
38
 
17
- def shipped_to_rubygems?(gem_name, gem_version, gem_platform)
18
- rubygems_url = "https://rubygems.org/api/v1/versions/#{gem_name}.json"
19
- gem_data = JSON.parse(%x(curl --silent #{rubygems_url}))
20
- gem = gem_data.select do |data|
21
- data['number'] == gem_version && data['platform'] == gem_platform
39
+ # Ship a Ruby gem file to a Nexus server, because
40
+ # you've lost the ability to feel joy anymore.
41
+ def ship_to_nexus(file)
42
+ write_nexus_config
43
+ cmd = "gem nexus #{file} --repo GEM_INTERNAL"
44
+ if ENV['DRYRUN']
45
+ puts "[DRY-RUN] #{cmd}"
46
+ else
47
+ stdout, _, _ = Pkg::Util::Execution.capture3(cmd, true)
48
+ # The `gem nexus` command always returns `0` regardless of what the
49
+ # command results in. In order to properly handle fail cases, this
50
+ # checks for the success case and fails otherwise. The `ex` command
51
+ # above will print any output, so the user should have enough info
52
+ # to debug the failure, and potentially update this fail case if
53
+ # needed.
54
+ fail unless stdout.include? "Created"
55
+ puts "#{file} pushed to nexus server at #{Pkg::Config.internal_nexus_host}"
22
56
  end
23
- return !gem.empty?
24
57
  rescue => e
25
- puts "Something went wrong searching for gem '#{gem_name}':"
58
+ puts "###########################################"
59
+ puts "# Nexus failed, ensure the nexus gem is installed,"
60
+ puts "# you have access to #{Pkg::Config.internal_nexus_host}"
61
+ puts "# and your settings in #{@nexus_config} are correct"
62
+ puts "###########################################"
63
+ puts
26
64
  puts e
27
- puts "Perhaps you're shipping '#{gem_name}' for the first time?"
28
- return false
65
+ raise e
66
+ end
67
+
68
+ # Ship a Ruby gem file to a Stickler server, because
69
+ # you've lost the ability to feel joy anymore.
70
+ def ship_to_stickler(file)
71
+ Pkg::Util::Tool.check_tool("stickler")
72
+ cmd = "stickler push #{file} --server=#{Pkg::Config.internal_stickler_host} 2>/dev/null"
73
+ if ENV['DRYRUN']
74
+ puts "[DRY-RUN] #{cmd}"
75
+ else
76
+ Pkg::Util::Execution.capture3(cmd)
77
+ puts "#{file} pushed to stickler server at #{Pkg::Config.internal_stickler_host}"
78
+ end
79
+ rescue => e
80
+ puts "###########################################"
81
+ puts "# Stickler failed, ensure it's installed"
82
+ puts "# and you have access to #{Pkg::Config.internal_stickler_host}"
83
+ puts "###########################################"
84
+ puts
85
+ puts e
86
+ raise e
87
+ end
88
+
89
+ # Use rsync to deploy a file and any associated detached signatures,
90
+ # checksums, or other glob-able artifacts to an external download server.
91
+ def rsync_to_downloads(file)
92
+ Pkg::Util.deprecate('Pkg::Gem.rsync_to_downloads', 'Pkg::Util::Ship.ship_pkgs')
93
+ Pkg::Util::Ship.ship_pkgs(["#{file}*"], Pkg::Config.gem_host, Pkg::Config.gem_path, platform_independent: true)
29
94
  end
30
95
 
31
96
  # Ship a Ruby gem file to rubygems.org. Requires the existence
32
97
  # of a ~/.gem/credentials file or else rubygems.org won't have
33
98
  # any idea who you are.
34
- def ship_to_rubygems(file, options = {})
35
- # rubygems uses 'ruby' as the platform when it's not a platform-specific
36
- # gem
37
- platform = file.match(/\w+-(?:\d+(?:\.)?)+-(.*)\.gem$/)
38
- unless platform.nil?
39
- gem_platform = platform[1]
40
- end
41
- gem_platform ||= 'ruby'
42
-
43
- if shipped_to_rubygems?(Pkg::Config.gem_name, Pkg::Config.gemversion, gem_platform)
44
- puts "#{file} has already been shipped to rubygems, skipping."
45
- return
46
- end
99
+ def ship_to_rubygems(file)
47
100
  Pkg::Util::File.file_exists?("#{ENV['HOME']}/.gem/credentials", :required => true)
48
- gem_push_command = "gem push #{file}"
49
- gem_push_command << " --host #{options[:host]}" if options[:host]
50
- gem_push_command << " --key #{options[:key]}" if options[:key]
51
- Pkg::Util::Execution.capture3(gem_push_command)
101
+ Pkg::Util::Execution.capture3("gem push #{file}")
52
102
  rescue => e
53
103
  puts "###########################################"
54
104
  puts "# Publishing to rubygems failed. Make sure your .gem/credentials"
@@ -58,13 +108,5 @@ module Pkg::Gem
58
108
  puts e
59
109
  raise e
60
110
  end
61
-
62
- def ship_to_internal_mirror(file)
63
- internal_mirror_api_key_name = 'artifactory_api_key'
64
- ship_to_rubygems(file, {
65
- host: Pkg::Config.internal_gem_host,
66
- key: internal_mirror_api_key_name
67
- })
68
- end
69
111
  end
70
112
  end