packaging 0.99.79 → 0.101.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/packaging/artifactory.rb +49 -20
- data/lib/packaging/platforms.rb +9 -1
- data/tasks/nightly_repos.rake +1 -1
- data/tasks/ship.rake +55 -34
- metadata +19 -19
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4a7e26777ec04e6bbb1824c27ec325110f49d6b52f8ca0dd84e933d3937f8fc3
|
4
|
+
data.tar.gz: 4b1932eb5b21ae8c1c3bb5ea6f21b5fb57479b3fe3e46273e8d075ba4beb86eb
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e7aca6b96f6e8886d0cfd8b5f381ec1e0bb92337dae30ed30601391feb11d141df89262c19996e9e01751434f318f6668570151421f3eadfd860f6db17a75632
|
7
|
+
data.tar.gz: b74047c635a19fbb0ecb308d0b45297d01c81437f77fccc3205c6434934f8e7378372f218d96aa90325496496a5b19f83aa5589be0082ce1a90d96e090e417dc
|
@@ -342,40 +342,69 @@ module Pkg
|
|
342
342
|
end
|
343
343
|
end
|
344
344
|
|
345
|
-
# Using the manifest provided by enterprise-dist, grab the
|
345
|
+
# Using the manifest provided by enterprise-dist, grab the
|
346
|
+
# appropropriate packages from artifactory based on md5sum
|
347
|
+
|
346
348
|
# @param staging_directory [String] location to download packages to
|
347
|
-
# @param manifest [File] JSON file
|
349
|
+
# @param manifest [File] JSON file listing which packages to download and
|
350
|
+
# the corresponding md5sums
|
348
351
|
# @param remote_path [String] Optional partial path on the remote host containing packages
|
349
352
|
# Used to specify which subdirectories packages will be downloaded from.
|
350
353
|
def download_packages(staging_directory, manifest, remote_path = '')
|
351
354
|
check_authorization
|
355
|
+
download_repositories = %w(rpm_enterprise__local debian_enterprise__local)
|
352
356
|
manifest.each do |dist, packages|
|
353
|
-
puts "Grabbing the #{dist} packages from artifactory"
|
354
357
|
packages.each do |name, info|
|
355
|
-
|
356
|
-
|
357
|
-
|
358
|
-
|
359
|
-
|
360
|
-
|
358
|
+
package_file_name = info['filename']
|
359
|
+
puts format(
|
360
|
+
"Searching Artifactory [%s]%s for %s (md5: %s)",
|
361
|
+
download_repositories.join(', '),
|
362
|
+
remote_path.empty? ? '' : "/#{remote_path}",
|
363
|
+
package_file_name,
|
364
|
+
info['md5']
|
365
|
+
)
|
366
|
+
artifacts = Artifactory::Resource::Artifact.checksum_search(
|
367
|
+
md5: info['md5'],
|
368
|
+
repos: download_repositories,
|
369
|
+
name: package_file_name
|
370
|
+
)
|
371
|
+
|
372
|
+
artifact_to_download = artifacts.select do |artifact|
|
373
|
+
artifact.download_uri.include? remote_path
|
374
|
+
end.first
|
375
|
+
|
376
|
+
# If we found matching artifacts, but not in the correct
|
377
|
+
# path, copy the artifact to the correct path This should
|
378
|
+
# help us keep repos up to date with the packages we are
|
379
|
+
# expecting to be there while helping us avoid 'could not
|
380
|
+
# find package' errors
|
361
381
|
if artifact_to_download.nil? && !artifacts.empty?
|
362
382
|
artifact_to_copy = artifacts.first
|
363
|
-
copy_artifact(artifact_to_copy, artifact_to_copy.repo,
|
364
|
-
|
365
|
-
|
383
|
+
copy_artifact(artifact_to_copy, artifact_to_copy.repo,
|
384
|
+
"#{remote_path}/#{dist}/#{package_file_name}")
|
385
|
+
|
386
|
+
# Now, search again to make sure we find them in the correct path.
|
387
|
+
artifacts = Artifactory::Resource::Artifact.checksum_search(
|
388
|
+
md5: info['md5'],
|
389
|
+
repos: download_repositories,
|
390
|
+
name: package_file_name
|
391
|
+
)
|
392
|
+
artifact_to_download = artifacts.select do |artifact|
|
393
|
+
artifact.download_uri.include? remote_path
|
394
|
+
end.first
|
366
395
|
end
|
367
396
|
|
368
397
|
if artifact_to_download.nil?
|
369
|
-
message = "Error:
|
370
|
-
|
371
|
-
|
372
|
-
end
|
398
|
+
message = "Error: could not find package #{package_file_name} " \
|
399
|
+
"with md5sum #{info['md5']}"
|
400
|
+
message += " in #{remote_path}" unless remote_path.empty?
|
373
401
|
raise message
|
374
|
-
else
|
375
|
-
full_staging_path = "#{staging_directory}/#{dist}"
|
376
|
-
puts "downloading #{artifact_to_download.download_uri} to #{File.join(full_staging_path, filename)}"
|
377
|
-
artifact_to_download.download(full_staging_path, filename: filename)
|
378
402
|
end
|
403
|
+
|
404
|
+
full_staging_path = "#{staging_directory}/#{dist}"
|
405
|
+
puts "Downloading #{artifact_to_download.download_uri} to " \
|
406
|
+
"#{File.join(full_staging_path, package_file_name)}"
|
407
|
+
artifact_to_download.download(full_staging_path, filename: package_file_name)
|
379
408
|
end
|
380
409
|
end
|
381
410
|
end
|
data/lib/packaging/platforms.rb
CHANGED
@@ -157,6 +157,14 @@ module Pkg
|
|
157
157
|
source_package_formats: ['src.rpm'],
|
158
158
|
signature_format: 'v3',
|
159
159
|
repo: true,
|
160
|
+
},
|
161
|
+
'8' => {
|
162
|
+
architectures: ['x86_64'],
|
163
|
+
source_architecture: 'SRPMS',
|
164
|
+
package_format: 'rpm',
|
165
|
+
source_package_formats: ['src.rpm'],
|
166
|
+
signature_format: 'v3',
|
167
|
+
repo: true,
|
160
168
|
}
|
161
169
|
},
|
162
170
|
|
@@ -219,7 +227,7 @@ module Pkg
|
|
219
227
|
},
|
220
228
|
'18.04' => {
|
221
229
|
codename: 'bionic',
|
222
|
-
architectures: ['amd64', 'ppc64el'],
|
230
|
+
architectures: ['amd64', 'ppc64el', 'aarch64'],
|
223
231
|
source_architecture: 'source',
|
224
232
|
package_format: 'deb',
|
225
233
|
source_package_formats: DEBIAN_SOURCE_FORMATS,
|
data/tasks/nightly_repos.rake
CHANGED
@@ -93,7 +93,7 @@ DOC
|
|
93
93
|
# The repo configs have Pkg::Config.builds_server used in them, but that
|
94
94
|
# is internal, so we need to replace it with our public server. We also
|
95
95
|
# want them only to see repos, and not signed repos, since the host is
|
96
|
-
# called nightlies.
|
96
|
+
# called nightlies.puppet.com. Here we replace those values in each
|
97
97
|
# config with the desired value.
|
98
98
|
Dir.glob("#{local_target}/repo_configs/**/*").select { |t_config| File.file?(t_config) }.each do |config|
|
99
99
|
new_contents = File.read(config).gsub(Pkg::Config.builds_server, target_host).gsub(/#{target_prefix}_repos/, "repos")
|
data/tasks/ship.rake
CHANGED
@@ -172,20 +172,10 @@ namespace :pl do
|
|
172
172
|
end
|
173
173
|
end
|
174
174
|
|
175
|
-
desc "Copy signed deb repos from #{Pkg::Config.apt_signing_server} to AWS S3"
|
176
|
-
task :deploy_apt_repo_to_s3 => 'pl:fetch' do
|
177
|
-
puts "Really run S3 sync to deploy Debian repos from #{Pkg::Config.apt_signing_server} to AWS S3? [y,n]"
|
178
|
-
if Pkg::Util.ask_yes_or_no
|
179
|
-
Pkg::Util::Execution.retry_on_fail(:times => 3) do
|
180
|
-
command = 'sudo /usr/local/bin/s3_repo_sync.sh apt.puppetlabs.com'
|
181
|
-
Pkg::Util::Net.remote_execute(Pkg::Config.apt_signing_server, command)
|
182
|
-
end
|
183
|
-
end
|
184
|
-
end
|
185
|
-
|
186
175
|
desc "Copy rpm repos from #{Pkg::Config.yum_staging_server} to #{Pkg::Config.yum_host}"
|
187
176
|
task deploy_yum_repo: 'pl:fetch' do
|
188
|
-
puts "Really run remote rsync to deploy yum repos from #{Pkg::Config.yum_staging_server}
|
177
|
+
puts "Really run remote rsync to deploy yum repos from #{Pkg::Config.yum_staging_server} " \
|
178
|
+
"to #{Pkg::Config.yum_host}? [y,n]"
|
189
179
|
if Pkg::Util.ask_yes_or_no
|
190
180
|
Pkg::Util::Execution.retry_on_fail(times: 3) do
|
191
181
|
Pkg::Rpm::Repo.deploy_repos(
|
@@ -198,27 +188,67 @@ namespace :pl do
|
|
198
188
|
end
|
199
189
|
end
|
200
190
|
|
201
|
-
|
191
|
+
##
|
192
|
+
## S3 / GCP syncing
|
193
|
+
S3_REPO_SYNC = 'sudo /usr/local/bin/s3_repo_sync.sh'
|
194
|
+
GCP_REPO_SYNC = '/usr/local/bin/gcp_repo_sync'
|
195
|
+
|
196
|
+
desc "Sync signed apt repos from #{Pkg::Config.apt_signing_server} to AWS S3"
|
197
|
+
task :deploy_apt_repo_to_s3 => 'pl:fetch' do
|
198
|
+
sync_command = "#{S3_REPO_SYNC} apt.puppetlabs.com"
|
199
|
+
puts "Sync apt repos from #{Pkg::Config.apt_signing_server} to AWS S3? [y,n]"
|
200
|
+
next unless Pkg::Util.ask_yes_or_no
|
201
|
+
|
202
|
+
Pkg::Util::Execution.retry_on_fail(times: 3) do
|
203
|
+
Pkg::Util::Net.remote_execute(Pkg::Config.apt_signing_server, sync_command)
|
204
|
+
end
|
205
|
+
end
|
206
|
+
|
207
|
+
desc "Sync signed yum repos from #{Pkg::Config.yum_staging_server} to AWS S3"
|
202
208
|
task :deploy_yum_repo_to_s3 => 'pl:fetch' do
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
end
|
209
|
+
sync_command = "#{S3_REPO_SYNC} yum.puppetlabs.com"
|
210
|
+
puts "Sync yum repos from #{Pkg::Config.yum_staging_server} to AWS S3? [y,n]"
|
211
|
+
next unless Pkg::Util.ask_yes_or_no
|
212
|
+
Pkg::Util::Execution.retry_on_fail(times: 3) do
|
213
|
+
Pkg::Util::Net.remote_execute(Pkg::Config.yum_staging_server, sync_command)
|
209
214
|
end
|
210
215
|
end
|
211
216
|
|
212
217
|
desc "Sync downloads.puppetlabs.com from #{Pkg::Config.staging_server} to AWS S3"
|
213
218
|
task :deploy_downloads_to_s3 => 'pl:fetch' do
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
219
|
+
sync_command = "#{S3_REPO_SYNC} downloads.puppetlabs.com"
|
220
|
+
puts "Sync downloads.puppetlabs.com from #{Pkg::Config.staging_server} to AWS S3? [y,n]"
|
221
|
+
next unless Pkg::Util.ask_yes_or_no
|
222
|
+
Pkg::Util::Execution.retry_on_fail(times: 3) do
|
223
|
+
Pkg::Util::Net.remote_execute(Pkg::Config.staging_server, sync_command)
|
224
|
+
end
|
225
|
+
end
|
226
|
+
|
227
|
+
desc "Sync nightlies.puppet.com from #{Pkg::Config.staging_server} to AWS S3"
|
228
|
+
task :deploy_nightlies_to_s3 => 'pl:fetch' do
|
229
|
+
sync_command = "#{S3_REPO_SYNC} nightlies.puppet.com"
|
230
|
+
puts "Syncing nightly builds from #{Pkg::Config.staging_server} to AWS S3"
|
231
|
+
Pkg::Util::Execution.retry_on_fail(times: 3) do
|
232
|
+
Pkg::Util::Net.remote_execute(Pkg::Config.staging_server, sync_command)
|
233
|
+
end
|
234
|
+
end
|
235
|
+
|
236
|
+
desc "Sync signed apt repos from #{Pkg::Config.apt_signing_server} to Google Cloud Platform"
|
237
|
+
task :sync_apt_repo_to_gcp => 'pl:fetch' do
|
238
|
+
target_site = 'apt.repos.puppetlabs.com'
|
239
|
+
sync_command_puppet_6 = "#{GCP_REPO_SYNC} apt.repos.puppet.com puppet6"
|
240
|
+
sync_command_puppet_7 = "#{GCP_REPO_SYNC} apt.repos.puppet.com puppet7"
|
241
|
+
print "Sync apt repos from #{Pkg::Config.apt_signing_server} to #{target_site}? [y,n] "
|
242
|
+
next unless Pkg::Util.ask_yes_or_no
|
243
|
+
puts
|
244
|
+
|
245
|
+
Pkg::Util::Execution.retry_on_fail(times: 3) do
|
246
|
+
Pkg::Util::Net.remote_execute(Pkg::Config.apt_signing_server, sync_command_puppet_6)
|
247
|
+
Pkg::Util::Net.remote_execute(Pkg::Config.apt_signing_server, sync_command_puppet_7)
|
220
248
|
end
|
221
249
|
end
|
250
|
+
# Keep 'deploy' for backward compatibility
|
251
|
+
task :deploy_apt_repo_to_gcp => :sync_apt_repo_to_gcp
|
222
252
|
|
223
253
|
desc "Sync apt, yum, and downloads.pl.com to AWS S3"
|
224
254
|
task :deploy_final_builds_to_s3 => "pl:fetch" do
|
@@ -227,15 +257,6 @@ namespace :pl do
|
|
227
257
|
Rake::Task['pl:remote:deploy_downloads_to_s3'].invoke
|
228
258
|
end
|
229
259
|
|
230
|
-
desc "Sync nightlies.puppetlabs.com from #{Pkg::Config.staging_server} to AWS S3"
|
231
|
-
task :deploy_nightlies_to_s3 => 'pl:fetch' do
|
232
|
-
puts "Deploying nightly builds from #{Pkg::Config.staging_server} to AWS S3..."
|
233
|
-
Pkg::Util::Execution.retry_on_fail(:times => 3) do
|
234
|
-
command = 'sudo /usr/local/bin/s3_repo_sync.sh nightlies.puppet.com'
|
235
|
-
Pkg::Util::Net.remote_execute(Pkg::Config.staging_server, command)
|
236
|
-
end
|
237
|
-
end
|
238
|
-
|
239
260
|
desc "Sync yum and apt from #{Pkg::Config.staging_server} to rsync servers"
|
240
261
|
task :deploy_to_rsync_server => 'pl:fetch' do
|
241
262
|
# This task must run after the S3 sync has run, or else /opt/repo-s3-stage won't be up-to-date
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: packaging
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.101.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Puppet Labs
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-07
|
11
|
+
date: 2021-10-07 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rspec
|
@@ -260,28 +260,28 @@ signing_key:
|
|
260
260
|
specification_version: 4
|
261
261
|
summary: Puppet Labs' packaging automation
|
262
262
|
test_files:
|
263
|
-
- spec/lib/packaging_spec.rb
|
264
|
-
- spec/lib/packaging/tar_spec.rb
|
265
|
-
- spec/lib/packaging/retrieve_spec.rb
|
266
|
-
- spec/lib/packaging/deb_spec.rb
|
267
|
-
- spec/lib/packaging/rpm/repo_spec.rb
|
268
263
|
- spec/lib/packaging/config_spec.rb
|
269
|
-
- spec/lib/packaging/
|
264
|
+
- spec/lib/packaging/sign_spec.rb
|
270
265
|
- spec/lib/packaging/platforms_spec.rb
|
271
|
-
- spec/lib/packaging/gem_spec.rb
|
272
|
-
- spec/lib/packaging/paths_spec.rb
|
273
|
-
- spec/lib/packaging/util/gpg_spec.rb
|
274
|
-
- spec/lib/packaging/util/ship_spec.rb
|
275
266
|
- spec/lib/packaging/util/rake_utils_spec.rb
|
276
|
-
- spec/lib/packaging/util/execution_spec.rb
|
277
|
-
- spec/lib/packaging/util/version_spec.rb
|
278
|
-
- spec/lib/packaging/util/misc_spec.rb
|
279
267
|
- spec/lib/packaging/util/jenkins_spec.rb
|
280
|
-
- spec/lib/packaging/util/
|
268
|
+
- spec/lib/packaging/util/execution_spec.rb
|
281
269
|
- spec/lib/packaging/util/net_spec.rb
|
282
|
-
- spec/lib/packaging/util/git_spec.rb
|
283
|
-
- spec/lib/packaging/util/os_spec.rb
|
284
270
|
- spec/lib/packaging/util/file_spec.rb
|
285
|
-
- spec/lib/packaging/
|
271
|
+
- spec/lib/packaging/util/misc_spec.rb
|
272
|
+
- spec/lib/packaging/util/gpg_spec.rb
|
273
|
+
- spec/lib/packaging/util/version_spec.rb
|
274
|
+
- spec/lib/packaging/util/os_spec.rb
|
275
|
+
- spec/lib/packaging/util/git_spec.rb
|
276
|
+
- spec/lib/packaging/util/git_tag_spec.rb
|
277
|
+
- spec/lib/packaging/util/ship_spec.rb
|
278
|
+
- spec/lib/packaging/tar_spec.rb
|
286
279
|
- spec/lib/packaging/repo_spec.rb
|
280
|
+
- spec/lib/packaging/deb_spec.rb
|
281
|
+
- spec/lib/packaging/paths_spec.rb
|
282
|
+
- spec/lib/packaging/rpm/repo_spec.rb
|
283
|
+
- spec/lib/packaging/gem_spec.rb
|
284
|
+
- spec/lib/packaging/retrieve_spec.rb
|
287
285
|
- spec/lib/packaging/artifactory_spec.rb
|
286
|
+
- spec/lib/packaging/deb/repo_spec.rb
|
287
|
+
- spec/lib/packaging_spec.rb
|