deb-s3-lock-fix 0.11.8.fix1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/README.md +284 -0
- data/bin/deb-s3 +10 -0
- data/lib/deb/s3/cli.rb +763 -0
- data/lib/deb/s3/lock.rb +125 -0
- data/lib/deb/s3/manifest.rb +144 -0
- data/lib/deb/s3/package.rb +309 -0
- data/lib/deb/s3/release.rb +161 -0
- data/lib/deb/s3/templates/package.erb +66 -0
- data/lib/deb/s3/templates/release.erb +20 -0
- data/lib/deb/s3/utils.rb +115 -0
- data/lib/deb/s3.rb +6 -0
- metadata +111 -0
data/lib/deb/s3/cli.rb
ADDED
@@ -0,0 +1,763 @@
|
|
1
|
+
# -*- encoding : utf-8 -*-
|
2
|
+
require "aws-sdk-s3"
|
3
|
+
require "thor"
|
4
|
+
|
5
|
+
# Hack: aws requires this!
|
6
|
+
require "json"
|
7
|
+
|
8
|
+
require "deb/s3"
|
9
|
+
require "deb/s3/utils"
|
10
|
+
require "deb/s3/manifest"
|
11
|
+
require "deb/s3/package"
|
12
|
+
require "deb/s3/release"
|
13
|
+
require "deb/s3/lock"
|
14
|
+
|
15
|
+
class Deb::S3::CLI < Thor
|
16
|
+
class_option :bucket,
|
17
|
+
:type => :string,
|
18
|
+
:aliases => "-b",
|
19
|
+
:desc => "The name of the S3 bucket to upload to."
|
20
|
+
|
21
|
+
class_option :prefix,
|
22
|
+
:type => :string,
|
23
|
+
:desc => "The path prefix to use when storing on S3."
|
24
|
+
|
25
|
+
class_option :origin,
|
26
|
+
:type => :string,
|
27
|
+
:aliases => "-o",
|
28
|
+
:desc => "The origin to use in the repository Release file."
|
29
|
+
|
30
|
+
class_option :suite,
|
31
|
+
:type => :string,
|
32
|
+
:desc => "The suite to use in the repository Release file."
|
33
|
+
|
34
|
+
class_option :codename,
|
35
|
+
:default => "stable",
|
36
|
+
:type => :string,
|
37
|
+
:aliases => "-c",
|
38
|
+
:desc => "The codename of the APT repository."
|
39
|
+
|
40
|
+
class_option :component,
|
41
|
+
:default => "main",
|
42
|
+
:type => :string,
|
43
|
+
:aliases => "-m",
|
44
|
+
:desc => "The component of the APT repository."
|
45
|
+
|
46
|
+
class_option :section,
|
47
|
+
:type => :string,
|
48
|
+
:aliases => "-s",
|
49
|
+
:hide => true
|
50
|
+
|
51
|
+
class_option :access_key_id,
|
52
|
+
:type => :string,
|
53
|
+
:desc => "The access key for connecting to S3."
|
54
|
+
|
55
|
+
class_option :secret_access_key,
|
56
|
+
:type => :string,
|
57
|
+
:desc => "The secret key for connecting to S3."
|
58
|
+
|
59
|
+
class_option :session_token,
|
60
|
+
:type => :string,
|
61
|
+
:desc => "The (optional) session token for connecting to S3."
|
62
|
+
|
63
|
+
class_option :endpoint,
|
64
|
+
:type => :string,
|
65
|
+
:desc => "The URL endpoint to the S3 API."
|
66
|
+
|
67
|
+
class_option :s3_region,
|
68
|
+
:type => :string,
|
69
|
+
:desc => "The region for connecting to S3.",
|
70
|
+
:default => ENV["AWS_DEFAULT_REGION"] || "us-east-1"
|
71
|
+
|
72
|
+
class_option :force_path_style,
|
73
|
+
:default => false,
|
74
|
+
:type => :boolean,
|
75
|
+
:desc => "Use S3 path style instead of subdomains."
|
76
|
+
|
77
|
+
class_option :proxy_uri,
|
78
|
+
:type => :string,
|
79
|
+
:desc => "The URI of the proxy to send service requests through."
|
80
|
+
|
81
|
+
class_option :visibility,
|
82
|
+
:default => "public",
|
83
|
+
:type => :string,
|
84
|
+
:aliases => "-v",
|
85
|
+
:desc => "The access policy for the uploaded files. " +
|
86
|
+
"Can be public, private, or authenticated."
|
87
|
+
|
88
|
+
class_option :sign,
|
89
|
+
:repeatable => true,
|
90
|
+
:desc => "GPG Sign the Release file when uploading a package, " +
|
91
|
+
"or when verifying it after removing a package. " +
|
92
|
+
"Use --sign with your GPG key ID to use a specific key (--sign=6643C242C18FE05B)." +
|
93
|
+
"Can be specified multiple times for multiple singing keys."
|
94
|
+
|
95
|
+
class_option :gpg_options,
|
96
|
+
:default => "",
|
97
|
+
:type => :string,
|
98
|
+
:desc => "Additional command line options to pass to GPG when signing."
|
99
|
+
|
100
|
+
class_option :encryption,
|
101
|
+
:default => false,
|
102
|
+
:type => :boolean,
|
103
|
+
:aliases => "-e",
|
104
|
+
:desc => "Use S3 server side encryption."
|
105
|
+
|
106
|
+
class_option :quiet,
|
107
|
+
:type => :boolean,
|
108
|
+
:aliases => "-q",
|
109
|
+
:desc => "Doesn't output information, just returns status appropriately."
|
110
|
+
|
111
|
+
class_option :cache_control,
|
112
|
+
:type => :string,
|
113
|
+
:aliases => "-C",
|
114
|
+
:desc => "Add cache-control headers to S3 objects."
|
115
|
+
|
116
|
+
desc "upload FILES",
|
117
|
+
"Uploads the given files to a S3 bucket as an APT repository."
|
118
|
+
|
119
|
+
option :arch,
|
120
|
+
:type => :string,
|
121
|
+
:aliases => "-a",
|
122
|
+
:desc => "The architecture of the package in the APT repository."
|
123
|
+
|
124
|
+
option :preserve_versions,
|
125
|
+
:default => false,
|
126
|
+
:type => :boolean,
|
127
|
+
:aliases => "-p",
|
128
|
+
:desc => "Whether to preserve other versions of a package " +
|
129
|
+
"in the repository when uploading one."
|
130
|
+
|
131
|
+
option :lock,
|
132
|
+
:default => false,
|
133
|
+
:type => :boolean,
|
134
|
+
:aliases => "-l",
|
135
|
+
:desc => "Whether to check for an existing lock on the repository " +
|
136
|
+
"to prevent simultaneous updates "
|
137
|
+
|
138
|
+
option :fail_if_exists,
|
139
|
+
:default => false,
|
140
|
+
:type => :boolean,
|
141
|
+
:desc => "Whether to overwrite any existing package that has the same " +
|
142
|
+
"filename in the pool or the same name and version in the manifest but " +
|
143
|
+
"different contents."
|
144
|
+
|
145
|
+
option :skip_package_upload,
|
146
|
+
:default => false,
|
147
|
+
:type => :boolean,
|
148
|
+
:desc => "Whether to skip all package uploads." +
|
149
|
+
"This is useful when hosting .deb files outside of the bucket."
|
150
|
+
|
151
|
+
def upload(*files)
|
152
|
+
if files.nil? || files.empty?
|
153
|
+
error("You must specify at least one file to upload")
|
154
|
+
end
|
155
|
+
|
156
|
+
# make sure all the files exists
|
157
|
+
if missing_file = files.find { |pattern| Dir.glob(pattern).empty? }
|
158
|
+
error("File '#{missing_file}' doesn't exist")
|
159
|
+
end
|
160
|
+
|
161
|
+
# configure AWS::S3
|
162
|
+
configure_s3_client
|
163
|
+
|
164
|
+
begin
|
165
|
+
if options[:lock]
|
166
|
+
log("Checking for existing lock file")
|
167
|
+
log("Locking repository for updates")
|
168
|
+
Deb::S3::Lock.lock(options[:codename], component, options[:arch], options[:cache_control])
|
169
|
+
@lock_acquired = true
|
170
|
+
end
|
171
|
+
|
172
|
+
# retrieve the existing manifests
|
173
|
+
log("Retrieving existing manifests")
|
174
|
+
release = Deb::S3::Release.retrieve(options[:codename], options[:origin], options[:suite], options[:cache_control])
|
175
|
+
manifests = {}
|
176
|
+
release.architectures.each do |arch|
|
177
|
+
manifests[arch] = Deb::S3::Manifest.retrieve(options[:codename], component, arch, options[:cache_control], options[:fail_if_exists], options[:skip_package_upload])
|
178
|
+
end
|
179
|
+
|
180
|
+
packages_arch_all = []
|
181
|
+
|
182
|
+
# examine all the files
|
183
|
+
files.collect { |f| Dir.glob(f) }.flatten.each do |file|
|
184
|
+
log("Examining package file #{File.basename(file)}")
|
185
|
+
pkg = Deb::S3::Package.parse_file(file)
|
186
|
+
|
187
|
+
# copy over some options if they weren't given
|
188
|
+
arch = options[:arch] || pkg.architecture
|
189
|
+
|
190
|
+
# If they've specified an arch type that doesn't match the package let them know
|
191
|
+
if options.key?("arch") && options[:arch] != pkg.architecture
|
192
|
+
warn("You specified architecture #{options[:arch]} but package #{pkg.name} has architecture type of #{pkg.architecture}")
|
193
|
+
end
|
194
|
+
|
195
|
+
# validate we have them
|
196
|
+
error("No architcture given and unable to determine one for #{file}. " +
|
197
|
+
"Please specify one with --arch [i386|amd64|armhf].") unless arch
|
198
|
+
|
199
|
+
# If the arch is all and the list of existing manifests is none, then
|
200
|
+
# throw an error. This is mainly the case when initializing a brand new
|
201
|
+
# repository. With "all", we won't know which architectures they're using.
|
202
|
+
if arch == "all" && manifests.count == 0
|
203
|
+
manifests['amd64'] = Deb::S3::Manifest.retrieve(options[:codename], component,'amd64', options[:cache_control], options[:fail_if_exists], options[:skip_package_upload])
|
204
|
+
manifests['i386'] = Deb::S3::Manifest.retrieve(options[:codename], component,'i386', options[:cache_control], options[:fail_if_exists], options[:skip_package_upload])
|
205
|
+
manifests['armhf'] = Deb::S3::Manifest.retrieve(options[:codename], component,'armhf', options[:cache_control], options[:fail_if_exists], options[:skip_package_upload])
|
206
|
+
|
207
|
+
# error("Package #{File.basename(file)} had architecture \"all\", " +
|
208
|
+
# "however noexisting package lists exist. This can often happen " +
|
209
|
+
# "if the first package you are add to a new repository is an " +
|
210
|
+
# "\"all\" architecture file. Please use --arch [i386|amd64|armhf] or " +
|
211
|
+
# "another platform type to upload the file.")
|
212
|
+
end
|
213
|
+
|
214
|
+
# retrieve the manifest for the arch if we don't have it already
|
215
|
+
manifests[arch] ||= Deb::S3::Manifest.retrieve(options[:codename], component, arch, options[:cache_control], options[:fail_if_exists], options[:skip_package_upload])
|
216
|
+
|
217
|
+
# add package in manifests
|
218
|
+
begin
|
219
|
+
manifests[arch].add(pkg, options[:preserve_versions])
|
220
|
+
rescue Deb::S3::Utils::AlreadyExistsError => e
|
221
|
+
error("Preparing manifest failed because: #{e}")
|
222
|
+
end
|
223
|
+
|
224
|
+
# If arch is all, we must add this package in all arch available
|
225
|
+
if arch == 'all'
|
226
|
+
packages_arch_all << pkg
|
227
|
+
end
|
228
|
+
end
|
229
|
+
|
230
|
+
manifests.each do |arch, manifest|
|
231
|
+
next if arch == 'all'
|
232
|
+
packages_arch_all.each do |pkg|
|
233
|
+
begin
|
234
|
+
manifest.add(pkg, options[:preserve_versions], false)
|
235
|
+
rescue Deb::S3::Utils::AlreadyExistsError => e
|
236
|
+
error("Preparing manifest failed because: #{e}")
|
237
|
+
end
|
238
|
+
end
|
239
|
+
end
|
240
|
+
|
241
|
+
# upload the manifest
|
242
|
+
log("Uploading packages and new manifests to S3")
|
243
|
+
manifests.each_value do |manifest|
|
244
|
+
begin
|
245
|
+
manifest.write_to_s3 { |f| sublog("Transferring #{f}") }
|
246
|
+
rescue Deb::S3::Utils::AlreadyExistsError => e
|
247
|
+
error("Uploading manifest failed because: #{e}")
|
248
|
+
end
|
249
|
+
release.update_manifest(manifest)
|
250
|
+
end
|
251
|
+
release.write_to_s3 { |f| sublog("Transferring #{f}") }
|
252
|
+
|
253
|
+
log("Update complete.")
|
254
|
+
ensure
|
255
|
+
if options[:lock] && @lock_acquired
|
256
|
+
Deb::S3::Lock.unlock(options[:codename], component, options[:arch], options[:cache_control])
|
257
|
+
log("Lock released.")
|
258
|
+
end
|
259
|
+
end
|
260
|
+
end
|
261
|
+
|
262
|
+
desc "list", "Lists packages in given codename, component, and optionally architecture"
|
263
|
+
|
264
|
+
option :long,
|
265
|
+
:type => :boolean,
|
266
|
+
:aliases => '-l',
|
267
|
+
:desc => "Shows all package information in original format.",
|
268
|
+
:default => false
|
269
|
+
|
270
|
+
option :arch,
|
271
|
+
:type => :string,
|
272
|
+
:aliases => "-a",
|
273
|
+
:desc => "The architecture of the package in the APT repository."
|
274
|
+
|
275
|
+
def list
|
276
|
+
configure_s3_client
|
277
|
+
|
278
|
+
release = Deb::S3::Release.retrieve(options[:codename])
|
279
|
+
archs = release.architectures
|
280
|
+
archs &= [options[:arch]] if options[:arch] && options[:arch] != "all"
|
281
|
+
widths = [0, 0]
|
282
|
+
rows = archs.map { |arch|
|
283
|
+
manifest = Deb::S3::Manifest.retrieve(options[:codename], component,
|
284
|
+
arch, options[:cache_control],
|
285
|
+
false, false)
|
286
|
+
manifest.packages.map do |package|
|
287
|
+
if options[:long]
|
288
|
+
package.generate(options[:codename])
|
289
|
+
else
|
290
|
+
[package.name, package.full_version, package.architecture].tap do |row|
|
291
|
+
row.each_with_index do |col, i|
|
292
|
+
widths[i] = [widths[i], col.size].max if widths[i]
|
293
|
+
end
|
294
|
+
end
|
295
|
+
end
|
296
|
+
end
|
297
|
+
}.flatten(1)
|
298
|
+
|
299
|
+
if options[:long]
|
300
|
+
$stdout.puts rows.join("\n")
|
301
|
+
else
|
302
|
+
rows.each do |row|
|
303
|
+
$stdout.puts "% -#{widths[0]}s % -#{widths[1]}s %s" % row
|
304
|
+
end
|
305
|
+
end
|
306
|
+
end
|
307
|
+
|
308
|
+
desc "show PACKAGE VERSION ARCH", "Shows information about a package."
|
309
|
+
|
310
|
+
def show(package_name, version, arch)
|
311
|
+
if version.nil?
|
312
|
+
error "You must specify the name of the package to show."
|
313
|
+
end
|
314
|
+
if version.nil?
|
315
|
+
error "You must specify the version of the package to show."
|
316
|
+
end
|
317
|
+
if arch.nil?
|
318
|
+
error "You must specify the architecture of the package to show."
|
319
|
+
end
|
320
|
+
|
321
|
+
configure_s3_client
|
322
|
+
|
323
|
+
# retrieve the existing manifests
|
324
|
+
manifest = Deb::S3::Manifest.retrieve(options[:codename], component, arch,
|
325
|
+
options[:cache_control], false, false)
|
326
|
+
package = manifest.packages.detect { |p|
|
327
|
+
p.name == package_name && p.full_version == version
|
328
|
+
}
|
329
|
+
if package.nil?
|
330
|
+
error "No such package found."
|
331
|
+
end
|
332
|
+
|
333
|
+
puts package.generate(options[:codename])
|
334
|
+
end
|
335
|
+
|
336
|
+
desc "copy PACKAGE TO_CODENAME TO_COMPONENT ",
|
337
|
+
"Copy the package named PACKAGE to given codename and component. If --versions is not specified, copy all versions of PACKAGE. Otherwise, only the specified versions will be copied. Source codename and component is given by --codename and --component options."
|
338
|
+
|
339
|
+
option :cache_control,
|
340
|
+
:type => :string,
|
341
|
+
:aliases => "-C",
|
342
|
+
:desc => "Add cache-control headers to S3 objects."
|
343
|
+
|
344
|
+
option :arch,
|
345
|
+
:type => :string,
|
346
|
+
:aliases => "-a",
|
347
|
+
:desc => "The architecture of the package in the APT repository."
|
348
|
+
|
349
|
+
option :lock,
|
350
|
+
:default => false,
|
351
|
+
:type => :boolean,
|
352
|
+
:aliases => "-l",
|
353
|
+
:desc => "Whether to check for an existing lock on the repository " +
|
354
|
+
"to prevent simultaneous updates "
|
355
|
+
|
356
|
+
option :versions,
|
357
|
+
:default => nil,
|
358
|
+
:type => :array,
|
359
|
+
:desc => "The space-delimited versions of PACKAGE to delete. If not " +
|
360
|
+
"specified, ALL VERSIONS will be deleted. Fair warning. " +
|
361
|
+
"E.g. --versions \"0.1 0.2 0.3\""
|
362
|
+
|
363
|
+
option :preserve_versions,
|
364
|
+
:default => false,
|
365
|
+
:type => :boolean,
|
366
|
+
:aliases => "-p",
|
367
|
+
:desc => "Whether to preserve other versions of a package " +
|
368
|
+
"in the repository when uploading one."
|
369
|
+
|
370
|
+
option :fail_if_exists,
|
371
|
+
:default => true,
|
372
|
+
:type => :boolean,
|
373
|
+
:desc => "Whether to overwrite any existing package that has the same " +
|
374
|
+
"filename in the pool or the same name and version in the manifest."
|
375
|
+
|
376
|
+
def copy(package_name, to_codename, to_component)
|
377
|
+
if package_name.nil?
|
378
|
+
error "You must specify a package name."
|
379
|
+
end
|
380
|
+
if to_codename.nil?
|
381
|
+
error "You must specify a codename to copy to."
|
382
|
+
end
|
383
|
+
if to_component.nil?
|
384
|
+
error "You must specify a component to copy to."
|
385
|
+
end
|
386
|
+
|
387
|
+
arch = options[:arch]
|
388
|
+
if arch.nil?
|
389
|
+
error "You must specify the architecture of the package to copy."
|
390
|
+
end
|
391
|
+
|
392
|
+
versions = options[:versions]
|
393
|
+
if versions.nil?
|
394
|
+
warn "===> WARNING: Copying all versions of #{package_name}"
|
395
|
+
else
|
396
|
+
log "Versions to copy: #{versions.join(', ')}"
|
397
|
+
end
|
398
|
+
|
399
|
+
configure_s3_client
|
400
|
+
|
401
|
+
begin
|
402
|
+
if options[:lock]
|
403
|
+
log("Checking for existing lock file")
|
404
|
+
log("Locking repository for updates")
|
405
|
+
Deb::S3::Lock.lock(options[:codename], to_component, options[:arch], options[:cache_control])
|
406
|
+
@lock_acquired = true
|
407
|
+
end
|
408
|
+
|
409
|
+
# retrieve the existing manifests
|
410
|
+
log "Retrieving existing manifests"
|
411
|
+
from_manifest = Deb::S3::Manifest.retrieve(options[:codename],
|
412
|
+
component, arch,
|
413
|
+
options[:cache_control],
|
414
|
+
false, options[:skip_package_upload])
|
415
|
+
to_release = Deb::S3::Release.retrieve(to_codename)
|
416
|
+
to_manifest = Deb::S3::Manifest.retrieve(to_codename, to_component, arch,
|
417
|
+
options[:cache_control],
|
418
|
+
options[:fail_if_exists],
|
419
|
+
options[:skip_package_upload])
|
420
|
+
packages = from_manifest.packages.select { |p|
|
421
|
+
p.name == package_name &&
|
422
|
+
(versions.nil? || versions.include?(p.full_version))
|
423
|
+
}
|
424
|
+
if packages.size == 0
|
425
|
+
error "No packages found in repository."
|
426
|
+
end
|
427
|
+
|
428
|
+
packages.each do |package|
|
429
|
+
begin
|
430
|
+
to_manifest.add package, options[:preserve_versions], false
|
431
|
+
rescue Deb::S3::Utils::AlreadyExistsError => e
|
432
|
+
error("Preparing manifest failed because: #{e}")
|
433
|
+
end
|
434
|
+
end
|
435
|
+
|
436
|
+
begin
|
437
|
+
to_manifest.write_to_s3 { |f| sublog("Transferring #{f}") }
|
438
|
+
rescue Deb::S3::Utils::AlreadyExistsError => e
|
439
|
+
error("Copying manifest failed because: #{e}")
|
440
|
+
end
|
441
|
+
to_release.update_manifest(to_manifest)
|
442
|
+
to_release.write_to_s3 { |f| sublog("Transferring #{f}") }
|
443
|
+
|
444
|
+
log "Copy complete."
|
445
|
+
ensure
|
446
|
+
if options[:lock] && @lock_acquired
|
447
|
+
Deb::S3::Lock.unlock(options[:codename], component, options[:arch], options[:cache_control])
|
448
|
+
log("Lock released.")
|
449
|
+
end
|
450
|
+
end
|
451
|
+
end
|
452
|
+
|
453
|
+
desc "delete PACKAGE",
|
454
|
+
"Remove the package named PACKAGE. If --versions is not specified, delete" +
|
455
|
+
"all versions of PACKAGE. Otherwise, only the specified versions will be " +
|
456
|
+
"deleted."
|
457
|
+
|
458
|
+
option :arch,
|
459
|
+
:type => :string,
|
460
|
+
:aliases => "-a",
|
461
|
+
:desc => "The architecture of the package in the APT repository."
|
462
|
+
|
463
|
+
option :lock,
|
464
|
+
:default => false,
|
465
|
+
:type => :boolean,
|
466
|
+
:aliases => "-l",
|
467
|
+
:desc => "Whether to check for an existing lock on the repository " +
|
468
|
+
"to prevent simultaneous updates "
|
469
|
+
|
470
|
+
option :versions,
|
471
|
+
:default => nil,
|
472
|
+
:type => :array,
|
473
|
+
:desc => "The space-delimited versions of PACKAGE to delete. If not " +
|
474
|
+
"specified, ALL VERSIONS will be deleted. Fair warning. " +
|
475
|
+
"E.g. --versions \"0.1 0.2 0.3\""
|
476
|
+
|
477
|
+
def delete(package)
|
478
|
+
if package.nil?
|
479
|
+
error("You must specify a package name.")
|
480
|
+
end
|
481
|
+
|
482
|
+
versions = options[:versions]
|
483
|
+
if versions.nil?
|
484
|
+
warn("===> WARNING: Deleting all versions of #{package}")
|
485
|
+
else
|
486
|
+
log("Versions to delete: #{versions.join(', ')}")
|
487
|
+
end
|
488
|
+
|
489
|
+
arch = options[:arch]
|
490
|
+
if arch.nil?
|
491
|
+
error("You must specify the architecture of the package to remove.")
|
492
|
+
end
|
493
|
+
|
494
|
+
configure_s3_client
|
495
|
+
|
496
|
+
begin
|
497
|
+
if options[:lock]
|
498
|
+
log("Checking for existing lock file")
|
499
|
+
log("Locking repository for updates")
|
500
|
+
Deb::S3::Lock.lock(options[:codename], component, options[:arch], options[:cache_control])
|
501
|
+
@lock_acquired = true
|
502
|
+
end
|
503
|
+
|
504
|
+
# retrieve the existing manifests
|
505
|
+
log("Retrieving existing manifests")
|
506
|
+
release = Deb::S3::Release.retrieve(options[:codename], options[:origin], options[:suite])
|
507
|
+
if arch == 'all'
|
508
|
+
selected_arch = release.architectures
|
509
|
+
else
|
510
|
+
selected_arch = [arch]
|
511
|
+
end
|
512
|
+
all_found = 0
|
513
|
+
selected_arch.each { |ar|
|
514
|
+
manifest = Deb::S3::Manifest.retrieve(options[:codename], component, ar, options[:cache_control], false, options[:skip_package_upload])
|
515
|
+
|
516
|
+
deleted = manifest.delete_package(package, versions)
|
517
|
+
all_found += deleted.length
|
518
|
+
if deleted.length == 0
|
519
|
+
if versions.nil?
|
520
|
+
sublog("No packages were deleted. #{package} not found in arch #{ar}.")
|
521
|
+
next
|
522
|
+
else
|
523
|
+
sublog("No packages were deleted. #{package} versions #{versions.join(', ')} could not be found in arch #{ar}.")
|
524
|
+
next
|
525
|
+
end
|
526
|
+
else
|
527
|
+
deleted.each { |p|
|
528
|
+
sublog("Deleting #{p.name} version #{p.full_version} from arch #{ar}")
|
529
|
+
}
|
530
|
+
end
|
531
|
+
|
532
|
+
log("Uploading new manifests to S3")
|
533
|
+
manifest.write_to_s3 {|f| sublog("Transferring #{f}") }
|
534
|
+
release.update_manifest(manifest)
|
535
|
+
release.write_to_s3 {|f| sublog("Transferring #{f}") }
|
536
|
+
|
537
|
+
log("Update complete.")
|
538
|
+
}
|
539
|
+
if all_found == 0
|
540
|
+
if versions.nil?
|
541
|
+
error("No packages were deleted. #{package} not found.")
|
542
|
+
else
|
543
|
+
error("No packages were deleted. #{package} versions #{versions.join(', ')} could not be found.")
|
544
|
+
end
|
545
|
+
end
|
546
|
+
ensure
|
547
|
+
if options[:lock] && @lock_acquired
|
548
|
+
Deb::S3::Lock.unlock(options[:codename], component, options[:arch], options[:cache_control])
|
549
|
+
log("Lock released.")
|
550
|
+
end
|
551
|
+
end
|
552
|
+
end
|
553
|
+
|
554
|
+
|
555
|
+
desc "verify", "Verifies that the files in the package manifests exist"
|
556
|
+
|
557
|
+
option :fix_manifests,
|
558
|
+
:default => false,
|
559
|
+
:type => :boolean,
|
560
|
+
:aliases => "-f",
|
561
|
+
:desc => "Whether to fix problems in manifests when verifying."
|
562
|
+
|
563
|
+
def verify
|
564
|
+
configure_s3_client
|
565
|
+
|
566
|
+
log("Retrieving existing manifests")
|
567
|
+
release = Deb::S3::Release.retrieve(options[:codename], options[:origin], options[:suite])
|
568
|
+
|
569
|
+
release.architectures.each do |arch|
|
570
|
+
log("Checking for missing packages in: #{options[:codename]}/#{options[:component]} #{arch}")
|
571
|
+
manifest = Deb::S3::Manifest.retrieve(options[:codename], component,
|
572
|
+
arch, options[:cache_control], false,
|
573
|
+
options[:skip_package_upload])
|
574
|
+
missing_packages = []
|
575
|
+
|
576
|
+
manifest.packages.each do |p|
|
577
|
+
unless Deb::S3::Utils.s3_exists? p.url_filename(options[:codename])
|
578
|
+
sublog("The following packages are missing:\n\n") if missing_packages.empty?
|
579
|
+
puts(p.generate(options[:codename]))
|
580
|
+
puts("")
|
581
|
+
|
582
|
+
missing_packages << p
|
583
|
+
end
|
584
|
+
end
|
585
|
+
|
586
|
+
if options[:sign] || (options[:fix_manifests] && !missing_packages.empty?)
|
587
|
+
log("Removing #{missing_packages.length} package(s) from the manifest...")
|
588
|
+
missing_packages.each { |p| manifest.packages.delete(p) }
|
589
|
+
manifest.write_to_s3 { |f| sublog("Transferring #{f}") }
|
590
|
+
release.update_manifest(manifest)
|
591
|
+
release.write_to_s3 { |f| sublog("Transferring #{f}") }
|
592
|
+
|
593
|
+
log("Update complete.")
|
594
|
+
end
|
595
|
+
end
|
596
|
+
end
|
597
|
+
|
598
|
+
desc "clean", "Delete packages from the pool which are no longer referenced"
|
599
|
+
|
600
|
+
option :lock,
|
601
|
+
:default => false,
|
602
|
+
:type => :boolean,
|
603
|
+
:aliases => "-l",
|
604
|
+
:desc => "Whether to check for an existing lock on the repository " +
|
605
|
+
"to prevent simultaneous updates "
|
606
|
+
|
607
|
+
def clean
|
608
|
+
configure_s3_client
|
609
|
+
|
610
|
+
begin
|
611
|
+
if options[:lock]
|
612
|
+
log("Checking for existing lock file")
|
613
|
+
log("Locking repository for updates")
|
614
|
+
Deb::S3::Lock.lock(options[:codename], component, options[:arch], options[:cache_control])
|
615
|
+
@lock_acquired = true
|
616
|
+
end
|
617
|
+
|
618
|
+
log("Retrieving existing manifests")
|
619
|
+
|
620
|
+
# Enumerate objects under the dists/<codename>/ prefix to find any
|
621
|
+
# Packages files and load them....
|
622
|
+
|
623
|
+
req = Deb::S3::Utils.s3.list_objects_v2({
|
624
|
+
:bucket => Deb::S3::Utils.bucket,
|
625
|
+
:prefix => Deb::S3::Utils.s3_path("dists/#{ options[:codename] }/"),
|
626
|
+
})
|
627
|
+
|
628
|
+
manifests = []
|
629
|
+
req.contents.each do |object|
|
630
|
+
if match = object.key.match(/dists\/([^\/]+)\/([^\/]+)\/binary-([^\/]+)\/Packages$/)
|
631
|
+
codename, component, arch = match.captures
|
632
|
+
manifests.push(Deb::S3::Manifest.retrieve(codename, component, arch, options[:cache_control], options[:fail_if_exists], options[:skip_package_upload]))
|
633
|
+
end
|
634
|
+
end
|
635
|
+
|
636
|
+
# Iterate over the packages in each manifest and build a Set of all the
|
637
|
+
# referenced URLs (relative to bucket root)...
|
638
|
+
|
639
|
+
refd_urls = Set[]
|
640
|
+
manifests.each do |manifest|
|
641
|
+
manifest.packages.each do |package|
|
642
|
+
refd_urls.add(Deb::S3::Utils.s3_path(package.url_filename(manifest.codename)))
|
643
|
+
end
|
644
|
+
end
|
645
|
+
|
646
|
+
log("Searching for unreferenced packages")
|
647
|
+
|
648
|
+
# Enumerate objects under the pools/<codename> prefix and delete any that
|
649
|
+
# arent referenced by any of the manifests.
|
650
|
+
|
651
|
+
continuation_token = nil
|
652
|
+
while true
|
653
|
+
req = Deb::S3::Utils.s3.list_objects_v2({
|
654
|
+
:bucket => Deb::S3::Utils.bucket,
|
655
|
+
:prefix => Deb::S3::Utils.s3_path("pool/#{ options[:codename] }/"),
|
656
|
+
:continuation_token => continuation_token,
|
657
|
+
})
|
658
|
+
|
659
|
+
req.contents.each do |object|
|
660
|
+
if not refd_urls.include?(object.key)
|
661
|
+
sublog("Deleting #{ object.key }")
|
662
|
+
|
663
|
+
Deb::S3::Utils.s3.delete_object({
|
664
|
+
:bucket => Deb::S3::Utils.bucket,
|
665
|
+
:key => object.key,
|
666
|
+
})
|
667
|
+
end
|
668
|
+
end
|
669
|
+
|
670
|
+
if req.is_truncated
|
671
|
+
continuation_token = req.next_continuation_token
|
672
|
+
else
|
673
|
+
break
|
674
|
+
end
|
675
|
+
end
|
676
|
+
ensure
|
677
|
+
if options[:lock] && @lock_acquired
|
678
|
+
Deb::S3::Lock.unlock(options[:codename], component, options[:arch], options[:cache_control])
|
679
|
+
log("Lock released.")
|
680
|
+
end
|
681
|
+
end
|
682
|
+
end
|
683
|
+
|
684
|
+
private
|
685
|
+
|
686
|
+
def component
|
687
|
+
return @component if @component
|
688
|
+
@component = if (section = options[:section])
|
689
|
+
warn("===> WARNING: The --section/-s argument is " \
|
690
|
+
"deprecated, please use --component/-m.")
|
691
|
+
section
|
692
|
+
else
|
693
|
+
options[:component]
|
694
|
+
end
|
695
|
+
end
|
696
|
+
|
697
|
+
def puts(*args)
|
698
|
+
$stdout.puts(*args) unless options[:quiet]
|
699
|
+
end
|
700
|
+
|
701
|
+
def log(message)
|
702
|
+
puts ">> #{message}" unless options[:quiet]
|
703
|
+
end
|
704
|
+
|
705
|
+
def sublog(message)
|
706
|
+
puts " -- #{message}" unless options[:quiet]
|
707
|
+
end
|
708
|
+
|
709
|
+
def error(message)
|
710
|
+
$stderr.puts "!! #{message}" unless options[:quiet]
|
711
|
+
exit 1
|
712
|
+
end
|
713
|
+
|
714
|
+
def provider
|
715
|
+
access_key_id = options[:access_key_id]
|
716
|
+
secret_access_key = options[:secret_access_key]
|
717
|
+
session_token = options[:session_token]
|
718
|
+
|
719
|
+
if access_key_id.nil? ^ secret_access_key.nil?
|
720
|
+
error("If you specify one of --access-key-id or --secret-access-key, you must specify the other.")
|
721
|
+
end
|
722
|
+
static_credentials = {}
|
723
|
+
static_credentials[:access_key_id] = access_key_id if access_key_id
|
724
|
+
static_credentials[:secret_access_key] = secret_access_key if secret_access_key
|
725
|
+
static_credentials[:session_token] = session_token if session_token
|
726
|
+
|
727
|
+
static_credentials
|
728
|
+
end
|
729
|
+
|
730
|
+
def configure_s3_client
|
731
|
+
error("No value provided for required options '--bucket'") unless options[:bucket]
|
732
|
+
|
733
|
+
settings = {
|
734
|
+
:region => options[:s3_region],
|
735
|
+
:http_proxy => options[:proxy_uri],
|
736
|
+
:force_path_style => options[:force_path_style]
|
737
|
+
}
|
738
|
+
settings[:endpoint] = options[:endpoint] if options[:endpoint]
|
739
|
+
settings.merge!(provider)
|
740
|
+
|
741
|
+
Deb::S3::Utils.s3 = Aws::S3::Client.new(settings)
|
742
|
+
Deb::S3::Utils.bucket = options[:bucket]
|
743
|
+
Deb::S3::Utils.signing_key = options[:sign]
|
744
|
+
Deb::S3::Utils.gpg_options = options[:gpg_options]
|
745
|
+
Deb::S3::Utils.prefix = options[:prefix]
|
746
|
+
Deb::S3::Utils.encryption = options[:encryption]
|
747
|
+
|
748
|
+
# make sure we have a valid visibility setting
|
749
|
+
Deb::S3::Utils.access_policy =
|
750
|
+
case options[:visibility]
|
751
|
+
when "public"
|
752
|
+
"public-read"
|
753
|
+
when "private"
|
754
|
+
"private"
|
755
|
+
when "authenticated"
|
756
|
+
"authenticated-read"
|
757
|
+
when "bucket_owner"
|
758
|
+
"bucket-owner-full-control"
|
759
|
+
else
|
760
|
+
error("Invalid visibility setting given. Can be public, private, authenticated, or bucket_owner.")
|
761
|
+
end
|
762
|
+
end
|
763
|
+
end
|