deb-s3-lock-fix 0.11.8.fix1

Sign up to get free protection for your applications and to get access to all the features.
data/lib/deb/s3/cli.rb ADDED
@@ -0,0 +1,763 @@
1
+ # -*- encoding : utf-8 -*-
2
+ require "aws-sdk-s3"
3
+ require "thor"
4
+
5
+ # Hack: aws requires this!
6
+ require "json"
7
+
8
+ require "deb/s3"
9
+ require "deb/s3/utils"
10
+ require "deb/s3/manifest"
11
+ require "deb/s3/package"
12
+ require "deb/s3/release"
13
+ require "deb/s3/lock"
14
+
15
+ class Deb::S3::CLI < Thor
16
+ class_option :bucket,
17
+ :type => :string,
18
+ :aliases => "-b",
19
+ :desc => "The name of the S3 bucket to upload to."
20
+
21
+ class_option :prefix,
22
+ :type => :string,
23
+ :desc => "The path prefix to use when storing on S3."
24
+
25
+ class_option :origin,
26
+ :type => :string,
27
+ :aliases => "-o",
28
+ :desc => "The origin to use in the repository Release file."
29
+
30
+ class_option :suite,
31
+ :type => :string,
32
+ :desc => "The suite to use in the repository Release file."
33
+
34
+ class_option :codename,
35
+ :default => "stable",
36
+ :type => :string,
37
+ :aliases => "-c",
38
+ :desc => "The codename of the APT repository."
39
+
40
+ class_option :component,
41
+ :default => "main",
42
+ :type => :string,
43
+ :aliases => "-m",
44
+ :desc => "The component of the APT repository."
45
+
46
+ class_option :section,
47
+ :type => :string,
48
+ :aliases => "-s",
49
+ :hide => true
50
+
51
+ class_option :access_key_id,
52
+ :type => :string,
53
+ :desc => "The access key for connecting to S3."
54
+
55
+ class_option :secret_access_key,
56
+ :type => :string,
57
+ :desc => "The secret key for connecting to S3."
58
+
59
+ class_option :session_token,
60
+ :type => :string,
61
+ :desc => "The (optional) session token for connecting to S3."
62
+
63
+ class_option :endpoint,
64
+ :type => :string,
65
+ :desc => "The URL endpoint to the S3 API."
66
+
67
+ class_option :s3_region,
68
+ :type => :string,
69
+ :desc => "The region for connecting to S3.",
70
+ :default => ENV["AWS_DEFAULT_REGION"] || "us-east-1"
71
+
72
+ class_option :force_path_style,
73
+ :default => false,
74
+ :type => :boolean,
75
+ :desc => "Use S3 path style instead of subdomains."
76
+
77
+ class_option :proxy_uri,
78
+ :type => :string,
79
+ :desc => "The URI of the proxy to send service requests through."
80
+
81
+ class_option :visibility,
82
+ :default => "public",
83
+ :type => :string,
84
+ :aliases => "-v",
85
+ :desc => "The access policy for the uploaded files. " +
86
+ "Can be public, private, or authenticated."
87
+
88
+ class_option :sign,
89
+ :repeatable => true,
90
+ :desc => "GPG Sign the Release file when uploading a package, " +
91
+ "or when verifying it after removing a package. " +
92
+ "Use --sign with your GPG key ID to use a specific key (--sign=6643C242C18FE05B)." +
93
+ "Can be specified multiple times for multiple singing keys."
94
+
95
+ class_option :gpg_options,
96
+ :default => "",
97
+ :type => :string,
98
+ :desc => "Additional command line options to pass to GPG when signing."
99
+
100
+ class_option :encryption,
101
+ :default => false,
102
+ :type => :boolean,
103
+ :aliases => "-e",
104
+ :desc => "Use S3 server side encryption."
105
+
106
+ class_option :quiet,
107
+ :type => :boolean,
108
+ :aliases => "-q",
109
+ :desc => "Doesn't output information, just returns status appropriately."
110
+
111
+ class_option :cache_control,
112
+ :type => :string,
113
+ :aliases => "-C",
114
+ :desc => "Add cache-control headers to S3 objects."
115
+
116
+ desc "upload FILES",
117
+ "Uploads the given files to a S3 bucket as an APT repository."
118
+
119
+ option :arch,
120
+ :type => :string,
121
+ :aliases => "-a",
122
+ :desc => "The architecture of the package in the APT repository."
123
+
124
+ option :preserve_versions,
125
+ :default => false,
126
+ :type => :boolean,
127
+ :aliases => "-p",
128
+ :desc => "Whether to preserve other versions of a package " +
129
+ "in the repository when uploading one."
130
+
131
+ option :lock,
132
+ :default => false,
133
+ :type => :boolean,
134
+ :aliases => "-l",
135
+ :desc => "Whether to check for an existing lock on the repository " +
136
+ "to prevent simultaneous updates "
137
+
138
+ option :fail_if_exists,
139
+ :default => false,
140
+ :type => :boolean,
141
+ :desc => "Whether to overwrite any existing package that has the same " +
142
+ "filename in the pool or the same name and version in the manifest but " +
143
+ "different contents."
144
+
145
+ option :skip_package_upload,
146
+ :default => false,
147
+ :type => :boolean,
148
+ :desc => "Whether to skip all package uploads." +
149
+ "This is useful when hosting .deb files outside of the bucket."
150
+
151
+ def upload(*files)
152
+ if files.nil? || files.empty?
153
+ error("You must specify at least one file to upload")
154
+ end
155
+
156
+ # make sure all the files exists
157
+ if missing_file = files.find { |pattern| Dir.glob(pattern).empty? }
158
+ error("File '#{missing_file}' doesn't exist")
159
+ end
160
+
161
+ # configure AWS::S3
162
+ configure_s3_client
163
+
164
+ begin
165
+ if options[:lock]
166
+ log("Checking for existing lock file")
167
+ log("Locking repository for updates")
168
+ Deb::S3::Lock.lock(options[:codename], component, options[:arch], options[:cache_control])
169
+ @lock_acquired = true
170
+ end
171
+
172
+ # retrieve the existing manifests
173
+ log("Retrieving existing manifests")
174
+ release = Deb::S3::Release.retrieve(options[:codename], options[:origin], options[:suite], options[:cache_control])
175
+ manifests = {}
176
+ release.architectures.each do |arch|
177
+ manifests[arch] = Deb::S3::Manifest.retrieve(options[:codename], component, arch, options[:cache_control], options[:fail_if_exists], options[:skip_package_upload])
178
+ end
179
+
180
+ packages_arch_all = []
181
+
182
+ # examine all the files
183
+ files.collect { |f| Dir.glob(f) }.flatten.each do |file|
184
+ log("Examining package file #{File.basename(file)}")
185
+ pkg = Deb::S3::Package.parse_file(file)
186
+
187
+ # copy over some options if they weren't given
188
+ arch = options[:arch] || pkg.architecture
189
+
190
+ # If they've specified an arch type that doesn't match the package let them know
191
+ if options.key?("arch") && options[:arch] != pkg.architecture
192
+ warn("You specified architecture #{options[:arch]} but package #{pkg.name} has architecture type of #{pkg.architecture}")
193
+ end
194
+
195
+ # validate we have them
196
+ error("No architcture given and unable to determine one for #{file}. " +
197
+ "Please specify one with --arch [i386|amd64|armhf].") unless arch
198
+
199
+ # If the arch is all and the list of existing manifests is none, then
200
+ # throw an error. This is mainly the case when initializing a brand new
201
+ # repository. With "all", we won't know which architectures they're using.
202
+ if arch == "all" && manifests.count == 0
203
+ manifests['amd64'] = Deb::S3::Manifest.retrieve(options[:codename], component,'amd64', options[:cache_control], options[:fail_if_exists], options[:skip_package_upload])
204
+ manifests['i386'] = Deb::S3::Manifest.retrieve(options[:codename], component,'i386', options[:cache_control], options[:fail_if_exists], options[:skip_package_upload])
205
+ manifests['armhf'] = Deb::S3::Manifest.retrieve(options[:codename], component,'armhf', options[:cache_control], options[:fail_if_exists], options[:skip_package_upload])
206
+
207
+ # error("Package #{File.basename(file)} had architecture \"all\", " +
208
+ # "however noexisting package lists exist. This can often happen " +
209
+ # "if the first package you are add to a new repository is an " +
210
+ # "\"all\" architecture file. Please use --arch [i386|amd64|armhf] or " +
211
+ # "another platform type to upload the file.")
212
+ end
213
+
214
+ # retrieve the manifest for the arch if we don't have it already
215
+ manifests[arch] ||= Deb::S3::Manifest.retrieve(options[:codename], component, arch, options[:cache_control], options[:fail_if_exists], options[:skip_package_upload])
216
+
217
+ # add package in manifests
218
+ begin
219
+ manifests[arch].add(pkg, options[:preserve_versions])
220
+ rescue Deb::S3::Utils::AlreadyExistsError => e
221
+ error("Preparing manifest failed because: #{e}")
222
+ end
223
+
224
+ # If arch is all, we must add this package in all arch available
225
+ if arch == 'all'
226
+ packages_arch_all << pkg
227
+ end
228
+ end
229
+
230
+ manifests.each do |arch, manifest|
231
+ next if arch == 'all'
232
+ packages_arch_all.each do |pkg|
233
+ begin
234
+ manifest.add(pkg, options[:preserve_versions], false)
235
+ rescue Deb::S3::Utils::AlreadyExistsError => e
236
+ error("Preparing manifest failed because: #{e}")
237
+ end
238
+ end
239
+ end
240
+
241
+ # upload the manifest
242
+ log("Uploading packages and new manifests to S3")
243
+ manifests.each_value do |manifest|
244
+ begin
245
+ manifest.write_to_s3 { |f| sublog("Transferring #{f}") }
246
+ rescue Deb::S3::Utils::AlreadyExistsError => e
247
+ error("Uploading manifest failed because: #{e}")
248
+ end
249
+ release.update_manifest(manifest)
250
+ end
251
+ release.write_to_s3 { |f| sublog("Transferring #{f}") }
252
+
253
+ log("Update complete.")
254
+ ensure
255
+ if options[:lock] && @lock_acquired
256
+ Deb::S3::Lock.unlock(options[:codename], component, options[:arch], options[:cache_control])
257
+ log("Lock released.")
258
+ end
259
+ end
260
+ end
261
+
262
+ desc "list", "Lists packages in given codename, component, and optionally architecture"
263
+
264
+ option :long,
265
+ :type => :boolean,
266
+ :aliases => '-l',
267
+ :desc => "Shows all package information in original format.",
268
+ :default => false
269
+
270
+ option :arch,
271
+ :type => :string,
272
+ :aliases => "-a",
273
+ :desc => "The architecture of the package in the APT repository."
274
+
275
+ def list
276
+ configure_s3_client
277
+
278
+ release = Deb::S3::Release.retrieve(options[:codename])
279
+ archs = release.architectures
280
+ archs &= [options[:arch]] if options[:arch] && options[:arch] != "all"
281
+ widths = [0, 0]
282
+ rows = archs.map { |arch|
283
+ manifest = Deb::S3::Manifest.retrieve(options[:codename], component,
284
+ arch, options[:cache_control],
285
+ false, false)
286
+ manifest.packages.map do |package|
287
+ if options[:long]
288
+ package.generate(options[:codename])
289
+ else
290
+ [package.name, package.full_version, package.architecture].tap do |row|
291
+ row.each_with_index do |col, i|
292
+ widths[i] = [widths[i], col.size].max if widths[i]
293
+ end
294
+ end
295
+ end
296
+ end
297
+ }.flatten(1)
298
+
299
+ if options[:long]
300
+ $stdout.puts rows.join("\n")
301
+ else
302
+ rows.each do |row|
303
+ $stdout.puts "% -#{widths[0]}s % -#{widths[1]}s %s" % row
304
+ end
305
+ end
306
+ end
307
+
308
+ desc "show PACKAGE VERSION ARCH", "Shows information about a package."
309
+
310
+ def show(package_name, version, arch)
311
+ if version.nil?
312
+ error "You must specify the name of the package to show."
313
+ end
314
+ if version.nil?
315
+ error "You must specify the version of the package to show."
316
+ end
317
+ if arch.nil?
318
+ error "You must specify the architecture of the package to show."
319
+ end
320
+
321
+ configure_s3_client
322
+
323
+ # retrieve the existing manifests
324
+ manifest = Deb::S3::Manifest.retrieve(options[:codename], component, arch,
325
+ options[:cache_control], false, false)
326
+ package = manifest.packages.detect { |p|
327
+ p.name == package_name && p.full_version == version
328
+ }
329
+ if package.nil?
330
+ error "No such package found."
331
+ end
332
+
333
+ puts package.generate(options[:codename])
334
+ end
335
+
336
+ desc "copy PACKAGE TO_CODENAME TO_COMPONENT ",
337
+ "Copy the package named PACKAGE to given codename and component. If --versions is not specified, copy all versions of PACKAGE. Otherwise, only the specified versions will be copied. Source codename and component is given by --codename and --component options."
338
+
339
+ option :cache_control,
340
+ :type => :string,
341
+ :aliases => "-C",
342
+ :desc => "Add cache-control headers to S3 objects."
343
+
344
+ option :arch,
345
+ :type => :string,
346
+ :aliases => "-a",
347
+ :desc => "The architecture of the package in the APT repository."
348
+
349
+ option :lock,
350
+ :default => false,
351
+ :type => :boolean,
352
+ :aliases => "-l",
353
+ :desc => "Whether to check for an existing lock on the repository " +
354
+ "to prevent simultaneous updates "
355
+
356
+ option :versions,
357
+ :default => nil,
358
+ :type => :array,
359
+ :desc => "The space-delimited versions of PACKAGE to delete. If not " +
360
+ "specified, ALL VERSIONS will be deleted. Fair warning. " +
361
+ "E.g. --versions \"0.1 0.2 0.3\""
362
+
363
+ option :preserve_versions,
364
+ :default => false,
365
+ :type => :boolean,
366
+ :aliases => "-p",
367
+ :desc => "Whether to preserve other versions of a package " +
368
+ "in the repository when uploading one."
369
+
370
+ option :fail_if_exists,
371
+ :default => true,
372
+ :type => :boolean,
373
+ :desc => "Whether to overwrite any existing package that has the same " +
374
+ "filename in the pool or the same name and version in the manifest."
375
+
376
+ def copy(package_name, to_codename, to_component)
377
+ if package_name.nil?
378
+ error "You must specify a package name."
379
+ end
380
+ if to_codename.nil?
381
+ error "You must specify a codename to copy to."
382
+ end
383
+ if to_component.nil?
384
+ error "You must specify a component to copy to."
385
+ end
386
+
387
+ arch = options[:arch]
388
+ if arch.nil?
389
+ error "You must specify the architecture of the package to copy."
390
+ end
391
+
392
+ versions = options[:versions]
393
+ if versions.nil?
394
+ warn "===> WARNING: Copying all versions of #{package_name}"
395
+ else
396
+ log "Versions to copy: #{versions.join(', ')}"
397
+ end
398
+
399
+ configure_s3_client
400
+
401
+ begin
402
+ if options[:lock]
403
+ log("Checking for existing lock file")
404
+ log("Locking repository for updates")
405
+ Deb::S3::Lock.lock(options[:codename], to_component, options[:arch], options[:cache_control])
406
+ @lock_acquired = true
407
+ end
408
+
409
+ # retrieve the existing manifests
410
+ log "Retrieving existing manifests"
411
+ from_manifest = Deb::S3::Manifest.retrieve(options[:codename],
412
+ component, arch,
413
+ options[:cache_control],
414
+ false, options[:skip_package_upload])
415
+ to_release = Deb::S3::Release.retrieve(to_codename)
416
+ to_manifest = Deb::S3::Manifest.retrieve(to_codename, to_component, arch,
417
+ options[:cache_control],
418
+ options[:fail_if_exists],
419
+ options[:skip_package_upload])
420
+ packages = from_manifest.packages.select { |p|
421
+ p.name == package_name &&
422
+ (versions.nil? || versions.include?(p.full_version))
423
+ }
424
+ if packages.size == 0
425
+ error "No packages found in repository."
426
+ end
427
+
428
+ packages.each do |package|
429
+ begin
430
+ to_manifest.add package, options[:preserve_versions], false
431
+ rescue Deb::S3::Utils::AlreadyExistsError => e
432
+ error("Preparing manifest failed because: #{e}")
433
+ end
434
+ end
435
+
436
+ begin
437
+ to_manifest.write_to_s3 { |f| sublog("Transferring #{f}") }
438
+ rescue Deb::S3::Utils::AlreadyExistsError => e
439
+ error("Copying manifest failed because: #{e}")
440
+ end
441
+ to_release.update_manifest(to_manifest)
442
+ to_release.write_to_s3 { |f| sublog("Transferring #{f}") }
443
+
444
+ log "Copy complete."
445
+ ensure
446
+ if options[:lock] && @lock_acquired
447
+ Deb::S3::Lock.unlock(options[:codename], component, options[:arch], options[:cache_control])
448
+ log("Lock released.")
449
+ end
450
+ end
451
+ end
452
+
453
+ desc "delete PACKAGE",
454
+ "Remove the package named PACKAGE. If --versions is not specified, delete" +
455
+ "all versions of PACKAGE. Otherwise, only the specified versions will be " +
456
+ "deleted."
457
+
458
+ option :arch,
459
+ :type => :string,
460
+ :aliases => "-a",
461
+ :desc => "The architecture of the package in the APT repository."
462
+
463
+ option :lock,
464
+ :default => false,
465
+ :type => :boolean,
466
+ :aliases => "-l",
467
+ :desc => "Whether to check for an existing lock on the repository " +
468
+ "to prevent simultaneous updates "
469
+
470
+ option :versions,
471
+ :default => nil,
472
+ :type => :array,
473
+ :desc => "The space-delimited versions of PACKAGE to delete. If not " +
474
+ "specified, ALL VERSIONS will be deleted. Fair warning. " +
475
+ "E.g. --versions \"0.1 0.2 0.3\""
476
+
477
+ def delete(package)
478
+ if package.nil?
479
+ error("You must specify a package name.")
480
+ end
481
+
482
+ versions = options[:versions]
483
+ if versions.nil?
484
+ warn("===> WARNING: Deleting all versions of #{package}")
485
+ else
486
+ log("Versions to delete: #{versions.join(', ')}")
487
+ end
488
+
489
+ arch = options[:arch]
490
+ if arch.nil?
491
+ error("You must specify the architecture of the package to remove.")
492
+ end
493
+
494
+ configure_s3_client
495
+
496
+ begin
497
+ if options[:lock]
498
+ log("Checking for existing lock file")
499
+ log("Locking repository for updates")
500
+ Deb::S3::Lock.lock(options[:codename], component, options[:arch], options[:cache_control])
501
+ @lock_acquired = true
502
+ end
503
+
504
+ # retrieve the existing manifests
505
+ log("Retrieving existing manifests")
506
+ release = Deb::S3::Release.retrieve(options[:codename], options[:origin], options[:suite])
507
+ if arch == 'all'
508
+ selected_arch = release.architectures
509
+ else
510
+ selected_arch = [arch]
511
+ end
512
+ all_found = 0
513
+ selected_arch.each { |ar|
514
+ manifest = Deb::S3::Manifest.retrieve(options[:codename], component, ar, options[:cache_control], false, options[:skip_package_upload])
515
+
516
+ deleted = manifest.delete_package(package, versions)
517
+ all_found += deleted.length
518
+ if deleted.length == 0
519
+ if versions.nil?
520
+ sublog("No packages were deleted. #{package} not found in arch #{ar}.")
521
+ next
522
+ else
523
+ sublog("No packages were deleted. #{package} versions #{versions.join(', ')} could not be found in arch #{ar}.")
524
+ next
525
+ end
526
+ else
527
+ deleted.each { |p|
528
+ sublog("Deleting #{p.name} version #{p.full_version} from arch #{ar}")
529
+ }
530
+ end
531
+
532
+ log("Uploading new manifests to S3")
533
+ manifest.write_to_s3 {|f| sublog("Transferring #{f}") }
534
+ release.update_manifest(manifest)
535
+ release.write_to_s3 {|f| sublog("Transferring #{f}") }
536
+
537
+ log("Update complete.")
538
+ }
539
+ if all_found == 0
540
+ if versions.nil?
541
+ error("No packages were deleted. #{package} not found.")
542
+ else
543
+ error("No packages were deleted. #{package} versions #{versions.join(', ')} could not be found.")
544
+ end
545
+ end
546
+ ensure
547
+ if options[:lock] && @lock_acquired
548
+ Deb::S3::Lock.unlock(options[:codename], component, options[:arch], options[:cache_control])
549
+ log("Lock released.")
550
+ end
551
+ end
552
+ end
553
+
554
+
555
+ desc "verify", "Verifies that the files in the package manifests exist"
556
+
557
+ option :fix_manifests,
558
+ :default => false,
559
+ :type => :boolean,
560
+ :aliases => "-f",
561
+ :desc => "Whether to fix problems in manifests when verifying."
562
+
563
+ def verify
564
+ configure_s3_client
565
+
566
+ log("Retrieving existing manifests")
567
+ release = Deb::S3::Release.retrieve(options[:codename], options[:origin], options[:suite])
568
+
569
+ release.architectures.each do |arch|
570
+ log("Checking for missing packages in: #{options[:codename]}/#{options[:component]} #{arch}")
571
+ manifest = Deb::S3::Manifest.retrieve(options[:codename], component,
572
+ arch, options[:cache_control], false,
573
+ options[:skip_package_upload])
574
+ missing_packages = []
575
+
576
+ manifest.packages.each do |p|
577
+ unless Deb::S3::Utils.s3_exists? p.url_filename(options[:codename])
578
+ sublog("The following packages are missing:\n\n") if missing_packages.empty?
579
+ puts(p.generate(options[:codename]))
580
+ puts("")
581
+
582
+ missing_packages << p
583
+ end
584
+ end
585
+
586
+ if options[:sign] || (options[:fix_manifests] && !missing_packages.empty?)
587
+ log("Removing #{missing_packages.length} package(s) from the manifest...")
588
+ missing_packages.each { |p| manifest.packages.delete(p) }
589
+ manifest.write_to_s3 { |f| sublog("Transferring #{f}") }
590
+ release.update_manifest(manifest)
591
+ release.write_to_s3 { |f| sublog("Transferring #{f}") }
592
+
593
+ log("Update complete.")
594
+ end
595
+ end
596
+ end
597
+
598
+ desc "clean", "Delete packages from the pool which are no longer referenced"
599
+
600
+ option :lock,
601
+ :default => false,
602
+ :type => :boolean,
603
+ :aliases => "-l",
604
+ :desc => "Whether to check for an existing lock on the repository " +
605
+ "to prevent simultaneous updates "
606
+
607
+ def clean
608
+ configure_s3_client
609
+
610
+ begin
611
+ if options[:lock]
612
+ log("Checking for existing lock file")
613
+ log("Locking repository for updates")
614
+ Deb::S3::Lock.lock(options[:codename], component, options[:arch], options[:cache_control])
615
+ @lock_acquired = true
616
+ end
617
+
618
+ log("Retrieving existing manifests")
619
+
620
+ # Enumerate objects under the dists/<codename>/ prefix to find any
621
+ # Packages files and load them....
622
+
623
+ req = Deb::S3::Utils.s3.list_objects_v2({
624
+ :bucket => Deb::S3::Utils.bucket,
625
+ :prefix => Deb::S3::Utils.s3_path("dists/#{ options[:codename] }/"),
626
+ })
627
+
628
+ manifests = []
629
+ req.contents.each do |object|
630
+ if match = object.key.match(/dists\/([^\/]+)\/([^\/]+)\/binary-([^\/]+)\/Packages$/)
631
+ codename, component, arch = match.captures
632
+ manifests.push(Deb::S3::Manifest.retrieve(codename, component, arch, options[:cache_control], options[:fail_if_exists], options[:skip_package_upload]))
633
+ end
634
+ end
635
+
636
+ # Iterate over the packages in each manifest and build a Set of all the
637
+ # referenced URLs (relative to bucket root)...
638
+
639
+ refd_urls = Set[]
640
+ manifests.each do |manifest|
641
+ manifest.packages.each do |package|
642
+ refd_urls.add(Deb::S3::Utils.s3_path(package.url_filename(manifest.codename)))
643
+ end
644
+ end
645
+
646
+ log("Searching for unreferenced packages")
647
+
648
+ # Enumerate objects under the pools/<codename> prefix and delete any that
649
+ # arent referenced by any of the manifests.
650
+
651
+ continuation_token = nil
652
+ while true
653
+ req = Deb::S3::Utils.s3.list_objects_v2({
654
+ :bucket => Deb::S3::Utils.bucket,
655
+ :prefix => Deb::S3::Utils.s3_path("pool/#{ options[:codename] }/"),
656
+ :continuation_token => continuation_token,
657
+ })
658
+
659
+ req.contents.each do |object|
660
+ if not refd_urls.include?(object.key)
661
+ sublog("Deleting #{ object.key }")
662
+
663
+ Deb::S3::Utils.s3.delete_object({
664
+ :bucket => Deb::S3::Utils.bucket,
665
+ :key => object.key,
666
+ })
667
+ end
668
+ end
669
+
670
+ if req.is_truncated
671
+ continuation_token = req.next_continuation_token
672
+ else
673
+ break
674
+ end
675
+ end
676
+ ensure
677
+ if options[:lock] && @lock_acquired
678
+ Deb::S3::Lock.unlock(options[:codename], component, options[:arch], options[:cache_control])
679
+ log("Lock released.")
680
+ end
681
+ end
682
+ end
683
+
684
+ private
685
+
686
+ def component
687
+ return @component if @component
688
+ @component = if (section = options[:section])
689
+ warn("===> WARNING: The --section/-s argument is " \
690
+ "deprecated, please use --component/-m.")
691
+ section
692
+ else
693
+ options[:component]
694
+ end
695
+ end
696
+
697
+ def puts(*args)
698
+ $stdout.puts(*args) unless options[:quiet]
699
+ end
700
+
701
+ def log(message)
702
+ puts ">> #{message}" unless options[:quiet]
703
+ end
704
+
705
+ def sublog(message)
706
+ puts " -- #{message}" unless options[:quiet]
707
+ end
708
+
709
+ def error(message)
710
+ $stderr.puts "!! #{message}" unless options[:quiet]
711
+ exit 1
712
+ end
713
+
714
+ def provider
715
+ access_key_id = options[:access_key_id]
716
+ secret_access_key = options[:secret_access_key]
717
+ session_token = options[:session_token]
718
+
719
+ if access_key_id.nil? ^ secret_access_key.nil?
720
+ error("If you specify one of --access-key-id or --secret-access-key, you must specify the other.")
721
+ end
722
+ static_credentials = {}
723
+ static_credentials[:access_key_id] = access_key_id if access_key_id
724
+ static_credentials[:secret_access_key] = secret_access_key if secret_access_key
725
+ static_credentials[:session_token] = session_token if session_token
726
+
727
+ static_credentials
728
+ end
729
+
730
+ def configure_s3_client
731
+ error("No value provided for required options '--bucket'") unless options[:bucket]
732
+
733
+ settings = {
734
+ :region => options[:s3_region],
735
+ :http_proxy => options[:proxy_uri],
736
+ :force_path_style => options[:force_path_style]
737
+ }
738
+ settings[:endpoint] = options[:endpoint] if options[:endpoint]
739
+ settings.merge!(provider)
740
+
741
+ Deb::S3::Utils.s3 = Aws::S3::Client.new(settings)
742
+ Deb::S3::Utils.bucket = options[:bucket]
743
+ Deb::S3::Utils.signing_key = options[:sign]
744
+ Deb::S3::Utils.gpg_options = options[:gpg_options]
745
+ Deb::S3::Utils.prefix = options[:prefix]
746
+ Deb::S3::Utils.encryption = options[:encryption]
747
+
748
+ # make sure we have a valid visibility setting
749
+ Deb::S3::Utils.access_policy =
750
+ case options[:visibility]
751
+ when "public"
752
+ "public-read"
753
+ when "private"
754
+ "private"
755
+ when "authenticated"
756
+ "authenticated-read"
757
+ when "bucket_owner"
758
+ "bucket-owner-full-control"
759
+ else
760
+ error("Invalid visibility setting given. Can be public, private, authenticated, or bucket_owner.")
761
+ end
762
+ end
763
+ end