deb-s3-lock-fix 0.11.8.fix1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/lib/deb/s3/cli.rb ADDED
@@ -0,0 +1,763 @@
1
+ # -*- encoding : utf-8 -*-
2
+ require "aws-sdk-s3"
3
+ require "thor"
4
+
5
+ # Hack: aws requires this!
6
+ require "json"
7
+
8
+ require "deb/s3"
9
+ require "deb/s3/utils"
10
+ require "deb/s3/manifest"
11
+ require "deb/s3/package"
12
+ require "deb/s3/release"
13
+ require "deb/s3/lock"
14
+
15
+ class Deb::S3::CLI < Thor
16
+ class_option :bucket,
17
+ :type => :string,
18
+ :aliases => "-b",
19
+ :desc => "The name of the S3 bucket to upload to."
20
+
21
+ class_option :prefix,
22
+ :type => :string,
23
+ :desc => "The path prefix to use when storing on S3."
24
+
25
+ class_option :origin,
26
+ :type => :string,
27
+ :aliases => "-o",
28
+ :desc => "The origin to use in the repository Release file."
29
+
30
+ class_option :suite,
31
+ :type => :string,
32
+ :desc => "The suite to use in the repository Release file."
33
+
34
+ class_option :codename,
35
+ :default => "stable",
36
+ :type => :string,
37
+ :aliases => "-c",
38
+ :desc => "The codename of the APT repository."
39
+
40
+ class_option :component,
41
+ :default => "main",
42
+ :type => :string,
43
+ :aliases => "-m",
44
+ :desc => "The component of the APT repository."
45
+
46
+ class_option :section,
47
+ :type => :string,
48
+ :aliases => "-s",
49
+ :hide => true
50
+
51
+ class_option :access_key_id,
52
+ :type => :string,
53
+ :desc => "The access key for connecting to S3."
54
+
55
+ class_option :secret_access_key,
56
+ :type => :string,
57
+ :desc => "The secret key for connecting to S3."
58
+
59
+ class_option :session_token,
60
+ :type => :string,
61
+ :desc => "The (optional) session token for connecting to S3."
62
+
63
+ class_option :endpoint,
64
+ :type => :string,
65
+ :desc => "The URL endpoint to the S3 API."
66
+
67
+ class_option :s3_region,
68
+ :type => :string,
69
+ :desc => "The region for connecting to S3.",
70
+ :default => ENV["AWS_DEFAULT_REGION"] || "us-east-1"
71
+
72
+ class_option :force_path_style,
73
+ :default => false,
74
+ :type => :boolean,
75
+ :desc => "Use S3 path style instead of subdomains."
76
+
77
+ class_option :proxy_uri,
78
+ :type => :string,
79
+ :desc => "The URI of the proxy to send service requests through."
80
+
81
+ class_option :visibility,
82
+ :default => "public",
83
+ :type => :string,
84
+ :aliases => "-v",
85
+ :desc => "The access policy for the uploaded files. " +
86
+ "Can be public, private, or authenticated."
87
+
88
+ class_option :sign,
89
+ :repeatable => true,
90
+ :desc => "GPG Sign the Release file when uploading a package, " +
91
+ "or when verifying it after removing a package. " +
92
+ "Use --sign with your GPG key ID to use a specific key (--sign=6643C242C18FE05B)." +
93
+ "Can be specified multiple times for multiple singing keys."
94
+
95
+ class_option :gpg_options,
96
+ :default => "",
97
+ :type => :string,
98
+ :desc => "Additional command line options to pass to GPG when signing."
99
+
100
+ class_option :encryption,
101
+ :default => false,
102
+ :type => :boolean,
103
+ :aliases => "-e",
104
+ :desc => "Use S3 server side encryption."
105
+
106
+ class_option :quiet,
107
+ :type => :boolean,
108
+ :aliases => "-q",
109
+ :desc => "Doesn't output information, just returns status appropriately."
110
+
111
+ class_option :cache_control,
112
+ :type => :string,
113
+ :aliases => "-C",
114
+ :desc => "Add cache-control headers to S3 objects."
115
+
116
+ desc "upload FILES",
117
+ "Uploads the given files to a S3 bucket as an APT repository."
118
+
119
+ option :arch,
120
+ :type => :string,
121
+ :aliases => "-a",
122
+ :desc => "The architecture of the package in the APT repository."
123
+
124
+ option :preserve_versions,
125
+ :default => false,
126
+ :type => :boolean,
127
+ :aliases => "-p",
128
+ :desc => "Whether to preserve other versions of a package " +
129
+ "in the repository when uploading one."
130
+
131
+ option :lock,
132
+ :default => false,
133
+ :type => :boolean,
134
+ :aliases => "-l",
135
+ :desc => "Whether to check for an existing lock on the repository " +
136
+ "to prevent simultaneous updates "
137
+
138
+ option :fail_if_exists,
139
+ :default => false,
140
+ :type => :boolean,
141
+ :desc => "Whether to overwrite any existing package that has the same " +
142
+ "filename in the pool or the same name and version in the manifest but " +
143
+ "different contents."
144
+
145
+ option :skip_package_upload,
146
+ :default => false,
147
+ :type => :boolean,
148
+ :desc => "Whether to skip all package uploads." +
149
+ "This is useful when hosting .deb files outside of the bucket."
150
+
151
+ def upload(*files)
152
+ if files.nil? || files.empty?
153
+ error("You must specify at least one file to upload")
154
+ end
155
+
156
+ # make sure all the files exists
157
+ if missing_file = files.find { |pattern| Dir.glob(pattern).empty? }
158
+ error("File '#{missing_file}' doesn't exist")
159
+ end
160
+
161
+ # configure AWS::S3
162
+ configure_s3_client
163
+
164
+ begin
165
+ if options[:lock]
166
+ log("Checking for existing lock file")
167
+ log("Locking repository for updates")
168
+ Deb::S3::Lock.lock(options[:codename], component, options[:arch], options[:cache_control])
169
+ @lock_acquired = true
170
+ end
171
+
172
+ # retrieve the existing manifests
173
+ log("Retrieving existing manifests")
174
+ release = Deb::S3::Release.retrieve(options[:codename], options[:origin], options[:suite], options[:cache_control])
175
+ manifests = {}
176
+ release.architectures.each do |arch|
177
+ manifests[arch] = Deb::S3::Manifest.retrieve(options[:codename], component, arch, options[:cache_control], options[:fail_if_exists], options[:skip_package_upload])
178
+ end
179
+
180
+ packages_arch_all = []
181
+
182
+ # examine all the files
183
+ files.collect { |f| Dir.glob(f) }.flatten.each do |file|
184
+ log("Examining package file #{File.basename(file)}")
185
+ pkg = Deb::S3::Package.parse_file(file)
186
+
187
+ # copy over some options if they weren't given
188
+ arch = options[:arch] || pkg.architecture
189
+
190
+ # If they've specified an arch type that doesn't match the package let them know
191
+ if options.key?("arch") && options[:arch] != pkg.architecture
192
+ warn("You specified architecture #{options[:arch]} but package #{pkg.name} has architecture type of #{pkg.architecture}")
193
+ end
194
+
195
+ # validate we have them
196
+ error("No architcture given and unable to determine one for #{file}. " +
197
+ "Please specify one with --arch [i386|amd64|armhf].") unless arch
198
+
199
+ # If the arch is all and the list of existing manifests is none, then
200
+ # throw an error. This is mainly the case when initializing a brand new
201
+ # repository. With "all", we won't know which architectures they're using.
202
+ if arch == "all" && manifests.count == 0
203
+ manifests['amd64'] = Deb::S3::Manifest.retrieve(options[:codename], component,'amd64', options[:cache_control], options[:fail_if_exists], options[:skip_package_upload])
204
+ manifests['i386'] = Deb::S3::Manifest.retrieve(options[:codename], component,'i386', options[:cache_control], options[:fail_if_exists], options[:skip_package_upload])
205
+ manifests['armhf'] = Deb::S3::Manifest.retrieve(options[:codename], component,'armhf', options[:cache_control], options[:fail_if_exists], options[:skip_package_upload])
206
+
207
+ # error("Package #{File.basename(file)} had architecture \"all\", " +
208
+ # "however noexisting package lists exist. This can often happen " +
209
+ # "if the first package you are add to a new repository is an " +
210
+ # "\"all\" architecture file. Please use --arch [i386|amd64|armhf] or " +
211
+ # "another platform type to upload the file.")
212
+ end
213
+
214
+ # retrieve the manifest for the arch if we don't have it already
215
+ manifests[arch] ||= Deb::S3::Manifest.retrieve(options[:codename], component, arch, options[:cache_control], options[:fail_if_exists], options[:skip_package_upload])
216
+
217
+ # add package in manifests
218
+ begin
219
+ manifests[arch].add(pkg, options[:preserve_versions])
220
+ rescue Deb::S3::Utils::AlreadyExistsError => e
221
+ error("Preparing manifest failed because: #{e}")
222
+ end
223
+
224
+ # If arch is all, we must add this package in all arch available
225
+ if arch == 'all'
226
+ packages_arch_all << pkg
227
+ end
228
+ end
229
+
230
+ manifests.each do |arch, manifest|
231
+ next if arch == 'all'
232
+ packages_arch_all.each do |pkg|
233
+ begin
234
+ manifest.add(pkg, options[:preserve_versions], false)
235
+ rescue Deb::S3::Utils::AlreadyExistsError => e
236
+ error("Preparing manifest failed because: #{e}")
237
+ end
238
+ end
239
+ end
240
+
241
+ # upload the manifest
242
+ log("Uploading packages and new manifests to S3")
243
+ manifests.each_value do |manifest|
244
+ begin
245
+ manifest.write_to_s3 { |f| sublog("Transferring #{f}") }
246
+ rescue Deb::S3::Utils::AlreadyExistsError => e
247
+ error("Uploading manifest failed because: #{e}")
248
+ end
249
+ release.update_manifest(manifest)
250
+ end
251
+ release.write_to_s3 { |f| sublog("Transferring #{f}") }
252
+
253
+ log("Update complete.")
254
+ ensure
255
+ if options[:lock] && @lock_acquired
256
+ Deb::S3::Lock.unlock(options[:codename], component, options[:arch], options[:cache_control])
257
+ log("Lock released.")
258
+ end
259
+ end
260
+ end
261
+
262
+ desc "list", "Lists packages in given codename, component, and optionally architecture"
263
+
264
+ option :long,
265
+ :type => :boolean,
266
+ :aliases => '-l',
267
+ :desc => "Shows all package information in original format.",
268
+ :default => false
269
+
270
+ option :arch,
271
+ :type => :string,
272
+ :aliases => "-a",
273
+ :desc => "The architecture of the package in the APT repository."
274
+
275
+ def list
276
+ configure_s3_client
277
+
278
+ release = Deb::S3::Release.retrieve(options[:codename])
279
+ archs = release.architectures
280
+ archs &= [options[:arch]] if options[:arch] && options[:arch] != "all"
281
+ widths = [0, 0]
282
+ rows = archs.map { |arch|
283
+ manifest = Deb::S3::Manifest.retrieve(options[:codename], component,
284
+ arch, options[:cache_control],
285
+ false, false)
286
+ manifest.packages.map do |package|
287
+ if options[:long]
288
+ package.generate(options[:codename])
289
+ else
290
+ [package.name, package.full_version, package.architecture].tap do |row|
291
+ row.each_with_index do |col, i|
292
+ widths[i] = [widths[i], col.size].max if widths[i]
293
+ end
294
+ end
295
+ end
296
+ end
297
+ }.flatten(1)
298
+
299
+ if options[:long]
300
+ $stdout.puts rows.join("\n")
301
+ else
302
+ rows.each do |row|
303
+ $stdout.puts "% -#{widths[0]}s % -#{widths[1]}s %s" % row
304
+ end
305
+ end
306
+ end
307
+
308
+ desc "show PACKAGE VERSION ARCH", "Shows information about a package."
309
+
310
+ def show(package_name, version, arch)
311
+ if version.nil?
312
+ error "You must specify the name of the package to show."
313
+ end
314
+ if version.nil?
315
+ error "You must specify the version of the package to show."
316
+ end
317
+ if arch.nil?
318
+ error "You must specify the architecture of the package to show."
319
+ end
320
+
321
+ configure_s3_client
322
+
323
+ # retrieve the existing manifests
324
+ manifest = Deb::S3::Manifest.retrieve(options[:codename], component, arch,
325
+ options[:cache_control], false, false)
326
+ package = manifest.packages.detect { |p|
327
+ p.name == package_name && p.full_version == version
328
+ }
329
+ if package.nil?
330
+ error "No such package found."
331
+ end
332
+
333
+ puts package.generate(options[:codename])
334
+ end
335
+
336
+ desc "copy PACKAGE TO_CODENAME TO_COMPONENT ",
337
+ "Copy the package named PACKAGE to given codename and component. If --versions is not specified, copy all versions of PACKAGE. Otherwise, only the specified versions will be copied. Source codename and component is given by --codename and --component options."
338
+
339
+ option :cache_control,
340
+ :type => :string,
341
+ :aliases => "-C",
342
+ :desc => "Add cache-control headers to S3 objects."
343
+
344
+ option :arch,
345
+ :type => :string,
346
+ :aliases => "-a",
347
+ :desc => "The architecture of the package in the APT repository."
348
+
349
+ option :lock,
350
+ :default => false,
351
+ :type => :boolean,
352
+ :aliases => "-l",
353
+ :desc => "Whether to check for an existing lock on the repository " +
354
+ "to prevent simultaneous updates "
355
+
356
+ option :versions,
357
+ :default => nil,
358
+ :type => :array,
359
+ :desc => "The space-delimited versions of PACKAGE to delete. If not " +
360
+ "specified, ALL VERSIONS will be deleted. Fair warning. " +
361
+ "E.g. --versions \"0.1 0.2 0.3\""
362
+
363
+ option :preserve_versions,
364
+ :default => false,
365
+ :type => :boolean,
366
+ :aliases => "-p",
367
+ :desc => "Whether to preserve other versions of a package " +
368
+ "in the repository when uploading one."
369
+
370
+ option :fail_if_exists,
371
+ :default => true,
372
+ :type => :boolean,
373
+ :desc => "Whether to overwrite any existing package that has the same " +
374
+ "filename in the pool or the same name and version in the manifest."
375
+
376
+ def copy(package_name, to_codename, to_component)
377
+ if package_name.nil?
378
+ error "You must specify a package name."
379
+ end
380
+ if to_codename.nil?
381
+ error "You must specify a codename to copy to."
382
+ end
383
+ if to_component.nil?
384
+ error "You must specify a component to copy to."
385
+ end
386
+
387
+ arch = options[:arch]
388
+ if arch.nil?
389
+ error "You must specify the architecture of the package to copy."
390
+ end
391
+
392
+ versions = options[:versions]
393
+ if versions.nil?
394
+ warn "===> WARNING: Copying all versions of #{package_name}"
395
+ else
396
+ log "Versions to copy: #{versions.join(', ')}"
397
+ end
398
+
399
+ configure_s3_client
400
+
401
+ begin
402
+ if options[:lock]
403
+ log("Checking for existing lock file")
404
+ log("Locking repository for updates")
405
+ Deb::S3::Lock.lock(options[:codename], to_component, options[:arch], options[:cache_control])
406
+ @lock_acquired = true
407
+ end
408
+
409
+ # retrieve the existing manifests
410
+ log "Retrieving existing manifests"
411
+ from_manifest = Deb::S3::Manifest.retrieve(options[:codename],
412
+ component, arch,
413
+ options[:cache_control],
414
+ false, options[:skip_package_upload])
415
+ to_release = Deb::S3::Release.retrieve(to_codename)
416
+ to_manifest = Deb::S3::Manifest.retrieve(to_codename, to_component, arch,
417
+ options[:cache_control],
418
+ options[:fail_if_exists],
419
+ options[:skip_package_upload])
420
+ packages = from_manifest.packages.select { |p|
421
+ p.name == package_name &&
422
+ (versions.nil? || versions.include?(p.full_version))
423
+ }
424
+ if packages.size == 0
425
+ error "No packages found in repository."
426
+ end
427
+
428
+ packages.each do |package|
429
+ begin
430
+ to_manifest.add package, options[:preserve_versions], false
431
+ rescue Deb::S3::Utils::AlreadyExistsError => e
432
+ error("Preparing manifest failed because: #{e}")
433
+ end
434
+ end
435
+
436
+ begin
437
+ to_manifest.write_to_s3 { |f| sublog("Transferring #{f}") }
438
+ rescue Deb::S3::Utils::AlreadyExistsError => e
439
+ error("Copying manifest failed because: #{e}")
440
+ end
441
+ to_release.update_manifest(to_manifest)
442
+ to_release.write_to_s3 { |f| sublog("Transferring #{f}") }
443
+
444
+ log "Copy complete."
445
+ ensure
446
+ if options[:lock] && @lock_acquired
447
+ Deb::S3::Lock.unlock(options[:codename], component, options[:arch], options[:cache_control])
448
+ log("Lock released.")
449
+ end
450
+ end
451
+ end
452
+
453
+ desc "delete PACKAGE",
454
+ "Remove the package named PACKAGE. If --versions is not specified, delete" +
455
+ "all versions of PACKAGE. Otherwise, only the specified versions will be " +
456
+ "deleted."
457
+
458
+ option :arch,
459
+ :type => :string,
460
+ :aliases => "-a",
461
+ :desc => "The architecture of the package in the APT repository."
462
+
463
+ option :lock,
464
+ :default => false,
465
+ :type => :boolean,
466
+ :aliases => "-l",
467
+ :desc => "Whether to check for an existing lock on the repository " +
468
+ "to prevent simultaneous updates "
469
+
470
+ option :versions,
471
+ :default => nil,
472
+ :type => :array,
473
+ :desc => "The space-delimited versions of PACKAGE to delete. If not " +
474
+ "specified, ALL VERSIONS will be deleted. Fair warning. " +
475
+ "E.g. --versions \"0.1 0.2 0.3\""
476
+
477
+ def delete(package)
478
+ if package.nil?
479
+ error("You must specify a package name.")
480
+ end
481
+
482
+ versions = options[:versions]
483
+ if versions.nil?
484
+ warn("===> WARNING: Deleting all versions of #{package}")
485
+ else
486
+ log("Versions to delete: #{versions.join(', ')}")
487
+ end
488
+
489
+ arch = options[:arch]
490
+ if arch.nil?
491
+ error("You must specify the architecture of the package to remove.")
492
+ end
493
+
494
+ configure_s3_client
495
+
496
+ begin
497
+ if options[:lock]
498
+ log("Checking for existing lock file")
499
+ log("Locking repository for updates")
500
+ Deb::S3::Lock.lock(options[:codename], component, options[:arch], options[:cache_control])
501
+ @lock_acquired = true
502
+ end
503
+
504
+ # retrieve the existing manifests
505
+ log("Retrieving existing manifests")
506
+ release = Deb::S3::Release.retrieve(options[:codename], options[:origin], options[:suite])
507
+ if arch == 'all'
508
+ selected_arch = release.architectures
509
+ else
510
+ selected_arch = [arch]
511
+ end
512
+ all_found = 0
513
+ selected_arch.each { |ar|
514
+ manifest = Deb::S3::Manifest.retrieve(options[:codename], component, ar, options[:cache_control], false, options[:skip_package_upload])
515
+
516
+ deleted = manifest.delete_package(package, versions)
517
+ all_found += deleted.length
518
+ if deleted.length == 0
519
+ if versions.nil?
520
+ sublog("No packages were deleted. #{package} not found in arch #{ar}.")
521
+ next
522
+ else
523
+ sublog("No packages were deleted. #{package} versions #{versions.join(', ')} could not be found in arch #{ar}.")
524
+ next
525
+ end
526
+ else
527
+ deleted.each { |p|
528
+ sublog("Deleting #{p.name} version #{p.full_version} from arch #{ar}")
529
+ }
530
+ end
531
+
532
+ log("Uploading new manifests to S3")
533
+ manifest.write_to_s3 {|f| sublog("Transferring #{f}") }
534
+ release.update_manifest(manifest)
535
+ release.write_to_s3 {|f| sublog("Transferring #{f}") }
536
+
537
+ log("Update complete.")
538
+ }
539
+ if all_found == 0
540
+ if versions.nil?
541
+ error("No packages were deleted. #{package} not found.")
542
+ else
543
+ error("No packages were deleted. #{package} versions #{versions.join(', ')} could not be found.")
544
+ end
545
+ end
546
+ ensure
547
+ if options[:lock] && @lock_acquired
548
+ Deb::S3::Lock.unlock(options[:codename], component, options[:arch], options[:cache_control])
549
+ log("Lock released.")
550
+ end
551
+ end
552
+ end
553
+
554
+
555
+ desc "verify", "Verifies that the files in the package manifests exist"
556
+
557
+ option :fix_manifests,
558
+ :default => false,
559
+ :type => :boolean,
560
+ :aliases => "-f",
561
+ :desc => "Whether to fix problems in manifests when verifying."
562
+
563
+ def verify
564
+ configure_s3_client
565
+
566
+ log("Retrieving existing manifests")
567
+ release = Deb::S3::Release.retrieve(options[:codename], options[:origin], options[:suite])
568
+
569
+ release.architectures.each do |arch|
570
+ log("Checking for missing packages in: #{options[:codename]}/#{options[:component]} #{arch}")
571
+ manifest = Deb::S3::Manifest.retrieve(options[:codename], component,
572
+ arch, options[:cache_control], false,
573
+ options[:skip_package_upload])
574
+ missing_packages = []
575
+
576
+ manifest.packages.each do |p|
577
+ unless Deb::S3::Utils.s3_exists? p.url_filename(options[:codename])
578
+ sublog("The following packages are missing:\n\n") if missing_packages.empty?
579
+ puts(p.generate(options[:codename]))
580
+ puts("")
581
+
582
+ missing_packages << p
583
+ end
584
+ end
585
+
586
+ if options[:sign] || (options[:fix_manifests] && !missing_packages.empty?)
587
+ log("Removing #{missing_packages.length} package(s) from the manifest...")
588
+ missing_packages.each { |p| manifest.packages.delete(p) }
589
+ manifest.write_to_s3 { |f| sublog("Transferring #{f}") }
590
+ release.update_manifest(manifest)
591
+ release.write_to_s3 { |f| sublog("Transferring #{f}") }
592
+
593
+ log("Update complete.")
594
+ end
595
+ end
596
+ end
597
+
598
+ desc "clean", "Delete packages from the pool which are no longer referenced"
599
+
600
+ option :lock,
601
+ :default => false,
602
+ :type => :boolean,
603
+ :aliases => "-l",
604
+ :desc => "Whether to check for an existing lock on the repository " +
605
+ "to prevent simultaneous updates "
606
+
607
+ def clean
608
+ configure_s3_client
609
+
610
+ begin
611
+ if options[:lock]
612
+ log("Checking for existing lock file")
613
+ log("Locking repository for updates")
614
+ Deb::S3::Lock.lock(options[:codename], component, options[:arch], options[:cache_control])
615
+ @lock_acquired = true
616
+ end
617
+
618
+ log("Retrieving existing manifests")
619
+
620
+ # Enumerate objects under the dists/<codename>/ prefix to find any
621
+ # Packages files and load them....
622
+
623
+ req = Deb::S3::Utils.s3.list_objects_v2({
624
+ :bucket => Deb::S3::Utils.bucket,
625
+ :prefix => Deb::S3::Utils.s3_path("dists/#{ options[:codename] }/"),
626
+ })
627
+
628
+ manifests = []
629
+ req.contents.each do |object|
630
+ if match = object.key.match(/dists\/([^\/]+)\/([^\/]+)\/binary-([^\/]+)\/Packages$/)
631
+ codename, component, arch = match.captures
632
+ manifests.push(Deb::S3::Manifest.retrieve(codename, component, arch, options[:cache_control], options[:fail_if_exists], options[:skip_package_upload]))
633
+ end
634
+ end
635
+
636
+ # Iterate over the packages in each manifest and build a Set of all the
637
+ # referenced URLs (relative to bucket root)...
638
+
639
+ refd_urls = Set[]
640
+ manifests.each do |manifest|
641
+ manifest.packages.each do |package|
642
+ refd_urls.add(Deb::S3::Utils.s3_path(package.url_filename(manifest.codename)))
643
+ end
644
+ end
645
+
646
+ log("Searching for unreferenced packages")
647
+
648
+ # Enumerate objects under the pools/<codename> prefix and delete any that
649
+ # arent referenced by any of the manifests.
650
+
651
+ continuation_token = nil
652
+ while true
653
+ req = Deb::S3::Utils.s3.list_objects_v2({
654
+ :bucket => Deb::S3::Utils.bucket,
655
+ :prefix => Deb::S3::Utils.s3_path("pool/#{ options[:codename] }/"),
656
+ :continuation_token => continuation_token,
657
+ })
658
+
659
+ req.contents.each do |object|
660
+ if not refd_urls.include?(object.key)
661
+ sublog("Deleting #{ object.key }")
662
+
663
+ Deb::S3::Utils.s3.delete_object({
664
+ :bucket => Deb::S3::Utils.bucket,
665
+ :key => object.key,
666
+ })
667
+ end
668
+ end
669
+
670
+ if req.is_truncated
671
+ continuation_token = req.next_continuation_token
672
+ else
673
+ break
674
+ end
675
+ end
676
+ ensure
677
+ if options[:lock] && @lock_acquired
678
+ Deb::S3::Lock.unlock(options[:codename], component, options[:arch], options[:cache_control])
679
+ log("Lock released.")
680
+ end
681
+ end
682
+ end
683
+
684
+ private
685
+
686
+ def component
687
+ return @component if @component
688
+ @component = if (section = options[:section])
689
+ warn("===> WARNING: The --section/-s argument is " \
690
+ "deprecated, please use --component/-m.")
691
+ section
692
+ else
693
+ options[:component]
694
+ end
695
+ end
696
+
697
+ def puts(*args)
698
+ $stdout.puts(*args) unless options[:quiet]
699
+ end
700
+
701
+ def log(message)
702
+ puts ">> #{message}" unless options[:quiet]
703
+ end
704
+
705
+ def sublog(message)
706
+ puts " -- #{message}" unless options[:quiet]
707
+ end
708
+
709
+ def error(message)
710
+ $stderr.puts "!! #{message}" unless options[:quiet]
711
+ exit 1
712
+ end
713
+
714
+ def provider
715
+ access_key_id = options[:access_key_id]
716
+ secret_access_key = options[:secret_access_key]
717
+ session_token = options[:session_token]
718
+
719
+ if access_key_id.nil? ^ secret_access_key.nil?
720
+ error("If you specify one of --access-key-id or --secret-access-key, you must specify the other.")
721
+ end
722
+ static_credentials = {}
723
+ static_credentials[:access_key_id] = access_key_id if access_key_id
724
+ static_credentials[:secret_access_key] = secret_access_key if secret_access_key
725
+ static_credentials[:session_token] = session_token if session_token
726
+
727
+ static_credentials
728
+ end
729
+
730
+ def configure_s3_client
731
+ error("No value provided for required options '--bucket'") unless options[:bucket]
732
+
733
+ settings = {
734
+ :region => options[:s3_region],
735
+ :http_proxy => options[:proxy_uri],
736
+ :force_path_style => options[:force_path_style]
737
+ }
738
+ settings[:endpoint] = options[:endpoint] if options[:endpoint]
739
+ settings.merge!(provider)
740
+
741
+ Deb::S3::Utils.s3 = Aws::S3::Client.new(settings)
742
+ Deb::S3::Utils.bucket = options[:bucket]
743
+ Deb::S3::Utils.signing_key = options[:sign]
744
+ Deb::S3::Utils.gpg_options = options[:gpg_options]
745
+ Deb::S3::Utils.prefix = options[:prefix]
746
+ Deb::S3::Utils.encryption = options[:encryption]
747
+
748
+ # make sure we have a valid visibility setting
749
+ Deb::S3::Utils.access_policy =
750
+ case options[:visibility]
751
+ when "public"
752
+ "public-read"
753
+ when "private"
754
+ "private"
755
+ when "authenticated"
756
+ "authenticated-read"
757
+ when "bucket_owner"
758
+ "bucket-owner-full-control"
759
+ else
760
+ error("Invalid visibility setting given. Can be public, private, authenticated, or bucket_owner.")
761
+ end
762
+ end
763
+ end