deb-s3 0.8.0 → 0.9.1

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 7e1bc9874f303cb3ec4a83a5dd1edf8948bf62bd
4
- data.tar.gz: d2a16561f73b2dfd0c324f99a4336a6082f5a0e3
3
+ metadata.gz: 0fdcdc9b5f39dd0c26eaa1c9045dc5ce520697b4
4
+ data.tar.gz: 743c9ae0a9368dac628bbfdee71da2c6606b411b
5
5
  SHA512:
6
- metadata.gz: 54c58ff8308b6c8a1f219df3fbb21d6a9693917f2a853b1be44269443f84f5ffc4ff66e6270d23c98651adb8c430dddc9351eccca317b60279d7977112e2c272
7
- data.tar.gz: c1a01cae8a4b91996e8d919cf41a8a2c8a4e6abe4ceb2c90496081fa5f9f77430472bb39067f65feaa5bf40305bf84043aa6f74bff0b9db234af868708e929c1
6
+ metadata.gz: a2eacf16e2fd89c4af49c9229b0cf185d0de6efeaa70bbe5543b93a83c11d651abc838734560a36dd54703b56a51ad15a53cf87f479d3c7645e366714fe0340e
7
+ data.tar.gz: b3d97c88ef06bd7f908173e6e6ab466a98482231400f46f7e914d9a4f3a9564eab80e610c3df7939e1e15557828d1cf0f83deb11fd00312f2313406c6f7e9c5c
@@ -1,6 +1,6 @@
1
1
  # -*- encoding : utf-8 -*-
2
2
  module Deb
3
3
  module S3
4
- VERSION = "0.8.0"
4
+ VERSION = "0.9.1"
5
5
  end
6
6
  end
@@ -1,5 +1,5 @@
1
1
  # -*- encoding : utf-8 -*-
2
- require "aws"
2
+ require "aws-sdk"
3
3
  require "thor"
4
4
 
5
5
  # Hack: aws requires this!
@@ -56,10 +56,10 @@ class Deb::S3::CLI < Thor
56
56
  :type => :string,
57
57
  :desc => "The secret key for connecting to S3."
58
58
 
59
- class_option :endpoint,
59
+ class_option :s3_region,
60
60
  :type => :string,
61
- :desc => "The region endpoint for connecting to S3.",
62
- :default => "s3.amazonaws.com"
61
+ :desc => "The region for connecting to S3.",
62
+ :default => "us-east-1"
63
63
 
64
64
  class_option :force_path_style,
65
65
  :default => false,
@@ -70,10 +70,10 @@ class Deb::S3::CLI < Thor
70
70
  :type => :string,
71
71
  :desc => "The URI of the proxy to send service requests through."
72
72
 
73
- class_option :use_ssl,
74
- :default => true,
75
- :type => :boolean,
76
- :desc => "Whether to use HTTP or HTTPS for request transport."
73
+ #class_option :use_ssl,
74
+ #:default => true,
75
+ #:type => :boolean,
76
+ #:desc => "Whether to use HTTP or HTTPS for request transport."
77
77
 
78
78
  class_option :visibility,
79
79
  :default => "public",
@@ -137,6 +137,12 @@ class Deb::S3::CLI < Thor
137
137
  :desc => "Whether to overwrite any existing package that has the same " +
138
138
  "filename in the pool or the same name and version in the manifest."
139
139
 
140
+ option :skip_package_upload,
141
+ :default => false,
142
+ :type => :boolean,
143
+ :desc => "Whether to skip all package uploads." +
144
+ "This is useful when hosting .deb files outside of the bucket."
145
+
140
146
  def upload(*files)
141
147
  if files.nil? || files.empty?
142
148
  error("You must specify at least one file to upload")
@@ -169,7 +175,7 @@ class Deb::S3::CLI < Thor
169
175
  release = Deb::S3::Release.retrieve(options[:codename], options[:origin], options[:suite], options[:cache_control])
170
176
  manifests = {}
171
177
  release.architectures.each do |arch|
172
- manifests[arch] = Deb::S3::Manifest.retrieve(options[:codename], component, arch, options[:cache_control], options[:fail_if_exists])
178
+ manifests[arch] = Deb::S3::Manifest.retrieve(options[:codename], component, arch, options[:cache_control], options[:fail_if_exists], options[:skip_package_upload])
173
179
  end
174
180
 
175
181
  packages_arch_all = []
@@ -203,7 +209,7 @@ class Deb::S3::CLI < Thor
203
209
  end
204
210
 
205
211
  # retrieve the manifest for the arch if we don't have it already
206
- manifests[arch] ||= Deb::S3::Manifest.retrieve(options[:codename], component, arch, options[:cache_control], options[:fail_if_exists])
212
+ manifests[arch] ||= Deb::S3::Manifest.retrieve(options[:codename], component, arch, options[:cache_control], options[:fail_if_exists], options[:skip_package_upload])
207
213
 
208
214
  # add package in manifests
209
215
  begin
@@ -273,7 +279,7 @@ class Deb::S3::CLI < Thor
273
279
  rows = archs.map { |arch|
274
280
  manifest = Deb::S3::Manifest.retrieve(options[:codename], component,
275
281
  arch, options[:cache_control],
276
- false)
282
+ false, false)
277
283
  manifest.packages.map do |package|
278
284
  if options[:long]
279
285
  package.generate
@@ -313,7 +319,7 @@ class Deb::S3::CLI < Thor
313
319
 
314
320
  # retrieve the existing manifests
315
321
  manifest = Deb::S3::Manifest.retrieve(options[:codename], component, arch,
316
- options[:cache_control], false)
322
+ options[:cache_control], false, false)
317
323
  package = manifest.packages.detect { |p|
318
324
  p.name == package_name && p.full_version == version
319
325
  }
@@ -387,11 +393,12 @@ class Deb::S3::CLI < Thor
387
393
  from_manifest = Deb::S3::Manifest.retrieve(options[:codename],
388
394
  component, arch,
389
395
  options[:cache_control],
390
- false)
396
+ false, options[:skip_package_upload])
391
397
  to_release = Deb::S3::Release.retrieve(to_codename)
392
398
  to_manifest = Deb::S3::Manifest.retrieve(to_codename, to_component, arch,
393
399
  options[:cache_control],
394
- options[:fail_if_exists])
400
+ options[:fail_if_exists],
401
+ options[:skip_package_upload])
395
402
  packages = from_manifest.packages.select { |p|
396
403
  p.name == package_name &&
397
404
  (versions.nil? || versions.include?(p.full_version))
@@ -458,7 +465,7 @@ class Deb::S3::CLI < Thor
458
465
  # retrieve the existing manifests
459
466
  log("Retrieving existing manifests")
460
467
  release = Deb::S3::Release.retrieve(options[:codename], options[:origin], options[:suite])
461
- manifest = Deb::S3::Manifest.retrieve(options[:codename], component, options[:arch], options[:cache_control], false)
468
+ manifest = Deb::S3::Manifest.retrieve(options[:codename], component, options[:arch], options[:cache_control], false, options[:skip_package_upload])
462
469
 
463
470
  deleted = manifest.delete_package(package, versions)
464
471
  if deleted.length == 0
@@ -499,7 +506,8 @@ class Deb::S3::CLI < Thor
499
506
  release.architectures.each do |arch|
500
507
  log("Checking for missing packages in: #{options[:codename]}/#{options[:component]} #{arch}")
501
508
  manifest = Deb::S3::Manifest.retrieve(options[:codename], component,
502
- arch, options[:cache_control], false)
509
+ arch, options[:cache_control], false,
510
+ options[:skip_package_upload])
503
511
  missing_packages = []
504
512
 
505
513
  manifest.packages.each do |p|
@@ -561,26 +569,24 @@ class Deb::S3::CLI < Thor
561
569
  if access_key_id.nil? ^ secret_access_key.nil?
562
570
  error("If you specify one of --access-key-id or --secret-access-key, you must specify the other.")
563
571
  end
564
-
565
572
  static_credentials = {}
566
573
  static_credentials[:access_key_id] = access_key_id if access_key_id
567
574
  static_credentials[:secret_access_key] = secret_access_key if secret_access_key
568
575
 
569
- AWS::Core::CredentialProviders::DefaultProvider.new(static_credentials)
576
+ static_credentials
570
577
  end
571
578
 
572
579
  def configure_s3_client
573
580
  error("No value provided for required options '--bucket'") unless options[:bucket]
574
581
 
575
582
  settings = {
576
- :s3_endpoint => options[:endpoint],
577
- :proxy_uri => options[:proxy_uri],
578
- :use_ssl => options[:use_ssl],
579
- :s3_force_path_style => options[:force_path_style]
583
+ :region => options[:s3_region],
584
+ :http_proxy => options[:proxy_uri],
585
+ :force_path_style => options[:force_path_style]
580
586
  }
581
- settings.merge!(provider.credentials)
587
+ settings.merge!(provider)
582
588
 
583
- Deb::S3::Utils.s3 = AWS::S3.new(settings)
589
+ Deb::S3::Utils.s3 = Aws::S3::Client.new(settings)
584
590
  Deb::S3::Utils.bucket = options[:bucket]
585
591
  Deb::S3::Utils.signing_key = options[:sign]
586
592
  Deb::S3::Utils.gpg_options = options[:gpg_options]
@@ -591,13 +597,13 @@ class Deb::S3::CLI < Thor
591
597
  Deb::S3::Utils.access_policy =
592
598
  case options[:visibility]
593
599
  when "public"
594
- :public_read
600
+ "public-read"
595
601
  when "private"
596
- :private
602
+ "private"
597
603
  when "authenticated"
598
- :authenticated_read
604
+ "authenticated-read"
599
605
  when "bucket_owner"
600
- :bucket_owner_full_control
606
+ "bucket-owner-full-control"
601
607
  else
602
608
  error("Invalid visibility setting given. Can be public, private, authenticated, or bucket_owner.")
603
609
  end
@@ -12,6 +12,7 @@ class Deb::S3::Manifest
12
12
  attr_accessor :cache_control
13
13
  attr_accessor :architecture
14
14
  attr_accessor :fail_if_exists
15
+ attr_accessor :skip_package_upload
15
16
 
16
17
  attr_accessor :files
17
18
 
@@ -26,10 +27,11 @@ class Deb::S3::Manifest
26
27
  @files = {}
27
28
  @cache_control = ""
28
29
  @fail_if_exists = false
30
+ @skip_package_upload = false
29
31
  end
30
32
 
31
33
  class << self
32
- def retrieve(codename, component, architecture, cache_control, fail_if_exists)
34
+ def retrieve(codename, component, architecture, cache_control, fail_if_exists, skip_package_upload=false)
33
35
  m = if s = Deb::S3::Utils.s3_read("dists/#{codename}/#{component}/binary-#{architecture}/Packages")
34
36
  self.parse_packages(s)
35
37
  else
@@ -41,6 +43,7 @@ class Deb::S3::Manifest
41
43
  m.architecture = architecture
42
44
  m.cache_control = cache_control
43
45
  m.fail_if_exists = fail_if_exists
46
+ m.skip_package_upload = skip_package_upload
44
47
  m
45
48
  end
46
49
 
@@ -56,7 +59,15 @@ class Deb::S3::Manifest
56
59
 
57
60
  def add(pkg, preserve_versions, needs_uploading=true)
58
61
  if self.fail_if_exists
59
- packages.each { |p| raise AlreadyExistsError, "package #{pkg.name}_#{pkg.full_version} already exists with different filename (#{p.url_filename})" if p.name == pkg.name && p.full_version == pkg.full_version && File.basename(p.url_filename) != File.basename(pkg.filename) }
62
+ packages.each { |p|
63
+ next unless p.name == pkg.name && \
64
+ p.full_version == pkg.full_version && \
65
+ File.basename(p.url_filename) != \
66
+ File.basename(pkg.url_filename)
67
+ raise AlreadyExistsError,
68
+ "package #{pkg.name}_#{pkg.full_version} already exists " \
69
+ "with different filename (#{p.url_filename})"
70
+ }
60
71
  end
61
72
  if preserve_versions
62
73
  packages.delete_if { |p| p.name == pkg.name && p.full_version == pkg.full_version }
@@ -91,10 +102,12 @@ class Deb::S3::Manifest
91
102
  def write_to_s3
92
103
  manifest = self.generate
93
104
 
94
- # store any packages that need to be stored
95
- @packages_to_be_upload.each do |pkg|
96
- yield pkg.url_filename if block_given?
97
- s3_store(pkg.filename, pkg.url_filename, 'application/octet-stream; charset=binary', self.cache_control, self.fail_if_exists)
105
+ unless self.skip_package_upload
106
+ # store any packages that need to be stored
107
+ @packages_to_be_upload.each do |pkg|
108
+ yield pkg.url_filename if block_given?
109
+ s3_store(pkg.filename, pkg.url_filename, 'application/octet-stream; charset=binary', self.cache_control, self.fail_if_exists)
110
+ end
98
111
  end
99
112
 
100
113
  # generate the Packages file
@@ -31,10 +31,10 @@ class Deb::S3::Release
31
31
  rel = self.parse_release(s)
32
32
  else
33
33
  rel = self.new
34
- rel.codename = codename
35
- rel.origin = origin unless origin.nil?
36
- rel.suite = suite unless suite.nil?
37
34
  end
35
+ rel.codename = codename
36
+ rel.origin = origin unless origin.nil?
37
+ rel.suite = suite unless suite.nil?
38
38
  rel.cache_control = cache_control
39
39
  rel
40
40
  end
@@ -103,15 +103,28 @@ class Deb::S3::Release
103
103
  # sign the file, if necessary
104
104
  if Deb::S3::Utils.signing_key
105
105
  key_param = Deb::S3::Utils.signing_key != "" ? "--default-key=#{Deb::S3::Utils.signing_key}" : ""
106
- if system("gpg -a #{key_param} #{Deb::S3::Utils.gpg_options} -b #{release_tmp.path}")
107
- local_file = release_tmp.path+".asc"
108
- remote_file = self.filename+".gpg"
109
- yield remote_file if block_given?
110
- raise "Unable to locate Release signature file" unless File.exists?(local_file)
111
- s3_store(local_file, remote_file, 'application/pgp-signature; charset=us-ascii', self.cache_control)
112
- File.unlink(local_file)
106
+ if self.codename == "xenial"
107
+ if system("gpg -a #{key_param} #{Deb::S3::Utils.gpg_options} -s --clearsign #{release_tmp.path}")
108
+ local_file = release_tmp.path+".asc"
109
+ remote_file = "dists/#{@codename}/InRelease"
110
+ yield remote_file if block_given?
111
+ raise "Unable to locate InRelease file" unless File.exists?(local_file)
112
+ s3_store(local_file, remote_file, 'application/pgp-signature; charset=us-ascii', self.cache_control)
113
+ File.unlink(local_file)
114
+ else
115
+ raise "Signing the InRelease file failed."
116
+ end
113
117
  else
114
- raise "Signing the Release file failed."
118
+ if system("gpg -a #{key_param} #{Deb::S3::Utils.gpg_options} -b #{release_tmp.path}")
119
+ local_file = release_tmp.path+".asc"
120
+ remote_file = self.filename+".gpg"
121
+ yield remote_file if block_given?
122
+ raise "Unable to locate Release signature file" unless File.exists?(local_file)
123
+ s3_store(local_file, remote_file, 'application/pgp-signature; charset=us-ascii', self.cache_control)
124
+ File.unlink(local_file)
125
+ else
126
+ raise "Signing the Release file failed."
127
+ end
115
128
  end
116
129
  else
117
130
  # remove an existing Release.gpg, if it was there
@@ -56,27 +56,42 @@ module Deb::S3::Utils
56
56
  end
57
57
 
58
58
  def s3_exists?(path)
59
- Deb::S3::Utils.s3.buckets[Deb::S3::Utils.bucket].objects[s3_path(path)].exists?
59
+ Deb::S3::Utils.s3.head_object(
60
+ :bucket => Deb::S3::Utils.bucket,
61
+ :key => s3_path(path),
62
+ )
63
+ rescue Aws::S3::Errors::NotFound
64
+ false
60
65
  end
61
66
 
62
67
  def s3_read(path)
63
- return nil unless s3_exists?(path)
64
- Deb::S3::Utils.s3.buckets[Deb::S3::Utils.bucket].objects[s3_path(path)].read
68
+ Deb::S3::Utils.s3.get_object(
69
+ :bucket => Deb::S3::Utils.bucket,
70
+ :key => s3_path(path),
71
+ )[:body].read
72
+ rescue Aws::S3::Errors::NoSuchKey
73
+ false
65
74
  end
66
75
 
67
76
  def s3_store(path, filename=nil, content_type='application/octet-stream; charset=binary', cache_control=nil, fail_if_exists=false)
68
77
  filename = File.basename(path) unless filename
69
- obj = Deb::S3::Utils.s3.buckets[Deb::S3::Utils.bucket].objects[s3_path(filename)]
78
+ obj = s3_exists?(path)
70
79
 
71
80
  file_md5 = Digest::MD5.file(path)
72
81
 
73
82
  # check if the object already exists
74
- if obj.exists?
75
- return if (file_md5.to_s == obj.etag.gsub('"', '') or file_md5.to_s == obj.metadata['md5'])
83
+ if obj != false
84
+ return if (file_md5.to_s == obj[:etag].gsub('"', '') or file_md5.to_s == obj[:metadata]['md5'])
76
85
  raise AlreadyExistsError, "file #{obj.public_url} already exists with different contents" if fail_if_exists
77
86
  end
78
87
 
79
- options = {:acl => Deb::S3::Utils.access_policy, :content_type => content_type, :metadata => {'md5' => file_md5}}
88
+ options = {
89
+ :bucket => Deb::S3::Utils.bucket,
90
+ :key => s3_path(filename),
91
+ :acl => Deb::S3::Utils.access_policy,
92
+ :content_type => content_type,
93
+ :metadata => { "md5" => file_md5.to_s },
94
+ }
80
95
  if !cache_control.nil?
81
96
  options[:cache_control] = cache_control
82
97
  end
@@ -85,10 +100,18 @@ module Deb::S3::Utils
85
100
  options[:server_side_encryption] = :aes256 if Deb::S3::Utils.encryption
86
101
 
87
102
  # upload the file
88
- obj.write(Pathname.new(path), options)
103
+ File.open(path) do |f|
104
+ options[:body] = f
105
+ Deb::S3::Utils.s3.put_object(options)
106
+ end
89
107
  end
90
108
 
91
109
  def s3_remove(path)
92
- Deb::S3::Utils.s3.buckets[Deb::S3::Utils.bucket].objects[s3_path(path)].delete if s3_exists?(path)
110
+ if s3_exists?(path)
111
+ Deb::S3::Utils.s3.delete_object(
112
+ :bucket =>Deb::S3::Utils.bucket,
113
+ :key => s3_path(path),
114
+ )
115
+ end
93
116
  end
94
117
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: deb-s3
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.8.0
4
+ version: 0.9.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ken Robertson
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-04-07 00:00:00.000000000 Z
11
+ date: 2017-08-08 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: thor
@@ -30,14 +30,14 @@ dependencies:
30
30
  requirements:
31
31
  - - "~>"
32
32
  - !ruby/object:Gem::Version
33
- version: '1.66'
33
+ version: '2'
34
34
  type: :runtime
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
38
  - - "~>"
39
39
  - !ruby/object:Gem::Version
40
- version: '1.66'
40
+ version: '2'
41
41
  - !ruby/object:Gem::Dependency
42
42
  name: minitest
43
43
  requirement: !ruby/object:Gem::Requirement
@@ -104,7 +104,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
104
104
  version: '0'
105
105
  requirements: []
106
106
  rubyforge_project:
107
- rubygems_version: 2.4.5.1
107
+ rubygems_version: 2.5.2
108
108
  signing_key:
109
109
  specification_version: 4
110
110
  summary: Easily create and manage an APT repository on S3.