deb-s3 0.8.0 → 0.9.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7e1bc9874f303cb3ec4a83a5dd1edf8948bf62bd
-  data.tar.gz: d2a16561f73b2dfd0c324f99a4336a6082f5a0e3
+  metadata.gz: 0fdcdc9b5f39dd0c26eaa1c9045dc5ce520697b4
+  data.tar.gz: 743c9ae0a9368dac628bbfdee71da2c6606b411b
 SHA512:
-  metadata.gz: 54c58ff8308b6c8a1f219df3fbb21d6a9693917f2a853b1be44269443f84f5ffc4ff66e6270d23c98651adb8c430dddc9351eccca317b60279d7977112e2c272
-  data.tar.gz: c1a01cae8a4b91996e8d919cf41a8a2c8a4e6abe4ceb2c90496081fa5f9f77430472bb39067f65feaa5bf40305bf84043aa6f74bff0b9db234af868708e929c1
+  metadata.gz: a2eacf16e2fd89c4af49c9229b0cf185d0de6efeaa70bbe5543b93a83c11d651abc838734560a36dd54703b56a51ad15a53cf87f479d3c7645e366714fe0340e
+  data.tar.gz: b3d97c88ef06bd7f908173e6e6ab466a98482231400f46f7e914d9a4f3a9564eab80e610c3df7939e1e15557828d1cf0f83deb11fd00312f2313406c6f7e9c5c

data/lib/deb/s3.rb

@@ -1,6 +1,6 @@
 # -*- encoding : utf-8 -*-
 module Deb
   module S3
-    VERSION = "0.8.0"
+    VERSION = "0.9.1"
   end
 end

data/lib/deb/s3/cli.rb

@@ -1,5 +1,5 @@
 # -*- encoding : utf-8 -*-
-require "aws"
+require "aws-sdk"
 require "thor"
 
 # Hack: aws requires this!
@@ -56,10 +56,10 @@ class Deb::S3::CLI < Thor
   :type     => :string,
   :desc     => "The secret key for connecting to S3."
 
-  class_option :endpoint,
+  class_option :s3_region,
   :type     => :string,
-  :desc     => "The region endpoint for connecting to S3.",
-  :default  => "s3.amazonaws.com"
+  :desc     => "The region for connecting to S3.",
+  :default  => "us-east-1"
 
   class_option :force_path_style,
   :default  => false,
@@ -70,10 +70,10 @@ class Deb::S3::CLI < Thor
   :type     => :string,
   :desc     => "The URI of the proxy to send service requests through."
 
-  class_option :use_ssl,
-  :default  => true,
-  :type     => :boolean,
-  :desc     => "Whether to use HTTP or HTTPS for request transport."
+  #class_option :use_ssl,
+  #:default  => true,
+  #:type     => :boolean,
+  #:desc     => "Whether to use HTTP or HTTPS for request transport."
 
   class_option :visibility,
   :default  => "public",
@@ -137,6 +137,12 @@ class Deb::S3::CLI < Thor
   :desc     => "Whether to overwrite any existing package that has the same " +
     "filename in the pool or the same name and version in the manifest."
 
+  option :skip_package_upload,
+  :default  => false,
+  :type     => :boolean,
+  :desc     => "Whether to skip all package uploads." +
+    "This is useful when hosting .deb files outside of the bucket."
+
   def upload(*files)
     if files.nil? || files.empty?
       error("You must specify at least one file to upload")
@@ -169,7 +175,7 @@ class Deb::S3::CLI < Thor
       release  = Deb::S3::Release.retrieve(options[:codename], options[:origin], options[:suite], options[:cache_control])
       manifests = {}
       release.architectures.each do |arch|
-        manifests[arch] = Deb::S3::Manifest.retrieve(options[:codename], component, arch, options[:cache_control], options[:fail_if_exists])
+        manifests[arch] = Deb::S3::Manifest.retrieve(options[:codename], component, arch, options[:cache_control], options[:fail_if_exists], options[:skip_package_upload])
       end
 
       packages_arch_all = []
@@ -203,7 +209,7 @@ class Deb::S3::CLI < Thor
         end
 
         # retrieve the manifest for the arch if we don't have it already
-        manifests[arch] ||= Deb::S3::Manifest.retrieve(options[:codename], component, arch, options[:cache_control], options[:fail_if_exists])
+        manifests[arch] ||= Deb::S3::Manifest.retrieve(options[:codename], component, arch, options[:cache_control], options[:fail_if_exists], options[:skip_package_upload])
 
         # add package in manifests
         begin
@@ -273,7 +279,7 @@ class Deb::S3::CLI < Thor
     rows = archs.map { |arch|
       manifest = Deb::S3::Manifest.retrieve(options[:codename], component,
                                             arch, options[:cache_control],
-                                            false)
+                                            false, false)
       manifest.packages.map do |package|
         if options[:long]
           package.generate
@@ -313,7 +319,7 @@ class Deb::S3::CLI < Thor
 
     # retrieve the existing manifests
     manifest = Deb::S3::Manifest.retrieve(options[:codename], component, arch,
-                                          options[:cache_control], false)
+                                          options[:cache_control], false, false)
     package = manifest.packages.detect { |p|
       p.name == package_name && p.full_version == version
     }
@@ -387,11 +393,12 @@ class Deb::S3::CLI < Thor
     from_manifest = Deb::S3::Manifest.retrieve(options[:codename],
                                                component, arch,
                                                options[:cache_control],
-                                               false)
+                                               false, options[:skip_package_upload])
     to_release = Deb::S3::Release.retrieve(to_codename)
     to_manifest = Deb::S3::Manifest.retrieve(to_codename, to_component, arch,
                                              options[:cache_control],
-                                             options[:fail_if_exists])
+                                             options[:fail_if_exists],
+                                             options[:skip_package_upload])
     packages = from_manifest.packages.select { |p|
       p.name == package_name &&
         (versions.nil? || versions.include?(p.full_version))
@@ -458,7 +465,7 @@ class Deb::S3::CLI < Thor
     # retrieve the existing manifests
     log("Retrieving existing manifests")
     release  = Deb::S3::Release.retrieve(options[:codename], options[:origin], options[:suite])
-    manifest = Deb::S3::Manifest.retrieve(options[:codename], component, options[:arch], options[:cache_control], false)
+    manifest = Deb::S3::Manifest.retrieve(options[:codename], component, options[:arch], options[:cache_control], false, options[:skip_package_upload])
 
     deleted = manifest.delete_package(package, versions)
     if deleted.length == 0
@@ -499,7 +506,8 @@ class Deb::S3::CLI < Thor
     release.architectures.each do |arch|
       log("Checking for missing packages in: #{options[:codename]}/#{options[:component]} #{arch}")
       manifest = Deb::S3::Manifest.retrieve(options[:codename], component,
-                                            arch, options[:cache_control], false)
+                                            arch, options[:cache_control], false,
+                                            options[:skip_package_upload])
       missing_packages = []
 
       manifest.packages.each do |p|
@@ -561,26 +569,24 @@ class Deb::S3::CLI < Thor
     if access_key_id.nil? ^ secret_access_key.nil?
       error("If you specify one of --access-key-id or --secret-access-key, you must specify the other.")
     end
-
     static_credentials = {}
     static_credentials[:access_key_id]     = access_key_id     if access_key_id
     static_credentials[:secret_access_key] = secret_access_key if secret_access_key
 
-    AWS::Core::CredentialProviders::DefaultProvider.new(static_credentials)
+    static_credentials
   end
 
   def configure_s3_client
     error("No value provided for required options '--bucket'") unless options[:bucket]
 
     settings = {
-      :s3_endpoint => options[:endpoint],
-      :proxy_uri   => options[:proxy_uri],
-      :use_ssl     => options[:use_ssl],
-      :s3_force_path_style => options[:force_path_style]
+      :region => options[:s3_region],
+      :http_proxy   => options[:proxy_uri],
+      :force_path_style => options[:force_path_style]
     }
-    settings.merge!(provider.credentials)
+    settings.merge!(provider)
 
-    Deb::S3::Utils.s3          = AWS::S3.new(settings)
+    Deb::S3::Utils.s3          = Aws::S3::Client.new(settings)
     Deb::S3::Utils.bucket      = options[:bucket]
     Deb::S3::Utils.signing_key = options[:sign]
     Deb::S3::Utils.gpg_options = options[:gpg_options]
@@ -591,13 +597,13 @@ class Deb::S3::CLI < Thor
     Deb::S3::Utils.access_policy =
       case options[:visibility]
       when "public"
-        :public_read
+        "public-read"
       when "private"
-        :private
+        "private"
       when "authenticated"
-        :authenticated_read
+        "authenticated-read"
       when "bucket_owner"
-        :bucket_owner_full_control
+        "bucket-owner-full-control"
       else
         error("Invalid visibility setting given. Can be public, private, authenticated, or bucket_owner.")
       end

data/lib/deb/s3/manifest.rb

@@ -12,6 +12,7 @@ class Deb::S3::Manifest
   attr_accessor :cache_control
   attr_accessor :architecture
   attr_accessor :fail_if_exists
+  attr_accessor :skip_package_upload
 
   attr_accessor :files
 
@@ -26,10 +27,11 @@ class Deb::S3::Manifest
     @files = {}
     @cache_control = ""
     @fail_if_exists = false
+    @skip_package_upload = false
   end
 
   class << self
-    def retrieve(codename, component, architecture, cache_control, fail_if_exists)
+    def retrieve(codename, component, architecture, cache_control, fail_if_exists, skip_package_upload=false)
       m = if s = Deb::S3::Utils.s3_read("dists/#{codename}/#{component}/binary-#{architecture}/Packages")
         self.parse_packages(s)
       else
@@ -41,6 +43,7 @@ class Deb::S3::Manifest
       m.architecture = architecture
       m.cache_control = cache_control
       m.fail_if_exists = fail_if_exists
+      m.skip_package_upload = skip_package_upload
       m
     end
 
@@ -56,7 +59,15 @@ class Deb::S3::Manifest
 
   def add(pkg, preserve_versions, needs_uploading=true)
     if self.fail_if_exists
-      packages.each { |p| raise AlreadyExistsError, "package #{pkg.name}_#{pkg.full_version} already exists with different filename (#{p.url_filename})" if p.name == pkg.name && p.full_version == pkg.full_version && File.basename(p.url_filename) != File.basename(pkg.filename) }
+      packages.each { |p|
+        next unless p.name == pkg.name && \
+                    p.full_version == pkg.full_version && \
+                    File.basename(p.url_filename) != \
+                    File.basename(pkg.url_filename)
+        raise AlreadyExistsError,
+              "package #{pkg.name}_#{pkg.full_version} already exists " \
+              "with different filename (#{p.url_filename})"
+      }
     end
     if preserve_versions
       packages.delete_if { |p| p.name == pkg.name && p.full_version == pkg.full_version }
@@ -91,10 +102,12 @@ class Deb::S3::Manifest
   def write_to_s3
     manifest = self.generate
 
-    # store any packages that need to be stored
-    @packages_to_be_upload.each do |pkg|
-      yield pkg.url_filename if block_given?
-      s3_store(pkg.filename, pkg.url_filename, 'application/octet-stream; charset=binary', self.cache_control, self.fail_if_exists)
+    unless self.skip_package_upload
+      # store any packages that need to be stored
+      @packages_to_be_upload.each do |pkg|
+        yield pkg.url_filename if block_given?
+        s3_store(pkg.filename, pkg.url_filename, 'application/octet-stream; charset=binary', self.cache_control, self.fail_if_exists)
+      end
     end
 
     # generate the Packages file

data/lib/deb/s3/release.rb

@@ -31,10 +31,10 @@ class Deb::S3::Release
         rel = self.parse_release(s)
       else
         rel = self.new
-        rel.codename = codename
-        rel.origin = origin unless origin.nil?
-        rel.suite = suite unless suite.nil?
       end
+      rel.codename = codename
+      rel.origin = origin unless origin.nil?
+      rel.suite = suite unless suite.nil?
       rel.cache_control = cache_control
       rel
     end
@@ -103,15 +103,28 @@ class Deb::S3::Release
     # sign the file, if necessary
     if Deb::S3::Utils.signing_key
       key_param = Deb::S3::Utils.signing_key != "" ? "--default-key=#{Deb::S3::Utils.signing_key}" : ""
-      if system("gpg -a #{key_param} #{Deb::S3::Utils.gpg_options} -b #{release_tmp.path}")
-        local_file = release_tmp.path+".asc"
-        remote_file = self.filename+".gpg"
-        yield remote_file if block_given?
-        raise "Unable to locate Release signature file" unless File.exists?(local_file)
-        s3_store(local_file, remote_file, 'application/pgp-signature; charset=us-ascii', self.cache_control)
-        File.unlink(local_file)
+      if self.codename == "xenial"
+        if system("gpg -a #{key_param} #{Deb::S3::Utils.gpg_options} -s --clearsign #{release_tmp.path}")
+          local_file = release_tmp.path+".asc"
+          remote_file = "dists/#{@codename}/InRelease"
+          yield remote_file if block_given?
+          raise "Unable to locate InRelease file" unless File.exists?(local_file)
+          s3_store(local_file, remote_file, 'application/pgp-signature; charset=us-ascii', self.cache_control)
+          File.unlink(local_file)
+        else
+          raise "Signing the InRelease file failed."
+        end
       else
-        raise "Signing the Release file failed."
+        if system("gpg -a #{key_param} #{Deb::S3::Utils.gpg_options} -b #{release_tmp.path}")
+          local_file = release_tmp.path+".asc"
+          remote_file = self.filename+".gpg"
+          yield remote_file if block_given?
+          raise "Unable to locate Release signature file" unless File.exists?(local_file)
+          s3_store(local_file, remote_file, 'application/pgp-signature; charset=us-ascii', self.cache_control)
+          File.unlink(local_file)
+        else
+          raise "Signing the Release file failed."
+        end
       end
     else
       # remove an existing Release.gpg, if it was there

data/lib/deb/s3/utils.rb

@@ -56,27 +56,42 @@ module Deb::S3::Utils
   end
 
   def s3_exists?(path)
-    Deb::S3::Utils.s3.buckets[Deb::S3::Utils.bucket].objects[s3_path(path)].exists?
+    Deb::S3::Utils.s3.head_object(
+      :bucket => Deb::S3::Utils.bucket,
+      :key => s3_path(path),
+    )
+  rescue Aws::S3::Errors::NotFound
+    false
   end
 
   def s3_read(path)
-    return nil unless s3_exists?(path)
-    Deb::S3::Utils.s3.buckets[Deb::S3::Utils.bucket].objects[s3_path(path)].read
+    Deb::S3::Utils.s3.get_object(
+      :bucket => Deb::S3::Utils.bucket,
+      :key => s3_path(path),
+    )[:body].read
+  rescue Aws::S3::Errors::NoSuchKey
+    false
   end
 
   def s3_store(path, filename=nil, content_type='application/octet-stream; charset=binary', cache_control=nil, fail_if_exists=false)
     filename = File.basename(path) unless filename
-    obj = Deb::S3::Utils.s3.buckets[Deb::S3::Utils.bucket].objects[s3_path(filename)]
+    obj = s3_exists?(path)
 
     file_md5 = Digest::MD5.file(path)
 
     # check if the object already exists
-    if obj.exists?
-      return if (file_md5.to_s == obj.etag.gsub('"', '') or file_md5.to_s == obj.metadata['md5'])
+    if obj != false
+      return if (file_md5.to_s == obj[:etag].gsub('"', '') or file_md5.to_s == obj[:metadata]['md5'])
       raise AlreadyExistsError, "file #{obj.public_url} already exists with different contents" if fail_if_exists
     end
 
-    options = {:acl => Deb::S3::Utils.access_policy, :content_type => content_type, :metadata => {'md5' => file_md5}}
+    options = {
+      :bucket => Deb::S3::Utils.bucket,
+      :key => s3_path(filename),
+      :acl => Deb::S3::Utils.access_policy,
+      :content_type => content_type,
+      :metadata => { "md5" => file_md5.to_s },
+    }
     if !cache_control.nil?
       options[:cache_control] = cache_control
     end
@@ -85,10 +100,18 @@ module Deb::S3::Utils
     options[:server_side_encryption] = :aes256 if Deb::S3::Utils.encryption
 
     # upload the file
-    obj.write(Pathname.new(path), options)
+    File.open(path) do |f|
+      options[:body] = f
+      Deb::S3::Utils.s3.put_object(options)
+    end
   end
 
   def s3_remove(path)
-    Deb::S3::Utils.s3.buckets[Deb::S3::Utils.bucket].objects[s3_path(path)].delete if s3_exists?(path)
+    if s3_exists?(path)
+      Deb::S3::Utils.s3.delete_object(
+        :bucket =>Deb::S3::Utils.bucket,
+        :key => s3_path(path),
+      )
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deb-s3
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 0.9.1
 platform: ruby
 authors:
 - Ken Robertson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-04-07 00:00:00.000000000 Z
+date: 2017-08-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.66'
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.66'
+        version: '2'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
@@ -104,7 +104,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5.1
+rubygems_version: 2.5.2
 signing_key: 
 specification_version: 4
 summary: Easily create and manage an APT repository on S3.