gcloud 0.10.0 → 0.11.0

data/lib/gcloud/storage/connection.rb

@@ -19,6 +19,7 @@ require "gcloud/backoff"
 require "gcloud/upload"
 require "google/api_client"
 require "mime/types"
+require "digest/sha2"
 
 module Gcloud
   module Storage
@@ -52,7 +53,7 @@ module Gcloud
         params["pageToken"]  = options[:token]  if options[:token]
         params["maxResults"] = options[:max]    if options[:max]
 
-        @client.execute(
+        execute(
           api_method: @storage.buckets.list,
           parameters: params
         )
@@ -61,7 +62,7 @@ module Gcloud
       ##
       # Retrieves bucket by name.
       def get_bucket bucket_name
-        @client.execute(
+        execute(
           api_method: @storage.buckets.get,
           parameters: { bucket: bucket_name }
         )
@@ -74,13 +75,11 @@ module Gcloud
                    predefinedDefaultObjectAcl: options[:default_acl]
                  }.delete_if { |_, v| v.nil? }
 
-        incremental_backoff options do
-          @client.execute(
-            api_method: @storage.buckets.insert,
-            parameters: params,
-            body_object: insert_bucket_request(bucket_name, options)
-          )
-        end
+        execute(
+          api_method: @storage.buckets.insert,
+          parameters: params,
+          body_object: insert_bucket_request(bucket_name, options)
+        )
       end
 
       ##
@@ -91,7 +90,7 @@ module Gcloud
                    predefinedDefaultObjectAcl: options[:predefined_default_acl]
                  }.delete_if { |_, v| v.nil? }
 
-        @client.execute(
+        execute(
           api_method: @storage.buckets.patch,
           parameters: params,
           body_object: patch_bucket_request(options)
@@ -100,19 +99,17 @@ module Gcloud
 
       ##
       # Permanently deletes an empty bucket.
-      def delete_bucket bucket_name, opts = {}
-        incremental_backoff opts do
-          @client.execute(
-            api_method: @storage.buckets.delete,
-            parameters: { bucket: bucket_name }
-          )
-        end
+      def delete_bucket bucket_name
+        execute(
+          api_method: @storage.buckets.delete,
+          parameters: { bucket: bucket_name }
+        )
       end
 
       ##
       # Retrieves a list of ACLs for the given bucket.
       def list_bucket_acls bucket_name
-        @client.execute(
+        execute(
           api_method: @storage.bucket_access_controls.list,
           parameters: { bucket: bucket_name }
         )
@@ -121,7 +118,7 @@ module Gcloud
       ##
       # Creates a new bucket ACL.
       def insert_bucket_acl bucket_name, entity, role
-        @client.execute(
+        execute(
           api_method: @storage.bucket_access_controls.insert,
           parameters: { bucket: bucket_name },
           body_object: { entity: entity, role: role }
@@ -131,7 +128,7 @@ module Gcloud
       ##
       # Permanently deletes a bucket ACL.
       def delete_bucket_acl bucket_name, entity
-        @client.execute(
+        execute(
           api_method: @storage.bucket_access_controls.delete,
           parameters: { bucket: bucket_name, entity: entity }
         )
@@ -140,7 +137,7 @@ module Gcloud
       ##
       # Retrieves a list of default ACLs for the given bucket.
       def list_default_acls bucket_name
-        @client.execute(
+        execute(
           api_method: @storage.default_object_access_controls.list,
           parameters: { bucket: bucket_name }
         )
@@ -149,7 +146,7 @@ module Gcloud
       ##
       # Creates a new default ACL.
       def insert_default_acl bucket_name, entity, role
-        @client.execute(
+        execute(
           api_method: @storage.default_object_access_controls.insert,
           parameters: { bucket: bucket_name },
           body_object: { entity: entity, role: role }
@@ -159,7 +156,7 @@ module Gcloud
       ##
       # Permanently deletes a default ACL.
       def delete_default_acl bucket_name, entity
-        @client.execute(
+        execute(
           api_method: @storage.default_object_access_controls.delete,
           parameters: { bucket: bucket_name, entity: entity }
         )
@@ -177,7 +174,7 @@ module Gcloud
           versions:   options[:versions]
         }.delete_if { |_, v| v.nil? }
 
-        @client.execute(
+        execute(
           api_method: @storage.objects.list,
           parameters: params
         )
@@ -198,7 +195,7 @@ module Gcloud
         result = insert_file resumable, bucket_name, upload_path, media, options
         return result unless resumable
         upload = result.resumable_upload
-        result = @client.execute upload while upload.resumable?
+        result = execute upload while upload.resumable?
         result
       end
 
@@ -208,17 +205,20 @@ module Gcloud
         query = { bucket: bucket_name, object: file_path }
         query[:generation] = options[:generation] if options[:generation]
 
-        @client.execute(
+        request_options = {
           api_method: @storage.objects.get,
           parameters: query
-        )
+        }
+
+        request_options = add_headers request_options, options
+        execute request_options
       end
 
       ## Copy a file from source bucket/object to a
       # destination bucket/object.
       def copy_file source_bucket_name, source_file_path,
                     destination_bucket_name, destination_file_path, options = {}
-        @client.execute(
+        request_options = {
           api_method: @storage.objects.copy,
           parameters: { sourceBucket: source_bucket_name,
                         sourceObject: source_file_path,
@@ -226,18 +226,24 @@ module Gcloud
                         destinationBucket: destination_bucket_name,
                         destinationObject: destination_file_path,
                         predefinedAcl: options[:acl]
-                      }.delete_if { |_, v| v.nil? })
+                      }.delete_if { |_, v| v.nil? }
+        }
+        request_options = add_headers request_options, options
+        execute request_options
       end
 
       ##
       # Download contents of a file.
-      def download_file bucket_name, file_path
-        @client.execute(
+
+      def download_file bucket_name, file_path, options = {}
+        request_options = {
           api_method: @storage.objects.get,
           parameters: { bucket: bucket_name,
                         object: file_path,
                         alt: :media }
-        )
+        }
+        request_options = add_headers request_options, options
+        execute request_options
       end
 
       ##
@@ -248,7 +254,7 @@ module Gcloud
                    predefinedAcl: options[:predefined_acl]
                  }.delete_if { |_, v| v.nil? }
 
-        @client.execute(
+        execute(
           api_method: @storage.objects.patch,
           parameters: params,
           body_object: patch_file_request(options)
@@ -258,7 +264,7 @@ module Gcloud
       ##
       # Permanently deletes a file.
       def delete_file bucket_name, file_path
-        @client.execute(
+        execute(
           api_method: @storage.objects.delete,
           parameters: { bucket: bucket_name,
                         object: file_path }
@@ -268,7 +274,7 @@ module Gcloud
       ##
       # Retrieves a list of ACLs for the given file.
       def list_file_acls bucket_name, file_name
-        @client.execute(
+        execute(
           api_method: @storage.object_access_controls.list,
           parameters: { bucket: bucket_name, object: file_name }
         )
@@ -280,7 +286,7 @@ module Gcloud
         query = { bucket: bucket_name, object: file_name }
         query[:generation] = options[:generation] if options[:generation]
 
-        @client.execute(
+        execute(
           api_method: @storage.object_access_controls.insert,
           parameters: query,
           body_object: { entity: entity, role: role }
@@ -293,7 +299,7 @@ module Gcloud
         query = { bucket: bucket_name, object: file_name, entity: entity }
         query[:generation] = options[:generation] if options[:generation]
 
-        @client.execute(
+        execute(
           api_method: @storage.object_access_controls.delete,
           parameters: query
         )
@@ -374,10 +380,14 @@ module Gcloud
                    predefinedAcl: options[:acl]
         }.delete_if { |_, v| v.nil? }
 
-        @client.execute api_method: @storage.objects.insert,
-                        media: media,
-                        parameters: params,
-                        body_object: insert_file_request(options)
+        request_options = {
+          api_method: @storage.objects.insert,
+          media: media,
+          parameters: params,
+          body_object: insert_file_request(options)
+        }
+        request_options = add_headers request_options, options
+        execute request_options
       end
 
       def file_media local_path, options, resumable
@@ -410,9 +420,23 @@ module Gcloud
         }.delete_if { |_, v| v.nil? }
       end
 
-      def incremental_backoff options = {}
-        Gcloud::Backoff.new(options).execute_gapi do
-          yield
+      def add_headers request_options, options
+        headers = (request_options[:headers] ||= {})
+        if options[:encryption_key] || options[:encryption_key_sha256]
+          headers["x-goog-encryption-algorithm"] = "AES256"
+        end
+        if (key = options.delete(:encryption_key))
+          headers["x-goog-encryption-key"] = Base64.encode64 key
+        end
+        if (key_sha256 = options.delete(:encryption_key_sha256))
+          headers["x-goog-encryption-key-sha256"] = Base64.encode64 key_sha256
+        end
+        request_options
+      end
+
+      def execute options
+        Gcloud::Backoff.new.execute_gapi do
+          @client.execute options
         end
       end
     end

data/lib/gcloud/storage/errors.rb

@@ -45,9 +45,15 @@ module Gcloud
 
       # @private
       def self.from_response resp
-        new resp.data["error"]["message"],
-            resp.data["error"]["code"],
-            resp.data["error"]["errors"]
+        if resp.data && resp.data["error"]
+          new resp.data["error"]["message"],
+              resp.data["error"]["code"],
+              resp.data["error"]["errors"]
+        else
+          new resp.error_message,
+              resp.status,
+              nil
+        end
       end
     end
 

data/lib/gcloud/storage/file.rb

@@ -245,6 +245,17 @@ module Gcloud
         patch_gapi! metadata: metadata
       end
 
+      ##
+      # An [RFC 4648](https://tools.ietf.org/html/rfc4648#section-4)
+      # Base64-encoded string of the SHA256 hash of the [customer-supplied
+      # encryption key](https://cloud.google.com/storage/docs/encryption#customer-supplied).
+      # You can use this SHA256 hash to uniquely identify the AES-256 encryption
+      # key required to decrypt this file.
+      def encryption_key_sha256
+        return nil unless @gapi["customerEncryption"]
+        Base64.decode64 @gapi["customerEncryption"]["keySha256"]
+      end
+
       ##
       # Updates the file with changes made in the given block in a single
       # PATCH request. The following attributes may be set: {#cache_control=},
@@ -285,6 +296,11 @@ module Gcloud
       #
       # By default, the download is verified by calculating the MD5 digest.
       #
+      # If a [customer-supplied encryption
+      # key](https://cloud.google.com/storage/docs/encryption#customer-supplied)
+      # was used with {#create_file}, the `encryption_key` and
+      # `encryption_key_sha256` options must be provided.
+      #
       # @param [String] path The path on the local file system to write the data
       #   to. The path provided must be writable.
       # @param [Symbol] verify The verification algoruthm used to ensure the
@@ -297,6 +313,15 @@ module Gcloud
       #   * `all` - Perform all available file content verification.
       #   * `none` - Don't perform file content verification.
       #
+      # @param [String] encryption_key Optional. The customer-supplied, AES-256
+      #   encryption key used to encrypt the file, if one was provided to
+      #   {#create_file}. Must be provided if `encryption_key_sha256` is
+      #   provided.
+      # @param [String] encryption_key_sha256 Optional. The SHA256 hash of the
+      #   customer-supplied, AES-256 encryption key used to encrypt the file, if
+      #   one was provided to {#create_file}. Must be provided if
+      #   `encryption_key` is provided.
+      #
       # @return [File] Returns a `::File` object on the local file system
       #
       # @example
@@ -343,9 +368,12 @@ module Gcloud
       #   file = bucket.file "path/to/my-file.ext"
       #   file.download "path/to/downloaded/file.ext", verify: :none
       #
-      def download path, verify: :md5
+      def download path, verify: :md5, encryption_key: nil,
+                   encryption_key_sha256: nil
         ensure_connection!
-        resp = connection.download_file bucket, name
+        options = { encryption_key: encryption_key,
+                    encryption_key_sha256: encryption_key_sha256 }
+        resp = connection.download_file bucket, name, options
         if resp.success?
           ::File.open path, "wb+" do |f|
             f.write resp.body
@@ -359,6 +387,11 @@ module Gcloud
       ##
       # Copy the file to a new location.
       #
+      # If a [customer-supplied encryption
+      # key](https://cloud.google.com/storage/docs/encryption#customer-supplied)
+      # was used with {#create_file}, the `encryption_key` and
+      # `encryption_key_sha256` options must be provided.
+      #
       # @param [String] dest_bucket_or_path Either the bucket to copy the file
       #   to, or the path to copy the file to in the current bucket.
       # @param [String] dest_path If a bucket was provided in the first
@@ -383,6 +416,14 @@ module Gcloud
       #     access, and allUsers get READER access.
       # @param [Integer] generation Select a specific revision of the file to
       #   copy. The default is the latest version.
+      # @param [String] encryption_key Optional. The customer-supplied, AES-256
+      #   encryption key used to encrypt the file, if one was provided to
+      #   {#create_file}. Must be provided if `encryption_key_sha256` is
+      #   provided.
+      # @param [String] encryption_key_sha256 Optional. The SHA256 hash of the
+      #   customer-supplied, AES-256 encryption key used to encrypt the file, if
+      #   one was provided to {#create_file}. Must be provided if
+      #   `encryption_key` is provided.
       #
       # @return [Gcloud::Storage::File]
       #
@@ -413,9 +454,12 @@ module Gcloud
       #   file.copy "copy/of/previous/generation/file.ext",
       #             generation: 123456
       #
-      def copy dest_bucket_or_path, dest_path = nil, acl: nil, generation: nil
+      def copy dest_bucket_or_path, dest_path = nil, acl: nil, generation: nil,
+               encryption_key: nil, encryption_key_sha256: nil
         ensure_connection!
-        options = { acl: acl, generation: generation }
+        options = { acl: acl, generation: generation,
+                    encryption_key: encryption_key,
+                    encryption_key_sha256: encryption_key_sha256 }
         dest_bucket, dest_path, options = fix_copy_args dest_bucket_or_path,
                                                         dest_path, options
 

data/lib/gcloud/storage/file/list.rb

@@ -22,32 +22,168 @@ module Gcloud
       # File::List is a special case Array with additional values.
       class List < DelegateClass(::Array)
         ##
-        # If not empty, indicates that there are more buckets
-        # that match the request and this value should be passed to
-        # the next Gcloud::Storage::Bucket#files to continue.
+        # If not empty, indicates that there are more files that match the
+        # request and this value should be passed to the next
+        # {Gcloud::Storage::Bucket#files} to continue.
         attr_accessor :token
 
-        # The list of prefixes of objects matching-but-not-listed up
-        # to and including the requested delimiter.
+        # The list of prefixes of objects matching-but-not-listed up to and
+        # including the requested delimiter.
         attr_accessor :prefixes
 
         ##
-        # Create a new File::List with an array of values.
-        def initialize arr = [], token = nil, prefixes = []
+        # @private Create a new File::List with an array of values.
+        def initialize arr = []
           super arr
-          @token = token
-          @prefixes = prefixes
+        end
+
+        ##
+        # Whether there is a next page of files.
+        #
+        # @return [Boolean]
+        #
+        # @example
+        #   require "gcloud"
+        #
+        #   gcloud = Gcloud.new
+        #   storage = gcloud.storage
+        #
+        #   bucket = storage.bucket "my-bucket"
+        #   files = bucket.files
+        #   if files.next?
+        #     next_files = files.next
+        #   end
+        #
+        def next?
+          !token.nil?
+        end
+
+        ##
+        # Retrieve the next page of files.
+        #
+        # @return [File::List]
+        #
+        # @example
+        #   require "gcloud"
+        #
+        #   gcloud = Gcloud.new
+        #   storage = gcloud.storage
+        #
+        #   bucket = storage.bucket "my-bucket"
+        #   files = bucket.files
+        #   if files.next?
+        #     next_files = files.next
+        #   end
+        #
+        def next
+          return nil unless next?
+          ensure_connection!
+          options = {
+            prefix: @prefix, delimiter: @delimiter, token: @token, max: @max,
+            versions: @versions
+          }
+          resp = @connection.list_files @bucket, options
+          fail ApiError.from_response(resp) unless resp.success?
+          File::List.from_response resp, @connection, @bucket, @prefix,
+                                   @delimiter, @max, @versions
+        end
+
+        ##
+        # Retrieves all files by repeatedly loading {#next} until {#next?}
+        # returns `false`. Calls the given block once for each file, which is
+        # passed as the parameter.
+        #
+        # An Enumerator is returned if no block is given.
+        #
+        # This method may make several API calls until all files are retrieved.
+        # Be sure to use as narrow a search criteria as possible. Please use
+        # with caution.
+        #
+        # @param [Integer] request_limit The upper limit of API requests to make
+        #   to load all files. Default is no limit.
+        # @yield [file] The block for accessing each file.
+        # @yieldparam [File] file The file object.
+        #
+        # @return [Enumerator]
+        #
+        # @example Iterating each file by passing a block:
+        #   require "gcloud"
+        #
+        #   gcloud = Gcloud.new
+        #   storage = gcloud.storage
+        #
+        #   bucket = storage.bucket "my-bucket"
+        #   files = bucket.files
+        #   files.all do |file|
+        #     puts file.name
+        #   end
+        #
+        # @example Using the enumerator by not passing a block:
+        #   require "gcloud"
+        #
+        #   gcloud = Gcloud.new
+        #   storage = gcloud.storage
+        #
+        #   bucket = storage.bucket "my-bucket"
+        #   files = bucket.files
+        #
+        #   all_names = files.all.map do |file|
+        #     file.name
+        #   end
+        #
+        # @example Limit the number of API calls made:
+        #   require "gcloud"
+        #
+        #   gcloud = Gcloud.new
+        #   storage = gcloud.storage
+        #
+        #   bucket = storage.bucket "my-bucket"
+        #   files = bucket.files
+        #   files.all(request_limit: 10) do |file|
+        #     puts file.name
+        #   end
+        #
+        def all request_limit: nil
+          request_limit = request_limit.to_i if request_limit
+          unless block_given?
+            return enum_for(:all, request_limit: request_limit)
+          end
+          results = self
+          loop do
+            results.each { |r| yield r }
+            if request_limit
+              request_limit -= 1
+              break if request_limit < 0
+            end
+            break unless results.next?
+            results = results.next
+          end
         end
 
         ##
         # @private New File::List from a response object.
-        def self.from_response resp, conn
-          buckets = Array(resp.data["items"]).map do |gapi_object|
+        def self.from_response resp, conn, bucket = nil, prefix = nil,
+                               delimiter = nil, max = nil, versions = nil
+          files = new(Array(resp.data["items"]).map do |gapi_object|
             File.from_gapi gapi_object, conn
-          end
-          token = resp.data["nextPageToken"]
-          prefixes = Array resp.data["prefixes"]
-          new buckets, token, prefixes
+          end)
+          files.instance_variable_set "@token", resp.data["nextPageToken"]
+          files.instance_variable_set "@prefixes", Array(resp.data["prefixes"])
+          files.instance_variable_set "@connection", conn
+          files.instance_variable_set "@bucket", bucket
+          files.instance_variable_set "@prefix", prefix
+          files.instance_variable_set "@delimiter", delimiter
+          files.instance_variable_set "@max", max
+          files.instance_variable_set "@versions", versions
+          files
+        end
+
+        protected
+
+        ##
+        # Raise an error unless an active connection is available.
+        def ensure_connection!
+          fail "Must have active connection" unless @connection
         end
       end
     end