gcloud 0.10.0 → 0.11.0

data/lib/gcloud/storage.rb

@@ -240,6 +240,45 @@ module Gcloud
   #                    "avatars/heidi/400x400.png"
   # ```
   #
+  # ### Customer-supplied encryption keys
+  #
+  # By default, Google Cloud Storage manages server-side encryption keys on
+  # your behalf. However, a [customer-supplied encryption
+  # key](https://cloud.google.com/storage/docs/encryption#customer-supplied)
+  # can be provided with the `encryption_key` and `encryption_key_sha256`
+  # options. If given, the same key and SHA256 hash also must be provided to
+  # subsequently download or copy the file. If you use customer-supplied
+  # encryption keys, you must securely manage your keys and ensure that they are
+  # not lost. Also, please note that file metadata is not encrypted, with the
+  # exception of the CRC32C checksum and MD5 hash. The names of files and
+  # buckets are also not encrypted, and you can read or update the metadata of
+  # an encrypted file without providing the encryption key.
+  #
+  # ```ruby
+  # require "gcloud"
+  # require "digest/sha2"
+  #
+  # gcloud = Gcloud.new
+  # storage = gcloud.storage
+  # bucket = storage.bucket "my-todo-app"
+  #
+  # # Key generation shown for example purposes only. Write your own.
+  # cipher = OpenSSL::Cipher.new "aes-256-cfb"
+  # cipher.encrypt
+  # key = cipher.random_key
+  # key_hash = Digest::SHA256.digest key
+  #
+  # bucket.create_file "/var/todo-app/avatars/heidi/400x400.png",
+  #                    "avatars/heidi/400x400.png",
+  #                    encryption_key: key,
+  #                    encryption_key_sha256: key_hash
+  #
+  # # Store your key and hash securely for later use.
+  # file = bucket.file "avatars/heidi/400x400.png",
+  #                    encryption_key: key,
+  #                    encryption_key_sha256: key_hash
+  # ```
+  #
   # ### A note about large uploads
   #
   # You may encounter a Broken pipe (Errno::EPIPE) error when attempting to
@@ -405,6 +444,30 @@ module Gcloud
   # file.acl.public!
   # ```
   #
+  # ## Configuring Backoff
+  #
+  # The {Gcloud::Backoff} class allows users to globally configure how Cloud API
+  # requests are automatically retried in the case of some errors, such as a
+  # `500` or `503` status code, or a specific internal error code such as
+  # `rateLimitExceeded`.
+  #
+  # If an API call fails, the response will be inspected to see if the call
+  # should be retried. If the response matches the criteria, then the request
+  # will be retried after a delay. If another error occurs, the delay will be
+  # increased incrementally before a subsequent attempt. The first retry will be
+  # delayed one second, the second retry two seconds, and so on.
+  #
+  # ```ruby
+  # require "gcloud"
+  # require "gcloud/backoff"
+  #
+  # Gcloud::Backoff.retries = 5 # Raise the maximum number of retries from 3
+  # ```
+  #
+  # See the [Storage status and error
+  # codes](https://cloud.google.com/storage/docs/json_api/v1/status-codes)
+  # for a list of error conditions.
+  #
   module Storage
   end
 end

data/lib/gcloud/storage/bucket.rb

@@ -326,11 +326,7 @@ module Gcloud
       # The bucket must be empty before it can be deleted.
       #
       # The API call to delete the bucket may be retried under certain
-      # conditions. See {Gcloud::Backoff} to control this behavior, or
-      # specify the wanted behavior using the `retries` option.
-      #
-      # @param [Integer] retries The number of times the API call should be
-      #   retried. Default is Gcloud::Backoff.retries.
+      # conditions. See {Gcloud::Backoff} to control this behavior.
       #
       # @return [Boolean] Returns `true` if the bucket was deleted.
       #
@@ -343,19 +339,9 @@ module Gcloud
       #   bucket = storage.bucket "my-bucket"
       #   bucket.delete
       #
-      # @example Specify the number of retries to attempt:
-      #   require "gcloud"
-      #
-      #   gcloud = Gcloud.new
-      #   storage = gcloud.storage
-      #
-      #   bucket = storage.bucket "my-bucket"
-      #   bucket.delete retries: 5
-      #
-      def delete retries: nil
+      def delete
         ensure_connection!
-        options = { retries: retries }
-        resp = connection.delete_bucket name, options
+        resp = connection.delete_bucket name
         if resp.success?
           true
         else
@@ -399,24 +385,16 @@ module Gcloud
       #     puts file.name
       #   end
       #
-      # @example With pagination: (See {File::List#token})
+      # @example Retrieve all files: (See {File::List#all})
       #   require "gcloud"
       #
       #   gcloud = Gcloud.new
       #   storage = gcloud.storage
       #
       #   bucket = storage.bucket "my-bucket"
-      #
-      #   all_files = []
-      #   tmp_files = bucket.files
-      #   while tmp_files.any? do
-      #     tmp_files.each do |file|
-      #       all_files << file
-      #     end
-      #     # break loop if no more buckets available
-      #     break if tmp_files.token.nil?
-      #     # get the next group of files
-      #     tmp_files = bucket.files token: tmp_files.token
+      #   files = bucket.files
+      #   files.all do |file|
+      #     puts file.name
       #   end
       #
       def files prefix: nil, delimiter: nil, token: nil, max: nil, versions: nil
@@ -429,20 +407,32 @@ module Gcloud
           versions:  versions
         }
         resp = connection.list_files name, options
-        if resp.success?
-          File::List.from_response resp, connection
-        else
-          fail ApiError.from_response(resp)
-        end
+        fail ApiError.from_response(resp) unless resp.success?
+        File::List.from_response resp, connection, name, prefix, delimiter, max,
+                                 versions
       end
       alias_method :find_files, :files
 
       ##
       # Retrieves a file matching the path.
       #
+      # If a [customer-supplied encryption
+      # key](https://cloud.google.com/storage/docs/encryption#customer-supplied)
+      # was used with {#create_file}, the `encryption_key` and
+      # `encryption_key_sha256` options must be provided or else the file's
+      # CRC32C checksum and MD5 hash will not be returned.
+      #
       # @param [String] path Name (path) of the file.
       # @param [Integer] generation When present, selects a specific revision of
       #   this object. Default is the latest version.
+      # @param [String] encryption_key Optional. The customer-supplied, AES-256
+      #   encryption key used to encrypt the file, if one was provided to
+      #   {#create_file}. Must be provided if `encryption_key_sha256` is
+      #   provided.
+      # @param [String] encryption_key_sha256 Optional. The SHA256 hash of the
+      #   customer-supplied, AES-256 encryption key used to encrypt the file, if
+      #   one was provided to {#create_file}. Must be provided if
+      #   `encryption_key` is provided.
       #
       # @return [Gcloud::Storage::File, nil] Returns nil if file does not exist
       #
@@ -457,9 +447,11 @@ module Gcloud
       #   file = bucket.file "path/to/my-file.ext"
       #   puts file.name
       #
-      def file path, generation: nil
+      def file path, generation: nil, encryption_key: nil,
+               encryption_key_sha256: nil
         ensure_connection!
-        options = { generation: generation }
+        options = { generation: generation, encryption_key: encryption_key,
+                    encryption_key_sha256: encryption_key_sha256 }
         resp = connection.get_file name, path, options
         if resp.success?
           File.from_gapi resp.data, connection
@@ -470,8 +462,8 @@ module Gcloud
       alias_method :find_file, :file
 
       ##
-      # Create a new File object by providing a path to a local file to upload
-      # and the path to store it with in the bucket.
+      # Creates a new {File} object by providing a path to a local file to
+      # upload and the path to store it with in the bucket.
       #
       # A `chunk_size` value can be provided in the options to be used
       # in resumable uploads. This value is the number of bytes per
@@ -479,6 +471,43 @@ module Gcloud
       # by 256KB then it will be lowered to the nearest acceptable
       # value.
       #
+      # #### Customer-supplied encryption keys
+      #
+      # By default, Google Cloud Storage manages server-side encryption keys on
+      # your behalf. However, a [customer-supplied encryption
+      # key](https://cloud.google.com/storage/docs/encryption#customer-supplied)
+      # can be provided with the `encryption_key` and `encryption_key_sha256`
+      # options. If given, the same key and SHA256 hash also must be provided to
+      # subsequently download or copy the file. If you use customer-supplied
+      # encryption keys, you must securely manage your keys and ensure that they
+      # are not lost. Also, please note that file metadata is not encrypted,
+      # with the exception of the CRC32C checksum and MD5 hash. The names of
+      # files and buckets are also not encrypted, and you can read or update the
+      # metadata of an encrypted file without providing the encryption key.
+      #
+      # #### Troubleshooting large uploads
+      #
+      # You may encounter errors while attempting to upload large files. Below
+      # are a couple of common cases and their solutions.
+      #
+      # ##### Handling memory errors
+      #
+      # If you encounter a memory error such as `NoMemoryError`, try performing
+      # a resumable upload and setting the `chunk_size` option to a value that
+      # works for your environment, as explained in the final example above.
+      #
+      # ##### Handling broken pipe errors
+      #
+      # To avoid broken pipe (`Errno::EPIPE`) errors when uploading, add the
+      # [httpclient](https://rubygems.org/gems/httpclient) gem to your project,
+      # and the configuration shown below. These lines must execute after you
+      # require gcloud but before you make your first gcloud connection. The
+      # first statement configures [Faraday](https://rubygems.org/gems/faraday)
+      # to use httpclient. The second statement, which should only be added if
+      # you are using a version of Faraday at or above 0.9.2, is a workaround
+      # for [this gzip
+      # issue](https://github.com/GoogleCloudPlatform/gcloud-ruby/issues/367).
+      #
       # @param [String] file Path of the file on the filesystem to upload.
       # @param [String] path Path to store the file in Google Cloud Storage.
       # @param [String] acl A predefined set of access controls to apply to this
@@ -532,6 +561,12 @@ module Gcloud
       # @param [Hash] metadata A hash of custom, user-provided web-safe keys and
       #   arbitrary string values that will returned with requests for the file
       #   as "x-goog-meta-" response headers.
+      # @param [String] encryption_key Optional. A customer-supplied, AES-256
+      #   encryption key that will be used to encrypt the file. Must be provided
+      #   if `encryption_key_sha256` is provided.
+      # @param [String] encryption_key_sha256 Optional. The SHA256 hash of the
+      #   customer-supplied, AES-256 encryption key that will be used to encrypt
+      #   the file. Must be provided if `encryption_key` is provided.
       #
       # @return [Gcloud::Storage::File]
       #
@@ -545,7 +580,7 @@ module Gcloud
       #
       #   bucket.create_file "path/to/local.file.ext"
       #
-      # @example Additionally, a destination path can be specified.
+      # @example Specifying a destination path:
       #   require "gcloud"
       #
       #   gcloud = Gcloud.new
@@ -556,7 +591,7 @@ module Gcloud
       #   bucket.create_file "path/to/local.file.ext",
       #                      "destination/path/file.ext"
       #
-      # @example Specify the chunk size as a number of bytes:
+      # @example Specifying the chunk size as a number of bytes:
       #   require "gcloud"
       #
       #   gcloud = Gcloud.new
@@ -568,30 +603,31 @@ module Gcloud
       #                      "destination/path/file.ext",
       #                      chunk_size: 1024*1024 # 1 MB chunk
       #
-      # #### Troubleshooting large uploads
+      # @example Providing a customer-supplied encryption key:
+      #   require "gcloud"
+      #   require "digest/sha2"
       #
-      # You may encounter errors while attempting to upload large files. Below
-      # are a couple of common cases and their solutions.
+      #   gcloud = Gcloud.new
+      #   storage = gcloud.storage
+      #   bucket = storage.bucket "my-bucket"
       #
-      # ##### Handling memory errors
+      #   # Key generation shown for example purposes only. Write your own.
+      #   cipher = OpenSSL::Cipher.new "aes-256-cfb"
+      #   cipher.encrypt
+      #   key = cipher.random_key
+      #   key_hash = Digest::SHA256.digest key
       #
-      # If you encounter a memory error such as `NoMemoryError`, try performing
-      # a resumable upload and setting the `chunk_size` option to a value that
-      # works for your environment, as explained in the final example above.
-      #
-      # ##### Handling broken pipe errors
+      #   bucket.create_file "path/to/local.file.ext",
+      #                      "destination/path/file.ext",
+      #                      encryption_key: key,
+      #                      encryption_key_sha256: key_hash
       #
-      # To avoid broken pipe (`Errno::EPIPE`) errors when uploading, add the
-      # [httpclient](https://rubygems.org/gems/httpclient) gem to your project,
-      # and the configuration shown below. These lines must execute after you
-      # require gcloud but before you make your first gcloud connection. The
-      # first statement configures [Faraday](https://rubygems.org/gems/faraday)
-      # to use httpclient. The second statement, which should only be added if
-      # you are using a version of Faraday at or above 0.9.2, is a workaround
-      # for [this gzip
-      # issue](https://github.com/GoogleCloudPlatform/gcloud-ruby/issues/367).
+      #   # Store your key and hash securely for later use.
+      #   file = bucket.file "destination/path/file.ext",
+      #                      encryption_key: key,
+      #                      encryption_key_sha256: key_hash
       #
-      # @example
+      # @example Avoiding broken pipe errors with large uploads:
       #   require "gcloud"
       #
       #   # Use httpclient to avoid broken pipe errors with large uploads
@@ -608,13 +644,16 @@ module Gcloud
       def create_file file, path = nil, acl: nil, cache_control: nil,
                       content_disposition: nil, content_encoding: nil,
                       content_language: nil, content_type: nil, chunk_size: nil,
-                      crc32c: nil, md5: nil, metadata: nil
+                      crc32c: nil, md5: nil, metadata: nil, encryption_key: nil,
+                      encryption_key_sha256: nil
         ensure_connection!
         options = { acl: File::Acl.predefined_rule_for(acl), md5: md5,
                     cache_control: cache_control, content_type: content_type,
                     content_disposition: content_disposition, crc32c: crc32c,
                     content_encoding: content_encoding, chunk_size: chunk_size,
-                    content_language: content_language, metadata: metadata }
+                    content_language: content_language, metadata: metadata,
+                    encryption_key: encryption_key,
+                    encryption_key_sha256: encryption_key_sha256 }
         ensure_file_exists! file
         resumable = resumable_upload?(file)
         resp = @connection.upload_file resumable, name, file, path, options

data/lib/gcloud/storage/bucket/list.rb

@@ -24,23 +24,147 @@ module Gcloud
         ##
         # If not empty, indicates that there are more buckets
         # that match the request and this value should be passed to
-        # the next Gcloud::Storage::Project#buckets to continue.
+        # the next {Gcloud::Storage::Project#buckets} to continue.
         attr_accessor :token
 
         ##
-        # Create a new Bucket::List with an array of values.
-        def initialize arr = [], token = nil
+        # @private Create a new Bucket::List with an array of values.
+        def initialize arr = []
           super arr
-          @token = token
+        end
+
+        ##
+        # Whether there is a next page of buckets.
+        #
+        # @return [Boolean]
+        #
+        # @example
+        #   require "gcloud"
+        #
+        #   gcloud = Gcloud.new
+        #   storage = gcloud.storage
+        #
+        #   buckets = storage.buckets
+        #   if buckets.next?
+        #     next_buckets = buckets.next
+        #   end
+        #
+        def next?
+          !token.nil?
+        end
+
+        ##
+        # Retrieve the next page of buckets.
+        #
+        # @return [Bucket::List]
+        #
+        # @example
+        #   require "gcloud"
+        #
+        #   gcloud = Gcloud.new
+        #   storage = gcloud.storage
+        #
+        #   buckets = storage.buckets
+        #   if buckets.next?
+        #     next_buckets = buckets.next
+        #   end
+        #
+        def next
+          return nil unless next?
+          ensure_connection!
+          options = { prefix: @prefix, token: @token, max: @max }
+          resp = @connection.list_buckets options
+          fail ApiError.from_response(resp) unless resp.success?
+          Bucket::List.from_response resp, @connection, @prefix, @max
+        end
+
+        ##
+        # Retrieves all buckets by repeatedly loading {#next} until {#next?}
+        # returns `false`. Calls the given block once for each bucket, which is
+        # passed as the parameter.
+        #
+        # An Enumerator is returned if no block is given.
+        #
+        # This method may make several API calls until all buckets are
+        # retrieved. Be sure to use as narrow a search criteria as possible.
+        # Please use with caution.
+        #
+        # @param [Integer] request_limit The upper limit of API requests to make
+        #   to load all buckets. Default is no limit.
+        # @yield [bucket] The block for accessing each bucket.
+        # @yieldparam [Bucket] bucket The bucket object.
+        #
+        # @return [Enumerator]
+        #
+        # @example Iterating each bucket by passing a block:
+        #   require "gcloud"
+        #
+        #   gcloud = Gcloud.new
+        #   storage = gcloud.storage
+        #
+        #   buckets = storage.buckets
+        #   buckets.all do |bucket|
+        #     puts bucket.name
+        #   end
+        #
+        # @example Using the enumerator by not passing a block:
+        #   require "gcloud"
+        #
+        #   gcloud = Gcloud.new
+        #   storage = gcloud.storage
+        #
+        #   buckets = storage.buckets
+        #   all_names = buckets.all.map do |bucket|
+        #     bucket.name
+        #   end
+        #
+        # @example Limit the number of API calls made:
+        #   require "gcloud"
+        #
+        #   gcloud = Gcloud.new
+        #   storage = gcloud.storage
+        #
+        #   buckets = storage.buckets
+        #   buckets.all(request_limit: 10) do |bucket|
+        #     puts bucket.name
+        #   end
+        #
+        def all request_limit: nil
+          request_limit = request_limit.to_i if request_limit
+          unless block_given?
+            return enum_for(:all, request_limit: request_limit)
+          end
+          results = self
+          loop do
+            results.each { |r| yield r }
+            if request_limit
+              request_limit -= 1
+              break if request_limit < 0
+            end
+            break unless results.next?
+            results = results.next
+          end
         end
 
         ##
         # @private New Bucket::List from a response object.
-        def self.from_response resp, conn
-          buckets = Array(resp.data["items"]).map do |gapi_object|
+        def self.from_response resp, conn, prefix = nil, max = nil
+          buckets = new(Array(resp.data["items"]).map do |gapi_object|
             Bucket.from_gapi gapi_object, conn
-          end
-          new buckets, resp.data["nextPageToken"]
+          end)
+          buckets.instance_variable_set "@token", resp.data["nextPageToken"]
+          buckets.instance_variable_set "@connection", conn
+          buckets.instance_variable_set "@prefix", prefix
+          buckets.instance_variable_set "@max", max
+          buckets
+        end
+
+        protected
+
+        ##
+        # Raise an error unless an active connection is available.
+        def ensure_connection!
+          fail "Must have active connection" unless @connection
         end
       end
     end