google-cloud-storage 0.20.0

data/lib/google/cloud/storage/file/verifier.rb

@@ -0,0 +1,69 @@
+# Copyright 2014 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+require "pathname"
+require "digest/md5"
+require "digest/crc32c"
+require "google/cloud/storage/errors"
+
+module Google
+  module Cloud
+    module Storage
+      class File
+        ##
+        # @private
+        # Verifies downloaded files by creating an MD5 or CRC32c hash digest
+        # and comparing the value to the one from the Storage API.
+        module Verifier
+          def self.verify_md5! gcloud_file, local_file
+            gcloud_digest = gcloud_file.md5
+            local_digest = md5_for local_file
+            if gcloud_digest != local_digest
+              fail FileVerificationError.for_md5(gcloud_digest, local_digest)
+            end
+          end
+
+          def self.verify_crc32c! gcloud_file, local_file
+            gcloud_digest = gcloud_file.crc32c
+            local_digest = crc32c_for local_file
+            if gcloud_digest != local_digest
+              fail FileVerificationError.for_crc32c(gcloud_digest, local_digest)
+            end
+          end
+
+          def self.verify_md5 gcloud_file, local_file
+            gcloud_file.md5 == md5_for(local_file)
+          end
+
+          def self.verify_crc32c gcloud_file, local_file
+            gcloud_file.crc32c == crc32c_for(local_file)
+          end
+
+          def self.md5_for local_file
+            ::File.open(Pathname(local_file).to_path, "rb") do |f|
+              ::Digest::MD5.file(f).base64digest
+            end
+          end
+
+          def self.crc32c_for local_file
+            ::File.open(Pathname(local_file).to_path, "rb") do |f|
+              ::Digest::CRC32c.file(f).base64digest
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/google/cloud/storage/project.rb

@@ -0,0 +1,321 @@
+# Copyright 2014 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+require "google/cloud/core/gce"
+require "google/cloud/storage/errors"
+require "google/cloud/storage/service"
+require "google/cloud/storage/credentials"
+require "google/cloud/storage/bucket"
+require "google/cloud/storage/bucket/cors"
+require "google/cloud/storage/file"
+
+module Google
+  module Cloud
+    module Storage
+      ##
+      # # Project
+      #
+      # Represents the project that storage buckets and files belong to.
+      # All data in Google Cloud Storage belongs inside a project.
+      # A project consists of a set of users, a set of APIs, billing,
+      # authentication, and monitoring settings for those APIs.
+      #
+      # Google::Cloud::Storage::Project is the main object for interacting with
+      # Google Storage. {Google::Cloud::Storage::Bucket} objects are created,
+      # read, updated, and deleted by Google::Cloud::Storage::Project.
+      #
+      # See {Google::Cloud#storage}
+      #
+      # @example
+      #   require "google/cloud"
+      #
+      #   gcloud = Google::Cloud.new
+      #   storage = gcloud.storage
+      #
+      #   bucket = storage.bucket "my-bucket"
+      #   file = bucket.file "path/to/my-file.ext"
+      #
+      class Project
+        ##
+        # @private The Service object.
+        attr_accessor :service
+
+        ##
+        # @private Creates a new Project instance.
+        #
+        # See {Google::Cloud#storage}
+        def initialize service
+          @service = service
+        end
+
+        ##
+        # The Storage project connected to.
+        #
+        # @example
+        #   require "google/cloud"
+        #
+        #   gcloud = Google::Cloud.new "my-todo-project",
+        #                       "/path/to/keyfile.json"
+        #   storage = gcloud.storage
+        #
+        #   storage.project #=> "my-todo-project"
+        #
+        def project
+          service.project
+        end
+
+        ##
+        # @private Default project.
+        def self.default_project
+          ENV["STORAGE_PROJECT"] ||
+            ENV["GOOGLE_CLOUD_PROJECT"] ||
+            ENV["GCLOUD_PROJECT"] ||
+            Google::Cloud::Core::GCE.project_id
+        end
+
+        ##
+        # Retrieves a list of buckets for the given project.
+        #
+        # @param [String] prefix Filter results to buckets whose names begin
+        #   with this prefix.
+        # @param [String] token A previously-returned page token representing
+        #   part of the larger set of results to view.
+        # @param [Integer] max Maximum number of buckets to return.
+        #
+        # @return [Array<Google::Cloud::Storage::Bucket>] (See
+        #   {Google::Cloud::Storage::Bucket::List})
+        #
+        # @example
+        #   require "google/cloud"
+        #
+        #   gcloud = Google::Cloud.new
+        #   storage = gcloud.storage
+        #
+        #   buckets = storage.buckets
+        #   buckets.each do |bucket|
+        #     puts bucket.name
+        #   end
+        #
+        # @example Retrieve buckets with names that begin with a given prefix:
+        #   require "google/cloud"
+        #
+        #   gcloud = Google::Cloud.new
+        #   storage = gcloud.storage
+        #
+        #   user_buckets = storage.buckets prefix: "user-"
+        #   user_buckets.each do |bucket|
+        #     puts bucket.name
+        #   end
+        #
+        # @example Retrieve all buckets: (See {Bucket::List#all})
+        #   require "google/cloud"
+        #
+        #   gcloud = Google::Cloud.new
+        #   storage = gcloud.storage
+        #
+        #   buckets = storage.buckets
+        #   buckets.all do |bucket|
+        #     puts bucket.name
+        #   end
+        #
+        def buckets prefix: nil, token: nil, max: nil
+          options = { prefix: prefix, token: token, max: max }
+          gapi = service.list_buckets options
+          Bucket::List.from_gapi gapi, service, prefix, max
+        end
+        alias_method :find_buckets, :buckets
+
+        ##
+        # Retrieves bucket by name.
+        #
+        # @param [String] bucket_name Name of a bucket.
+        #
+        # @return [Google::Cloud::Storage::Bucket, nil] Returns nil if bucket
+        #   does not exist
+        #
+        # @example
+        #   require "google/cloud"
+        #
+        #   gcloud = Google::Cloud.new
+        #   storage = gcloud.storage
+        #
+        #   bucket = storage.bucket "my-bucket"
+        #   puts bucket.name
+        #
+        def bucket bucket_name
+          gapi = service.get_bucket bucket_name
+          Bucket.from_gapi gapi, service
+        rescue Google::Cloud::NotFoundError
+          nil
+        end
+        alias_method :find_bucket, :bucket
+
+        ##
+        # Creates a new bucket with optional attributes. Also accepts a block
+        # for defining the CORS configuration for a static website served from
+        # the bucket. See {Bucket::Cors} for details.
+        #
+        # The API call to create the bucket may be retried under certain
+        # conditions. See {Google::Cloud#storage} to control this behavior.
+        #
+        # You can pass [website
+        # settings](https://cloud.google.com/storage/docs/website-configuration)
+        # for the bucket, including a block that defines CORS rule. See
+        # {Bucket::Cors} for details.
+        #
+        # @see https://cloud.google.com/storage/docs/cross-origin Cross-Origin
+        #   Resource Sharing (CORS)
+        # @see https://cloud.google.com/storage/docs/website-configuration How
+        #   to Host a Static Website
+        #
+        # @param [String] bucket_name Name of a bucket.
+        # @param [String] acl Apply a predefined set of access controls to this
+        #   bucket.
+        #
+        #   Acceptable values are:
+        #
+        #   * `auth`, `auth_read`, `authenticated`, `authenticated_read`,
+        #     `authenticatedRead` - Project team owners get OWNER access, and
+        #     allAuthenticatedUsers get READER access.
+        #   * `private` - Project team owners get OWNER access.
+        #   * `project_private`, `projectPrivate` - Project team members get
+        #     access according to their roles.
+        #   * `public`, `public_read`, `publicRead` - Project team owners get
+        #     OWNER access, and allUsers get READER access.
+        #   * `public_write`, `publicReadWrite` - Project team owners get OWNER
+        #     access, and allUsers get WRITER access.
+        # @param [String] default_acl Apply a predefined set of default object
+        #   access controls to this bucket.
+        #
+        #   Acceptable values are:
+        #
+        #   * `auth`, `auth_read`, `authenticated`, `authenticated_read`,
+        #     `authenticatedRead` - File owner gets OWNER access, and
+        #     allAuthenticatedUsers get READER access.
+        #   * `owner_full`, `bucketOwnerFullControl` - File owner gets OWNER
+        #     access, and project team owners get OWNER access.
+        #   * `owner_read`, `bucketOwnerRead` - File owner gets OWNER access,
+        #     and project team owners get READER access.
+        #   * `private` - File owner gets OWNER access.
+        #   * `project_private`, `projectPrivate` - File owner gets OWNER
+        #     access, and project team members get access according to their
+        #     roles.
+        #   * `public`, `public_read`, `publicRead` - File owner gets OWNER
+        #     access, and allUsers get READER access.
+        # @param [String] location The location of the bucket. Object data for
+        #   objects in the bucket resides in physical storage within this
+        #   region. Possible values include `ASIA`, `EU`, and `US`. (See the
+        #   [developer's
+        #   guide](https://cloud.google.com/storage/docs/bucket-locations) for
+        #   the authoritative list. The default value is `US`.
+        # @param [String] logging_bucket The destination bucket for the bucket's
+        #   logs. For more information, see [Access
+        #   Logs](https://cloud.google.com/storage/docs/access-logs).
+        # @param [String] logging_prefix The prefix used to create log object
+        #   names for the bucket. It can be at most 900 characters and must be a
+        #   [valid object
+        #   name](https://cloud.google.com/storage/docs/bucket-naming#objectnames)
+        #   . By default, the object prefix is the name of the bucket for which
+        #   the logs are enabled. For more information, see [Access
+        #   Logs](https://cloud.google.com/storage/docs/access-logs).
+        # @param [Symbol, String] storage_class Defines how objects in the
+        #   bucket are stored and determines the SLA and the cost of storage.
+        #   Values include `:standard`, `:nearline`, and `:dra` (Durable Reduced
+        #   Availability), as well as the strings returned by
+        #   Bucket#storage_class. For more information, see [Storage
+        #   Classes](https://cloud.google.com/storage/docs/storage-classes). The
+        #   default value is `:standard`.
+        # @param [Boolean] versioning Whether [Object
+        #   Versioning](https://cloud.google.com/storage/docs/object-versioning)
+        #   is to be enabled for the bucket. The default value is `false`.
+        # @param [String] website_main The index page returned from a static
+        #   website served from the bucket when a site visitor requests the top
+        #   level directory. For more information, see [How to Host a Static
+        #   Website
+        #   ](https://cloud.google.com/storage/docs/website-configuration#step4).
+        # @param [String] website_404 The page returned from a static website
+        #   served from the bucket when a site visitor requests a resource that
+        #   does not exist. For more information, see [How to Host a Static
+        #   Website
+        #   ](https://cloud.google.com/storage/docs/website-configuration#step4).
+        # @yield [bucket] a block for configuring the bucket before it is
+        #   created
+        # @yieldparam [Bucket] cors the bucket object to be configured
+        #
+        # @return [Google::Cloud::Storage::Bucket]
+        #
+        # @example
+        #   require "google/cloud"
+        #
+        #   gcloud = Google::Cloud.new
+        #   storage = gcloud.storage
+        #
+        #   bucket = storage.create_bucket "my-bucket"
+        #
+        # @example Configure the bucket in a block:
+        #   require "google/cloud"
+        #
+        #   gcloud = Google::Cloud.new
+        #   storage = gcloud.storage
+        #
+        #   bucket = storage.create_bucket "my-bucket" do |b|
+        #     b.website_main = "index.html"
+        #     b.website_404 = "not_found.html"
+        #     b.cors.add_rule ["http://example.org", "https://example.org"],
+        #                      "*",
+        #                      response_headers: ["X-My-Custom-Header"],
+        #                      max_age: 300
+        #   end
+        #
+        def create_bucket bucket_name, acl: nil, default_acl: nil,
+                          location: nil, storage_class: nil,
+                          logging_bucket: nil, logging_prefix: nil,
+                          website_main: nil, website_404: nil, versioning: nil
+          new_bucket = Google::Apis::StorageV1::Bucket.new({
+            name: bucket_name,
+            location: location,
+            storage_class: storage_class_for(storage_class)
+          }.delete_if { |_, v| v.nil? })
+          updater = Bucket::Updater.new(new_bucket).tap do |b|
+            b.logging_bucket = logging_bucket unless logging_bucket.nil?
+            b.logging_prefix = logging_prefix unless logging_prefix.nil?
+            b.website_main = website_main unless website_main.nil?
+            b.website_404 = website_404 unless website_404.nil?
+            b.versioning = versioning unless versioning.nil?
+          end
+          yield updater if block_given?
+          updater.check_for_mutable_cors!
+          gapi = service.insert_bucket \
+            new_bucket, acl: acl_rule(acl), default_acl: acl_rule(default_acl)
+          Bucket.from_gapi gapi, service
+        end
+
+        protected
+
+        def acl_rule option_name
+          Bucket::Acl.predefined_rule_for option_name
+        end
+
+        def storage_class_for str
+          { "durable_reduced_availability" => "DURABLE_REDUCED_AVAILABILITY",
+            "dra" => "DURABLE_REDUCED_AVAILABILITY",
+            "durable" => "DURABLE_REDUCED_AVAILABILITY",
+            "nearline" => "NEARLINE",
+            "standard" => "STANDARD" }[str.to_s.downcase]
+        end
+      end
+    end
+  end
+end

data/lib/google/cloud/storage/service.rb

@@ -0,0 +1,310 @@
+# Copyright 2014 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+require "google/cloud/storage/version"
+require "google/cloud/core/grpc_backoff"
+require "google/apis/storage_v1"
+require "digest"
+require "mime/types"
+require "pathname"
+
+module Google
+  module Cloud
+    module Storage
+      ##
+      # @private Represents the connection to Storage,
+      # as well as expose the API calls.
+      class Service
+        ##
+        # Alias to the Google Client API module
+        API = Google::Apis::StorageV1
+
+        # @private
+        attr_accessor :project
+
+        # @private
+        attr_accessor :credentials
+
+        ##
+        # Creates a new Service instance.
+        def initialize project, credentials, retries: nil, timeout: nil
+          @project = project
+          @credentials = credentials
+          @credentials = credentials
+          @service = API::StorageService.new
+          @service.client_options.application_name    = "google-cloud-storage"
+          @service.client_options.application_version = \
+            Google::Cloud::Storage::VERSION
+          @service.request_options.retries = retries || 3
+          @service.request_options.timeout_sec = timeout if timeout
+          @service.authorization = @credentials.client
+        end
+
+        def service
+          return mocked_service if mocked_service
+          @service
+        end
+        attr_accessor :mocked_service
+
+        ##
+        # Retrieves a list of buckets for the given project.
+        def list_buckets prefix: nil, token: nil, max: nil
+          execute do
+            service.list_buckets @project, prefix: prefix, page_token: token,
+                                           max_results: max
+          end
+        end
+
+        ##
+        # Retrieves bucket by name.
+        # Returns Google::Apis::StorageV1::Bucket.
+        def get_bucket bucket_name
+          execute { service.get_bucket bucket_name }
+        end
+
+        ##
+        # Creates a new bucket.
+        # Returns Google::Apis::StorageV1::Bucket.
+        def insert_bucket bucket_gapi, options = {}
+          execute do
+            service.insert_bucket \
+              @project, bucket_gapi,
+              predefined_acl: options[:acl],
+              predefined_default_object_acl: options[:default_acl]
+          end
+        end
+
+        ##
+        # Updates a bucket, including its ACL metadata.
+        def patch_bucket bucket_name, bucket_gapi = nil, predefined_acl: nil,
+                         predefined_default_acl: nil
+          bucket_gapi ||= Google::Apis::StorageV1::Bucket.new
+          bucket_gapi.acl = nil if predefined_acl
+          bucket_gapi.default_object_acl = nil if predefined_default_acl
+
+          execute do
+            service.patch_bucket \
+              bucket_name, bucket_gapi,
+              predefined_acl: predefined_acl,
+              predefined_default_object_acl: predefined_default_acl
+          end
+        end
+
+        ##
+        # Permanently deletes an empty bucket.
+        def delete_bucket bucket_name
+          execute { service.delete_bucket bucket_name }
+        end
+
+        ##
+        # Retrieves a list of ACLs for the given bucket.
+        def list_bucket_acls bucket_name
+          execute { service.list_bucket_access_controls bucket_name }
+        end
+
+        ##
+        # Creates a new bucket ACL.
+        def insert_bucket_acl bucket_name, entity, role
+          new_acl = Google::Apis::StorageV1::BucketAccessControl.new \
+            entity: entity, role: role
+          execute { service.insert_bucket_access_control bucket_name, new_acl }
+        end
+
+        ##
+        # Permanently deletes a bucket ACL.
+        def delete_bucket_acl bucket_name, entity
+          execute { service.delete_bucket_access_control bucket_name, entity }
+        end
+
+        ##
+        # Retrieves a list of default ACLs for the given bucket.
+        def list_default_acls bucket_name
+          execute { service.list_default_object_access_controls bucket_name }
+        end
+
+        ##
+        # Creates a new default ACL.
+        def insert_default_acl bucket_name, entity, role
+          new_acl = Google::Apis::StorageV1::ObjectAccessControl.new \
+            entity: entity, role: role
+          execute do
+            service.insert_default_object_access_control bucket_name, new_acl
+          end
+        end
+
+        ##
+        # Permanently deletes a default ACL.
+        def delete_default_acl bucket_name, entity
+          execute do
+            service.delete_default_object_access_control bucket_name, entity
+          end
+        end
+
+        ##
+        # Retrieves a list of files matching the criteria.
+        def list_files bucket_name, options = {}
+          execute do
+            service.list_objects \
+              bucket_name, delimiter: options[:delimiter],
+                           max_results: options[:max],
+                           page_token: options[:token],
+                           prefix: options[:prefix],
+                           versions: options[:versions]
+          end
+        end
+
+        ##
+        # Inserts a new file for the given bucket
+        def insert_file bucket_name, source, path = nil, acl: nil,
+                        cache_control: nil, content_disposition: nil,
+                        content_encoding: nil, content_language: nil,
+                        content_type: nil, crc32c: nil, md5: nil, metadata: nil,
+                        key: nil, key_sha256: nil
+          file_obj = Google::Apis::StorageV1::Object.new \
+            cache_control: cache_control, content_type: content_type,
+            content_disposition: content_disposition, md5_hash: md5,
+            content_encoding: content_encoding, crc32c: crc32c,
+            content_language: content_language, metadata: metadata
+          content_type ||= mime_type_for(Pathname(source).to_path)
+          execute do
+            service.insert_object \
+              bucket_name, file_obj,
+              name: path, predefined_acl: acl, upload_source: source,
+              content_encoding: content_encoding, content_type: content_type,
+              options: key_options(key: key, key_sha256: key_sha256)
+          end
+        end
+
+        ##
+        # Retrieves an object or its metadata.
+        def get_file bucket_name, file_path, generation: nil, key: nil,
+                     key_sha256: nil
+          execute do
+            service.get_object \
+              bucket_name, file_path,
+              generation: generation,
+              options: key_options(key: key, key_sha256: key_sha256)
+          end
+        end
+
+        ## Copy a file from source bucket/object to a
+        # destination bucket/object.
+        def copy_file source_bucket_name, source_file_path,
+                      destination_bucket_name, destination_file_path,
+                      options = {}
+          execute do
+            service.copy_object \
+              source_bucket_name, source_file_path,
+              destination_bucket_name, destination_file_path,
+              destination_predefined_acl: options[:acl],
+              source_generation: options[:generation],
+              options: key_options(key: options[:key],
+                                   key_sha256: options[:key_sha256])
+          end
+        end
+
+        ##
+        # Download contents of a file.
+        def download_file bucket_name, file_path, target_path, generation: nil,
+                          key: nil, key_sha256: nil
+          execute do
+            service.get_object \
+              bucket_name, file_path,
+              download_dest: target_path, generation: generation,
+              options: key_options(key: key, key_sha256: key_sha256)
+          end
+        end
+
+        ##
+        # Updates a file's metadata.
+        def patch_file bucket_name, file_path, file_gapi = nil,
+                       predefined_acl: nil
+          file_gapi ||= Google::Apis::StorageV1::Object.new
+          execute do
+            service.patch_object \
+              bucket_name, file_path, file_gapi,
+              predefined_acl: predefined_acl
+          end
+        end
+
+        ##
+        # Permanently deletes a file.
+        def delete_file bucket_name, file_path
+          execute { service.delete_object bucket_name, file_path }
+        end
+
+        ##
+        # Retrieves a list of ACLs for the given file.
+        def list_file_acls bucket_name, file_name
+          execute { service.list_object_access_controls bucket_name, file_name }
+        end
+
+        ##
+        # Creates a new file ACL.
+        def insert_file_acl bucket_name, file_name, entity, role, options = {}
+          new_acl = Google::Apis::StorageV1::ObjectAccessControl.new \
+            entity: entity, role: role
+          execute do
+            service.insert_object_access_control \
+              bucket_name, file_name, new_acl, generation: options[:generation]
+          end
+        end
+
+        ##
+        # Permanently deletes a file ACL.
+        def delete_file_acl bucket_name, file_name, entity, options = {}
+          execute do
+            service.delete_object_access_control \
+              bucket_name, file_name, entity, generation: options[:generation]
+          end
+        end
+
+        ##
+        # Retrieves the mime-type for a file path.
+        # An empty string is returned if no mime-type can be found.
+        def mime_type_for path
+          MIME::Types.of(path).first.to_s
+        end
+
+        # @private
+        def inspect
+          "#{self.class}(#{@project})"
+        end
+
+        protected
+
+        def key_options key: nil, key_sha256: nil
+          options = {}
+          if key
+            headers = {}
+            headers["x-goog-encryption-algorithm"] = "AES256"
+            headers["x-goog-encryption-key"] = Base64.strict_encode64 key
+            key_sha256 ||= Digest::SHA256.digest key
+            headers["x-goog-encryption-key-sha256"] = \
+              Base64.strict_encode64 key_sha256
+            options[:header] = headers
+          end
+          options
+        end
+
+        def execute
+          yield
+        rescue Google::Apis::Error => e
+          raise Google::Cloud::Error.from_error(e)
+        end
+      end
+    end
+  end
+end