shrine-google_cloud_storage 0.2.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: '08d6183e134fdb573cb7a432c55489f17a59d05a'
-  data.tar.gz: 4c0604286ce8695f96509d6f332f5b6045c408c4
+  metadata.gz: 2e36b4d1de28a447a15aab4f23369f8d580d3486
+  data.tar.gz: 8857d5c06e26ec8c96b4cd0e69672db095eab118
 SHA512:
-  metadata.gz: fa358fd9e4a7a53ba56da9612860eee38998f544858e3010103b492c6fa9ee1ebb8937e39e42e42572daf29aea0e8a9a1d799ac39ebdf5b31248536e07ba8f1d
-  data.tar.gz: 963493555038ef7918db7b129a1b5e837edb59212570ae936cc8fa78db4fd99fe9082d7395d87c47b9f6553297f976575523e3a82cbe5d5d4dcfecfa14ad7c0f
+  metadata.gz: e68c8763ebc015e2c65a8b523b4b56c26115efd9aaad97cd64cea85928abfce73f54a7cc96f8de189c3f70f3bce5b862b304a5ba16d79a6f50525ef02055f1af
+  data.tar.gz: 0e9322682a9934641d37ecfbff19eb9f69318727e989cb6c33eb7e24b6dc36b111c47cd5ec91a818b7e85a4479d54f6d8a87f20e29a7014052ec8032011fc49d

data/README.md

@@ -12,13 +12,13 @@ gem "shrine-google_cloud_storage"
 
 ## Authentication
 
-The GCS plugin uses Google's [Application Default Credentials]. Please check
+The GCS plugin uses Google's [Project and Credential Lookup]. Please check
 documentation for the various ways to provide credentials.
 
 ## Usage
 
 ```rb
-require "shrine/storage/gcs"
+require "shrine/storage/google_cloud_storage"
 
 Shrine.storages = {
   cache: Shrine::Storage::GoogleCloudStorage.new(bucket: "cache"),
@@ -41,25 +41,60 @@ Shrine::Storage::GoogleCloudStorage.new(
 
 ## Contributing
 
-Firstly you need to create an `.env` file with a dedicated GCS bucket:
+### Test setup
+
+#### Option 1 - use the script
+
+Review the script `test/create_test_environment.sh`. It will:
+- create a Google Cloud project
+- associate it with your billing account
+- create a service account
+- add the `roles/storage.admin` iam policy
+- download the json credentials
+- create a test bucket
+- add the needed variables to your `.env` file
+
+To run, it assumes you have already run `gcloud auth login`.
+It also needs a `.env` file in the project root containing the project name
+and the billing account to use:
 
 ```sh
-# .env
-GCS_BUCKET="..."
+cp .env.sample .env
+# Edit .env to fill in your project and billing accounts
+./test/create_test_environment.sh
 ```
 
-Warning: all content of the bucket is cleared between tests, create a new one only for this usage!
+#### Option 2 - manual setup
 
-Afterwards you can run the tests:
+Create your own bucket and provide variables that allow for [project and credential lookup](http://googlecloudplatform.github.io/google-cloud-ruby/#/docs/google-cloud-storage/v1.6.0/guides/authentication#projectandcredentiallookup).
+For example:
+
+```sh
+GCS_BUCKET=shrine-gcs-test-my-project
+GOOGLE_CLOUD_PROJECT=my-project
+GOOGLE_CLOUD_KEYFILE=/Users/user/.gcp/my-project/shrine-gcs-test.json
+```
+
+**Warning**: all content of the bucket is cleared between tests, create a new one only for this usage!
+
+### Running tests
+
+After setting up your bucket, run the tests:
 
 ```sh
 $ bundle exec rake test
 ```
 
+For additional debug, add the following to your `.env` file:
+
+```sh
+GCS_DEBUG=true
+```
+
 ## License
 
 [MIT](http://opensource.org/licenses/MIT)
 
 [Google Cloud Storage]: https://cloud.google.com/storage/
 [Shrine]: https://github.com/janko-m/shrine
-[Application Default Credentials]: https://developers.google.com/identity/protocols/application-default-credentials
+[Project and Credential Lookup]: http://googlecloudplatform.github.io/google-cloud-ruby/#/docs/google-cloud-storage/master/guides/authentication#projectandcredentiallookup

data/lib/shrine/storage/google_cloud_storage.rb

@@ -1,129 +1,124 @@
 require "shrine"
-require "googleauth"
-require "google/apis/storage_v1"
+require "google/cloud/storage"
+require "down/chunked_io"
 
 class Shrine
   module Storage
     class GoogleCloudStorage
       attr_reader :bucket, :prefix, :host
 
-      def initialize(bucket:, prefix: nil, host: nil, default_acl: nil, object_options: {})
+      # Initialize a Shrine::Storage for GCS allowing for auto-discovery of the Google::Cloud::Storage client.
+      # @param [String] project Provide if not using auto discovery
+      # @see http://googlecloudplatform.github.io/google-cloud-ruby/#/docs/google-cloud-storage/v1.6.0/guides/authentication#environmentvariables for information on discovery
+      def initialize(project: nil, bucket:, prefix: nil, host: nil, default_acl: nil, object_options: {})
+        @project = project
         @bucket = bucket
         @prefix = prefix
         @host = host
         @default_acl = default_acl
         @object_options = object_options
+        @storage = nil
       end
 
-      def upload(io, id, shrine_metadata: {}, **_options)
-        # uploads `io` to the location `id`
-
-        object = Google::Apis::StorageV1::Object.new @object_options.merge(bucket: @bucket, name: object_name(id))
-
+      # If the file is an UploadFile from GCS, issues a copy command, otherwise it uploads a file.
+      # @param [IO] io - io like object
+      # @param [String] id - location
+      def upload(io, id, shrine_metadata: {}, **options)
         if copyable?(io)
-          storage_api.copy_object(
-            io.storage.bucket,
-            io.storage.object_name(io.id),
-            @bucket,
-            object_name(id),
-            object,
-            destination_predefined_acl: @default_acl,
-          )
+          existing_file = get_bucket(io.storage.bucket).file(io.storage.object_name(io.id))
+          file = existing_file.copy(
+              @bucket, # dest_bucket_or_path - the bucket to copy the file to
+              object_name(id), # dest_path - the path to copy the file to in the given bucket
+              acl: @default_acl
+          ) do |f|
+            # update the additional options
+            @object_options.merge(options).each_pair do |key, value|
+              f.send("#{key}=", value)
+            end
+          end
+          file
         else
-          storage_api.insert_object(
-            @bucket,
-            object,
-            content_type: shrine_metadata["mime_type"],
-            upload_source: io.to_io,
-            options: { uploadType: 'multipart' },
-            predefined_acl: @default_acl,
+          get_bucket.create_file(
+              prepare_io(io), # file -  IO object, or IO-ish object like StringIO
+              object_name(id), # path
+              @object_options.merge(
+                  content_type: shrine_metadata["mime_type"],
+                  acl: @default_acl
+              ).merge(options)
           )
         end
       end
 
-      def url(id, **_options)
-        # URL to the remote file, accepts options for customizing the URL
-        host = @host || "storage.googleapis.com/#{@bucket}"
-
-        "https://#{host}/#{object_name(id)}"
+      # URL to the remote file, accepts options for customizing the URL
+      def url(id, **options)
+        if(options.key? :expires)
+          signed_url = presign(id, options).url
+          if @host.nil?
+            signed_url
+          else
+            signed_url.gsub(/storage.googleapis.com\/#{@bucket}/, @host)
+          end
+        else
+          host = @host || "storage.googleapis.com/#{@bucket}"
+          "https://#{host}/#{object_name(id)}"
+        end
       end
 
+      # Downloads the file from GCS, and returns a `Tempfile`.
       def download(id)
         tempfile = Tempfile.new(["googlestorage", File.extname(id)], binmode: true)
-        storage_api.get_object(@bucket, object_name(id), download_dest: tempfile)
+        get_file(id).download tempfile.path
         tempfile.tap(&:open)
       end
 
+      # Opens the remote file and returns it as `Down::ChunkedIO` object.
+      # @return [Down::ChunkedIO] object
+      # @see https://github.com/janko-m/down#downchunkedio
       def open(id)
-        # returns the remote file as an IO-like object
-        io = storage_api.get_object(@bucket, object_name(id), download_dest: StringIO.new)
-        io.rewind
-        io
-      end
+        file = get_file(id)
 
-      def exists?(id)
-        # checks if the file exists on the storage
-        storage_api.get_object(@bucket, object_name(id)) do |_, err|
-          if err
-            if err.status_code == 404
-              false
-            else
-              raise err
-            end
-          else
-            true
-          end
+        # create enumerator which lazily yields chunks of downloaded content
+        chunks = Enumerator.new do |yielder|
+          # trick to get google client to stream the download
+          proc_io = ProcIO.new { |data| yielder << data }
+          file.download(proc_io, verify: :none)
         end
-      end
 
-      def delete(id)
-        # deletes the file from the storage
-        storage_api.delete_object(@bucket, object_name(id))
-
-      rescue Google::Apis::ClientError => e
-        # The object does not exist, Shrine expects us to be ok
-        return true if e.status_code == 404
+        # wrap chunks in an IO-like object which downloads when needed
+        Down::ChunkedIO.new(
+          chunks: chunks,
+          size:   file.size,
+          data:   { file: file }
+        )
+      end
 
-        raise e
+      # checks if the file exists on the storage
+      def exists?(id)
+        file = get_file(id)
+        return false if file.nil?
+        file.exists?
       end
 
-      def multi_delete(ids)
-        batch_delete(ids.map { |i| object_name(i) })
+      # deletes the file from the storage
+      def delete(id)
+        file = get_file(id)
+        file.delete unless file.nil?
       end
 
+      # Otherwise deletes all objects from the storage.
       def clear!
-        all_objects = storage_api.fetch_all do |token, s|
-          prefix = "#{@prefix}/" if @prefix
-          s.list_objects(
-            @bucket,
-            prefix: prefix,
-            fields: "items/name",
-            page_token: token,
-          )
+        prefix = "#{@prefix}/" if @prefix
+        files = get_bucket.files prefix: prefix
+        batch_delete(files.lazy.map(&:name))
+        loop do
+          break if !files.next?
+          batch_delete(files.next.lazy.map(&:name))
         end
-
-        batch_delete(all_objects.lazy.map(&:name))
       end
 
       def presign(id, **options)
-        method = options[:method] || "GET"
-        content_md5 = options[:content_md5] || ""
-        content_type = options[:content_type] || ""
-        expires = (Time.now.utc + (options[:expires] || 300)).to_i
-        headers = nil
-        path = "/#{@bucket}/" + object_name(id)
-
-        to_sign = [method, content_md5, content_type, expires, headers, path].compact.join("\n")
-
-        signing_key = options[:signing_key]
-        signing_key = OpenSSL::PKey::RSA.new(signing_key) unless signing_key.respond_to?(:sign)
-        signature = Base64.strict_encode64(signing_key.sign(OpenSSL::Digest::SHA256.new, to_sign)).delete("\n")
-
-        signed_url = "https://storage.googleapis.com#{path}?GoogleAccessId=#{options[:issuer]}" \
-          "&Expires=#{expires}&Signature=#{CGI.escape(signature)}"
-
         OpenStruct.new(
-          url: signed_url,
+          url: storage.signed_url(@bucket, object_name(id), options),
           fields: {},
         )
       end
@@ -134,33 +129,70 @@ class Shrine
 
       private
 
+      def get_file(id)
+        get_bucket.file(object_name(id))
+      end
+
+      def get_bucket(bucket_name = @bucket)
+        storage.bucket(bucket_name, skip_lookup: true)
+      end
+
+      # @see http://googlecloudplatform.github.io/google-cloud-ruby/#/docs/google-cloud-storage/v1.6.0/guides/authentication
+      def storage
+        @storage ||= if @project.nil?
+                       Google::Cloud::Storage.new
+                     else
+                       Google::Cloud::Storage.new(project: @project)
+                     end
+      end
+
       def copyable?(io)
         io.is_a?(UploadedFile) &&
           io.storage.is_a?(Storage::GoogleCloudStorage)
         # TODO: add a check for the credentials
       end
 
+      # Google cloud storage client only accepts IO|IOString|Tempfile instances
+      # or else it will raise a "Google::Apis::ClientError, 'Invalid upload source"
+      #
+      # We need to convert our file to an IO.
+      j
+      # see:
+      # https://github.com/google/google-api-ruby-client/blob/1c2cf5d57fd5e606c03f2aecab88469f46b8f3b2/lib/google/apis/core/upload.rb#L77
+      def prepare_io(io)
+        if io.respond_to?(:to_io)
+          io.to_io
+        elsif io.respond_to?(:tempfile)
+          io.tempfile
+        else
+          io
+        end
+      end
+
       def batch_delete(object_names)
-        # Batches are limited to 100 operations
-        object_names.each_slice(100) do |names|
-          storage_api.batch do |storage|
-            names.each do |name|
-              storage.delete_object(@bucket, name)
-            end
-          end
+        bucket = get_bucket
+        object_names.each do |name|
+          bucket.file(name).delete
         end
       end
 
-      def storage_api
-        if !@storage_api || @storage_api.authorization.expired?
-          service = Google::Apis::StorageV1::StorageService.new
-          scopes = ['https://www.googleapis.com/auth/devstorage.read_write']
-          authorization = Google::Auth.get_application_default(scopes)
-          authorization.fetch_access_token!
-          service.authorization = authorization
-          @storage_api = service
+      # This class provides a writable IO wrapper around a proc object, with
+      # #write simply calling the proc, which we can pass in as the destination
+      # IO for download.
+      class ProcIO
+        def initialize(&proc)
+          @proc = proc
+        end
+
+        def write(data)
+          @proc.call(data)
+          data.bytesize # match return value of other IO objects
+        end
+
+        # TODO: Remove this once google/google-api-ruby-client#638 is merged.
+        def flush
+          # google-api-client calls this method
         end
-        @storage_api
       end
     end
   end

data/shrine-google_cloud_storage.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |gem|
   gem.name          = "shrine-google_cloud_storage"
-  gem.version       = "0.2.0"
+  gem.version       = "1.0.0"
 
   gem.required_ruby_version = ">= 2.1"
 
@@ -14,7 +14,7 @@ Gem::Specification.new do |gem|
   gem.require_path = "lib"
 
   gem.add_dependency "shrine", "~> 2.0"
-  gem.add_dependency "google-api-client", "~> 0.13.0"
+  gem.add_dependency "google-cloud-storage", "~> 1.6"
 
   gem.add_development_dependency "rake"
   gem.add_development_dependency "minitest"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: shrine-google_cloud_storage
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 1.0.0
 platform: ruby
 authors:
 - Renaud Chaput
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-06-23 00:00:00.000000000 Z
+date: 2017-12-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: shrine
@@ -25,19 +25,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.0'
 - !ruby/object:Gem::Dependency
-  name: google-api-client
+  name: google-cloud-storage
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.13.0
+        version: '1.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.13.0
+        version: '1.6'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -110,7 +110,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.2
+rubygems_version: 2.6.14
 signing_key: 
 specification_version: 4
 summary: Provides Google Cloud Storage storage for Shrine.