fog-google 1.10.0 → 1.14.0

data/lib/fog/storage/google_json/real.rb

@@ -13,7 +13,14 @@ module Fog
           options[:google_api_scope_url] = GOOGLE_STORAGE_JSON_API_SCOPE_URLS.join(" ")
           @host = options[:host] || "storage.googleapis.com"
 
+          # TODO(temikus): Do we even need this client?
           @client = initialize_google_client(options)
+          # IAM client used for SignBlob API
+          @iam_service = ::Google::Apis::IamcredentialsV1::IAMCredentialsService.new
+          apply_client_options(@iam_service, {
+                                 google_api_scope_url: GOOGLE_STORAGE_JSON_IAM_API_SCOPE_URLS.join(" ")
+                               })
+
           @storage_json = ::Google::Apis::StorageV1::StorageService.new
           apply_client_options(@storage_json, options)
 
@@ -56,12 +63,108 @@ DATA
           canonical_resource.chop!
           string_to_sign << canonical_resource.to_s
 
-          key = OpenSSL::PKey::RSA.new(@client.signing_key)
-          digest = OpenSSL::Digest::SHA256.new
-          signed_string = key.sign(digest, string_to_sign)
+          # TODO(temikus): make signer configurable or add ability to supply your own via lambda
+          if !@storage_json.authorization.signing_key.nil?
+            signed_string = default_signer(string_to_sign)
+          else
+            # If client doesn't contain signing key attempt to auth via IAM SignBlob API
+            signed_string = iam_signer(string_to_sign)
+          end
 
           Base64.encode64(signed_string).chomp!
         end
+
+        private
+
+        def google_access_id
+          @google_access_id ||= get_google_access_id
+        end
+
+        ##
+        # Fetches the google service account name
+        #
+        # @return [String] Service account name, typically needed for GoogleAccessId, e.g.
+        #   my-account@project.iam.gserviceaccount
+        # @raises [Fog::Errors::Error] If authorisation is incorrect or inapplicable for current action
+        def get_google_access_id
+          if @storage_json.authorization.is_a?(::Google::Auth::UserRefreshCredentials)
+            raise Fog::Errors::Error.new("User / Application Default Credentials are not supported for storage"\
+                                         "url signing, please use a service account or metadata authentication.")
+          end
+
+          if !@storage_json.authorization.issuer.nil?
+            return @storage_json.authorization.issuer
+          else
+            get_access_id_from_metadata
+          end
+        end
+
+        ##
+        # Attempts to fetch the google service account name from metadata using Google::Cloud::Env
+        #
+        # @return [String] Service account name, typically needed for GoogleAccessId, e.g.
+        #   my-account@project.iam.gserviceaccount
+        # @raises [Fog::Errors::Error] If Metadata service is not available or returns an invalid response
+        def get_access_id_from_metadata
+          if @google_cloud_env.metadata?
+            access_id = @google_cloud_env.lookup_metadata("instance", "service-accounts/default/email")
+          else
+            raise Fog::Errors::Error.new("Metadata service not available, unable to retrieve service account info.")
+          end
+
+          if access_id.nil?
+            raise Fog::Errors::Error.new("Metadata service found but didn't return data." \
+               "Please file a bug: https://github.com/fog/fog-google")
+          end
+
+          return access_id
+        end
+
+        ##
+        # Default url signer using service account keys
+        #
+        # @param [String] string_to_sign Special collection of headers and options for V2 storage signing, e.g.:
+        #
+        #   StringToSign = HTTP_Verb + "\n" +
+        #                  Content_MD5 + "\n" +
+        #                  Content_Type + "\n" +
+        #                  Expires + "\n" +
+        #                  Canonicalized_Extension_Headers +
+        #                  Canonicalized_Resource
+        #
+        #   See https://cloud.google.com/storage/docs/access-control/signed-urls-v2
+        # @return [String] Signature binary blob
+        def default_signer(string_to_sign)
+          key = OpenSSL::PKey::RSA.new(@storage_json.authorization.signing_key)
+          digest = OpenSSL::Digest::SHA256.new
+          return key.sign(digest, string_to_sign)
+        end
+
+        ##
+        # Fallback URL signer using the IAM SignServiceAccountBlob API, see
+        #   Google::Apis::IamcredentialsV1::IAMCredentialsService#sign_service_account_blob
+        #
+        # @param [String] string_to_sign Special collection of headers and options for V2 storage signing, e.g.:
+        #
+        #   StringToSign = HTTP_Verb + "\n" +
+        #                  Content_MD5 + "\n" +
+        #                  Content_Type + "\n" +
+        #                  Expires + "\n" +
+        #                  Canonicalized_Extension_Headers +
+        #                  Canonicalized_Resource
+        #
+        #   See https://cloud.google.com/storage/docs/access-control/signed-urls-v2
+        # @return [String] Signature binary blob
+        def iam_signer(string_to_sign)
+          request = {
+            "payload": string_to_sign
+          }
+
+          resource = "projects/-/serviceAccounts/#{google_access_id}"
+          response = @iam_service.sign_service_account_blob resource, request, {}
+
+          return response.signed_blob
+        end
       end
     end
   end

data/lib/fog/storage/google_json/requests/copy_object.rb

@@ -14,9 +14,20 @@ module Fog
         def copy_object(source_bucket, source_object,
                         target_bucket, target_object, options = {})
           request_options = ::Google::Apis::RequestOptions.default.merge(options)
+
+          object = ::Google::Apis::StorageV1::Object.new(**options)
+
           @storage_json.copy_object(source_bucket, source_object,
                                     target_bucket, target_object,
-                                    request_options)
+                                    object, options: request_options, **filter_keyword_args(options))
+        end
+
+        private
+
+        def filter_keyword_args(options)
+          method = @storage_json.method(:copy_object)
+          allowed = method.parameters.filter { |param| %i(key keyreq).include?(param[0]) }.map { |param| param[1] }.compact
+          options.filter { |key, _| allowed.include?(key) }
         end
       end
 

data/lib/fog/storage/google_json/requests/get_object.rb

@@ -39,6 +39,7 @@ module Fog
           raise ArgumentError.new("object_name is required") unless object_name
 
           buf = Tempfile.new("fog-google-storage-temp")
+          buf.unlink
 
           # Two requests are necessary, first for metadata, then for content.
           # google-api-ruby-client doesn't allow fetching both metadata and content
@@ -53,18 +54,19 @@ module Fog
             :options => request_options
           }
 
-          object = @storage_json.get_object(bucket_name, object_name, all_opts).to_h
+          object = @storage_json.get_object(bucket_name, object_name, **all_opts).to_h
           @storage_json.get_object(
             bucket_name,
             object_name,
-            all_opts.merge(:download_dest => buf.path)
+            **all_opts.merge(:download_dest => buf)
           )
 
+          buf.seek(0)
+
           if block_given?
             yield buf.read, nil, nil
           else
             object[:body] = buf.read
-            buf.unlink
           end
 
           object

data/lib/fog/storage/google_json/requests/list_buckets.rb

@@ -11,10 +11,10 @@ module Fog
                          prefix: nil, projection: nil)
           @storage_json.list_buckets(
             @project,
-            :max_results => max_results,
-            :page_token => page_token,
-            :prefix => prefix,
-            :projection => projection
+            max_results: max_results,
+            page_token: page_token,
+            prefix: prefix,
+            projection: projection
           )
         end
       end

data/lib/fog/storage/google_json/requests/list_objects.rb

@@ -31,7 +31,7 @@ module Fog
 
           @storage_json.list_objects(
             bucket,
-            options.select { |k, _| allowed_opts.include? k }
+            **options.select { |k, _| allowed_opts.include? k }
           )
         end
       end

data/lib/fog/storage/google_json/requests/put_bucket.rb

@@ -19,7 +19,7 @@ module Fog
                        predefined_default_object_acl: nil,
                        **options)
           bucket = ::Google::Apis::StorageV1::Bucket.new(
-            options.merge(:name => bucket_name)
+            **options.merge(:name => bucket_name)
           )
 
           @storage_json.insert_bucket(

data/lib/fog/storage/google_json/requests/put_bucket_acl.rb

@@ -12,7 +12,7 @@ module Fog
           raise ArgumentError.new("bucket_name is required") unless bucket_name
           raise ArgumentError.new("acl is required") unless acl
 
-          acl_update = ::Google::Apis::StorageV1::BucketAccessControl.new(acl)
+          acl_update = ::Google::Apis::StorageV1::BucketAccessControl.new(**acl)
           @storage_json.insert_bucket_access_control(bucket_name, acl_update)
         end
       end

data/lib/fog/storage/google_json/requests/put_object.rb

@@ -51,7 +51,7 @@ module Fog
           data, options = normalize_data(data, options)
 
           object_config = ::Google::Apis::StorageV1::Object.new(
-            options.merge(:name => object_name)
+            **options.merge(:name => object_name)
           )
 
           @storage_json.insert_object(

data/lib/fog/storage/google_json/requests/put_object_acl.rb

@@ -15,7 +15,7 @@ module Fog
           raise ArgumentError.new("object_name is required") unless object_name
           raise ArgumentError.new("acl is required") unless acl
 
-          acl_object = ::Google::Apis::StorageV1::ObjectAccessControl.new(acl)
+          acl_object = ::Google::Apis::StorageV1::ObjectAccessControl.new(**acl)
 
           @storage_json.insert_object_access_control(
             bucket_name, object_name, acl_object

data/lib/fog/storage/google_json/utils.rb

@@ -19,7 +19,11 @@ module Fog
 
         def host_path_query(params, expires)
           params[:headers]["Date"] = expires.to_i
-          params[:path] = URI.encode(params[:path]).gsub("%2F", "/")
+          # implementation from CGI.escape, but without ' ' to  '+' conversion
+          params[:path] = params[:path].b.gsub(/([^a-zA-Z0-9_.\-~]+)/) { |m|
+            '%' + m.unpack('H2' * m.bytesize).join('%').upcase
+          }.gsub("%2F", "/")
+
           query = []
 
           if params[:query]
@@ -27,7 +31,7 @@ module Fog
             query = filtered.map { |k, v| [k.to_s, Fog::Google.escape(v)].join("=") }
           end
 
-          query << "GoogleAccessId=#{@client.issuer}"
+          query << "GoogleAccessId=#{google_access_id}"
           query << "Signature=#{CGI.escape(signature(params))}"
           query << "Expires=#{params[:headers]['Date']}"
           "#{params[:host]}/#{params[:path]}?#{query.join('&')}"

data/lib/fog/storage/google_xml/models/file.rb

@@ -51,7 +51,7 @@ module Fog
 
         remove_method :metadata
         def metadata
-          attributes.reject { |key, _value| key.to_s =~ /^x-goog-meta-/ }
+          attributes.select { |key, _value| key.to_s =~ /^x-goog-meta-/ }
         end
 
         remove_method :metadata=
@@ -109,7 +109,7 @@ module Fog
           options["Expires"] = expires if expires
           options.merge!(metadata)
 
-          data = service.put_object(directory.key, key, body, options)
+          data = service.put_object(directory.key, key, body, **options)
           merge_attributes(data.headers.reject { |key, _value| ["Content-Length", "Content-Type"].include?(key) })
           self.content_length = Fog::Storage.get_body_size(body)
           self.content_type ||= Fog::Storage.get_content_type(body)

data/lib/fog/storage/google_xml/requests/get_bucket.rb

@@ -1,4 +1,3 @@
-require "pp"
 module Fog
   module Storage
     class GoogleXML

data/lib/fog/storage/google_xml/requests/head_object.rb

@@ -33,12 +33,13 @@ module Fog
           headers["If-Modified-Since"] = Fog::Time.at(options["If-Modified-Since"].to_i).to_date_header if options["If-Modified-Since"]
           headers["If-Unmodified-Since"] = Fog::Time.at(options["If-Unmodified-Since"].to_i).to_date_header if options["If-Modified-Since"]
           headers.merge!(options)
-          request(:expects  => [200, 206],
-                  :headers  => headers,
-                  :host     => "#{bucket_name}.#{@host}",
-                  :method   => "HEAD",
-                  :path     => Fog::Google.escape(object_name),
-                  :query    => query)
+          request(:expects    => [200, 206],
+                  :headers    => headers,
+                  :host       => "#{bucket_name}.#{@host}",
+                  :idempotent => true,
+                  :method     => "HEAD",
+                  :path       => Fog::Google.escape(object_name),
+                  :query      => query)
         end
       end
 

data/tasks/changelog.rake

@@ -20,11 +20,20 @@ end
 
 # Extracts all changes that have been made after the latest pushed tag
 def changes_since_last_tag
-  `git --no-pager log $(git describe --tags --abbrev=0)..HEAD --oneline`
+  `git --no-pager log $(git describe --tags --abbrev=0)..HEAD --grep="Merge" --pretty=format:"%t - %s%n%b%n"`
+end
+
+# Extracts all github users contributed since last tag
+def users_since_last_tag
+  `git --no-pager log $(git describe --tags --abbrev=0)..HEAD --grep="Merge" --pretty=format:"%s" | cut -d' ' -f 6 | cut -d/ -f1 | sort | uniq`
 end
 
 namespace :changelog do
   task :generate do
     insert_after_line("CHANGELOG.md", changes_since_last_tag, /^## Next/)
+    contributors = users_since_last_tag.split("\n").map { |name| "@" + name }
+    printf("Huge thanks to all our contributors 🎆\n")
+    printf("Special thanks to: " + contributors.join(" ") + "\n")
+    printf("\nI'll merge this and release the gem once all tests pass.\n")
   end
 end

data/test/helpers/integration_test_helper.rb

@@ -1,5 +1,6 @@
 require "helpers/test_helper"
 require "helpers/test_collection"
+require "retriable"
 
 # Use :test credentials in ~/.fog for live integration testing
 # XXX not sure if this will work on Travis CI or not
@@ -22,7 +23,7 @@ TEST_IMAGE_FAMILY = "debian-9".freeze
 
 # XXX This depends on a public image in gs://fog-test-bucket; there may be a better way to do this
 # The image was created like so: https://cloud.google.com/compute/docs/images#export_an_image_to_google_cloud_storage
-TEST_RAW_DISK_SOURCE = "http://storage.googleapis.com/fog-test-bucket/fog-test-raw-disk-source.image.tar.gz".freeze
+TEST_RAW_DISK_SOURCE = "https://storage.googleapis.com/fog-testing-bucket/fog-test-raw-disk-source.image.tar.gz".freeze
 
 TEST_SQL_TIER = "db-n1-standard-1".freeze
 TEST_SQL_REGION = TEST_REGION
@@ -87,8 +88,21 @@ wLjafhPTSAIS0jijglJ7uzaSbFUW11fw1/EIqIFNe0R0Xf9lsyPxFA==
 -----END RSA PRIVATE KEY-----
 KEY
 
+# Retry module config - standard retriable gem settings
+# see: https://github.com/kamui/retriable
+RETRIABLE_TRIES = 3
+RETRIABLE_BASE_INTERVAL = 50
+
 class FogIntegrationTest < MiniTest::Test
   def namespaced_name
     "#{self.class}_#{name}"
   end
+
+  def retry_on(klass)
+    Retriable.retriable(on: klass,
+                        tries: RETRIABLE_TRIES,
+                        base_interval: RETRIABLE_BASE_INTERVAL) do
+      yield
+    end
+  end
 end

data/test/integration/monitoring/test_metric_descriptors.rb

@@ -68,7 +68,7 @@ class TestMetricDescriptors < FogIntegrationTest
       :labels => [label]
     }
 
-    created = @client.create_metric_descriptor(options)
+    created = @client.create_metric_descriptor(**options)
 
     # Check created metric descriptor
     assert_equal(_full_name(metric_type), created.name)

data/test/integration/monitoring/test_timeseries.rb

@@ -1,12 +1,11 @@
 require "helpers/integration_test_helper"
-require "retriable"
 
 class TestMetricDescriptors < FogIntegrationTest
-  # Retriable is used to wrap each request in this test due to Stackdriver API being slow with
-  # metric propagation (sometimes 80+ seconds) and client returning
-  # Google::Apis::ClientError: badRequest if the metric hasn't yet been created instead of a 404.
-  RETRIABLE_TRIES = 3
-  RETRIABLE_BASE_INTERVAL = 50
+  # This module has a decent amount of retry wrappers as the Stackdriver API
+  # can be quite slow with metric propagation (sometimes 80+ seconds) and client
+  # returning errors, e.g. Google::Apis::ClientError: badRequest if the metric
+  # hasn't yet been created instead of a 404.
+
   TEST_METRIC_PREFIX = "custom.googleapis.com/fog-google-test/timeseries".freeze
   LABEL_DESCRIPTORS = [
     {
@@ -51,10 +50,7 @@ class TestMetricDescriptors < FogIntegrationTest
     assert_empty(resp.to_h)
 
     # Wait for metric to be created
-    # TODO: Dedup retries into a helper method
-    Retriable.retriable(on: Google::Apis::ClientError,
-                        tries: RETRIABLE_TRIES,
-                        base_interval: RETRIABLE_BASE_INTERVAL) do
+    retry_on(Google::Apis::ClientError) do
       @client.list_timeseries(
           :filter => "metric.type = \"#{metric_type}\"",
           :interval => {
@@ -67,9 +63,7 @@ class TestMetricDescriptors < FogIntegrationTest
       ).time_series
     end
 
-    series = Retriable.retriable(on: Google::Apis::ClientError,
-                                 tries: RETRIABLE_TRIES,
-                                 base_interval: RETRIABLE_BASE_INTERVAL) do
+    series = retry_on(Google::Apis::ClientError) do
       @client.timeseries_collection.all(
         :filter => "metric.type = \"#{metric_type}\"",
         :interval => {
@@ -115,9 +109,7 @@ class TestMetricDescriptors < FogIntegrationTest
       _some_timeseries(start_time, metric_type, labels)
     end
 
-    Retriable.retriable(on: Google::Apis::ServerError,
-                        tries: RETRIABLE_TRIES,
-                        base_interval: RETRIABLE_BASE_INTERVAL) do
+    retry_on(Google::Apis::ServerError) do
       @client.create_timeseries(:timeseries => timeseries)
     end
     interval = {
@@ -132,9 +124,7 @@ class TestMetricDescriptors < FogIntegrationTest
     # Wait for metric to be created
     # Retriable is used instead of wait_for due to API client returning Google::Apis::ClientError: badRequest if the
     # metric hasn't yet been created
-    Retriable.retriable(on: Google::Apis::ClientError,
-                        tries: RETRIABLE_TRIES,
-                        base_interval: RETRIABLE_BASE_INTERVAL) do
+    retry_on(Google::Apis::ClientError) do
       @client.list_timeseries(
         :filter => "metric.type = \"#{metric_type}\"",
         :interval => interval
@@ -142,9 +132,7 @@ class TestMetricDescriptors < FogIntegrationTest
     end
 
     # Test page size
-    resp = Retriable.retriable(on: Google::Apis::ClientError,
-                               tries: RETRIABLE_TRIES,
-                               base_interval: RETRIABLE_BASE_INTERVAL) do
+    resp = retry_on(Google::Apis::ClientError) do
       @client.list_timeseries(
         :filter => "metric.type = \"#{metric_type}\"",
         :interval => interval,
@@ -170,9 +158,7 @@ class TestMetricDescriptors < FogIntegrationTest
            "expected different timeseries when using page_token")
 
     # Test filter
-    series = Retriable.retriable(on: Google::Apis::ClientError,
-                                 tries: RETRIABLE_TRIES,
-                                 base_interval: RETRIABLE_BASE_INTERVAL) do
+    series = retry_on(Google::Apis::ClientError) do
       @client.timeseries_collection.all(
         :filter => %[
           metric.type = "#{metric_type}" AND