google-cloud-storage 1.25.1 → 1.28.0

data/lib/google/cloud/storage/file/list.rb

@@ -77,11 +77,13 @@ module Google
           def next
             return nil unless next?
             ensure_service!
-            options = {
-              prefix: @prefix, delimiter: @delimiter, token: @token, max: @max,
-              versions: @versions, user_project: @user_project
-            }
-            gapi = @service.list_files @bucket, options
+
+            gapi = @service.list_files @bucket, prefix: @prefix,
+                                                delimiter: @delimiter,
+                                                token: @token,
+                                                max: @max,
+                                                versions: @versions,
+                                                user_project: @user_project
             File::List.from_gapi gapi, @service, @bucket, @prefix,
                                  @delimiter, @max, @versions,
                                  user_project: @user_project

data/lib/google/cloud/storage/file/signer_v2.rb

@@ -77,13 +77,21 @@ module Google
           end
 
           def determine_signing_key options = {}
-            options[:signing_key] || options[:private_key] ||
-              @service.credentials.signing_key
+            signing_key = options[:signing_key] || options[:private_key] ||
+                          options[:signer] || @service.credentials.signing_key
+            raise SignedUrlUnavailable, error_msg("signing_key (private_key, signer)") unless signing_key
+            signing_key
           end
 
           def determine_issuer options = {}
-            options[:issuer] || options[:client_email] ||
-              @service.credentials.issuer
+            issuer = options[:issuer] || options[:client_email] || @service.credentials.issuer
+            raise SignedUrlUnavailable, error_msg("issuer (client_email)") unless issuer
+            issuer
+          end
+
+          def error_msg attr_name
+            "Service account credentials '#{attr_name}' is missing. To generate service account credentials " \
+            "see https://cloud.google.com/iam/docs/service-accounts"
           end
 
           def post_object options
@@ -99,8 +107,6 @@ module Google
             i = determine_issuer options
             s = determine_signing_key options
 
-            raise SignedUrlUnavailable unless i && s
-
             policy_str = p.to_json
             policy = Base64.strict_encode64(policy_str).delete "\n"
 
@@ -119,18 +125,21 @@ module Google
             i = determine_issuer options
             s = determine_signing_key options
 
-            raise SignedUrlUnavailable unless i && s
-
             sig = generate_signature s, signature_str(options)
             generate_signed_url i, sig, options[:expires], options[:query]
           end
 
           def generate_signature signing_key, secret
-            unless signing_key.respond_to? :sign
-              signing_key = OpenSSL::PKey::RSA.new signing_key
+            unencoded_signature = ""
+            if signing_key.is_a? Proc
+              unencoded_signature = signing_key.call secret
+            else
+              unless signing_key.respond_to? :sign
+                signing_key = OpenSSL::PKey::RSA.new signing_key
+              end
+              unencoded_signature = signing_key.sign OpenSSL::Digest::SHA256.new, secret
             end
-            signature = signing_key.sign OpenSSL::Digest::SHA256.new, secret
-            Base64.strict_encode64(signature).delete "\n"
+            Base64.strict_encode64(unencoded_signature).delete "\n"
           end
 
           def generate_signed_url issuer, signed_string, expires, query

data/lib/google/cloud/storage/file/signer_v4.rb

@@ -39,37 +39,83 @@ module Google
             new bucket.name, file_name, bucket.service
           end
 
-          def signed_url method: "GET", expires: nil, headers: nil,
-                         issuer: nil, client_email: nil, signing_key: nil,
-                         private_key: nil, query: nil
-            issuer, signer = issuer_and_signer issuer, client_email,
-                                               signing_key, private_key
+          def post_object issuer: nil,
+                          client_email: nil,
+                          signing_key: nil,
+                          private_key: nil,
+                          signer: nil,
+                          expires: nil,
+                          fields: nil,
+                          conditions: nil,
+                          scheme: "https",
+                          virtual_hosted_style: nil,
+                          bucket_bound_hostname: nil
+            i = determine_issuer issuer, client_email
+            s = determine_signing_key signing_key, private_key, signer
+
+            now = Time.now.utc
+            base_fields = required_fields i, now
+            post_fields = fields.dup || {}
+            post_fields.merge! base_fields
+
+            p = {}
+            p["conditions"] = policy_conditions base_fields, conditions, fields
+            expires ||= 60*60*24
+            p["expiration"] = (now + expires).strftime "%Y-%m-%dT%H:%M:%SZ"
+
+            policy_str = escape_characters p.to_json
+
+            policy = Base64.strict_encode64(policy_str).force_encoding "utf-8"
+            signature = generate_signature s, policy
+
+            post_fields["x-goog-signature"] = signature
+            post_fields["policy"] = policy
+            url = post_object_ext_url scheme, virtual_hosted_style, bucket_bound_hostname
+            hostname = "#{url}#{bucket_path path_style?(virtual_hosted_style, bucket_bound_hostname)}"
+            Google::Cloud::Storage::PostObject.new hostname, post_fields
+          end
+
+          def signed_url method: "GET",
+                         expires: nil,
+                         headers: nil,
+                         issuer: nil,
+                         client_email: nil,
+                         signing_key: nil,
+                         private_key: nil,
+                         signer: nil,
+                         query: nil,
+                         scheme: "https",
+                         virtual_hosted_style: nil,
+                         bucket_bound_hostname: nil
+            raise ArgumentError, "method is required" unless method
+            issuer, signer = issuer_and_signer issuer, client_email, signing_key, private_key, signer
             datetime_now = Time.now.utc
             goog_date = datetime_now.strftime "%Y%m%dT%H%M%SZ"
             datestamp = datetime_now.strftime "%Y%m%d"
             # goog4_request is not checked.
             scope = "#{datestamp}/auto/storage/goog4_request"
 
-            canonical_headers_str, signed_headers_str = \
-              canonical_and_signed_headers headers
+            canonical_headers_str, signed_headers_str = canonical_and_signed_headers \
+              headers, virtual_hosted_style, bucket_bound_hostname
 
             algorithm = "GOOG4-RSA-SHA256"
             expires = determine_expires expires
-            credential = CGI.escape issuer + "/" + scope
-            canonical_query_str = canonical_query query, algorithm,
-                                                  credential, goog_date,
-                                                  expires, signed_headers_str
+            credential = issuer + "/" + scope
+            canonical_query_str = canonical_query query, algorithm, credential, goog_date, expires, signed_headers_str
 
             # From AWS: You don't include a payload hash in the Canonical
             # Request, because when you create a presigned URL, you don't know
             # the payload content because the URL is used to upload an arbitrary
             # payload. Instead, you use a constant string UNSIGNED-PAYLOAD.
+            payload = headers&.key?("X-Goog-Content-SHA256") ? headers["X-Goog-Content-SHA256"] : "UNSIGNED-PAYLOAD"
+
             canonical_request = [method,
-                                 ext_path,
+                                 file_path(!(virtual_hosted_style || bucket_bound_hostname)),
                                  canonical_query_str,
                                  canonical_headers_str,
                                  signed_headers_str,
-                                 "UNSIGNED-PAYLOAD"].join("\n")
+                                 payload].join("\n")
+
             # Construct string to sign
             req_sha = Digest::SHA256.hexdigest canonical_request
             string_to_sign = [algorithm, goog_date, scope, req_sha].join "\n"
@@ -78,66 +124,129 @@ module Google
             signature = signer.call string_to_sign
 
             # Construct signed URL
-            "#{ext_url}?#{canonical_query_str}&X-Goog-Signature=#{signature}"
+            hostname = signed_url_hostname scheme, virtual_hosted_style, bucket_bound_hostname
+            "#{hostname}?#{canonical_query_str}&X-Goog-Signature=#{signature}"
+          end
+
+          # methods below are public visibility only for unit testing
+          def escape_characters str
+            str.split("").map do |s|
+              if !s.ascii_only?
+                escape_special_unicode s
+              else
+                case s
+                when "\\"
+                  '\\'
+                when "\b"
+                  '\b'
+                when "\f"
+                  '\f'
+                when "\n"
+                  '\n'
+                when "\r"
+                  '\r'
+                when "\t"
+                  '\t'
+                when "\v"
+                  '\v'
+                else
+                  s
+                end
+              end
+            end.join
+          end
+
+          def escape_special_unicode str
+            str.unpack("U*").map { |i| '\u' + i.to_s(16).rjust(4, "0") }.join
           end
 
           protected
 
+          def required_fields issuer, time
+            {
+              "key" => @file_name,
+              "x-goog-date" => time.strftime("%Y%m%dT%H%M%SZ"),
+              "x-goog-credential" => "#{issuer}/#{time.strftime '%Y%m%d'}/auto/storage/goog4_request",
+              "x-goog-algorithm" => "GOOG4-RSA-SHA256"
+            }.freeze
+          end
+
+          def policy_conditions base_fields, user_conditions, user_fields
+            # Convert each pair in base_fields hash to a single-entry hash in an array.
+            conditions = base_fields.to_a.map { |f| Hash[*f] }
+            # Add the bucket to the head of the base_fields. This is not returned in the PostObject fields.
+            conditions.unshift "bucket" => @bucket_name
+            # Add user-provided conditions to the head of the conditions array.
+            conditions.unshift user_conditions if user_conditions && !user_conditions.empty?
+            if user_fields
+              # Convert each pair in fields hash to a single-entry hash and add it to the head of the conditions array.
+              user_fields.to_a.reverse.each { |f| conditions.unshift Hash[*f] }
+            end
+            conditions.freeze
+          end
+
+          def signed_url_hostname scheme, virtual_hosted_style, bucket_bound_hostname
+            url = ext_url scheme, virtual_hosted_style, bucket_bound_hostname
+            "#{url}#{file_path path_style?(virtual_hosted_style, bucket_bound_hostname)}"
+          end
+
           def determine_issuer issuer, client_email
             # Parse the Service Account and get client id and private key
             issuer = issuer || client_email || @service.credentials.issuer
-            unless issuer
-              raise SignedUrlUnavailable, "issuer (client_email) missing"
-            end
+            raise SignedUrlUnavailable, error_msg("issuer (client_email)") unless issuer
             issuer
           end
 
-          def determine_signing_key signing_key, private_key
-            signing_key = signing_key || private_key ||
-                          @service.credentials.signing_key
-            unless signing_key
-              raise SignedUrlUnavailable, "signing_key (private_key) missing"
-            end
+          def determine_signing_key signing_key, private_key, signer
+            signing_key = signing_key || private_key || signer || @service.credentials.signing_key
+            raise SignedUrlUnavailable, error_msg("signing_key (private_key, signer)") unless signing_key
             signing_key
           end
 
+          def error_msg attr_name
+            "Service account credentials '#{attr_name}' is missing. To generate service account credentials " \
+            "see https://cloud.google.com/iam/docs/service-accounts"
+          end
+
           def service_account_signer signer
-            unless signer.respond_to? :sign
-              signer = OpenSSL::PKey::RSA.new signer
-            end
-            # Sign string to sign
-            lambda do |string_to_sign|
-              sig = signer.sign OpenSSL::Digest::SHA256.new, string_to_sign
-              sig.unpack("H*").first
+            if signer.is_a? Proc
+              lambda do |string_to_sign|
+                sig = signer.call string_to_sign
+                sig.unpack("H*").first
+              end
+            else
+              signer = OpenSSL::PKey::RSA.new signer unless signer.respond_to? :sign
+              # Sign string to sign
+              lambda do |string_to_sign|
+                sig = signer.sign OpenSSL::Digest::SHA256.new, string_to_sign
+                sig.unpack("H*").first
+              end
             end
           end
 
-          def issuer_and_signer issuer, client_email, signing_key, private_key
+          def issuer_and_signer issuer, client_email, signing_key, private_key, signer
             issuer = determine_issuer issuer, client_email
-            signing_key = determine_signing_key signing_key, private_key
+            signing_key = determine_signing_key signing_key, private_key, signer
             signer = service_account_signer signing_key
             [issuer, signer]
           end
 
-          def canonical_and_signed_headers headers
-            # Headers needs to be in alpha order.
+          def canonical_and_signed_headers headers, virtual_hosted_style, bucket_bound_hostname
+            if virtual_hosted_style && bucket_bound_hostname
+              raise "virtual_hosted_style: #{virtual_hosted_style} and bucket_bound_hostname: " \
+                    "#{bucket_bound_hostname} params cannot both be passed together"
+            end
+
             canonical_headers = headers || {}
             headers_arr = canonical_headers.map do |k, v|
-              [k.downcase, v.strip.gsub(/[^\S\t]+/, " ").gsub(/\t+/, "\t")]
+              [k.downcase, v.strip.gsub(/[^\S\t]+/, " ").gsub(/\t+/, " ")]
             end
             canonical_headers = Hash[headers_arr]
-            canonical_headers["host"] = "storage.googleapis.com"
-
-            canonical_headers = canonical_headers.sort_by do |k, _|
-              k.downcase
-            end.to_h
-            canonical_headers_str = ""
-            canonical_headers.each do |k, v|
-              canonical_headers_str += "#{k}:#{v}\n"
-            end
-            signed_headers_str = ""
-            canonical_headers.each_key { |k| signed_headers_str += "#{k};" }
-            signed_headers_str = signed_headers_str.chomp ";"
+            canonical_headers["host"] = host_name virtual_hosted_style, bucket_bound_hostname
+
+            canonical_headers = canonical_headers.sort_by(&:first).to_h
+            canonical_headers_str = canonical_headers.map { |k, v| "#{k}:#{v}\n" }.join
+            signed_headers_str = canonical_headers.keys.join ";"
             [canonical_headers_str, signed_headers_str]
           end
 
@@ -149,32 +258,108 @@ module Google
             expires
           end
 
-          def canonical_query query, algorithm, credential, goog_date, expires,
-                              signed_headers_str
+          def canonical_query query, algorithm, credential, goog_date, expires, signed_headers_str
             query ||= {}
             query["X-Goog-Algorithm"] = algorithm
             query["X-Goog-Credential"] = credential
             query["X-Goog-Date"] = goog_date
             query["X-Goog-Expires"] = expires
-            query["X-Goog-SignedHeaders"] = CGI.escape signed_headers_str
-            query = query.sort_by { |k, _| k.to_s.downcase }.to_h
-            canonical_query_str = ""
-            query.each { |k, v| canonical_query_str += "#{k}=#{v}&" }
-            canonical_query_str.chomp "&"
+            query["X-Goog-SignedHeaders"] = signed_headers_str
+            query = query.map { |k, v| [escape_query_param(k), escape_query_param(v)] }.sort_by(&:first).to_h
+            query.map { |k, v| "#{k}=#{v}" }.join "&"
+          end
+
+          ##
+          # Only the characters in the regex set [A-Za-z0-9.~_-] must be left un-escaped; all others must be
+          # percent-encoded using %XX UTF-8 style.
+          def escape_query_param str
+            CGI.escape(str.to_s).gsub("%7E", "~")
+          end
+
+          def host_name virtual_hosted_style, bucket_bound_hostname
+            return bucket_bound_hostname if bucket_bound_hostname
+            virtual_hosted_style ? "#{@bucket_name}.storage.googleapis.com" : "storage.googleapis.com"
           end
 
           ##
           # The URI-encoded (percent encoded) external path to the file.
-          def ext_path
-            path = "/#{@bucket_name}"
-            path += "/#{String(@file_name)}" if @file_name && !@file_name.empty?
-            Addressable::URI.escape path
+          def file_path path_style
+            path = []
+            path << "/#{@bucket_name}" if path_style
+            path << "/#{String(@file_name)}" if @file_name && !@file_name.empty?
+            CGI.escape(path.join).gsub "%2F", "/"
+          end
+
+          ##
+          # The external path to the bucket, with trailing slash.
+          def bucket_path path_style
+            return "/#{@bucket_name}/" if path_style
+          end
+
+          ##
+          # The external url to the file.
+          def ext_url scheme, virtual_hosted_style, bucket_bound_hostname
+            url = GOOGLEAPIS_URL.dup
+            if virtual_hosted_style
+              parts = url.split "//"
+              parts[1] = "#{@bucket_name}.#{parts[1]}"
+              parts.join "//"
+            elsif bucket_bound_hostname
+              raise ArgumentError, "scheme is required" unless scheme
+              URI "#{scheme.to_s.downcase}://#{bucket_bound_hostname}"
+            else
+              url
+            end
+          end
+
+          def path_style? virtual_hosted_style, bucket_bound_hostname
+            !(virtual_hosted_style || bucket_bound_hostname)
+          end
+
+          ##
+          # The external path to the file, URI-encoded.
+          # Will not URI encode the special `${filename}` variable.
+          # "You can also use the ${filename} variable..."
+          # https://cloud.google.com/storage/docs/xml-api/post-object
+          #
+          def post_object_ext_path
+            path = "/#{@bucket_name}/#{@file_name}"
+            escaped = Addressable::URI.escape path
+            special_var = "${filename}"
+            # Restore the unencoded `${filename}` variable, if present.
+            if path.include? special_var
+              return escaped.gsub "$%7Bfilename%7D", special_var
+            end
+            escaped
           end
 
           ##
           # The external url to the file.
-          def ext_url
-            "#{GOOGLEAPIS_URL}#{ext_path}"
+          def post_object_ext_url scheme, virtual_hosted_style, bucket_bound_hostname
+            url = GOOGLEAPIS_URL.dup
+            if virtual_hosted_style
+              parts = url.split "//"
+              parts[1] = "#{@bucket_name}.#{parts[1]}/"
+              parts.join "//"
+            elsif bucket_bound_hostname
+              raise ArgumentError, "scheme is required" unless scheme
+              URI "#{scheme.to_s.downcase}://#{bucket_bound_hostname}/"
+            else
+              url
+            end
+          end
+
+          def generate_signature signing_key, data
+            packed_signature = nil
+            if signing_key.is_a? Proc
+              packed_signature = signing_key.call data
+            else
+              unless signing_key.respond_to? :sign
+                signing_key = OpenSSL::PKey::RSA.new signing_key
+              end
+              packed_signature = signing_key.sign OpenSSL::Digest::SHA256.new, data
+            end
+            packed_signature.unpack("H*").first.force_encoding "utf-8"
           end
         end
       end