google-apis-core 0.7.2 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 38cee6b3beff5b42204b1b518e742b3fe8925bc58d7a64565dd6744300193bb1
-  data.tar.gz: f2b1ebe5ff08156d56867e1eb51520a6328c6654847ded469df9b90efc438208
+  metadata.gz: 0e3d4a94e6fc85bf5b6a94cc7d579c3e32be968dfa8df1c56e71232ee3030bbe
+  data.tar.gz: d8a593392db24783f8e31785996f90108f67d13d950d7c7f82e669caae65c0a8
 SHA512:
-  metadata.gz: a888eda52341a233166b95641dd0bea997fff51f7242e5be0a1df449ffd19c9afdca57fb2d6fbad33f44aa7710a53d6939dd88538b60415d94b1f61087d8a78d
-  data.tar.gz: bed80fd1c5b86c02ffcec60c5ca1fbe8c1cd434c62e96935b32f447dee61cbb5f3359d6580f82f0269b79028a1817c39c3470ebc8658cb98c7e56c45eab272cd
+  metadata.gz: 286116c08612e28eedc00886eee6bdf96b85fd6b7e8b2fa7287b66504c84458bc98936cb9893451db7843cbce6c4bdc797e726678c16241832f1ce467ab01eb0
+  data.tar.gz: 81c3a46b26d25a587c558d089f8f2f2d36ddceffd381df969d4296d111b75f54dff592c59680171ebbf8c6641723f5bfeb2613bd939a54c28d1b556acd42e550

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Release History
 
+### 0.8.0 (2022-09-16)
+
+#### Features
+
+* Add storage upload to move away from unified upload protocol ([#11508](https://github.com/googleapis/google-api-ruby-client/issues/11508)) 
+
 ### 0.7.2 (2022-09-15)
 
 #### Bug Fixes

data/lib/google/apis/core/base_service.rb

@@ -19,6 +19,7 @@ require 'google/apis/core/version'
 require 'google/apis/core/api_command'
 require 'google/apis/core/batch'
 require 'google/apis/core/upload'
+require 'google/apis/core/storage_upload'
 require 'google/apis/core/download'
 require 'google/apis/core/storage_download'
 require 'google/apis/options'
@@ -318,6 +319,26 @@ module Google
           command
         end
 
+        # Create a new storage upload command.
+        # This is specifically for storage because we are moving to a new upload protocol.
+        # Ref: https://cloud.google.com/storage/docs/performing-resumable-uploads
+        #
+        # @param [Symbol] method
+        #  HTTP method for uploading. The initial request to initiate a resumable session
+        #  is :post and the subsequent chunks uploaded to the session are :put
+        # @param [String] path
+        #  Additional path to upload endpoint, appended to API base path
+        # @param [Hash, Google::Apis::RequestOptions] options
+        #  Request-specific options
+        # @return [Google::Apis::Core::StorageUploadCommand]
+        def make_storage_upload_command(method, path, options)
+          template = Addressable::Template.new(root_url + upload_path + path)
+          command = StorageUploadCommand.new(method, template, client_version: client_version)
+          command.options = request_options.merge(options)
+          apply_command_defaults(command)
+          command
+        end
+
         # Create a new download command.
         #
         # @param [symbol] method

data/lib/google/apis/core/batch.rb

@@ -28,6 +28,7 @@
 require 'google/apis/core/multipart'
 require 'google/apis/core/http_command'
 require 'google/apis/core/upload'
+require 'google/apis/core/storage_upload'
 require 'google/apis/core/download'
 require 'google/apis/core/composite_io'
 require 'addressable/uri'
@@ -120,7 +121,7 @@ module Google
         end
 
         def ensure_valid_command(command)
-          if command.is_a?(Google::Apis::Core::BaseUploadCommand) || command.is_a?(Google::Apis::Core::DownloadCommand) || command.is_a?(Google::Apis::Core::StorageDownloadCommand)
+          if command.is_a?(Google::Apis::Core::BaseUploadCommand) || command.is_a?(Google::Apis::Core::DownloadCommand) || command.is_a?(Google::Apis::Core::StorageDownloadCommand) || command.is_a?(Google::Apis::Core::StorageUploadCommand)
             fail Google::Apis::ClientError, 'Can not include media requests in batch'
           end
           fail Google::Apis::ClientError, 'Invalid command object' unless command.is_a?(HttpCommand)

data/lib/google/apis/core/http_command.rb

@@ -98,9 +98,17 @@ module Google
         # @raise [Google::Apis::ServerError] An error occurred on the server and the request can be retried
         # @raise [Google::Apis::ClientError] The request is invalid and should not be retried without modification
         # @raise [Google::Apis::AuthorizationError] Authorization is required
-        def execute(client)
+        def execute(client, &block)
           prepare!
           opencensus_begin_span
+          do_retry :execute_once, client, &block
+        ensure
+          opencensus_end_span
+          @http_res = nil
+          release!
+        end
+
+        def do_retry func, client
           begin
             Retriable.retriable tries: options.retries + 1,
                                 max_elapsed_time: options.max_elapsed_time,
@@ -115,7 +123,7 @@ module Google
               Retriable.retriable tries: auth_tries,
                                   on: [Google::Apis::AuthorizationError, Signet::AuthorizationError, Signet::RemoteServerError, Signet::UnexpectedStatusError],
                                   on_retry: proc { |*| refresh_authorization } do
-                execute_once(client).tap do |result|
+                send(func, client).tap do |result|
                   if block_given?
                     yield result, nil
                   end
@@ -129,10 +137,6 @@ module Google
               raise e
             end
           end
-        ensure
-          opencensus_end_span
-          @http_res = nil
-          release!
         end
 
         # Refresh the authorization authorization after a 401 error
@@ -216,7 +220,7 @@ module Google
         def check_status(status, header = nil, body = nil, message = nil)
           # TODO: 304 Not Modified depends on context...
           case status
-          when 200...300
+          when 200...300, 308
             nil
           when 301, 302, 303, 307
             message ||= sprintf('Redirect to %s', header['Location'])

data/lib/google/apis/core/storage_upload.rb

@@ -0,0 +1,181 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'google/apis/core/http_command'
+require 'google/apis/core/api_command'
+require 'google/apis/errors'
+require 'tempfile'
+require 'mini_mime'
+
+module Google
+  module Apis
+    module Core
+      # Base upload command. Not intended to be used directly
+      # @private
+      class StorageUploadCommand < ApiCommand
+        CONTENT_LENGTH_HEADER = "Content-Length"
+        CONTENT_TYPE_HEADER = "Content-Type"
+        UPLOAD_CONTENT_TYPE_HEADER = "X-Upload-Content-Type"
+        LOCATION_HEADER = "Location"
+        CONTENT_RANGE_HEADER = "Content-Range"
+        RESUMABLE = "resumable"
+        OK_STATUS = 200
+        CHUNK_SIZE = 8 * 1024 * 1024 # 8 MB
+
+        # File name or IO containing the content to upload
+        # @return [String, File, #read]
+        attr_accessor :upload_source
+
+        # Content type of the upload material
+        # @return [String]
+        attr_accessor :upload_content_type
+
+        # Content, as UploadIO
+        # @return [Google::Apis::Core::UploadIO]
+        attr_accessor :upload_io
+
+        # Ensure the content is readable and wrapped in an IO instance.
+        #
+        # @return [void]
+        # @raise [Google::Apis::ClientError] if upload source is invalid
+        def prepare!
+          @upload_url = nil
+          @offset = 0
+          @upload_incomplete = true
+          # Prevent the command from populating the body with form encoding, by
+          # asserting that it already has a body. Form encoding is never used
+          # by upload requests.
+          self.body = '' unless self.body
+
+          super
+          if streamable?(upload_source)
+            self.upload_io = upload_source
+            @close_io_on_finish = false
+          elsif self.upload_source.is_a?(String)
+            self.upload_io = File.new(upload_source, 'r')
+            if self.upload_content_type.nil?
+              type = MiniMime.lookup_by_filename(upload_source)
+              self.upload_content_type = type&.content_type
+            end
+            @close_io_on_finish = true
+          else
+            fail Google::Apis::ClientError, 'Invalid upload source'
+          end
+        end
+
+        # Close IO stream when command done. Only closes the stream if it was opened by the command.
+        def release!
+          upload_io.close if @close_io_on_finish
+        end
+
+        # Execute the command, retrying as necessary
+        #
+        # @param [HTTPClient] client
+        #   HTTP client
+        # @yield [result, err] Result or error if block supplied
+        # @return [Object]
+        # @raise [Google::Apis::ServerError] An error occurred on the server and the request can be retried
+        # @raise [Google::Apis::ClientError] The request is invalid and should not be retried without modification
+        # @raise [Google::Apis::AuthorizationError] Authorization is required
+        def execute(client)
+          prepare!
+          opencensus_begin_span
+
+          do_retry :initiate_resumable_upload, client
+          while @upload_incomplete
+            res = do_retry :send_upload_command, client
+          end
+          res
+        ensure
+          opencensus_end_span
+          @http_res = nil
+          release!
+        end
+
+        def initiate_resumable_upload(client)
+          logger.debug { sprintf('Intiating resumable upload command to %s', url) }
+
+          request_header = header.dup
+          apply_request_options(request_header)
+
+          request_query = query.dup
+          request_query['uploadType'] = RESUMABLE
+
+          request_header[CONTENT_LENGTH_HEADER] = upload_io.size.to_s
+          request_header[CONTENT_TYPE_HEADER] = JSON_CONTENT_TYPE
+          request_header[UPLOAD_CONTENT_TYPE_HEADER] = upload_content_type unless upload_content_type.nil?
+
+          response = client.post(url.to_s, query: request_query,
+                         body: body,
+                         header: request_header,
+                         follow_redirect: true)
+          result = process_response(response.status_code, response.header, response.body)
+          success(result)
+        rescue => e
+          error(e, rethrow: true)
+        end
+
+        # Send the actual content
+        #
+        # @param [HTTPClient] client
+        #   HTTP client
+        # @return [HTTP::Message]
+        # @raise [Google::Apis::ServerError] Unable to send the request
+        def send_upload_command(client)
+          logger.debug { sprintf('Sending upload command to %s', @upload_url) }
+
+          remaining_content_size = upload_io.size - @offset
+          current_chunk_size = remaining_content_size < CHUNK_SIZE ? remaining_content_size : CHUNK_SIZE
+
+          request_header = header.dup
+          request_header[CONTENT_RANGE_HEADER] = sprintf('bytes %d-%d/%d', @offset, @offset+current_chunk_size-1, upload_io.size)
+          request_header[CONTENT_LENGTH_HEADER] = current_chunk_size
+          chunk_body = upload_io.read(current_chunk_size)
+
+          response = client.put(@upload_url, body: chunk_body, header: request_header, follow_redirect: true)
+
+          result = process_response(response.status_code, response.header, response.body)
+          @upload_incomplete = false if response.status_code.eql? OK_STATUS
+          @offset += current_chunk_size if @upload_incomplete
+          success(result)
+        rescue => e
+          upload_io.pos = @offset
+          error(e, rethrow: true)
+        end
+
+        # Check the to see if the upload is complete or needs to be resumed.
+        #
+        # @param [Integer] status
+        #   HTTP status code of response
+        # @param [HTTP::Message::Headers] header
+        #   Response headers
+        # @param [String, #read] body
+        #  Response body
+        # @return [Object]
+        #   Response object
+        # @raise [Google::Apis::ServerError] An error occurred on the server and the request can be retried
+        # @raise [Google::Apis::ClientError] The request is invalid and should not be retried without modification
+        # @raise [Google::Apis::AuthorizationError] Authorization is required
+        def process_response(status, header, body)
+          @upload_url = header[LOCATION_HEADER].first unless header[LOCATION_HEADER].empty?
+          super(status, header, body)
+        end
+
+        def streamable?(upload_source)
+          upload_source.is_a?(IO) || upload_source.is_a?(StringIO) || upload_source.is_a?(Tempfile)
+        end
+      end
+    end
+  end
+end

data/lib/google/apis/core/version.rb

@@ -16,7 +16,7 @@ module Google
   module Apis
     module Core
       # Core version
-      VERSION = "0.7.2".freeze
+      VERSION = "0.8.0".freeze
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: google-apis-core
 version: !ruby/object:Gem::Version
-  version: 0.7.2
+  version: 0.8.0
 platform: ruby
 authors:
 - Google LLC
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-09-15 00:00:00.000000000 Z
+date: 2022-09-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: representable
@@ -175,6 +175,7 @@ files:
 - lib/google/apis/core/logging.rb
 - lib/google/apis/core/multipart.rb
 - lib/google/apis/core/storage_download.rb
+- lib/google/apis/core/storage_upload.rb
 - lib/google/apis/core/upload.rb
 - lib/google/apis/core/version.rb
 - lib/google/apis/errors.rb
@@ -185,7 +186,7 @@ licenses:
 metadata:
   bug_tracker_uri: https://github.com/googleapis/google-api-ruby-client/issues
   changelog_uri: https://github.com/googleapis/google-api-ruby-client/tree/main/google-apis-core/CHANGELOG.md
-  documentation_uri: https://googleapis.dev/ruby/google-apis-core/v0.7.2
+  documentation_uri: https://googleapis.dev/ruby/google-apis-core/v0.8.0
   source_code_uri: https://github.com/googleapis/google-api-ruby-client/tree/main/google-apis-core
 post_install_message: 
 rdoc_options: []