google-apis-core 0.7.1 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4f02ec5d554c69a4510fa74b0b1b4ddf67f7a34fb0d968374952905e44a4a663
-  data.tar.gz: 2d08a648b8ebdd6679bd602ccef761ee3e1b2418cd008514880912a532d0be33
+  metadata.gz: 39e8c0ee69956045fad35dd99b60187bee9a559e66c31141784f69c938b828da
+  data.tar.gz: 2bc2f65430050943360436b018b2149f876bdb190d8d637e42d1ac5c908aee41
 SHA512:
-  metadata.gz: 7f8bd3f598d3631677a518b3a744bc49ac25913a094678fb930a935e75aae6be445c0a5639a59b72ae33a6e5f8efd1dbafabcfaa210c554aaad325392ccdd7c1
-  data.tar.gz: 973119aa7df5d7eef5e4a870cd57bcb0980657faaddf408f95beb8962bec169d79811fbd75f3291f73ed12223625cd78833f789b0258ee26ddf42151e1aa2931
+  metadata.gz: 26a81d876d54cd42e9e84a4c057297ccd77a1cfebe481de646504f8e30d08496b72141581532dfeae03d3dcb9ff955330b7e94c81bc9bb555130e501fc2879f2
+  data.tar.gz: 2c1242fd64e8e66e82b04aa2a4f4c0fb0979e7a0a53f8162836a90a58242f1d46d4b8660101a15c9e82b6b6e48819cdffe5053f90894a9cf1dd5040de2c21db0

data/CHANGELOG.md

@@ -1,5 +1,23 @@
 # Release History
 
+### 0.9.0 (2022-09-18)
+
+#### Features
+
+* add support to have invocation-id header ([#11655](https://github.com/googleapis/google-api-ruby-client/issues/11655)) 
+
+### 0.8.0 (2022-09-16)
+
+#### Features
+
+* Add storage upload to move away from unified upload protocol ([#11508](https://github.com/googleapis/google-api-ruby-client/issues/11508)) 
+
+### 0.7.2 (2022-09-15)
+
+#### Bug Fixes
+
+* do not reset query_values in case of form encoding ([#11654](https://github.com/googleapis/google-api-ruby-client/issues/11654)) 
+
 ### 0.7.1 (2022-09-14)
 
 #### Bug Fixes

data/lib/google/apis/core/api_command.rb

@@ -18,6 +18,7 @@ require 'google/apis/core/http_command'
 require 'google/apis/errors'
 require 'json'
 require 'retriable'
+require "securerandom"
 
 module Google
   module Apis
@@ -161,6 +162,7 @@ module Google
           xgac = "gl-ruby/#{RUBY_VERSION} gdcl/#{munged_client_version}"
           xgac = old_xgac.empty? ? xgac : "#{old_xgac} #{xgac}"
           header.delete_if { |k, v| k.downcase == 'x-goog-api-client' }
+          xgac.concat(" ",invocation_id_header) if options.add_invocation_id_header
           header['X-Goog-Api-Client'] = xgac
         end
 
@@ -172,6 +174,10 @@ module Google
           header['X-Goog-User-Project'] = quota_project_id if quota_project_id
         end
 
+        def invocation_id_header
+          "gccl-invocation-id/#{SecureRandom.uuid}"
+        end
+
         # Attempt to parse a JSON error message
         # @param [String] body
         #  HTTP response body

data/lib/google/apis/core/base_service.rb

@@ -19,6 +19,7 @@ require 'google/apis/core/version'
 require 'google/apis/core/api_command'
 require 'google/apis/core/batch'
 require 'google/apis/core/upload'
+require 'google/apis/core/storage_upload'
 require 'google/apis/core/download'
 require 'google/apis/core/storage_download'
 require 'google/apis/options'
@@ -318,6 +319,26 @@ module Google
           command
         end
 
+        # Create a new storage upload command.
+        # This is specifically for storage because we are moving to a new upload protocol.
+        # Ref: https://cloud.google.com/storage/docs/performing-resumable-uploads
+        #
+        # @param [Symbol] method
+        #  HTTP method for uploading. The initial request to initiate a resumable session
+        #  is :post and the subsequent chunks uploaded to the session are :put
+        # @param [String] path
+        #  Additional path to upload endpoint, appended to API base path
+        # @param [Hash, Google::Apis::RequestOptions] options
+        #  Request-specific options
+        # @return [Google::Apis::Core::StorageUploadCommand]
+        def make_storage_upload_command(method, path, options)
+          template = Addressable::Template.new(root_url + upload_path + path)
+          command = StorageUploadCommand.new(method, template, client_version: client_version)
+          command.options = request_options.merge(options)
+          apply_command_defaults(command)
+          command
+        end
+
         # Create a new download command.
         #
         # @param [symbol] method

data/lib/google/apis/core/batch.rb

@@ -28,6 +28,7 @@
 require 'google/apis/core/multipart'
 require 'google/apis/core/http_command'
 require 'google/apis/core/upload'
+require 'google/apis/core/storage_upload'
 require 'google/apis/core/download'
 require 'google/apis/core/composite_io'
 require 'addressable/uri'
@@ -120,7 +121,7 @@ module Google
         end
 
         def ensure_valid_command(command)
-          if command.is_a?(Google::Apis::Core::BaseUploadCommand) || command.is_a?(Google::Apis::Core::DownloadCommand) || command.is_a?(Google::Apis::Core::StorageDownloadCommand)
+          if command.is_a?(Google::Apis::Core::BaseUploadCommand) || command.is_a?(Google::Apis::Core::DownloadCommand) || command.is_a?(Google::Apis::Core::StorageDownloadCommand) || command.is_a?(Google::Apis::Core::StorageUploadCommand)
             fail Google::Apis::ClientError, 'Can not include media requests in batch'
           end
           fail Google::Apis::ClientError, 'Invalid command object' unless command.is_a?(HttpCommand)

data/lib/google/apis/core/http_command.rb

@@ -98,9 +98,17 @@ module Google
         # @raise [Google::Apis::ServerError] An error occurred on the server and the request can be retried
         # @raise [Google::Apis::ClientError] The request is invalid and should not be retried without modification
         # @raise [Google::Apis::AuthorizationError] Authorization is required
-        def execute(client)
+        def execute(client, &block)
           prepare!
           opencensus_begin_span
+          do_retry :execute_once, client, &block
+        ensure
+          opencensus_end_span
+          @http_res = nil
+          release!
+        end
+
+        def do_retry func, client
           begin
             Retriable.retriable tries: options.retries + 1,
                                 max_elapsed_time: options.max_elapsed_time,
@@ -115,7 +123,7 @@ module Google
               Retriable.retriable tries: auth_tries,
                                   on: [Google::Apis::AuthorizationError, Signet::AuthorizationError, Signet::RemoteServerError, Signet::UnexpectedStatusError],
                                   on_retry: proc { |*| refresh_authorization } do
-                execute_once(client).tap do |result|
+                send(func, client).tap do |result|
                   if block_given?
                     yield result, nil
                   end
@@ -129,10 +137,6 @@ module Google
               raise e
             end
           end
-        ensure
-          opencensus_end_span
-          @http_res = nil
-          release!
         end
 
         # Refresh the authorization authorization after a 401 error
@@ -168,7 +172,6 @@ module Google
             @form_encoded = true
             self.body = Addressable::URI.form_encode(url.query_values(Array))
             self.header['Content-Type'] = 'application/x-www-form-urlencoded; charset=UTF-8'
-            self.url.query_values = {}
           else
             @form_encoded = false
           end
@@ -217,7 +220,7 @@ module Google
         def check_status(status, header = nil, body = nil, message = nil)
           # TODO: 304 Not Modified depends on context...
           case status
-          when 200...300
+          when 200...300, 308
             nil
           when 301, 302, 303, 307
             message ||= sprintf('Redirect to %s', header['Location'])

data/lib/google/apis/core/storage_upload.rb

@@ -0,0 +1,181 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'google/apis/core/http_command'
+require 'google/apis/core/api_command'
+require 'google/apis/errors'
+require 'tempfile'
+require 'mini_mime'
+
+module Google
+  module Apis
+    module Core
+      # Base upload command. Not intended to be used directly
+      # @private
+      class StorageUploadCommand < ApiCommand
+        CONTENT_LENGTH_HEADER = "Content-Length"
+        CONTENT_TYPE_HEADER = "Content-Type"
+        UPLOAD_CONTENT_TYPE_HEADER = "X-Upload-Content-Type"
+        LOCATION_HEADER = "Location"
+        CONTENT_RANGE_HEADER = "Content-Range"
+        RESUMABLE = "resumable"
+        OK_STATUS = 200
+        CHUNK_SIZE = 8 * 1024 * 1024 # 8 MB
+
+        # File name or IO containing the content to upload
+        # @return [String, File, #read]
+        attr_accessor :upload_source
+
+        # Content type of the upload material
+        # @return [String]
+        attr_accessor :upload_content_type
+
+        # Content, as UploadIO
+        # @return [Google::Apis::Core::UploadIO]
+        attr_accessor :upload_io
+
+        # Ensure the content is readable and wrapped in an IO instance.
+        #
+        # @return [void]
+        # @raise [Google::Apis::ClientError] if upload source is invalid
+        def prepare!
+          @upload_url = nil
+          @offset = 0
+          @upload_incomplete = true
+          # Prevent the command from populating the body with form encoding, by
+          # asserting that it already has a body. Form encoding is never used
+          # by upload requests.
+          self.body = '' unless self.body
+
+          super
+          if streamable?(upload_source)
+            self.upload_io = upload_source
+            @close_io_on_finish = false
+          elsif self.upload_source.is_a?(String)
+            self.upload_io = File.new(upload_source, 'r')
+            if self.upload_content_type.nil?
+              type = MiniMime.lookup_by_filename(upload_source)
+              self.upload_content_type = type&.content_type
+            end
+            @close_io_on_finish = true
+          else
+            fail Google::Apis::ClientError, 'Invalid upload source'
+          end
+        end
+
+        # Close IO stream when command done. Only closes the stream if it was opened by the command.
+        def release!
+          upload_io.close if @close_io_on_finish
+        end
+
+        # Execute the command, retrying as necessary
+        #
+        # @param [HTTPClient] client
+        #   HTTP client
+        # @yield [result, err] Result or error if block supplied
+        # @return [Object]
+        # @raise [Google::Apis::ServerError] An error occurred on the server and the request can be retried
+        # @raise [Google::Apis::ClientError] The request is invalid and should not be retried without modification
+        # @raise [Google::Apis::AuthorizationError] Authorization is required
+        def execute(client)
+          prepare!
+          opencensus_begin_span
+
+          do_retry :initiate_resumable_upload, client
+          while @upload_incomplete
+            res = do_retry :send_upload_command, client
+          end
+          res
+        ensure
+          opencensus_end_span
+          @http_res = nil
+          release!
+        end
+
+        def initiate_resumable_upload(client)
+          logger.debug { sprintf('Intiating resumable upload command to %s', url) }
+
+          request_header = header.dup
+          apply_request_options(request_header)
+
+          request_query = query.dup
+          request_query['uploadType'] = RESUMABLE
+
+          request_header[CONTENT_LENGTH_HEADER] = upload_io.size.to_s
+          request_header[CONTENT_TYPE_HEADER] = JSON_CONTENT_TYPE
+          request_header[UPLOAD_CONTENT_TYPE_HEADER] = upload_content_type unless upload_content_type.nil?
+
+          response = client.post(url.to_s, query: request_query,
+                         body: body,
+                         header: request_header,
+                         follow_redirect: true)
+          result = process_response(response.status_code, response.header, response.body)
+          success(result)
+        rescue => e
+          error(e, rethrow: true)
+        end
+
+        # Send the actual content
+        #
+        # @param [HTTPClient] client
+        #   HTTP client
+        # @return [HTTP::Message]
+        # @raise [Google::Apis::ServerError] Unable to send the request
+        def send_upload_command(client)
+          logger.debug { sprintf('Sending upload command to %s', @upload_url) }
+
+          remaining_content_size = upload_io.size - @offset
+          current_chunk_size = remaining_content_size < CHUNK_SIZE ? remaining_content_size : CHUNK_SIZE
+
+          request_header = header.dup
+          request_header[CONTENT_RANGE_HEADER] = sprintf('bytes %d-%d/%d', @offset, @offset+current_chunk_size-1, upload_io.size)
+          request_header[CONTENT_LENGTH_HEADER] = current_chunk_size
+          chunk_body = upload_io.read(current_chunk_size)
+
+          response = client.put(@upload_url, body: chunk_body, header: request_header, follow_redirect: true)
+
+          result = process_response(response.status_code, response.header, response.body)
+          @upload_incomplete = false if response.status_code.eql? OK_STATUS
+          @offset += current_chunk_size if @upload_incomplete
+          success(result)
+        rescue => e
+          upload_io.pos = @offset
+          error(e, rethrow: true)
+        end
+
+        # Check the to see if the upload is complete or needs to be resumed.
+        #
+        # @param [Integer] status
+        #   HTTP status code of response
+        # @param [HTTP::Message::Headers] header
+        #   Response headers
+        # @param [String, #read] body
+        #  Response body
+        # @return [Object]
+        #   Response object
+        # @raise [Google::Apis::ServerError] An error occurred on the server and the request can be retried
+        # @raise [Google::Apis::ClientError] The request is invalid and should not be retried without modification
+        # @raise [Google::Apis::AuthorizationError] Authorization is required
+        def process_response(status, header, body)
+          @upload_url = header[LOCATION_HEADER].first unless header[LOCATION_HEADER].empty?
+          super(status, header, body)
+        end
+
+        def streamable?(upload_source)
+          upload_source.is_a?(IO) || upload_source.is_a?(StringIO) || upload_source.is_a?(Tempfile)
+        end
+      end
+    end
+  end
+end

data/lib/google/apis/core/version.rb

@@ -16,7 +16,7 @@ module Google
   module Apis
     module Core
       # Core version
-      VERSION = "0.7.1".freeze
+      VERSION = "0.9.0".freeze
     end
   end
 end

data/lib/google/apis/options.rb

@@ -39,7 +39,8 @@ module Google
       :api_format_version,
       :use_opencensus,
       :quota_project,
-      :query)
+      :query,
+      :add_invocation_id_header)
 
     # General client options
     class ClientOptions
@@ -95,6 +96,8 @@ module Google
       #   @return [String] Project ID to charge quota, or `nil` to default to the credentials-specified project.
       # @!attribute [rw] query
       #   @return [Hash<String,String>] Additional HTTP URL query parameters to include in requests.
+      # @!attribute [rw] add_invocation_id_header
+      #   @return [Boolean] True if the header gccl-invocation-id need to be set
 
       # Get the default options
       # @return [Google::Apis::RequestOptions]
@@ -129,5 +132,6 @@ module Google
     RequestOptions.default.api_format_version = nil
     RequestOptions.default.use_opencensus = true
     RequestOptions.default.quota_project = nil
+    RequestOptions.default.add_invocation_id_header = false
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: google-apis-core
 version: !ruby/object:Gem::Version
-  version: 0.7.1
+  version: 0.9.0
 platform: ruby
 authors:
 - Google LLC
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-09-14 00:00:00.000000000 Z
+date: 2022-09-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: representable
@@ -175,6 +175,7 @@ files:
 - lib/google/apis/core/logging.rb
 - lib/google/apis/core/multipart.rb
 - lib/google/apis/core/storage_download.rb
+- lib/google/apis/core/storage_upload.rb
 - lib/google/apis/core/upload.rb
 - lib/google/apis/core/version.rb
 - lib/google/apis/errors.rb
@@ -185,7 +186,7 @@ licenses:
 metadata:
   bug_tracker_uri: https://github.com/googleapis/google-api-ruby-client/issues
   changelog_uri: https://github.com/googleapis/google-api-ruby-client/tree/main/google-apis-core/CHANGELOG.md
-  documentation_uri: https://googleapis.dev/ruby/google-apis-core/v0.7.1
+  documentation_uri: https://googleapis.dev/ruby/google-apis-core/v0.9.0
   source_code_uri: https://github.com/googleapis/google-api-ruby-client/tree/main/google-apis-core
 post_install_message: 
 rdoc_options: []