google-apis-core 0.1.0

data/lib/google/apis/core/batch.rb

@@ -0,0 +1,236 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# Copyright 2015 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'google/apis/core/multipart'
+require 'google/apis/core/http_command'
+require 'google/apis/core/upload'
+require 'google/apis/core/download'
+require 'google/apis/core/composite_io'
+require 'addressable/uri'
+require 'securerandom'
+
+module Google
+  module Apis
+    module Core
+      # Wrapper request for batching multiple calls in a single server request
+      class BatchCommand < HttpCommand
+        MULTIPART_MIXED = 'multipart/mixed'
+
+        # @param [symbol] method
+        #   HTTP method
+        # @param [String,Addressable::URI, Addressable::Template] url
+        #   HTTP URL or template
+        def initialize(method, url)
+          super(method, url)
+          @calls = []
+          @base_id = SecureRandom.uuid
+        end
+
+        ##
+        # Add a new call to the batch request.
+        #
+        # @param [Google::Apis::Core::HttpCommand] call API Request to add
+        # @yield [result, err] Result & error when response available
+        # @return [Google::Apis::Core::BatchCommand] self
+        def add(call, &block)
+          ensure_valid_command(call)
+          @calls << [call, block]
+          self
+        end
+
+        protected
+
+        ##
+        # Deconstruct the batch response and process the individual results
+        #
+        # @param [String] content_type
+        #  Content type of body
+        # @param [String, #read] body
+        #  Response body
+        # @return [Object]
+        #   Response object
+        def decode_response_body(content_type, body)
+          m = /.*boundary=(.+)/.match(content_type)
+          if m
+            parts = split_parts(body, m[1])
+            deserializer = CallDeserializer.new
+            parts.each_index do |index|
+              response = deserializer.to_http_response(parts[index])
+              outer_header = response.shift
+              call_id = header_to_id(outer_header['Content-ID'].first) || index
+              call, callback = @calls[call_id]
+              begin
+                result = call.process_response(*response) unless call.nil?
+                success(result, &callback)
+              rescue => e
+                error(e, &callback)
+              end
+            end
+          end
+          nil
+        end
+
+        def split_parts(body, boundary)
+          parts = body.split(/\r?\n?--#{Regexp.escape(boundary)}/)
+          parts[1...-1]
+        end
+
+        # Encode the batch request
+        # @return [void]
+        # @raise [Google::Apis::BatchError] if batch is empty
+        def prepare!
+          fail BatchError, 'Cannot make an empty batch request' if @calls.empty?
+
+          serializer = CallSerializer.new
+          multipart = Multipart.new(content_type: MULTIPART_MIXED)
+          @calls.each_index do |index|
+            call, _ = @calls[index]
+            content_id = id_to_header(index)
+            io = serializer.to_part(call)
+            multipart.add_upload(io, content_type: 'application/http', content_id: content_id)
+          end
+          self.body = multipart.assemble
+
+          header['Content-Type'] = multipart.content_type
+          super
+        end
+
+        def ensure_valid_command(command)
+          if command.is_a?(Google::Apis::Core::BaseUploadCommand) || command.is_a?(Google::Apis::Core::DownloadCommand)
+            fail Google::Apis::ClientError, 'Can not include media requests in batch'
+          end
+          fail Google::Apis::ClientError, 'Invalid command object' unless command.is_a?(HttpCommand)
+        end
+
+        def id_to_header(call_id)
+          return sprintf('<%s+%i>', @base_id, call_id)
+        end
+
+        def header_to_id(content_id)
+          match = /<response-.*\+(\d+)>/.match(content_id)
+          return match[1].to_i if match
+          return nil
+        end
+
+      end
+
+      # Wrapper request for batching multiple uploads in a single server request
+      class BatchUploadCommand < BatchCommand
+        def ensure_valid_command(command)
+          fail Google::Apis::ClientError, 'Can only include upload commands in batch' \
+            unless command.is_a?(Google::Apis::Core::BaseUploadCommand)
+        end
+
+        def prepare!
+          header['X-Goog-Upload-Protocol'] = 'batch'
+          super
+        end
+      end
+
+      # Serializes a command for embedding in a multipart batch request
+      # @private
+      class CallSerializer
+        ##
+        # Serialize a single batched call for assembling the multipart message
+        #
+        # @param [Google::Apis::Core::HttpCommand] call
+        #   the call to serialize.
+        # @return [IO]
+        #   the serialized request
+        def to_part(call)
+          call.prepare!
+          # This will add the Authorization header if needed.
+          call.apply_request_options(call.header)
+          parts = []
+          parts << build_head(call)
+          parts << build_body(call) unless call.body.nil?
+          length = parts.inject(0) { |a, e| a + e.length }
+          Google::Apis::Core::CompositeIO.new(*parts)
+        end
+
+        protected
+
+        def build_head(call)
+          request_head = "#{call.method.to_s.upcase} #{Addressable::URI.parse(call.url).request_uri} HTTP/1.1"
+          call.header.each do |key, value|
+            request_head << sprintf("\r\n%s: %s", key, value)
+          end
+          request_head << sprintf("\r\nHost: %s", call.url.host)
+          request_head << "\r\n\r\n"
+          StringIO.new(request_head)
+        end
+
+        def build_body(call)
+          return nil if call.body.nil?
+          return call.body if call.body.respond_to?(:read)
+          StringIO.new(call.body)
+        end
+      end
+
+      # Deconstructs a raw HTTP response part
+      # @private
+      class CallDeserializer
+        # Parse a batched response.
+        #
+        # @param [String] call_response
+        #   the response to parse.
+        # @return [Array<(Fixnum, Hash, String)>]
+        #   Status, header, and response body.
+        def to_http_response(call_response)
+          outer_header, outer_body = split_header_and_body(call_response)
+          status_line, payload = outer_body.split(/\n/, 2)
+          _, status = status_line.split(' ', 3)
+
+          header, body = split_header_and_body(payload)
+          [outer_header, status.to_i, header, body]
+        end
+
+        protected
+
+        # Auxiliary method to split the header from the body in an HTTP response.
+        #
+        # @param [String] response
+        #   the response to parse.
+        # @return [Array<(HTTP::Message::Headers, String)>]
+        #   the header and the body, separately.
+        def split_header_and_body(response)
+          header = HTTP::Message::Headers.new
+          payload = response.lstrip
+          while payload
+            line, payload = payload.split(/\n/, 2)
+            line.sub!(/\s+\z/, '')
+            break if line.empty?
+            match = /\A([^:]+):\s*/.match(line)
+            fail BatchError, sprintf('Invalid header line in response: %s', line) if match.nil?
+            header[match[1]] = match.post_match
+          end
+          [header, payload]
+        end
+      end
+    end
+  end
+end

data/lib/google/apis/core/composite_io.rb

@@ -0,0 +1,97 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# Copyright 2015 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'google/apis/core/http_command'
+require 'google/apis/core/upload'
+require 'google/apis/core/download'
+require 'addressable/uri'
+require 'securerandom'
+module Google
+  module Apis
+    module Core
+      class CompositeIO
+        def initialize(*ios)
+          @ios = ios.flatten
+          @pos = 0
+          @index = 0
+          @sizes = @ios.map(&:size)
+        end
+
+        def read(length = nil, buf = nil)
+          buf = buf ? buf.replace('') : ''
+
+          begin
+            io = @ios[@index]
+            break if io.nil?
+            result = io.read(length)
+            if result
+              buf << result
+              if length
+                length -= result.length
+                break if length == 0
+              end
+            end
+            @index += 1
+          end while @index < @ios.length
+          buf.length > 0 ? buf : nil
+        end
+
+        def size
+          @sizes.reduce(:+)
+        end
+
+        alias_method :length, :size
+
+        def pos
+          @pos
+        end
+
+        def pos=(pos)
+          fail ArgumentError, "Position can not be negative" if pos < 0
+          @pos = pos
+          new_index = nil
+          @ios.each_with_index do |io,idx|
+            size = io.size
+            if pos <= size
+              new_index ||= idx
+              io.pos = pos
+              pos = 0
+            else
+              io.pos = size
+              pos -= size
+            end
+          end
+          @index = new_index unless new_index.nil?
+        end
+
+        def rewind
+          self.pos = 0
+        end
+      end
+    end
+  end
+end

data/lib/google/apis/core/download.rb

@@ -0,0 +1,118 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'google/apis/core/api_command'
+require 'google/apis/errors'
+require 'addressable/uri'
+
+module Google
+  module Apis
+    module Core
+      # Streaming/resumable media download support
+      class DownloadCommand < ApiCommand
+        RANGE_HEADER = 'Range'
+        OK_STATUS = [200, 201, 206]
+
+        # File or IO to write content to
+        # @return [String, File, #write]
+        attr_accessor :download_dest
+
+        # Ensure the download destination is a writable stream.
+        #
+        # @return [void]
+        def prepare!
+          @state = :start
+          @download_url = nil
+          @offset = 0
+          if download_dest.respond_to?(:write)
+            @download_io = download_dest
+            @close_io_on_finish = false
+          elsif download_dest.is_a?(String)
+            @download_io = File.open(download_dest, 'wb')
+            @close_io_on_finish = true
+          else
+            @download_io = StringIO.new('', 'wb')
+            @close_io_on_finish = false
+          end
+          super
+        end
+
+        # Close IO stream when command done. Only closes the stream if it was opened by the command.
+        def release!
+          @download_io.close if @close_io_on_finish
+        end
+
+        # Execute the upload request once. Overrides the default implementation to handle streaming/chunking
+        # of file content.
+        #
+        # @private
+        # @param [HTTPClient] client
+        #   HTTP client
+        # @yield [result, err] Result or error if block supplied
+        # @return [Object]
+        # @raise [Google::Apis::ServerError] An error occurred on the server and the request can be retried
+        # @raise [Google::Apis::ClientError] The request is invalid and should not be retried without modification
+        # @raise [Google::Apis::AuthorizationError] Authorization is required
+        def execute_once(client, &block)
+          request_header = header.dup
+          apply_request_options(request_header)
+          download_offset = nil
+
+          if @offset > 0
+            logger.debug { sprintf('Resuming download from offset %d', @offset) }
+            request_header[RANGE_HEADER] = sprintf('bytes=%d-', @offset)
+          end
+
+          http_res = client.get(url.to_s,
+                     query: query,
+                     header: request_header,
+                     follow_redirect: true) do |res, chunk|
+            status = res.http_header.status_code.to_i
+            next unless OK_STATUS.include?(status)
+
+            download_offset ||= (status == 206 ? @offset : 0)
+            download_offset  += chunk.bytesize
+
+            if download_offset - chunk.bytesize == @offset
+              next_chunk = chunk
+            else
+              # Oh no! Requested a chunk, but received the entire content
+              chunk_index = @offset - (download_offset - chunk.bytesize)
+              next_chunk = chunk.byteslice(chunk_index..-1)
+              next if next_chunk.nil?
+            end
+
+            # logger.debug { sprintf('Writing chunk (%d bytes, %d total)', chunk.length, bytes_read) }
+            @download_io.write(next_chunk)
+
+            @offset += next_chunk.bytesize
+          end
+
+         @download_io.flush if @download_io.respond_to?(:flush)
+
+          if @close_io_on_finish
+            result = nil
+          else
+            result = @download_io
+          end
+          check_status(http_res.status.to_i, http_res.header, http_res.body)
+          success(result, &block)
+        rescue => e
+          @download_io.flush if @download_io.respond_to?(:flush)
+          error(e, rethrow: true, &block)
+        end
+      end
+    end
+  end
+end