baidubce-sdk 0.9.0

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/baidubce-sdk.gemspec

@@ -0,0 +1,31 @@
+# coding: utf-8
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'baidubce/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "baidubce-sdk"
+  spec.version       = Baidubce::VERSION
+  spec.authors       = ["xiaoyong"]
+  spec.email         = ["xiaoyong@baidu.com"]
+
+  spec.summary       = 'BaiduBce BOS Ruby SDK'
+  spec.description   = 'The official Ruby sdk used to accessing BaiduBce Object Storage Service'
+  spec.homepage      = "https://github.com/baidubce/bce-sdk-ruby"
+  spec.license       = "Apache-2.0"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "lib/baidubce"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.15"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "rest-client", "~> 2.0", ">= 2.0.2"
+  spec.add_development_dependency "logger", "~> 1.2", ">= 1.2.8"
+  spec.add_development_dependency "mimemagic", "~> 0.3", ">= 0.3.2"
+  spec.required_ruby_version = ">= 2.0.0"
+end

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "baidubce/bce"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/baidubce/auth/bce_credentials.rb

@@ -0,0 +1,31 @@
+# Copyright (c) 2017 Baidu.com, Inc. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+# except in compliance with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the
+# License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+# either express or implied. See the License for the specific language governing permissions
+# and limitations under the License.
+
+=begin
+Provides access to the BCE credentials used for accessing BCE services: BCE access key ID and
+secret access key.
+These credentials are used to securely sign requests to BCE services.
+=end
+
+module Baidubce
+    module Auth
+        class BceCredentials
+            attr_accessor :access_key_id, :secret_access_key, :security_token
+
+            def initialize(access_key_id, secret_access_key, security_token="")
+                @access_key_id = access_key_id
+                @secret_access_key = secret_access_key
+                @security_token = security_token
+            end
+        end
+    end
+end

data/lib/baidubce/auth/bce_v1_signer.rb

@@ -0,0 +1,76 @@
+# Copyright (c) 2017 Baidu.com, Inc. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+# except in compliance with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the
+# License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+# either express or implied. See the License for the specific language governing permissions
+# and limitations under the License.
+
+# This module provides authentication functions for bce services.
+
+require 'time'
+require 'openssl'
+
+require_relative '../utils/utils'
+
+module Baidubce
+    module Auth
+        class BceV1Signer
+
+            def get_canonical_headers(headers, headers_to_sign = nil)
+                default = false
+                if headers_to_sign.to_a.empty?
+                    default = true
+                    headers_to_sign = ["host", "content-md5", "content-length", "content-type"]
+                end
+
+                ret_arr = []
+                headers_arr = []
+                headers.each do |key, value|
+                    next if value.to_s.strip.empty?
+                    if headers_to_sign.include?(key.downcase) ||
+                            (default && key.downcase.to_s.start_with?(Http::BCE_PREFIX))
+                        str = ERB::Util.url_encode(key.downcase) + ":" + ERB::Util.url_encode(value.to_s.strip)
+                        ret_arr << str
+                        headers_arr << key.downcase
+                    end
+                end
+                ret_arr.sort!
+                headers_arr.sort!
+                return ret_arr.join("\n"), headers_arr
+            end
+
+            def get_canonical_uri_path(path)
+                return '/' if path.to_s.empty?
+                encoded_path = Utils.url_encode_except_slash(path)
+                return path[0] == '/' ? encoded_path : '/' + encoded_path
+            end
+
+            # Create the authorization.
+            def sign(credentials, http_method, path, headers, params,
+                     timestamp=nil, expiration_in_seconds=1800, headers_to_sign=nil)
+
+                timestamp = Time.now.to_i if timestamp.nil?
+                sign_key_info = sprintf('bce-auth-v1/%s/%s/%d',
+                                        credentials.access_key_id,
+                                        Time.at(timestamp).utc.iso8601,
+                                        expiration_in_seconds)
+                sign_key = OpenSSL::HMAC.hexdigest(OpenSSL::Digest.new('sha256'),
+                                                   credentials.secret_access_key, sign_key_info)
+                canonical_uri = get_canonical_uri_path(path)
+                canonical_querystring = Utils.get_canonical_querystring(params, true)
+                canonical_headers, headers_to_sign = get_canonical_headers(headers, headers_to_sign)
+                canonical_request = [http_method, canonical_uri, canonical_querystring, canonical_headers].join("\n")
+                signature = OpenSSL::HMAC.hexdigest(OpenSSL::Digest.new('sha256'),
+                                                    sign_key, canonical_request)
+
+                headers_str = headers_to_sign.join(';') unless headers_to_sign.nil?
+                sign_key_info + '/' + headers_str + '/' + signature
+            end
+        end
+    end
+end

data/lib/baidubce/bce_base_client.rb

@@ -0,0 +1,52 @@
+# Copyright 2017 Baidu, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+# except in compliance with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the
+# License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+# either express or implied. See the License for the specific language governing permissions
+# and limitations under the License.
+
+# This module provide base class for BCE service clients.
+
+require_relative 'auth/bce_v1_signer'
+require_relative 'auth/bce_credentials'
+require_relative 'http/base_http_client'
+require_relative 'bce_client_configuration'
+
+module Baidubce
+
+    class BceBaseClient
+
+        include Http
+        include Auth
+
+        def initialize(config, service_id="", region_supported=true)
+            @config = config
+            @service_id = service_id
+            @region_supported = region_supported
+            @config.endpoint = compute_endpoint if @config.endpoint.to_s.empty?
+            @http_client = BaseHttpClient.new()
+            @signer = BceV1Signer.new()
+        end
+
+        def compute_endpoint
+            if @region_supported
+                return sprintf('%s://%s.%s.%s',
+                               @config.protocol,
+                               @service_id,
+                               @config.region,
+                               DEFAULT_SERVICE_DOMAIN)
+            else
+                return sprintf('%s://%s.%s',
+                               @config.protocol,
+                               @service_id,
+                               DEFAULT_SERVICE_DOMAIN)
+            end
+
+        end
+    end
+end

data/lib/baidubce/bce_client_configuration.rb

@@ -0,0 +1,47 @@
+# Copyright 2017 Baidu, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+# except in compliance with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the
+# License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+# either express or implied. See the License for the specific language governing permissions
+# and limitations under the License.
+
+# This module defines a common configuration class for BCE.
+
+require_relative 'retry_policy'
+
+module Baidubce
+
+    DEFAULT_PROTOCOL = "http"
+    DEFAULT_REGION = "bj"
+    DEFAULT_OPEN_TIMEOUT_IN_MILLIS = 60 * 1000
+    DEFAULT_READ_TIMEOUT_IN_MILLIS = 10 * 60 * 1000
+    DEFAULT_SEND_BUF_SIZE = 1024 * 1024
+    DEFAULT_RECV_BUF_SIZE = 10 * 1024 * 1024
+
+    class BceClientConfiguration
+        attr_accessor :credentials, :endpoint, :protocol, :region, :open_timeout_in_millis,
+                      :read_timeout_in_millis, :send_buf_size, :recv_buf_size, :retry_policy
+
+        def initialize(credentials,
+                       endpoint,
+                       options={})
+
+            @credentials = credentials
+            @endpoint = endpoint
+            @protocol = options['protocol'] || DEFAULT_PROTOCOL
+            @region = options['region'] || DEFAULT_REGION
+            @open_timeout_in_millis = options['open_timeout_in_millis'] ||
+                DEFAULT_OPEN_TIMEOUT_IN_MILLIS
+            @read_timeout_in_millis = options['read_timeout_in_millis'] || DEFAULT_READ_TIMEOUT_IN_MILLIS
+            @send_buf_size = options['send_buf_size'] || DEFAULT_SEND_BUF_SIZE
+            @recv_buf_size = options['recv_buf_size'] || DEFAULT_RECV_BUF_SIZE
+            @retry_policy = options['retry_policy'] || BackOffRetryPolicy.new
+        end
+
+    end
+end

data/lib/baidubce/bce_constants.rb

@@ -0,0 +1,20 @@
+# Copyright 2017 Baidu, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+# except in compliance with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the
+# License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+# either express or implied. See the License for the specific language governing permissions
+# and limitations under the License.
+
+# This module provide constants for BCE sdk.
+
+module Baidubce
+
+    DEFAULT_SERVICE_DOMAIN = 'baidubce.com'
+    DEFAULT_ENCODING = 'UTF-8'
+
+end

data/lib/baidubce/exception.rb

@@ -0,0 +1,34 @@
+class BceClientException < RuntimeError
+   attr_accessor :message
+
+   def initialize(message)
+      @message = message
+   end
+
+end
+
+class BceServerException < RuntimeError
+   attr_accessor :status_code
+   attr_accessor :message
+
+   def initialize(status_code, message)
+      @status_code = status_code
+      @message = message
+   end
+
+end
+
+class BceHttpException < RuntimeError
+    attr_accessor :http_code
+    attr_accessor :http_headers
+    attr_accessor :http_body
+    attr_accessor :message
+
+    def initialize(http_code, http_headers, http_body, message)
+        @http_code = http_code
+        @http_headers = http_headers
+        @http_body = http_body
+        @message = message
+    end
+
+end

data/lib/baidubce/http/base_http_client.rb

@@ -0,0 +1,259 @@
+# Copyright 2017 Baidu, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+# except in compliance with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the
+# License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+# either express or implied. See the License for the specific language governing permissions
+# and limitations under the License.
+
+# This module provide http request function for bce services.
+
+require 'rest-client'
+require 'time'
+require 'objspace'
+require 'fiber'
+
+require_relative '../version'
+require_relative '../exception'
+require_relative '../retry_policy'
+require_relative '../utils/utils'
+require_relative '../utils/log'
+require_relative '../bce_constants'
+require_relative 'http_constants'
+
+module Baidubce
+    module Http
+
+        class BaseHttpClient
+
+            include Log
+            # Send request to BCE services.
+            def send_request(config, signer, http_method, path, params, headers, body, save_path=nil, &block)
+                headers[USER_AGENT] = sprintf(
+                    'bce-sdk-ruby/%s/%s/%s',
+                    VERSION,
+                    RUBY_VERSION,
+                    RUBY_PLATFORM
+                )
+
+                should_get_new_date = headers.has_key?(BCE_DATE) ? false : true
+
+                url, headers[HOST] = Utils.parse_url_host(config)
+                url += Utils.url_encode_except_slash(path)
+                query_str = Utils.get_canonical_querystring(params, false)
+                url += "?#{query_str}" unless query_str.to_s.empty?
+
+                logger.info("url: #{url}, params: #{params}")
+                set_content_length_header(headers, body, &block)
+                headers[STS_SECURITY_TOKEN] = config.credentials.security_token unless config.credentials.security_token.to_s.empty?
+
+                retries_attempted = 0
+                while true
+                    headers[BCE_DATE] = Time.now.utc.iso8601 if should_get_new_date
+                    headers[AUTHORIZATION] = signer.sign(config.credentials, http_method,
+                                                         path, headers, params)
+
+                    logger.debug("Request headers: #{headers}")
+                    args = { method: http_method,
+                            url: url,
+                            headers: headers,
+                            payload: body,
+                            open_timeout: config.open_timeout_in_millis / 1000.0,
+                            read_timeout: config.read_timeout_in_millis / 1000.0
+                    }
+                    args[:payload] = BufWriter.new(&block) if block_given?
+
+                    begin
+                        if save_path
+                            logger.debug("Response save file path: #{save_path}")
+                            resp_headers = {}
+                            File.open(save_path, 'w+') { |f|
+                                block = proc { |response|
+                                    response.read_body { |chunk| f << chunk }
+                                    resp_headers = response.to_hash
+                                    resp_headers.each { |k, v| resp_headers[k]=v[0] }
+                                    raise BceHttpException.new(response.code.to_i,
+                                        resp_headers, '', 'get_object_to_file exception') if response.code.to_i > 300
+                                }
+                                block_arg = { block_response: block }
+                                args.merge! block_arg
+                                RestClient::Request.new(args).execute
+                                return '', resp_headers
+                            }
+                        else
+                            resp = RestClient::Request.new(args).execute
+                            logger.debug("Response code: #{resp.code}")
+                            logger.debug("Response headers: #{resp.headers.to_s}")
+                            return resp.body, resp.headers
+                        end
+                    rescue BceHttpException, RestClient::ExceptionWithResponse => err
+                        logger.debug("ExceptionWithResponse: #{err.http_code}, #{err.http_body}, #{err.http_headers}, #{err.message}")
+                        if config.retry_policy.should_retry(err.http_code, retries_attempted)
+                            delay_in_millis = config.retry_policy.get_delay_before_next_retry_in_millis(retries_attempted)
+                            sleep(delay_in_millis / 1000.0)
+                        else
+                            request_id = err.http_headers[:x_bce_request_id]
+                            if err.is_a?(BceHttpException)
+                                err.http_body = File.read(save_path)
+                                request_id = err.http_headers['x-bce-request-id']
+                            end
+                            msg = err.http_body
+                            if err.http_body.empty?
+                                msg = "{\"code\":\"#{err.http_code}\",\"message\":\"#{err.message}\",\"requestId\":\"#{request_id}\"}"
+                            end
+                            raise BceServerException.new(err.http_code, msg)
+                        end
+                    end
+
+                    retries_attempted += 1
+                end
+
+            end
+
+            def set_content_length_header(headers, body, &block)
+                unless block_given?
+                    if body.to_s.empty?
+                        headers[CONTENT_LENGTH] = 0
+                    elsif body.instance_of?(String)
+                        body = body.encode('UTF-8') if body.encoding.name != 'UTF-8'
+                        headers[CONTENT_LENGTH] = body.bytesize
+                    elsif body.instance_of?(File)
+                        headers[CONTENT_LENGTH] = body.size()
+                    else
+                        headers[CONTENT_LENGTH] = ObjectSpace.memsize_of(body)
+                    end
+                end
+            end
+
+            class BufWriter
+
+                def initialize()
+                    @buffer = ""
+                    @producer = Fiber.new { yield self if block_given? }
+                    @producer.resume
+                end
+
+                def read(bytes = nil, outbuf = nil)
+                    ret = ""
+                    while true
+                        if bytes
+                            fail if bytes < 0
+                            piece = @buffer.slice!(0, bytes)
+                            if piece
+                                ret << piece
+                                bytes -= piece.size
+                                break if bytes == 0
+                            end
+                        else
+                            ret << @buffer
+                            @buffer.clear
+                        end
+
+                        if @producer.alive?
+                            @producer.resume
+                        else
+                            break
+                        end
+                    end
+
+                    if outbuf
+                        outbuf.clear
+                        outbuf << ret
+                    end
+
+                    return nil if ret.empty? && !bytes.nil? && bytes > 0
+                    ret
+                end
+
+                def write(chunk)
+                    @buffer << chunk.to_s
+                    Fiber.yield
+                    self
+                end
+
+                alias << write
+
+                def closed?
+                    false
+                end
+
+                def close
+                end
+
+                def inspect
+                    "@buffer: " + @buffer[0, 32].inspect + "...#{@buffer.size} bytes"
+                end
+            end
+        end
+    end
+end
+
+module RestClient
+  module Payload
+    class Base
+      def headers
+          ({'content-length' => size.to_s} if size) || {}
+      end
+    end
+  end
+end
+
+module Net
+    class HTTP
+        def transport_request(req)
+            count = 0
+            begin
+                begin_transport req
+                res = catch(:response) {
+                    req.exec @socket, @curr_http_version, edit_path(req.path)
+                    begin
+                        res = HTTPResponse.read_new(@socket)
+                        res.decode_content = req.decode_content
+                    end while res.kind_of?(HTTPContinue)
+
+                    res.uri = req.uri
+
+                    res
+                }
+                res.reading_body(@socket, req.response_body_permitted?) {
+                    yield res if block_given?
+                }
+            rescue Net::OpenTimeout
+                raise
+            rescue Net::ReadTimeout, IOError, EOFError,
+                Errno::ECONNRESET, Errno::ECONNABORTED, Errno::EPIPE,
+                # avoid a dependency on OpenSSL
+                defined?(OpenSSL::SSL) ? OpenSSL::SSL::SSLError : IOError,
+                Timeout::Error => exception
+                if count == 0 && IDEMPOTENT_METHODS_.include?(req.method)
+                    count += 1
+                    @socket.close if @socket and not @socket.closed?
+                    D "Conn close because of error #{exception}, and retry"
+                    if req.body_stream
+                        if req.body_stream.respond_to?(:rewind)
+                            req.body_stream.rewind
+                        else
+                            raise
+                        end
+                    end
+                    retry
+                end
+                D "Conn close because of error #{exception}"
+                @socket.close if @socket and not @socket.closed?
+                raise
+            end
+
+            end_transport req, res
+            res
+        rescue => exception
+            D "Conn close because of error #{exception}"
+            @socket.close if @socket and not @socket.closed?
+            raise exception
+        end
+    end
+end
+