qingstor-sdk 2.4.0 → 2.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cb83b84c4cd53131d0092b26061348057db9f2af57d1d4670bc5cef3b36c12b8
-  data.tar.gz: 0c2d01a2a0d134540e57e11cdeaef5f5547cb613b3b8a1ee5a2e531302391a15
+  metadata.gz: b4067ee8cebe68b7a6ff943ce45aab389887fec3eb3725311eb8fd0bbf44fa50
+  data.tar.gz: 97d3720cb4c6ac65b45fa66a1ac8b117300f96489894c19381972ed945b59c79
 SHA512:
-  metadata.gz: bf9f6e59304fa869921e5300b57560f5fae7728b9bb521a6ffc2d1d1d03783871dc03b68b9f08ffa875afef7a0210b237d518f8fd30d24260e6db14a766ce7ad
-  data.tar.gz: 394b4f14aea92f46e8b511dbbd283a4e08a9f7d28ef1f5f71e87e0eae63ee32c4bd30a172266638efd9a715fe81a70c91c7dec0c738dabec2eed1175c5376c45
+  metadata.gz: 2d09dc5817005d1645473bddb4f5a354a4fef382dcf34775dee845cbffe1188d28d8684df73daf7440d58a83d6aa3e26d1c1e2199dacedf294f40fae915f25ec
+  data.tar.gz: a0a940c7419f8acc356977c9d336a7c982b67282624baa0e0b44e0f8b31a37cf3dd4b6e4d3f5b19ca84c6bf11a7d14c0169a522fc44d3390d297d29598be675b

data/README.md

@@ -14,7 +14,9 @@ This Gem uses Ruby's _keyword arguments_ feature, thus Ruby v2.1.5 or higher is
 required.  See [this article](https://robots.thoughtbot.com/ruby-2-keyword-arguments)
 for more details about _keyword arguments_.
 
-_Notice:_ If you are using Ruby v1.9.x, please checkout the [compatible] branch.
+_Notice:_ As of sdk v2.4, we no longer support Ruby 2.4 and earlier versions since Ruby 2.4 has been marked as EOL. 
+If you are still using Ruby 2.4 and earlier, you may not be able to use some new features of our sdk, 
+so please upgrade your version of Ruby as soon as possible.
 
 ### Install from RubyGems
 
@@ -122,6 +124,11 @@ server (in case of QingStor is deployed in private environment, thus has
 different endpoint with public QingStor) either in the config file, or in
 the program dynamically.
 
+**Notice:** config will not be checked available when `new`, `load` and `update`, 
+which will be done right before initializing `service` or `bucket`. 
+If you need to check config, you should call `config.check` whenever you want.
+If `check` failed, a `ConfigurationError` will be raised.
+
 ___Code Example:___
 
 ``` ruby
@@ -131,6 +138,7 @@ require 'qingstor/sdk'
 config = QingStor::SDK::Config.new.load_default_config
 
 # Create with default value
+# priority from high to low: param > env > user config > default config
 config = QingStor::SDK::Config.new({
   host:      'qingstor.dev',
   log_level: 'debug',
@@ -144,6 +152,13 @@ config = QingStor::SDK::Config.init ENV['ENV_ACCESS_KEY_ID'],
 config = QingStor::SDK::Config.new
 config = config.load_config_from_file '~/qingstor/config.yaml'
 
+# Load configuration from env variable:
+# QINGSTOR_ACCESS_KEY_ID for access_key_id
+# QINGSTOR_SECRET_ACCESS_KEY for secret_access_key
+# QINGSTOR_ENABLE_VIRTUAL_HOST_STYLE for enable_virtual_host_style
+# QINGSTOR_ENDPOINT for endpoint
+config = QingStor::SDK::Config.new.load_env_config
+
 # Change API server
 config.update({host: 'test.qingstor.com'})
 ```
@@ -159,15 +174,38 @@ secret_access_key: 'SECRET_ACCESS_KEY'
 host: 'qingstor.com'
 port: 443
 protocol: 'https'
+# or set endpoint directly
+# endpoint: https://qingstor.com:443
+
 connection_retries: 3
 
+# Additional User-Agent
+additional_user_agent: ""
+
 # Valid log levels are "debug", "info", "warn", "error", and "fatal".
-log_level: 'warn'
+log_level: warn
+
+# SDK will use virtual host style for all API calls if enabled
+enable_virtual_host_style: false
 ```
 
 ## Change Log
 All notable changes to QingStor SDK for Ruby will be documented here.
 
+### [v2.5.0] - 2021-02-09
+
+### Added
+
+- config: Add env variable support (#52)
+- signer: Add support for anonymous API call (#53)
+- config: Add support for endpoint (#55)
+- config: Add support for enable_vhost_style (#56)
+
+### Changed
+
+- Request: Modify Metadata in header to apply rfc 02 (#49)
+- config: Refactor config check (#54)
+
 ### [v2.4.0] - 2021-01-05
 
 ### Added
@@ -274,6 +312,7 @@ All notable changes to QingStor SDK for Ruby will be documented here.
 The Apache License (Version 2.0, January 2004).
 
 [compatible]: https://github.com/yunify/qingstor-sdk-ruby/tree/compatible
+[v2.5.0]: https://github.com/yunify/qingstor-sdk-ruby/compare/v2.4.0...v2.5.0
 [v2.4.0]: https://github.com/yunify/qingstor-sdk-ruby/compare/v2.3.0...v2.4.0
 [v2.3.0]: https://github.com/yunify/qingstor-sdk-ruby/compare/v2.2.3...v2.3.0
 [v2.2.3]: https://github.com/yunify/qingstor-sdk-ruby/compare/v2.2.2...v2.2.3

data/lib/qingstor/sdk/general/config.rb

@@ -15,10 +15,13 @@
 #  +-------------------------------------------------------------------------
 
 require 'fileutils'
+require 'ipaddr'
+require 'uri'
 require 'yaml'
 
 require 'active_support/core_ext/hash/keys'
 require 'active_support/core_ext/hash/deep_merge'
+require 'active_support/core_ext/object/blank'
 require 'net/http/persistent'
 
 module QingStor
@@ -26,6 +29,15 @@ module QingStor
     class Config < Hash
       attr_accessor :connection
 
+      DEFAULT_AS_HASH = {
+        host:                      'qingstor.com'.freeze,
+        port:                      443,
+        protocol:                  'https'.freeze,
+        connection_retries:        3,
+        log_level:                 'warn'.freeze,
+        enable_virtual_host_style: false
+      }
+
       def self.init(access_key_id, secret_access_key)
         initial_config = {
           access_key_id:     access_key_id,
@@ -36,23 +48,64 @@ module QingStor
 
       def initialize(initial_config = {})
         self.connection = Net::HTTP::Persistent.new
+        # load default config as basic
         load_default_config
+        # load from config, env path superior to ~/.qingstor/config.yaml
+        load_user_config_path_exist
+        # load envs, cover corresponding config if env exists
+        load_env_config
+        # cover by user's param
         update initial_config
       end
 
       def update(another_config = {})
         deep_merge! another_config.deep_symbolize_keys!
+        parse_boolean(:enable_virtual_host_style)
         Logger.set_level self[:log_level]
         self
       end
 
       def check
-        %i[access_key_id secret_access_key host port protocol].each do |x|
-          if !self[x] || self[x].to_s.empty?
-            raise ConfigurationError, "#{x.to_sym} not specified"
+        if self[:access_key_id].blank? && self[:secret_access_key].present? ||
+           self[:access_key_id].present? && self[:secret_access_key].blank?
+          raise ConfigurationError, 'ak and sk should be both both empty or not empty'
+        end
+
+        if self[:access_key_id].blank? && self[:secret_access_key].blank?
+          Logger.warn 'Both ak and sk not configured, will call api as anonymous user'
+        end
+
+        # check endpoint and host/port/protocol
+        if self[:endpoint].blank?
+          # host/port/protocol must set if endpoint not set
+          %i[host port protocol].each do |x|
+            if self[x].blank?
+              raise ConfigurationError, "#{x.to_sym} not specified"
+            end
+          end
+        else
+          # if endpoint set, host/port/protocol ignore, and warn
+          %i[host port protocol].each do |x|
+            if self[x].present? && !Config.is_default?(x, self[x])
+              Logger.warn "Endpoint configured, #{x.to_sym} will be ignored"
+            end
           end
         end
-        if self[:additional_user_agent] && !self[:additional_user_agent].empty?
+
+        # add ip check for vhost enabled
+        if self[:enable_virtual_host_style]
+          if self[:endpoint].present?
+            uri = Preprocessor.parse_endpoint self[:endpoint]
+            ip = uri.host
+          else
+            ip = self[:host]
+          end
+          if is_valid_ip? ip
+            raise ConfigurationError, 'ip host not allowed if vhost enabled'
+          end
+        end
+
+        if self[:additional_user_agent].present?
           self[:additional_user_agent].each_byte do |x|
             # Allow space(32) to ~(126) in ASCII Table, exclude "(34).
             if x < 32 || x > 126 || x == 32 || x == 34
@@ -64,12 +117,36 @@ module QingStor
       end
 
       def load_default_config
-        load_config_from_file Contract::DEFAULT_CONFIG_FILEPATH
+        update DEFAULT_AS_HASH
       end
 
       def load_user_config
-        install_default_user_config unless File.exist? Contract::USER_CONFIG_FILEPATH
-        load_config_from_file Contract::USER_CONFIG_FILEPATH
+        if !ENV[Contract::ENV_CONFIG_PATH].nil?
+          load_config_from_file ENV[Contract::ENV_CONFIG_PATH]
+        else
+          load_config_from_file Contract::USER_CONFIG_FILEPATH
+        end
+      end
+
+      def load_env_config
+        another_config = {}
+        unless ENV[Contract::ENV_ACCESS_KEY_ID].nil?
+          another_config[:access_key_id] =
+            ENV[Contract::ENV_ACCESS_KEY_ID]
+        end
+        unless ENV[Contract::ENV_SECRET_ACCESS_KEY].nil?
+          another_config[:secret_access_key] =
+            ENV[Contract::ENV_SECRET_ACCESS_KEY]
+        end
+        unless ENV[Contract::ENV_ENABLE_VIRTUAL_HOST_STYLE].nil?
+          another_config[:enable_virtual_host_style] =
+            ENV[Contract::ENV_ENABLE_VIRTUAL_HOST_STYLE]
+        end
+        unless ENV[Contract::ENV_ENDPOINT].nil?
+          another_config[:endpoint] =
+            ENV[Contract::ENV_ENDPOINT]
+        end
+        update another_config
       end
 
       def load_config_from_file(path)
@@ -79,10 +156,30 @@ module QingStor
 
       private
 
-      def install_default_user_config
-        Logger.warn "Installing default config file to #{Contract::USER_CONFIG_FILEPATH}"
-        FileUtils.mkdir_p Contract::USER_SUPPORT_DIRECTORY
-        FileUtils.copy Contract::DEFAULT_CONFIG_FILEPATH, Contract::USER_CONFIG_FILEPATH
+      # load user config if path exist, and skip check
+      def load_user_config_path_exist
+        # if env path configured, update from env; otherwise, if ~/.qingstor/config.yaml exists, update from this
+        if !ENV[Contract::ENV_CONFIG_PATH].nil? && File.exist?(ENV[Contract::ENV_CONFIG_PATH])
+          load_config_from_file ENV[Contract::ENV_CONFIG_PATH]
+        elsif File.exist? Contract::USER_CONFIG_FILEPATH
+          load_config_from_file Contract::USER_CONFIG_FILEPATH
+        end
+      end
+
+      def parse_boolean(key)
+        self[key.to_sym] = self[key.to_sym].to_s.downcase == 'true'
+      end
+
+      def is_valid_ip?(ip)
+        IPAddr.new ip
+        true
+      rescue
+        false
+      end
+
+      # @return boolean
+      def self.is_default?(key, value)
+        DEFAULT_AS_HASH[key.to_sym].present? && DEFAULT_AS_HASH[key.to_sym] == value
       end
     end
   end

data/lib/qingstor/sdk/general/contract.rb

@@ -23,9 +23,12 @@ module QingStor
 
       # GEM_DIRECTORY             = Gem::Specification.find_by_name('qingcloud-sdk').gem_dir
       # DEFAULT_SUPPORT_DIRECTORY = GEM_DIRECTORY + '/lib/qingcloud/sdk/commons/default'
-      DEFAULT_SUPPORT_DIRECTORY = File.expand_path(File.dirname(__FILE__) + '/./default')
-      DEFAULT_CONFIG_FILENAME   = 'config.yaml'.freeze
-      DEFAULT_CONFIG_FILEPATH   = "#{DEFAULT_SUPPORT_DIRECTORY}/#{DEFAULT_CONFIG_FILENAME}".freeze
+
+      ENV_ACCESS_KEY_ID             = 'QINGSTOR_ACCESS_KEY_ID'.freeze
+      ENV_SECRET_ACCESS_KEY         = 'QINGSTOR_SECRET_ACCESS_KEY'.freeze
+      ENV_CONFIG_PATH               = 'QINGSTOR_CONFIG_PATH'.freeze
+      ENV_ENABLE_VIRTUAL_HOST_STYLE = 'QINGSTOR_ENABLE_VIRTUAL_HOST_STYLE'.freeze
+      ENV_ENDPOINT                  = 'QINGSTOR_ENDPOINT'.freeze
     end
   end
 end

data/lib/qingstor/sdk/request/preprocessor.rb

@@ -17,8 +17,10 @@
 require 'cgi'
 require 'base64'
 require 'digest'
+require 'uri'
 
 require 'active_support/core_ext/hash/keys'
+require 'active_support/core_ext/object/blank'
 require 'mimemagic'
 
 module QingStor
@@ -41,6 +43,15 @@ module QingStor
       def self.request_endpoint(input)
         config = input[:config]
         zone = input[:properties] ? input[:properties][:zone] : nil
+        # handle endpoint directly if it exists
+        if config[:endpoint].present?
+          uri = parse_endpoint(config[:endpoint].to_s)
+          if zone
+            uri.host = "#{zone}.#{uri.host}"
+          end
+          return uri.to_s
+        end
+
         if zone
           "#{config[:protocol]}://#{zone}.#{config[:host]}:#{config[:port]}"
         else
@@ -49,6 +60,16 @@ module QingStor
       end
 
       def self.request_uri(input)
+        config = input[:config]
+        # if enable vhost, and uri property contains bucket-name
+        if config[:enable_virtual_host_style] && input[:properties].present? && input[:properties][:"bucket-name"]
+          uri = parse_endpoint input[:request_endpoint]
+          # modify host, add bucket before
+          uri.host = "#{input[:properties][:"bucket-name"]}.#{uri.host}"
+          input[:request_endpoint] = uri.to_s
+          # handle request_uri, remove prefix (bucket-name)
+          input[:request_uri].delete_prefix! URI_BUCKET_PREFIX if input[:request_uri].start_with? URI_BUCKET_PREFIX
+        end
         unless input[:properties].nil?
           input[:properties].each do |k, v|
             input[:request_uri].gsub! "<#{k}>", (escape v.to_s)
@@ -102,21 +123,18 @@ module QingStor
           input[:request_headers][:'Content-MD5'] = Base64.encode64(Digest::MD5.digest(input[:request_body])).strip
         end
 
+        # X-QS-MetaData used to handle meta data
         unless input[:request_headers][:'X-QS-MetaData'].nil?
           if input[:request_headers][:'X-QS-MetaData'].is_a?(Hash) && !input[:request_headers][:'X-QS-MetaData'].empty?
             prefix = 'x-qs-meta-'
             input[:request_headers][:'X-QS-MetaData'].each do |k, v|
               k = k.to_s
-              # check whether meta header has prefix
-              if k.downcase.start_with? prefix
-                suffix_len = prefix.size - k.size
-                k = prefix + k[suffix_len, -suffix_len]
-              else
-                k = prefix + k
-              end
+              # add prefix for meta data in header
+              k = prefix + k
               input[:request_headers][:"#{k}"] = v
             end
           end
+          # remove X-QS-MetaData from request header
           input[:request_headers].delete :'X-QS-MetaData'
         end
 
@@ -145,7 +163,7 @@ module QingStor
       def self.compact(object)
         object.each do |k, v|
           object[k] = compact v if v.is_a? Hash
-          object.delete k if v.nil? || v == '' || v == []
+          object.delete k if v.nil? || v == ''
         end
         object
       end
@@ -157,6 +175,32 @@ module QingStor
         origin.gsub! '+', '%20'
         origin
       end
+
+      # try to parse endpoint:
+      # if endpoint invalid, means ip host without scheme, like: 192.168.0.1:3000
+      # if endpoint parsed, and no scheme find, means host without scheme, like: qingstor.dev
+      # both above will add default schema at the start, and parse again
+      def self.parse_endpoint(endpoint)
+        if endpoint.blank?
+          raise 'endpoint should not be empty when parse'
+        end
+
+        begin
+          uri = URI.parse endpoint
+          unless uri.scheme.nil?
+            return uri
+          end
+        rescue URI::InvalidURIError
+          # Ignored and continue, add scheme prefix then
+        end
+
+        endpoint = "http://#{endpoint}" # add default scheme for endpoint
+        URI.parse endpoint
+      end
+
+      private
+
+      URI_BUCKET_PREFIX = '/<bucket-name>'.freeze
     end
   end
 end

data/lib/qingstor/sdk/request/request.rb

@@ -19,6 +19,7 @@ require 'json'
 require 'net/http'
 
 require 'active_support/core_ext/hash/keys'
+require 'active_support/core_ext/object/blank'
 
 module QingStor
   module SDK
@@ -52,11 +53,19 @@ module QingStor
 
       def sign
         check
+        if Signer.is_anonymous? input
+          Logger.warn 'anonymous api call, skip sign'
+          return
+        end
         self.input = Signer.sign input
       end
 
       def sign_query(timeout_seconds)
         check
+        if Signer.is_anonymous? input
+          Logger.warn 'anonymous api call, skip sign query'
+          return
+        end
         self.input = Signer.sign_query input, Time.now.to_i + timeout_seconds
       end
 
@@ -97,10 +106,11 @@ module QingStor
       private
 
       def check
-        unless !input[:config][:access_key_id].nil? && !input[:config][:access_key_id].empty?
+        # ak and sk should be both set or not set
+        if input[:config][:access_key_id].blank? && input[:config][:secret_access_key].present?
           raise SDKError, 'access key not provided'
         end
-        unless !input[:config][:secret_access_key].nil? && !input[:config][:secret_access_key].empty?
+        if input[:config][:secret_access_key].blank? && input[:config][:access_key_id].present?
           raise SDKError, 'secret access key not provided'
         end
       end

data/lib/qingstor/sdk/request/signer.rb

@@ -18,23 +18,31 @@ require 'base64'
 require 'cgi'
 require 'openssl'
 
+require 'active_support/core_ext/object/blank'
+
 module QingStor
   module SDK
     class Signer
       def self.sign(input)
-        authorization = "QS #{input[:config][:access_key_id]}:#{signature input}"
-        input[:request_headers][:Authorization] = authorization
+        unless is_anonymous? input
+          authorization = "QS #{input[:config][:access_key_id]}:#{signature input}"
+          input[:request_headers][:Authorization] = authorization
+
+          Logger.debug "QingStor request authorization: [#{input[:id]}] #{authorization}"
+        end
 
-        Logger.debug "QingStor request authorization: [#{input[:id]}] #{authorization}"
         input
       end
 
       def self.sign_query(input, expires)
-        input[:request_params][:signature] = query_signature input, expires
-        input[:request_params][:access_key_id] = input[:config][:access_key_id]
-        input[:request_params][:expires] = expires
+        unless is_anonymous? input
+          input[:request_params][:signature] = query_signature input, expires
+          input[:request_params][:access_key_id] = input[:config][:access_key_id]
+          input[:request_params][:expires] = expires
+
+          Logger.debug "QingStor query signature: [#{input[:id]}] #{input[:request_params][:signature]}"
+        end
 
-        Logger.debug "QingStor query signature: [#{input[:id]}] #{input[:request_params][:signature]}"
         input
       end
 
@@ -105,6 +113,14 @@ module QingStor
           response-content-language response-content-encoding response-content-disposition
         ].include? key
       end
+
+      def self.is_anonymous?(input)
+        if input[:config].nil?
+          return true
+        end
+
+        input[:config][:access_key_id].blank? && input[:config][:secret_access_key].blank?
+      end
     end
   end
 end

data/lib/qingstor/sdk/service/bucket.rb

@@ -22,6 +22,7 @@ module QingStor
       attr_accessor :config, :properties
 
       def initialize(config, properties)
+        config.check
         self.config     = config
         self.properties = properties.deep_symbolize_keys
       end
@@ -1058,10 +1059,6 @@ module QingStor
       def put_bucket_acl_input_validate(input)
         input.deep_stringify_keys!
 
-        unless !input['request_elements']['acl'].nil? && !input['request_elements']['acl'].empty?
-          raise ParameterRequiredError.new('acl', 'PutBucketACLInput')
-        end
-
         input['request_elements']['acl'].each do |x|
           unless x['grantee'].nil?
 

data/lib/qingstor/sdk/service/qingstor.rb

@@ -25,6 +25,7 @@ module QingStor
       attr_accessor :config, :properties
 
       def initialize(config)
+        config.check
         self.config = config
       end
 

data/lib/qingstor/sdk/version.rb

@@ -16,6 +16,6 @@
 
 module QingStor
   module SDK
-    VERSION = '2.4.0'.freeze
+    VERSION = '2.5.0'.freeze
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: qingstor-sdk
 version: !ruby/object:Gem::Version
-  version: 2.4.0
+  version: 2.5.0
 platform: ruby
 authors:
 - Yunify SDK Group
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-01-05 00:00:00.000000000 Z
+date: 2021-02-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -78,14 +78,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 5.2.0
+        version: 5.3.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 5.2.0
+        version: 5.3.0
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -157,7 +157,6 @@ files:
 - lib/qingstor/sdk.rb
 - lib/qingstor/sdk/general/config.rb
 - lib/qingstor/sdk/general/contract.rb
-- lib/qingstor/sdk/general/default/config.yaml
 - lib/qingstor/sdk/general/error.rb
 - lib/qingstor/sdk/general/logger.rb
 - lib/qingstor/sdk/request/preprocessor.rb
@@ -187,7 +186,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.8
+rubygems_version: 3.1.4
 signing_key:
 specification_version: 4
 summary: SDK for QingStor

data/lib/qingstor/sdk/general/default/config.yaml

@@ -1,15 +0,0 @@
-# QingStor services configuration
-
-#access_key_id: ACCESS_KEY_ID
-#secret_access_key: SECRET_ACCESS_KEY
-
-host: qingstor.com
-port: 443
-protocol: https
-connection_retries: 3
-
-# Additional User-Agent
-additional_user_agent: ""
-
-# Valid log levels are "debug", "info", "warn", "error", and "fatal".
-log_level: warn