qingstor-sdk 2.2.2 → 2.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: c4391cc24e2f51a70db0f75c7e99d735535b473b
-  data.tar.gz: 463094d6e75f6494f3016947b11c93b6a4fa2797
+SHA256:
+  metadata.gz: b4067ee8cebe68b7a6ff943ce45aab389887fec3eb3725311eb8fd0bbf44fa50
+  data.tar.gz: 97d3720cb4c6ac65b45fa66a1ac8b117300f96489894c19381972ed945b59c79
 SHA512:
-  metadata.gz: 7d0e092d7899da72e282d8a72943b771d28d4ad5d36b811835b5d335deaf39c61a6f3b9d287f4d343514687bc11eb5d751ea878997064e4412841e9405a58258
-  data.tar.gz: 1b7338a27c55f6545afc53b148e0d42c85f6555d87014649e4c76cb3f6d43041c274df1ec91bbe02355f6de1c487be5b85746d5389319e7997e21de61b48f44c
+  metadata.gz: 2d09dc5817005d1645473bddb4f5a354a4fef382dcf34775dee845cbffe1188d28d8684df73daf7440d58a83d6aa3e26d1c1e2199dacedf294f40fae915f25ec
+  data.tar.gz: a0a940c7419f8acc356977c9d336a7c982b67282624baa0e0b44e0f8b31a37cf3dd4b6e4d3f5b19ca84c6bf11a7d14c0169a522fc44d3390d297d29598be675b

data/README.md

@@ -1,11 +1,10 @@
 # QingStor::SDK
 
-<span style="display: inline-block">
-[![Build Status](https://travis-ci.org/yunify/qingstor-sdk-ruby.svg?branch=master)](https://travis-ci.org/yunify/qingstor-sdk-ruby)
+[![Build Status](https://github.com/qingstor/qingstor-sdk-ruby/workflows/Unit%20Test/badge.svg?branch=master)](https://github.com/qingstor/qingstor-sdk-ruby/actions?query=workflow%3A%22Unit+Test%22)
 [![Gem Version](https://badge.fury.io/rb/qingstor-sdk.svg)](http://badge.fury.io/rb/qingstor-sdk)
 [![API Reference](http://img.shields.io/badge/api-reference-green.svg)](https://docs.qingcloud.com/qingstor/)
 [![License](http://img.shields.io/badge/license-apache%20v2-blue.svg)](https://github.com/yunify/qingstor-sdk-ruby/blob/master/LICENSE)
-</span>
+[![Join the chat](https://img.shields.io/badge/chat-online-blue?style=flat&logo=zulip)](https://qingstor.zulipchat.com/join/odapi42t7xhqc7v4gb2wfgjx/)
 
 The official QingStor SDK for Ruby programming language.
 
@@ -15,7 +14,9 @@ This Gem uses Ruby's _keyword arguments_ feature, thus Ruby v2.1.5 or higher is
 required.  See [this article](https://robots.thoughtbot.com/ruby-2-keyword-arguments)
 for more details about _keyword arguments_.
 
-_Notice:_ If you are using Ruby v1.9.x, please checkout the [compatible] branch.
+_Notice:_ As of sdk v2.4, we no longer support Ruby 2.4 and earlier versions since Ruby 2.4 has been marked as EOL. 
+If you are still using Ruby 2.4 and earlier, you may not be able to use some new features of our sdk, 
+so please upgrade your version of Ruby as soon as possible.
 
 ### Install from RubyGems
 
@@ -123,6 +124,11 @@ server (in case of QingStor is deployed in private environment, thus has
 different endpoint with public QingStor) either in the config file, or in
 the program dynamically.
 
+**Notice:** config will not be checked available when `new`, `load` and `update`, 
+which will be done right before initializing `service` or `bucket`. 
+If you need to check config, you should call `config.check` whenever you want.
+If `check` failed, a `ConfigurationError` will be raised.
+
 ___Code Example:___
 
 ``` ruby
@@ -132,6 +138,7 @@ require 'qingstor/sdk'
 config = QingStor::SDK::Config.new.load_default_config
 
 # Create with default value
+# priority from high to low: param > env > user config > default config
 config = QingStor::SDK::Config.new({
   host:      'qingstor.dev',
   log_level: 'debug',
@@ -145,6 +152,13 @@ config = QingStor::SDK::Config.init ENV['ENV_ACCESS_KEY_ID'],
 config = QingStor::SDK::Config.new
 config = config.load_config_from_file '~/qingstor/config.yaml'
 
+# Load configuration from env variable:
+# QINGSTOR_ACCESS_KEY_ID for access_key_id
+# QINGSTOR_SECRET_ACCESS_KEY for secret_access_key
+# QINGSTOR_ENABLE_VIRTUAL_HOST_STYLE for enable_virtual_host_style
+# QINGSTOR_ENDPOINT for endpoint
+config = QingStor::SDK::Config.new.load_env_config
+
 # Change API server
 config.update({host: 'test.qingstor.com'})
 ```
@@ -160,26 +174,84 @@ secret_access_key: 'SECRET_ACCESS_KEY'
 host: 'qingstor.com'
 port: 443
 protocol: 'https'
+# or set endpoint directly
+# endpoint: https://qingstor.com:443
+
 connection_retries: 3
 
+# Additional User-Agent
+additional_user_agent: ""
+
 # Valid log levels are "debug", "info", "warn", "error", and "fatal".
-log_level: 'warn'
+log_level: warn
+
+# SDK will use virtual host style for all API calls if enabled
+enable_virtual_host_style: false
 ```
 
 ## Change Log
 All notable changes to QingStor SDK for Ruby will be documented here.
 
+### [v2.5.0] - 2021-02-09
+
+### Added
+
+- config: Add env variable support (#52)
+- signer: Add support for anonymous API call (#53)
+- config: Add support for endpoint (#55)
+- config: Add support for enable_vhost_style (#56)
+
+### Changed
+
+- Request: Modify Metadata in header to apply rfc 02 (#49)
+- config: Refactor config check (#54)
+
+### [v2.4.0] - 2021-01-05
+
+### Added
+
+- Support to set x-qs-metadata-directive header (#29)
+- Add support for bucket replication (#45)
+
+### Changed
+
+- ci: Transfer ci into github action (#30)
+
+### Fixed
+
+- Fix metadata added into signature when empty (#44)
+
+### [v2.3.0] - 2020-05-11
+
+### Added
+
+- Add support for lifecycle and notification.
+- Add support for bucket logging, bucket cname and append object. (#24)
+
+### Fixed
+
+- Modify content-type check for application/json. (#21)
+- Fix sub-resources not be recognized when generate signature. (#25)
+- Fix meta data not work as intended. (#26)
+- Reverse fix of empty map in template, should not compact empty hash when sign. (#28)
+
+### [v2.2.3] - 2017-03-28
+
+### Fixed
+
+- Fix status code of DELETE CORS API.
+
 ### [v2.2.2] - 2017-03-10
 
 ### Fixed
 
-- Resource is not mandatory in bucket policy statement
+- Resource is not mandatory in bucket policy statement.
 
 ### [v2.2.1] - 2017-03-10
 
 #### Added
 
-- Allow user to append additional info to User-Agent
+- Allow user to append additional info to User-Agent.
 
 ### [v2.2.0] - 2017-02-28
 
@@ -240,6 +312,10 @@ All notable changes to QingStor SDK for Ruby will be documented here.
 The Apache License (Version 2.0, January 2004).
 
 [compatible]: https://github.com/yunify/qingstor-sdk-ruby/tree/compatible
+[v2.5.0]: https://github.com/yunify/qingstor-sdk-ruby/compare/v2.4.0...v2.5.0
+[v2.4.0]: https://github.com/yunify/qingstor-sdk-ruby/compare/v2.3.0...v2.4.0
+[v2.3.0]: https://github.com/yunify/qingstor-sdk-ruby/compare/v2.2.3...v2.3.0
+[v2.2.3]: https://github.com/yunify/qingstor-sdk-ruby/compare/v2.2.2...v2.2.3
 [v2.2.2]: https://github.com/yunify/qingstor-sdk-ruby/compare/v2.2.1...v2.2.2
 [v2.2.1]: https://github.com/yunify/qingstor-sdk-ruby/compare/v2.2.0...v2.2.1
 [v2.2.0]: https://github.com/yunify/qingstor-sdk-ruby/compare/v2.1.1...v2.2.0

data/lib/qingstor/sdk/general/config.rb

@@ -15,10 +15,13 @@
 #  +-------------------------------------------------------------------------
 
 require 'fileutils'
+require 'ipaddr'
+require 'uri'
 require 'yaml'
 
 require 'active_support/core_ext/hash/keys'
 require 'active_support/core_ext/hash/deep_merge'
+require 'active_support/core_ext/object/blank'
 require 'net/http/persistent'
 
 module QingStor
@@ -26,33 +29,83 @@ module QingStor
     class Config < Hash
       attr_accessor :connection
 
+      DEFAULT_AS_HASH = {
+        host:                      'qingstor.com'.freeze,
+        port:                      443,
+        protocol:                  'https'.freeze,
+        connection_retries:        3,
+        log_level:                 'warn'.freeze,
+        enable_virtual_host_style: false
+      }
+
       def self.init(access_key_id, secret_access_key)
         initial_config = {
           access_key_id:     access_key_id,
-          secret_access_key: secret_access_key,
+          secret_access_key: secret_access_key
         }
         Config.new(initial_config)
       end
 
       def initialize(initial_config = {})
         self.connection = Net::HTTP::Persistent.new
+        # load default config as basic
         load_default_config
+        # load from config, env path superior to ~/.qingstor/config.yaml
+        load_user_config_path_exist
+        # load envs, cover corresponding config if env exists
+        load_env_config
+        # cover by user's param
         update initial_config
       end
 
       def update(another_config = {})
         deep_merge! another_config.deep_symbolize_keys!
+        parse_boolean(:enable_virtual_host_style)
         Logger.set_level self[:log_level]
         self
       end
 
       def check
-        [:access_key_id, :secret_access_key, :host, :port, :protocol].each do |x|
-          if !self[x] || self[x].to_s.empty?
-            raise ConfigurationError, "#{x.to_sym} not specified"
+        if self[:access_key_id].blank? && self[:secret_access_key].present? ||
+           self[:access_key_id].present? && self[:secret_access_key].blank?
+          raise ConfigurationError, 'ak and sk should be both both empty or not empty'
+        end
+
+        if self[:access_key_id].blank? && self[:secret_access_key].blank?
+          Logger.warn 'Both ak and sk not configured, will call api as anonymous user'
+        end
+
+        # check endpoint and host/port/protocol
+        if self[:endpoint].blank?
+          # host/port/protocol must set if endpoint not set
+          %i[host port protocol].each do |x|
+            if self[x].blank?
+              raise ConfigurationError, "#{x.to_sym} not specified"
+            end
+          end
+        else
+          # if endpoint set, host/port/protocol ignore, and warn
+          %i[host port protocol].each do |x|
+            if self[x].present? && !Config.is_default?(x, self[x])
+              Logger.warn "Endpoint configured, #{x.to_sym} will be ignored"
+            end
           end
         end
-        if self[:additional_user_agent] && !self[:additional_user_agent].empty?
+
+        # add ip check for vhost enabled
+        if self[:enable_virtual_host_style]
+          if self[:endpoint].present?
+            uri = Preprocessor.parse_endpoint self[:endpoint]
+            ip = uri.host
+          else
+            ip = self[:host]
+          end
+          if is_valid_ip? ip
+            raise ConfigurationError, 'ip host not allowed if vhost enabled'
+          end
+        end
+
+        if self[:additional_user_agent].present?
           self[:additional_user_agent].each_byte do |x|
             # Allow space(32) to ~(126) in ASCII Table, exclude "(34).
             if x < 32 || x > 126 || x == 32 || x == 34
@@ -64,12 +117,36 @@ module QingStor
       end
 
       def load_default_config
-        load_config_from_file Contract::DEFAULT_CONFIG_FILEPATH
+        update DEFAULT_AS_HASH
       end
 
       def load_user_config
-        install_default_user_config unless File.exist? Contract::USER_CONFIG_FILEPATH
-        load_config_from_file Contract::USER_CONFIG_FILEPATH
+        if !ENV[Contract::ENV_CONFIG_PATH].nil?
+          load_config_from_file ENV[Contract::ENV_CONFIG_PATH]
+        else
+          load_config_from_file Contract::USER_CONFIG_FILEPATH
+        end
+      end
+
+      def load_env_config
+        another_config = {}
+        unless ENV[Contract::ENV_ACCESS_KEY_ID].nil?
+          another_config[:access_key_id] =
+            ENV[Contract::ENV_ACCESS_KEY_ID]
+        end
+        unless ENV[Contract::ENV_SECRET_ACCESS_KEY].nil?
+          another_config[:secret_access_key] =
+            ENV[Contract::ENV_SECRET_ACCESS_KEY]
+        end
+        unless ENV[Contract::ENV_ENABLE_VIRTUAL_HOST_STYLE].nil?
+          another_config[:enable_virtual_host_style] =
+            ENV[Contract::ENV_ENABLE_VIRTUAL_HOST_STYLE]
+        end
+        unless ENV[Contract::ENV_ENDPOINT].nil?
+          another_config[:endpoint] =
+            ENV[Contract::ENV_ENDPOINT]
+        end
+        update another_config
       end
 
       def load_config_from_file(path)
@@ -79,10 +156,30 @@ module QingStor
 
       private
 
-      def install_default_user_config
-        Logger.warn "Installing default config file to #{Contract::USER_CONFIG_FILEPATH}"
-        FileUtils.mkdir_p Contract::USER_SUPPORT_DIRECTORY
-        FileUtils.copy Contract::DEFAULT_CONFIG_FILEPATH, Contract::USER_CONFIG_FILEPATH
+      # load user config if path exist, and skip check
+      def load_user_config_path_exist
+        # if env path configured, update from env; otherwise, if ~/.qingstor/config.yaml exists, update from this
+        if !ENV[Contract::ENV_CONFIG_PATH].nil? && File.exist?(ENV[Contract::ENV_CONFIG_PATH])
+          load_config_from_file ENV[Contract::ENV_CONFIG_PATH]
+        elsif File.exist? Contract::USER_CONFIG_FILEPATH
+          load_config_from_file Contract::USER_CONFIG_FILEPATH
+        end
+      end
+
+      def parse_boolean(key)
+        self[key.to_sym] = self[key.to_sym].to_s.downcase == 'true'
+      end
+
+      def is_valid_ip?(ip)
+        IPAddr.new ip
+        true
+      rescue
+        false
+      end
+
+      # @return boolean
+      def self.is_default?(key, value)
+        DEFAULT_AS_HASH[key.to_sym].present? && DEFAULT_AS_HASH[key.to_sym] == value
       end
     end
   end

data/lib/qingstor/sdk/general/contract.rb

@@ -23,9 +23,12 @@ module QingStor
 
       # GEM_DIRECTORY             = Gem::Specification.find_by_name('qingcloud-sdk').gem_dir
       # DEFAULT_SUPPORT_DIRECTORY = GEM_DIRECTORY + '/lib/qingcloud/sdk/commons/default'
-      DEFAULT_SUPPORT_DIRECTORY = File.expand_path(File.dirname(__FILE__) + '/./default')
-      DEFAULT_CONFIG_FILENAME   = 'config.yaml'.freeze
-      DEFAULT_CONFIG_FILEPATH   = "#{DEFAULT_SUPPORT_DIRECTORY}/#{DEFAULT_CONFIG_FILENAME}".freeze
+
+      ENV_ACCESS_KEY_ID             = 'QINGSTOR_ACCESS_KEY_ID'.freeze
+      ENV_SECRET_ACCESS_KEY         = 'QINGSTOR_SECRET_ACCESS_KEY'.freeze
+      ENV_CONFIG_PATH               = 'QINGSTOR_CONFIG_PATH'.freeze
+      ENV_ENABLE_VIRTUAL_HOST_STYLE = 'QINGSTOR_ENABLE_VIRTUAL_HOST_STYLE'.freeze
+      ENV_ENDPOINT                  = 'QINGSTOR_ENDPOINT'.freeze
     end
   end
 end

data/lib/qingstor/sdk/general/logger.rb

@@ -23,7 +23,7 @@ module QingStor
       @@level = :warn
 
       def self.set_level(level)
-        index = %w(debug info warn error fatal).find_index level.to_s
+        index = %w[debug info warn error fatal].find_index level.to_s
         @@logger.level = index.nil? ? 0 : index
         @@level = level.to_sym
       end

data/lib/qingstor/sdk/request/preprocessor.rb

@@ -17,8 +17,10 @@
 require 'cgi'
 require 'base64'
 require 'digest'
+require 'uri'
 
 require 'active_support/core_ext/hash/keys'
+require 'active_support/core_ext/object/blank'
 require 'mimemagic'
 
 module QingStor
@@ -41,6 +43,15 @@ module QingStor
       def self.request_endpoint(input)
         config = input[:config]
         zone = input[:properties] ? input[:properties][:zone] : nil
+        # handle endpoint directly if it exists
+        if config[:endpoint].present?
+          uri = parse_endpoint(config[:endpoint].to_s)
+          if zone
+            uri.host = "#{zone}.#{uri.host}"
+          end
+          return uri.to_s
+        end
+
         if zone
           "#{config[:protocol]}://#{zone}.#{config[:host]}:#{config[:port]}"
         else
@@ -49,6 +60,16 @@ module QingStor
       end
 
       def self.request_uri(input)
+        config = input[:config]
+        # if enable vhost, and uri property contains bucket-name
+        if config[:enable_virtual_host_style] && input[:properties].present? && input[:properties][:"bucket-name"]
+          uri = parse_endpoint input[:request_endpoint]
+          # modify host, add bucket before
+          uri.host = "#{input[:properties][:"bucket-name"]}.#{uri.host}"
+          input[:request_endpoint] = uri.to_s
+          # handle request_uri, remove prefix (bucket-name)
+          input[:request_uri].delete_prefix! URI_BUCKET_PREFIX if input[:request_uri].start_with? URI_BUCKET_PREFIX
+        end
         unless input[:properties].nil?
           input[:properties].each do |k, v|
             input[:request_uri].gsub! "<#{k}>", (escape v.to_s)
@@ -102,6 +123,21 @@ module QingStor
           input[:request_headers][:'Content-MD5'] = Base64.encode64(Digest::MD5.digest(input[:request_body])).strip
         end
 
+        # X-QS-MetaData used to handle meta data
+        unless input[:request_headers][:'X-QS-MetaData'].nil?
+          if input[:request_headers][:'X-QS-MetaData'].is_a?(Hash) && !input[:request_headers][:'X-QS-MetaData'].empty?
+            prefix = 'x-qs-meta-'
+            input[:request_headers][:'X-QS-MetaData'].each do |k, v|
+              k = k.to_s
+              # add prefix for meta data in header
+              k = prefix + k
+              input[:request_headers][:"#{k}"] = v
+            end
+          end
+          # remove X-QS-MetaData from request header
+          input[:request_headers].delete :'X-QS-MetaData'
+        end
+
         input[:request_headers].map do |k, v|
           input[:request_headers][k] = escape v.to_s unless v.to_s.ascii_only?
         end
@@ -139,6 +175,32 @@ module QingStor
         origin.gsub! '+', '%20'
         origin
       end
+
+      # try to parse endpoint:
+      # if endpoint invalid, means ip host without scheme, like: 192.168.0.1:3000
+      # if endpoint parsed, and no scheme find, means host without scheme, like: qingstor.dev
+      # both above will add default schema at the start, and parse again
+      def self.parse_endpoint(endpoint)
+        if endpoint.blank?
+          raise 'endpoint should not be empty when parse'
+        end
+
+        begin
+          uri = URI.parse endpoint
+          unless uri.scheme.nil?
+            return uri
+          end
+        rescue URI::InvalidURIError
+          # Ignored and continue, add scheme prefix then
+        end
+
+        endpoint = "http://#{endpoint}" # add default scheme for endpoint
+        URI.parse endpoint
+      end
+
+      private
+
+      URI_BUCKET_PREFIX = '/<bucket-name>'.freeze
     end
   end
 end