qiniu 6.0.1

data/lib/qiniu/fop.rb

@@ -0,0 +1,3 @@
+# -*- encoding: utf-8 -*-
+
+require 'qiniu/image'

data/lib/qiniu/image.rb

@@ -0,0 +1,38 @@
+# -*- encoding: utf-8 -*-
+
+module Qiniu
+  module Fop
+    module Image
+      class << self
+        include Utils
+
+        def info(url)
+          Utils.http_request url + '?imageInfo', nil, {:method => :get}
+        end # info
+
+        def exif(url)
+          Utils.http_request url + '?exif', nil, {:method => :get}
+        end # exif
+
+        def mogrify_preview_url(source_image_url, options)
+          source_image_url + '?' + generate_mogrify_params_string(options)
+        end
+
+        def generate_mogrify_params_string(options = {})
+          opts = {}
+          options.each do |k, v|
+            opts[k.to_s] = v
+          end
+          params_string = ""
+          keys = ["thumbnail", "gravity", "crop", "quality", "rotate", "format"]
+          keys.each do |key|
+            params_string += %Q(/#{key}/#{opts[key]}) unless opts[key].nil?
+          end
+          params_string += '/auto-orient' unless opts["auto_orient"].nil?
+          'imageMogr' + URI.escape(params_string)
+        end
+
+      end
+    end # module Image
+  end # module Fop
+end # module Qiniu

data/lib/qiniu/log.rb

@@ -0,0 +1,15 @@
+# -*- encoding: utf-8 -*-
+
+require 'logger'
+
+module Qiniu
+    module Log
+      class << self
+        attr_accessor :logger
+
+        def logger
+          @logger ||= Logger.new(STDERR)
+        end
+      end
+    end # module Log
+end # module Qiniu

data/lib/qiniu/management.rb

@@ -0,0 +1,114 @@
+# -*- encoding: utf-8 -*-
+
+module Qiniu
+    module Storage
+      class << self
+        include Utils
+
+        def buckets
+          Auth.request Config.settings[:rs_host] + '/buckets'
+        end # buckets
+
+        def mkbucket(bucket_name)
+          Auth.request Config.settings[:rs_host] + '/mkbucket/' + bucket_name
+        end # mkbucket
+
+        def stat(bucket, key)
+          Auth.request Config.settings[:rs_host] + '/stat/' + encode_entry_uri(bucket, key)
+        end # stat
+
+        def get(bucket, key, save_as = nil, expires_in = nil, version = nil)
+          url = Config.settings[:rs_host] + '/get/' + encode_entry_uri(bucket, key)
+          url += '/base/' + version unless version.nil?
+          url += '/attName/' + Utils.urlsafe_base64_encode(save_as) unless save_as.nil?
+          url += '/expires/' + expires_in.to_s if !expires_in.nil? && expires_in > 0
+          Auth.request url
+        end # get
+
+        def copy(source_bucket, source_key, target_bucket, target_key)
+          uri = _generate_cp_or_mv_opstr('copy', source_bucket, source_key, target_bucket, target_key)
+          Auth.request Config.settings[:rs_host] + uri
+        end # copy
+
+        def move(source_bucket, source_key, target_bucket, target_key)
+          uri = _generate_cp_or_mv_opstr('move', source_bucket, source_key, target_bucket, target_key)
+          Auth.request Config.settings[:rs_host] + uri
+        end # move
+
+        def delete(bucket, key)
+          Auth.request Config.settings[:rs_host] + '/delete/' + encode_entry_uri(bucket, key)
+        end # delete
+
+        def publish(domain, bucket)
+          encoded_domain = Utils.urlsafe_base64_encode(domain)
+          Auth.request Config.settings[:rs_host] + "/publish/#{encoded_domain}/from/#{bucket}"
+        end # publish
+
+        def unpublish(domain)
+          encoded_domain = Utils.urlsafe_base64_encode(domain)
+          Auth.request Config.settings[:rs_host] + "/unpublish/#{encoded_domain}"
+        end # unpublish
+
+        def drop(bucket)
+          Auth.request Config.settings[:rs_host] + "/drop/#{bucket}"
+        end # drop
+
+        def batch(command, bucket, keys)
+          execs = []
+          keys.each do |key|
+            encoded_uri = encode_entry_uri(bucket, key)
+            execs << "op=/#{command}/#{encoded_uri}"
+          end
+          Auth.request Config.settings[:rs_host] + "/batch", execs.join("&"), {:mime => "application/x-www-form-urlencoded" }
+        end # batch
+
+        def batch_get(bucket, keys)
+          batch("get", bucket, keys)
+        end # batch_get
+
+        def batch_stat(bucket, keys)
+          batch("stat", bucket, keys)
+        end # batch_stat
+
+        def batch_copy(*args)
+          _batch_cp_or_mv('copy', args)
+        end # batch_copy
+
+        def batch_move(*args)
+          _batch_cp_or_mv('move', args)
+        end # batch_move
+
+        def batch_delete(bucket, keys)
+          batch("delete", bucket, keys)
+        end # batch_delete
+
+        def save_as(bucket, key, source_url, op_params_string)
+          encoded_uri = encode_entry_uri(bucket, key)
+          save_as_string = '/save-as/' + encoded_uri
+          new_url = source_url + '?' + op_params_string + save_as_string
+          Auth.request new_url
+        end # save_as
+
+        def image_mogrify_save_as(bucket, key, source_image_url, options)
+          mogrify_params_string = Fop::Image.generate_mogrify_params_string(options)
+          save_as(bucket, key, source_image_url, mogrify_params_string)
+        end # image_mogrify_save_as
+
+        private
+
+        def _generate_cp_or_mv_opstr(command, source_bucket, source_key, target_bucket, target_key)
+          source_encoded_entry_uri = encode_entry_uri(source_bucket, source_key)
+          target_encoded_entry_uri = encode_entry_uri(target_bucket, target_key)
+          %Q(/#{command}/#{source_encoded_entry_uri}/#{target_encoded_entry_uri})
+        end # _generate_cp_or_mv_opstr
+
+        def _batch_cp_or_mv(command, *op_args)
+          execs = []
+          op_args.each do |e| 
+            execs << 'op=' + _generate_cp_or_mv_opstr(command, e[0], e[1], e[2], e[3]) if e.size == 4
+          end 
+          Auth.request Config.settings[:rs_host] + "/batch", execs.join("&"), {:mime => "application/x-www-form-urlencoded" }
+        end # _batch_cp_or_mv
+      end
+    end # module Storage
+end # module Qiniu

data/lib/qiniu/misc.rb

@@ -0,0 +1,35 @@
+# -*- encoding: utf-8 -*-
+
+module Qiniu
+    module Misc
+      class << self
+        include Utils
+
+        def set_protected(bucket, protected_mode)
+          host = Config.settings[:pub_host]
+          Auth.request %Q(#{host}/accessMode/#{bucket}/mode/#{protected_mode})
+        end
+
+        def set_separator(bucket, separator)
+          host = Config.settings[:pub_host]
+          encoded_separator = Utils.urlsafe_base64_encode(separator)
+          Auth.request %Q(#{host}/separator/#{bucket}/sep/#{encoded_separator})
+        end
+
+        def set_style(bucket, name, style)
+          host = Config.settings[:pub_host]
+          encoded_name = Utils.urlsafe_base64_encode(name)
+          encoded_style = Utils.urlsafe_base64_encode(style)
+          Auth.request %Q(#{host}/style/#{bucket}/name/#{encoded_name}/style/#{encoded_style})
+        end
+
+        def unset_style(bucket, name)
+          host = Config.settings[:pub_host]
+          encoded_name = Utils.urlsafe_base64_encode(name)
+          Auth.request %Q(#{host}/unstyle/#{bucket}/name/#{encoded_name})
+        end
+
+      end
+    end # module Misc
+end # module Qiniu
+

data/lib/qiniu/resumable_upload.rb

@@ -0,0 +1,306 @@
+# -*- encoding: utf-8 -*-
+
+require 'zlib'
+require 'yaml'
+require 'tmpdir'
+require 'fileutils'
+require 'mime/types'
+require 'digest/sha1'
+require 'qiniu/abstract'
+require 'qiniu/exceptions'
+
+module Qiniu
+    module Storage
+
+      module AbstractClass
+        class ChunkProgressNotifier
+          include Qiniu::Abstract
+          abstract_methods :notify
+          # def notify(block_index, block_put_progress); end
+        end
+
+        class BlockProgressNotifier
+          include Qiniu::Abstract
+          abstract_methods :notify
+          # def notify(block_index, checksum); end
+        end
+      end # module AbstractClass
+
+      class ChunkProgressNotifier < AbstractClass::ChunkProgressNotifier
+          def notify(index, progress)
+              logmsg = "chunk #{progress[:offset]/Config.settings[:chunk_size]} in block #{index} successfully uploaded.\n" + progress.to_s
+              Utils.debug(logmsg)
+          end
+      end # class ChunkProgressNotifier
+
+      class BlockProgressNotifier < AbstractClass::BlockProgressNotifier
+          def notify(index, checksum)
+              Utils.debug "block #{index}: {ctx: #{checksum}} successfully uploaded."
+              Utils.debug "block #{index}: {checksum: #{checksum}} successfully uploaded."
+          end
+      end # class BlockProgressNotifier
+
+      class << self
+        include Utils
+
+        def resumable_upload_with_token(uptoken,
+                              local_file,
+                              bucket,
+                              key = nil,
+                              mime_type = nil,
+                              custom_meta = nil,
+                              customer = nil,
+                              callback_params = nil,
+                              rotate = nil)
+          begin
+            ifile = File.open(local_file, 'rb')
+            fh = FileData.new(ifile)
+            fsize = fh.data_size
+            key = Digest::SHA1.hexdigest(local_file + fh.mtime.to_s) if key.nil?
+            if mime_type.nil? || mime_type.empty?
+              mime = MIME::Types.type_for local_file
+              mime_type = mime.empty? ? 'application/octet-stream' : mime[0].content_type
+            end
+            code, data = _resumable_upload(uptoken, fh, fsize, bucket, key, mime_type, custom_meta, customer, callback_params, rotate)
+            [code, data]
+          ensure
+            ifile.close unless ifile.nil?
+          end
+        end # resumable_upload_with_token
+
+        private
+
+        class FileData
+            attr_accessor :fh
+            def initialize(fh)
+                @fh = fh
+            end
+            def data_size
+                @fh.stat.size
+            end
+            def get_data(offset, length)
+                @fh.seek(offset)
+                @fh.read(length)
+            end
+            def path
+                @fh.path
+            end
+            def mtime
+                @fh.mtime
+            end
+            #delegate :path, :mtime, :to => :fh
+        end # class FileData
+
+        def _new_block_put_progress_data
+          {:ctx => nil, :offset => 0, :restsize => nil, :status_code => nil, :host => nil}
+        end # _new_block_put_progress_data
+
+        def _call_binary_with_token(uptoken, url, data, content_type = nil, retry_times = 0)
+          options = {
+              :method => :post,
+              :content_type => 'application/octet-stream',
+              :upload_signature_token => uptoken
+          }
+          options[:content_type] = content_type if !content_type.nil? && !content_type.empty?
+          code, data = http_request url, data, options
+          unless Utils.is_response_ok?(code)
+              retry_times += 1
+              if Config.settings[:auto_reconnect] && retry_times < Config.settings[:max_retry_times]
+                  return _call_binary_with_token(uptoken, url, data, options[:content_type], retry_times)
+              end
+          end
+          [code, data]
+        end # _call_binary_with_token
+
+        def _mkblock(uptoken, block_size, body)
+            url = Config.settings[:up_host] + "/mkblk/#{block_size}"
+            _call_binary_with_token(uptoken, url, body)
+        end # _mkblock
+
+        def _putblock(uphost, uptoken, ctx, offset, body)
+            url = uphost + "/bput/#{ctx}/#{offset}"
+            _call_binary_with_token(uptoken, url, body)
+        end # _putblock
+
+        def _resumable_put_block(uptoken,
+                                 fh,
+                                 block_index,
+                                 block_size,
+                                 chunk_size,
+                                 progress,
+                                 retry_times,
+                                 notifier)
+            code, data = 0, {}
+            fpath = fh.path
+
+            # this block has never been uploaded.
+            if progress[:ctx] == nil || progress[:ctx].empty?
+                progress[:offset] = 0
+                progress[:restsize] = block_size
+                # choose the smaller one
+                body_length = [block_size, chunk_size].min
+                for i in 1..retry_times
+                    seek_pos = block_index*Config.settings[:block_size]
+                    body = fh.get_data(seek_pos, body_length)
+                    result_length = body.length
+                    if result_length != body_length
+                        raise FileSeekReadError.new(fpath, block_index, seek_pos, body_length, result_length)
+                    end
+                    code, data = _mkblock(uptoken, block_size, body)
+                    body_crc32 = Zlib.crc32(body)
+                    if Utils.is_response_ok?(code) && data["crc32"] == body_crc32
+                        progress[:ctx] = data["ctx"]
+                        progress[:offset] = body_length
+                        progress[:restsize] = block_size - body_length
+                        progress[:status_code] = code
+                        progress[:host] = data["host"]
+                        if !notifier.nil? && notifier.respond_to?("notify")
+                            notifier.notify(block_index, progress)
+                        end
+                        break
+                    elsif i == retry_times && data["crc32"] != body_crc32
+                        Log.logger.error %Q(Uploading block error. Expected crc32: #{body_crc32}, but got: #{data["crc32"]})
+                    end
+                end
+            elsif progress[:offset] + progress[:restsize] != block_size
+                raise BlockSizeNotMathchError.new(fpath, block_index, progress[:offset], progress[:restsize], block_size)
+            end
+
+            # loop uploading other chunks except the first one
+            while progress[:restsize].to_i > 0 && progress[:restsize] < block_size
+                # choose the smaller one
+                body_length = [progress[:restsize], chunk_size].min
+                for i in 1..retry_times
+                    seek_pos = block_index*Config.settings[:block_size] + progress[:offset]
+                    body = fh.get_data(seek_pos, body_length)
+                    result_length = body.length
+                    if result_length != body_length
+                        raise FileSeekReadError.new(fpath, block_index, seek_pos, body_length, result_length)
+                    end
+                    code, data = _putblock(progress[:host], uptoken, progress[:ctx], progress[:offset], body)
+                    body_crc32 = Zlib.crc32(body)
+                    if Utils.is_response_ok?(code) && data["crc32"] == body_crc32
+                        progress[:ctx] = data["ctx"]
+                        progress[:offset] += body_length
+                        progress[:restsize] -= body_length
+                        progress[:status_code] = code
+                        progress[:host] = data["host"]
+                        if !notifier.nil? && notifier.respond_to?("notify")
+                            notifier.notify(block_index, progress)
+                        end
+                        break
+                    elsif i == retry_times && data["crc32"] != body_crc32
+                        Log.logger.error %Q(Uploading block error. Expected crc32: #{body_crc32}, but got: #{data["crc32"]})
+                    end
+                end
+            end
+            # return
+            return [code, data]
+        end # _resumable_put_block
+
+        def _block_count(fsize)
+            ((fsize + Config.settings[:block_size] - 1) / Config.settings[:block_size]).to_i
+        end # _block_count
+
+        def _resumable_put(uptoken,
+                           fh,
+                           checksums,
+                           progresses,
+                           block_notifier = nil,
+                           chunk_notifier = nil)
+            code, data = 0, {}
+            fsize = fh.data_size
+            block_count = _block_count(fsize)
+            checksum_count = checksums.length
+            progress_count = progresses.length
+            if checksum_count != block_count || progress_count != block_count
+                raise BlockCountNotMathchError.new(fh.path, block_count, checksum_count, progress_count)
+            end
+            0.upto(block_count-1).each do |block_index|
+                if checksums[block_index].nil? || checksums[block_index].empty?
+                    block_size = Config.settings[:block_size]
+                    if block_index == block_count - 1
+                        block_size = fsize - block_index*Config.settings[:block_size]
+                    end
+                    if progresses[block_index].nil?
+                        progresses[block_index] = _new_block_put_progress_data
+                    end
+                    #code, data = _resumable_put_block(uptoken, fh, block_index, block_size, Config.settings[:chunk_size], progresses[block_index], Config.settings[:max_retry_times], chunk_notifier)
+                    # Put the whole block as a chunk
+                    code, data = _resumable_put_block(uptoken, fh, block_index, block_size, block_size, progresses[block_index], Config.settings[:max_retry_times], chunk_notifier)
+                    if Utils.is_response_ok?(code)
+                        #checksums[block_index] = data["checksum"]
+                        checksums[block_index] = data["ctx"]
+                        if !block_notifier.nil? && block_notifier.respond_to?("notify")
+                            block_notifier.notify(block_index, checksums[block_index])
+                        end
+                    end
+                end
+            end
+            return [code, data]
+        end # _resumable_put
+
+        def _mkfile(uphost,
+                    uptoken,
+                    entry_uri,
+                    fsize,
+                    checksums,
+                    mime_type = nil,
+                    custom_meta = nil,
+                    customer = nil,
+                    callback_params = nil,
+                    rotate = nil)
+          path = '/rs-mkfile/' + Utils.urlsafe_base64_encode(entry_uri) + "/fsize/#{fsize}"
+          path += '/mimeType/' + Utils.urlsafe_base64_encode(mime_type) if !mime_type.nil? && !mime_type.empty?
+          path += '/meta/' + Utils.urlsafe_base64_encode(custom_meta) if !custom_meta.nil? && !custom_meta.empty?
+          path += '/customer/' + customer if !customer.nil? && !customer.empty?
+          callback_query_string = Utils.generate_query_string(callback_params) if !callback_params.nil? && !callback_params.empty?
+          path += '/params/' + Utils.urlsafe_base64_encode(callback_query_string) if !callback_query_string.nil? && !callback_query_string.empty?
+          path += '/rotate/' + rotate if !rotate.nil? && rotate.to_i >= 0
+          url = uphost + path
+          #body = ''
+          #checksums.each do |checksum|
+          #    body += Utils.urlsafe_base64_decode(checksum)
+          #end
+          body = checksums.join(',')
+          _call_binary_with_token(uptoken, url, body, 'text/plain')
+        end # _mkfile
+
+        def _resumable_upload(uptoken,
+                              fh,
+                              fsize,
+                              bucket,
+                              key,
+                              mime_type = nil,
+                              custom_meta = nil,
+                              customer = nil,
+                              callback_params = nil,
+                              rotate = nil)
+
+          block_count = _block_count(fsize)
+
+          chunk_notifier = ChunkProgressNotifier.new()
+          block_notifier = BlockProgressNotifier.new()
+
+          progresses = []
+          block_count.times{progresses << _new_block_put_progress_data}
+          checksums = []
+          block_count.times{checksums << ''}
+
+          code, data = _resumable_put(uptoken, fh, checksums, progresses, block_notifier, chunk_notifier)
+
+          if Utils.is_response_ok?(code)
+            uphost = data["host"]
+            entry_uri = bucket + ':' + key
+            code, data = _mkfile(uphost, uptoken, entry_uri, fsize, checksums, mime_type, custom_meta, customer, callback_params, rotate)
+          end
+
+          if Utils.is_response_ok?(code)
+            Utils.debug "File #{fh.path} {size: #{fsize}} successfully uploaded."
+          end
+
+          [code, data]
+        end # _resumable_upload
+      end # self class
+    end # module Storage
+end # module Qiniu