cos 0.1.0 → 0.1.1

data/lib/cos/download.rb

@@ -0,0 +1,252 @@
+# coding: utf-8
+
+module COS
+
+  # 大文件分片下载, 支持断点续传, 支持多线程
+  # Range Headers support in HTTP1.1(rfc2616)
+  class Download < Checkpoint
+
+    include Logging
+
+    # 默认分块大小
+    PART_SIZE = 5 * 1024 * 1024
+
+    # 默认文件读取大小
+    READ_SIZE = 16 * 1024
+
+    required_attrs :bucket, :cos_file, :file_store, :options
+    optional_attrs :progress
+
+    attr_accessor :cpt_file, :session
+
+    def initialize(opts = {})
+      super(opts)
+
+      @cpt_file = options[:cpt_file] || "#{File.expand_path(file_store)}.cpt"
+    end
+
+    # 开始下载
+    def download
+      logger.info("Begin download, file: #{file_store}, threads: #{@num_threads}")
+
+      # 重建断点续传
+      rebuild
+
+      # 文件分片
+      divide_parts if @parts.empty?
+
+      # 未完成的片段
+      @todo_parts = @parts.reject { |p| p[:done] }
+
+      # 多线程下载
+      (1..@num_threads).map do
+        logger.debug("#{@num_threads} Threads Downloads")
+
+        Thread.new do
+          logger.debug("Create Thread #{Thread.current.object_id}")
+
+          loop do
+            # 获取下一个未下载的片段
+            p = sync_get_todo_part
+            break unless p
+
+            # 下载片段
+            download_part(p)
+          end
+        end
+      end.map(&:join)
+
+      # 完成下载, 合并文件
+      complete
+
+      unless finish?
+        File.delete(file_store) if File.exist?(file_store)
+        raise DownloadError, 'File downloaded sha1 not match, deleted!'
+      end
+    end
+
+    # 断点续传状态记录
+    # @example
+    #   states = {
+    #     :session => 'session',
+    #     :file => 'file',
+    #     :file_meta => {
+    #       :sha1 => 'file sha1',
+    #       :size => 10000,
+    #     },
+    #     :parts => [
+    #       {:number => 1, :range => [0, 100], :done => false},
+    #       {:number => 2, :range => [100, 200], :done => true}
+    #     ],
+    #     :sha1 => 'checkpoint file sha1'
+    #   }
+    def checkpoint
+      logger.debug("Make checkpoint, options[:disable_cpt]: #{options[:disable_cpt] == true}")
+
+      parts = sync_get_all_parts
+      states = {
+          :session    => session,
+          :file       => file_store,
+          :file_meta  => @file_meta,
+          :parts      => parts
+      }
+
+      done = parts.count { |p| p[:done] }
+
+      # 下载进度回调
+      if progress
+        if done == 0 or parts.count == 0
+          progress.call(0.to_f)
+        else
+          percent = done.to_f / parts.size
+          progress.call(percent > 1 ? 1.to_f : percent)
+        end
+      end
+
+      write_checkpoint(states, cpt_file) unless options[:disable_cpt]
+
+      logger.debug("Download Parts #{done}/#{parts.size}")
+    end
+
+    private
+
+    # 是否完成下载并比对sha1
+    def finish?
+      @file_meta[:sha1].downcase == Util.file_sha1(@file_store)
+    end
+
+    def complete
+      # 返回100%的进度
+      progress.call(1.to_f) if progress
+
+      # 获取全部的分块
+      parts = sync_get_all_parts
+
+      # 合并分块文件
+      File.open(@file_store, 'w') do |w|
+        # 排序组合文件
+        parts.sort{ |x, y| x[:number] <=> y[:number] }.each do |p|
+          File.open(get_part_file(p)) do |r|
+            w.write(r.read(READ_SIZE)) until r.eof?
+          end
+        end
+      end
+
+      # 下载完成, 删除checkpoint文件
+      File.delete(cpt_file) unless options[:disable_cpt]
+      # 删除分块文件
+      parts.each{ |p| File.delete(get_part_file(p)) }
+
+      logger.info("Done download, file: #{@file_store}")
+    end
+
+    # 断点续传文件重建
+    def rebuild
+      logger.info("Begin rebuild session, checkpoint: #{cpt_file}")
+
+      # 是否启用断点续传并且记录文件存在
+      if options[:disable_cpt] || !File.exist?(cpt_file)
+        # 初始化
+        initiate
+      else
+        # 加载断点续传
+        states = load_checkpoint(cpt_file)
+
+        @session    = states[:session]
+        @file_meta  = states[:file_meta]
+        @parts      = states[:parts]
+      end
+
+      logger.info("Done rebuild session, Parts: #{@parts.count}")
+    end
+
+    def initiate
+      logger.info('Begin initiate session')
+
+      @session = "#{cos_file.bucket.bucket_name}-#{cos_file.path}-#{Time.now.to_i}"
+
+      @file_meta = {
+          :sha1  => cos_file.sha,
+          :size  => cos_file.filesize
+      }
+
+      # 保存断点
+      checkpoint
+
+      logger.info("Done initiate session: #{@session}")
+    end
+
+    # 下载片段
+    def download_part(p)
+      logger.debug("Begin download slice: #{p}")
+
+      part_file = get_part_file(p)
+
+      url = cos_file.url
+
+      # 下载
+      # Range:bytes=0-11
+      bucket.client.api.download(
+          url,
+          part_file,
+          headers: {Range: "bytes=#{p[:range].at(0)}-#{p[:range].at(1) - 1}"},
+          bucket: bucket.bucket_name
+      )
+
+      sync_update_part(p.merge(done: true))
+
+      checkpoint
+
+      logger.debug("Done download part: #{p}")
+    end
+
+    # 文件片段拆分
+    def divide_parts
+      logger.info("Begin divide parts, file: #{file_store}")
+
+      object_size = @file_meta[:size]
+      part_size   = @options[:part_size] || PART_SIZE
+      num_parts   = (object_size - 1) / part_size + 1
+
+      @parts = (1..num_parts).map do |i|
+        {
+            :number => i,
+            :range => [(i - 1) * part_size, [i * part_size, object_size].min],
+            :done => false
+        }
+      end
+
+      checkpoint
+
+      logger.info("Done divide parts, parts: #{@parts.size}")
+    end
+
+    # 同步获取下一片段
+    def sync_get_todo_part
+      @todo_mutex.synchronize {
+        @todo_parts.shift
+      }
+    end
+
+    # 同步更新片段
+    def sync_update_part(p)
+      @all_mutex.synchronize {
+        @parts[p[:number] - 1] = p
+      }
+    end
+
+    # 同步获取所有片段
+    def sync_get_all_parts
+      @all_mutex.synchronize {
+        @parts.dup
+      }
+    end
+
+    # 获取分块文件名
+    def get_part_file(p)
+      "#{@file_store}.part.#{p[:number]}"
+    end
+
+  end
+
+end

data/lib/cos/exception.rb

@@ -0,0 +1,62 @@
+# coding: utf-8
+
+require 'json'
+
+module COS
+
+  # 异常基类
+  class Exception < RuntimeError; end
+
+  # 服务端返回异常
+  # Code: -166, Message: 索引不存在, HttpCode: 400
+  # Code: -173, Message: 目录非空, HttpCode: 400
+  # Code: -180, Message: 非法路径, HttpCode: 400
+  # Code: -288, Message: process打包失败, HttpCode: 400
+  # Code: -4018, Message: 相同文件已上传过, HttpCode: 400
+  # Code: -5997, Message: 后端网络错误, HttpCode: 400
+  # Code: -5999, Message: 参数错误, HttpCode: 400
+  class ServerError < Exception
+
+    attr_reader :response, :http_code, :error_code, :message
+
+    def initialize(response)
+      @response = response
+      resp_obj  = JSON.parse(response.body)
+
+      @error_code = resp_obj['code']
+      @message    = resp_obj['message']
+      @http_code  = response.code
+    end
+
+    def message
+      @message || "UnknownError[#{http_code}]."
+    end
+
+    def to_s
+      "ServerError Code: #{error_code}, Message: #{message}, HttpCode: #{http_code}"
+    end
+
+  end
+
+  # 参数错误
+  class AttrError < Exception; end
+
+  # 客户端错误
+  class ClientError < Exception; end
+
+  # 文件不一致
+  class FileInconsistentError < Exception; end
+
+  # 断点续传记录损坏
+  class CheckpointBrokenError < Exception; end
+
+  # 下载错误
+  class DownloadError < Exception; end
+
+  # 文件上传未完成
+  class FileUploadNotComplete < Exception; end
+
+  # 本地目录不存在
+  class LocalPathNotExist < Exception; end
+
+end

data/lib/cos/file.rb

@@ -0,0 +1,152 @@
+# coding: utf-8
+
+module COS
+
+  # COS文件资源
+  class COSFile < ResourceOperator
+
+    STORAGE_UNITS = %w[B KB MB GB]
+    STORAGE_BASE = 1024
+
+    # 初始化
+    #
+    # @param [Hash] attrs 参数
+    # @option attrs [Bucket] :bucket COS::Bucket对象
+    # @option attrs [String] :path 存储路径
+    # @option attrs [String] :name 文件名
+    # @option attrs [String] :ctime 创建时间unix时间戳
+    # @option attrs [String] :mtime 修改时间unix时间戳
+    # @option attrs [String] :biz_attr 业务信息
+    # @option attrs [String] :filesize 文件存储大小
+    # @option attrs [String] :filelen 文件大小
+    # @option attrs [String] :sha 文件sha1值
+    # @option attrs [String] :access_url 文件访问地址
+    #
+    # @raise [AttrError] 缺少参数
+    #
+    # @return [COS::COSFile]
+    def initialize(attrs = {})
+      super(attrs)
+      @type = 'file'
+    end
+
+    # 获取文件存储大小并转为数值型
+    #
+    # @return [Integer] 文件存储大小
+    def filesize
+      @filesize.to_i
+    end
+
+    # 获取文件大小并转为数值型
+    #
+    # @return [Integer] 文件大小
+    def filelen
+      @filelen.to_i
+    end
+
+    # 判断文件sha1是否一致
+    #
+    # @return [Boolean] 是否一致
+    def sha1_match?(file)
+      file = File.expand_path(file)
+      File.exist?(file) and sha.upcase == Util.file_sha1(file).upcase
+    end
+
+    # 文件大小
+    #
+    # @alias filesize
+    def size
+      filesize.to_i
+    end
+
+    # 获取格式化的文件大小
+    #
+    # @example
+    #  1B 1KB 1.1MB 1.12GB...
+    #
+    # @return [String]
+    def format_size
+      if filesize.to_i < STORAGE_BASE
+        size_str = filesize.to_s + STORAGE_UNITS[0]
+      else
+        c_size = human_rep(filesize.to_i)
+        size_str = "%.2f" % c_size[:size].round(2)
+        size_str = "#{size_str}#{c_size[:unit]}"
+      end
+
+      size_str
+    end
+
+    # 文件是否完整, 是否上传完了
+    #
+    # @return [Boolean] 是否完整
+    def complete?
+      access_url != nil and filelen == filesize
+    end
+
+    # 获取文件的URL, 支持cname, https
+    #
+    # @note 私有读取的bucket会自动生成带签名的URL
+    #
+    # @param options [Hash] 高级参数
+    # @option options [String] :cname 在cos控制台设置的cname域名
+    # @option options [Boolean] :https 是否生成https的URL
+    # @option options [Integer] :expire_seconds 签名有效时间(秒,私有读取bucket时需要)
+    #
+    # @raise [ServerError] 服务端异常返回
+    #
+    # @return [String] 文件访问URL
+    def url(options = {})
+      bucket.url(self, options)
+    end
+
+    # 下载当前文件, 支持断点续传, 支持多线程
+    #
+    # @param file_store [String] 本地文件存储路径
+    # @param options [Hash] 高级参数
+    # @option options [Integer] :min_slice_size 完整下载最小文件大小,
+    #  超过此大小将会使用分片多线程断点续传
+    # @option options [Integer] :download_retry 下载重试次数, 默认10
+    # @option options [Boolean] :disable_cpt 是否禁用checkpoint功能，如
+    #  果设置为true，则在下载的过程中不会写checkpoint文件，这意味着
+    #  下载失败后不能断点续传，而只能重新下载整个文件。如果这个值为
+    #  true，则:cpt_file会被忽略。
+    # @option options [Integer] :threads 多线程下载线程数, 默认为10
+    # @option options [Integer] :slice_size 设置分片下载时每个分片的大小
+    #  默认为5 MB。
+    # @option options [String] :cpt_file 断点续传的checkpoint文件，如果
+    #  指定的cpt文件不存在，则会在file所在目录创建一个默认的cpt文件，
+    #  命名方式为：file.cpt，其中file是用户要下载的文件名。在下载的过
+    #  程中会不断更新此文件，成功完成下载后会删除此文件；如果指定的
+    #  cpt文件已存在，则从cpt文件中记录的点继续下载。
+    #
+    # @yield [Float] 下载进度百分比回调, 进度值是一个0-1之间的小数
+    #
+    # @raise [ServerError] 服务端异常返回
+    #
+    # @return [String]
+    #
+    # @see Bucket#download
+    def download(file_store, options = {}, &block)
+      bucket.download(self, file_store, options, &block)
+    end
+
+    private
+
+    # 计算文件大小格式化单位
+    def human_rep(bytes)
+      number   = Float(bytes)
+      max_exp  = STORAGE_UNITS.size - 1
+
+      exponent = (Math.log(bytes) / Math.log(STORAGE_BASE)).to_i
+      exponent = max_exp if exponent > max_exp
+
+      number   /= STORAGE_BASE ** exponent
+      unit     = STORAGE_UNITS[exponent]
+
+      { size: number, unit: unit }
+    end
+
+  end
+
+end