qiniu2upyun 0.0.5

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b51e2995913626a163c38b05723c98395d812c21
+  data.tar.gz: 0fc4a3acf7422edbb0f080584d6a7895a780f07e
+SHA512:
+  metadata.gz: 6356edc529786256f5bb9a8e500fa218300d3baabe0ab92a4cec067221630d05f228c412aea7df5a7a89951a6767089c9e25779ad1a9edd28df8ec98a2b152a3
+  data.tar.gz: c21b8e691fa3fdc5f09b0576f36861c8170bed517856c030eaa4176fbde373c5827fbcc7a6bafc6f41a6d2cbbefe52f3003ec9f76a00751cda3748bccafd45d1

data/.gitignore

@@ -0,0 +1,12 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+*.swp
+.DS_Store

data/Gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in qiniu2upyun.gemspec
+gemspec
+
+gem 'ruby-progressbar'

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 zhouguangming
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,65 @@
+# Qiniu2Upyun
+
+Migrate Qiniu to Upyun.
+
+## Installation
+
+    $ gem install qiniu2upyun
+
+## Usage
+
+    qiniu2upyun --help
+
+    Usage: qiniu2upyun [options]
+        -C, --config PATH                Load PATH as a config file
+                                         (default: $HOME/.qiniu2upyun.yml)
+        -V, --version                    Show version
+        -h, --help                       Show help
+
+## Config
+
+    # $HOME/.qiniu2upyun.yml 
+
+    qiniu:
+      access_key: <your_access_key>
+      secret_key: <your_secret_key>
+      bucket:     <your_bucket>
+    upyun:
+      username:   <your_username>
+      password:   <your_password>
+      bucket:     <your_bucket>
+
+## Snap
+
+    $ qiniu2upyun
+
+    Fetch '001.jpg` from qiniu:
+        Already exists in upyun and skip migrating.
+    
+    Fetch '002.jpg` from qiniu:
+        Already exists in upyun and skip migrating.
+
+    Fetch '003.jpg` from qiniu:
+        Download |========================================| 100%
+          Upload |========================================| 100%
+                                                (2926510B/12.3s)
+    
+    Fetch '004.jpg` from qiniu:
+        Download |========================================| 100%
+          Upload |========================================| 100%
+                                                  (758846B/2.8s)
+    
+    Fetch '005.jpg` from qiniu:
+        Download |========================================| 100%
+          Upload |========================================| 100%
+                                                 (343525B/1.73s)
+    
+    Complete, migrated 3 files and skipped 2 files in 17.24 seconds.
+
+## Contributing
+
+1. Fork it ( http://gitcafe.com/zgm/QINIU-to-UPYUN/)
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/qiniu2upyun

@@ -0,0 +1,5 @@
+#! /usr/bin/env ruby
+
+require File.expand_path('../../lib/qiniu2upyun/cli', __FILE__)
+
+Qiniu2Upyun::CLI.new.run!

data/lib/qiniu2upyun.rb

@@ -0,0 +1,32 @@
+require 'net/http'
+require 'uri'
+require 'base64'
+require 'openssl'
+require 'yaml'
+require 'json'
+require 'digest'
+require 'time'
+require 'tempfile'
+require 'stringio'
+require 'ruby-progressbar'
+
+module Qiniu2Upyun
+  require_relative 'qiniu2upyun/version'
+  require_relative 'qiniu2upyun/qiniu'
+  require_relative 'qiniu2upyun/upyun'
+  require_relative 'qiniu2upyun/migrating'
+  require_relative 'qiniu2upyun/config'
+  require_relative 'qiniu2upyun/error'
+  require_relative 'qiniu2upyun/data'
+  require_relative 'qiniu2upyun/record'
+  require_relative 'qiniu2upyun/utils'
+
+  def self.migrate!(options={})
+    Config.load!(options.delete(:config))
+
+    qiniu = Qiniu.new(Config.qiniu)
+    upyun = Upyun.new(Config.upyun)
+
+    Migrating.new(qiniu, upyun).perform!
+  end
+end

data/lib/qiniu2upyun/cli.rb

@@ -0,0 +1,48 @@
+require 'optparse'
+
+module Qiniu2Upyun
+  class CLI
+    DEFAULT_OPTIONS = {
+      config: "#{ENV['HOME']}/.qiniu2upyun.yml"
+    }
+
+    def parse
+      options = {}
+
+      parser = OptionParser.new do |opts|
+        opts.banner = "Usage: qiniu2upyun [options]"
+
+        opts.on("-C", "--config PATH", 
+                "Load PATH as a config file",
+                "(default: #{DEFAULT_OPTIONS[:config]})") do |c|
+
+          options[:config] = c
+        end
+
+        opts.on("-V", "--version", "Show version") do |v|
+          require File.expand_path('../version', __FILE__)
+
+          puts "qiniu2upyun version #{Qiniu2Upyun::VERSION}"
+
+          exit
+        end
+
+        opts.on("-h", "--help", "Show help") do |h|
+          puts opts
+
+          exit
+        end
+      end.parse!
+
+      options
+    end
+
+    def run!
+      options = DEFAULT_OPTIONS.merge(parse)
+
+      require File.expand_path('../../qiniu2upyun', __FILE__)
+
+      Qiniu2Upyun.migrate!(options)
+    end
+  end
+end

data/lib/qiniu2upyun/config.rb

@@ -0,0 +1,41 @@
+module Qiniu2Upyun
+  class Config
+    class Qiniu
+      attr_reader :access_key, :secret_key, :bucket, :domain
+
+      def initialize(options)
+        @access_key = options['access_key']
+        @secret_key = options['secret_key']
+        @bucket     = options['bucket']
+        @domain     = options['domain'] || default_domain
+      end
+
+      def default_domain
+        "http://#{bucket}.qiniudn.com"
+      end
+    end
+
+    class Upyun
+      attr_reader :username, :password, :bucket
+
+      def initialize(options)
+        @username = options['username']
+        @password = options['password']
+        @bucket   = options['bucket']
+      end
+    end
+
+    class << self
+      attr_reader :qiniu, :upyun
+
+      def load!(path)
+        raise NoConfigFileFoundError.new(path) unless File.exists?(path)
+
+        config = YAML.load_file(path)
+
+        @qiniu = Qiniu.new(config['qiniu'])
+        @upyun = Upyun.new(config['upyun'])
+      end
+    end
+  end
+end

data/lib/qiniu2upyun/data.rb

@@ -0,0 +1,34 @@
+module Qiniu2Upyun
+  class Data
+    attr_reader :io, :bar
+
+    def initialize(io, bar)
+      @io  = io
+      @bar = bar
+    end
+
+    def read(*args, &block)
+      result = io.read(*args, &block)
+
+      length = args.last.length
+
+      length > 0 ?  bar.progress += length : bar.finish
+
+      result
+    end
+
+    def write(*args, &block)
+      result = io.write(*args, &block)
+
+      bar.progress += result
+
+      bar.finish if bar.total == io.length
+
+      result
+    end
+
+    def method_missing(method, *args, &block)
+      io.send(method)
+    end
+  end
+end

data/lib/qiniu2upyun/error.rb

@@ -0,0 +1,24 @@
+module Qiniu2Upyun
+  class NoConfigFileFoundError < StandardError
+    def initialize(path)
+      @path = path
+    end
+
+    def message
+      "No config file found: #{@path}"
+    end
+  end
+
+  class ResponseError < StandardError
+    def initialize(response)
+      @response = response
+    end
+
+    def message
+      "<#{@response.code}: #{@response.body}>"
+    end
+  end
+
+  class NoActionError < StandardError
+  end
+end

data/lib/qiniu2upyun/migrating.rb

@@ -0,0 +1,88 @@
+module Qiniu2Upyun
+  class Migrating
+    attr_reader :from, :to, :record, :migrated_count, :skipped_count
+
+    def initialize(from, to)
+      @from           = from
+      @to             = to
+      @record         = Record.new
+      @migrated_count = 0
+      @skipped_count  = 0
+    end
+
+    def perform!
+      if from.is_a?(Qiniu) && to.is_a?(Upyun)
+        from_qiniu_to_upyun
+      end
+    end
+
+    private
+
+    def from_qiniu_to_upyun
+      record.recording do |init_marker, init_key|
+        time = Time.now
+
+        if init_key
+          sources = get_sources(init_marker)
+
+          marker, items = sources['marker'], cut_items(sources['items'], init_key)
+
+          perform_items(items, init_marker)
+        else
+          marker = init_marker
+        end
+
+        while marker && (_marker = marker.dup)
+          sources = get_sources(marker)
+
+          marker, items = sources['marker'], sources['items']
+
+          perform_items(items, _marker)
+        end
+
+        puts "\nComplete, migrated %s files and skipped %s files in %.2f seconds.\n\n" %
+          [migrated_count.to_s.green, skipped_count.to_s.red, Time.now-time]
+      end
+    end
+
+    def perform_items(items, marker)
+      items.each do |item|
+        record.set(marker, item['key']) do
+          perform_item(item['key']) ? @migrated_count+=1 : @skipped_count+=1
+        end
+      end
+    end
+
+    def perform_item(key)
+      if to.exist?(key)
+        puts "\nFetch '#{key}` from #{from.name}:".red
+      
+        puts "    Already exists in #{to.name} and skip migrating.".red
+
+        false
+      else
+        puts "\nFetch '#{key}` from #{from.name}:".green
+
+        time = Time.now
+
+        data = from.download(key)
+
+        to.upload(key, data)
+
+        puts ("%60s" % "(#{data.length}B/#{(Time.now-time).round(2)}s)").green
+
+        true
+      end
+    end
+
+    def get_sources(marker)
+      marker == Record::START_MARKER ? sources = from.list : sources = from.list(marker: marker)
+    end
+
+    def cut_items(items, key)
+      index = items.index { |item| item['key'] == key }
+
+      index ? items.drop(index+1) : items
+    end
+  end
+end

data/lib/qiniu2upyun/qiniu.rb

@@ -0,0 +1,116 @@
+module Qiniu2Upyun
+  class Qiniu
+    attr_reader :config
+
+    HOST = 'http://rsf.qbox.me'
+    NAME = 'qiniu'
+
+    def initialize(config)
+      @config = config
+    end
+
+    def name
+      NAME
+    end
+
+    def list(params={})
+      uri = get_list_uri(params)
+
+      Net::HTTP.start(uri.host) do |http|
+        request = Net::HTTP::Post.new(uri)
+
+        set_headers(request, uri)
+
+        response = http.request(request)
+
+        if response.is_a?(Net::HTTPOK)
+          JSON.parse(response.body)
+        else
+          raise ResponseError.new(response)
+        end
+      end
+    end
+
+    def download(key)
+      uri = get_download_uri(key)
+
+      Net::HTTP.start(uri.host) do |http|
+        request = Net::HTTP::Get.new(uri)
+
+        http.request(request) do |response|
+          data = get_data(key, response)
+
+          response.read_body { |chunk| data.write(chunk) }
+
+          data.io.rewind
+
+          return(data.io)
+        end
+      end
+    end
+
+    private
+
+    def set_headers(request, uri)
+      access_token = get_access_token(uri)
+
+      request['Authorization'] = access_token
+    end
+
+
+    def get_data(key, response)
+      length = response['Content-Length'].to_i
+
+      bar = Utils.get_bar(length, 'Download')
+
+      if length > 1024 * 1024
+        Data.new(Tempfile.new(key), bar)
+      else
+        Data.new(StringIO.new, bar)
+      end
+    end
+
+    def get_list_uri(params)
+      default_params = { 
+        bucket: config.bucket,
+        limit: 100,
+        marker: nil,
+        prefix: nil
+      }
+
+      query = URI.encode_www_form(default_params.merge(params))
+
+      uri = URI(URI.encode("#{HOST}/list?#{query}"))
+    end
+
+    def get_download_uri(key)
+      url = "#{config.domain}/#{key}?e=#{Time.now.utc.to_i + 3600}"
+
+      uri = URI(URI.encode(url))
+
+      download_token = get_download_token(uri)
+
+      url = url.concat("&token=#{download_token}")
+
+      uri = URI(URI.encode(url))
+    end
+
+    def get_download_token(uri)
+      get_sign(uri.to_s)
+    end
+
+    def get_access_token(uri, body=nil)
+      "QBox #{get_sign("#{uri.request_uri}\n#{body}")}"
+    end
+
+    def get_sign(signing_str)
+      digest = OpenSSL::Digest.new('sha1') 
+
+      sign = OpenSSL::HMAC.digest(digest, config.secret_key, signing_str)
+
+      encoding_sign = Base64.urlsafe_encode64(sign)
+
+      "#{config.access_key}:#{encoding_sign}"
+    end
+  end
+end

data/lib/qiniu2upyun/record.rb

@@ -0,0 +1,42 @@
+module Qiniu2Upyun
+  class Record
+    PATH = '/tmp/qiniu2upyun'
+    START_MARKER = '0'
+
+    def get
+      File.exists?(PATH) ? File.open(PATH, 'r').read.split(':') : [START_MARKER, nil]
+    end
+
+    def set(marker, key)
+      yield if block_given?
+
+      File.open(PATH, 'w').write("#{marker}:#{key}")
+    end
+
+    def delete
+      File.delete(PATH) if File.exists?(PATH)
+    end
+
+    def recording(&block)
+      marker, key = get
+
+      if key
+        puts "Migration is not completed on time.(marker: #{marker}, key: #{key})"
+        puts "    1: Continue"
+        puts "    2: Restart"
+        print "Input: "
+
+        case gets.chomp.to_i
+        when 1
+        when 2 then marker, key = START_MARKER, nil
+        else 
+          raise NoActionError
+        end
+      end
+
+      yield(marker, key)
+
+      delete
+    end
+  end
+end

data/lib/qiniu2upyun/upyun.rb

@@ -0,0 +1,88 @@
+module Qiniu2Upyun
+  class Upyun
+    attr_reader :config
+
+    HOST = 'http://v0.api.upyun.com'
+    NAME = 'upyun'
+
+    def initialize(config)
+      @config = config
+    end
+
+    def name
+      NAME
+    end
+
+    def exist?(key)
+      info(key)
+    rescue ResponseError
+      false
+    end
+
+    def upload(key, data)
+      uri = URI(URI.encode("#{HOST}/#{config.bucket}/#{key}"))
+
+      send_request(uri, 'PUT', data)
+    end
+
+    def info(key)
+      uri = URI(URI.encode("#{HOST}/#{config.bucket}/#{key}"))
+
+      response = send_request(uri, 'HEAD')
+
+      response.is_a?(Net::HTTPOK) or raise ResponseError.new(response)
+    end
+
+    private
+
+    def send_request(uri, method, data=nil, headers={})
+      Net::HTTP.start(uri.host) do |http|
+        request = get_request(method, uri)
+
+        set_headers(request, uri, method, data, headers)
+
+        if data.nil?
+          http.request(request)
+        else
+          bar = Utils.get_bar(data.length, 'Upload')
+
+          request.body_stream = Data.new(data, bar)
+
+          http.request(request)
+        end
+      end
+    end
+
+    def get_request(method, uri)
+      Net::HTTP.const_get(method[0].concat(method[1..-1].downcase)).new(uri)
+    end
+
+    def set_headers(request, uri, method, data, headers)
+      current = Time.now.utc.httpdate
+
+      content_length = data ? data.length : 0
+
+      sign = get_sign(method, current, uri, content_length)
+
+      default_headers = {
+        'Authorization'  => sign,
+        'Date'           => current,
+        'Content-Length' => content_length
+      }
+
+      headers = default_headers.merge(headers)
+
+      headers.each { |key, value| request[key] = value }
+    end
+
+    def get_sign(method, current, uri, content_length)
+      digested_password = Digest::MD5.hexdigest(config.password)
+
+      signing_str = "#{method}&#{uri.path}&#{current}&#{content_length}&#{digested_password}"
+
+      sign = Digest::MD5.hexdigest(signing_str)
+
+      "UpYun #{config.username}:#{sign}"
+    end
+  end
+end

data/lib/qiniu2upyun/utils.rb

@@ -0,0 +1,36 @@
+module Qiniu2Upyun
+  module Utils
+    class << self
+      def get_bar(total, title)
+        ProgressBar.create(
+          title: ("%12s" % title),
+          length: 60,
+          total: total,
+          format: "%t |%B| %p%".green
+        ) 
+      end
+    end
+  end
+end
+
+class String
+  def colorize(code)
+    "\e[#{code}m#{self}\e[0m"
+  end
+
+  def red
+    colorize(31)
+  end
+
+  def green
+    colorize(32)
+  end
+
+  def yellow
+    colorize(33)
+  end
+
+  def pink
+    colorize(35)
+  end
+end

data/lib/qiniu2upyun/version.rb

@@ -0,0 +1,4 @@
+module Qiniu2Upyun
+  VERSION = '0.0.5'
+end
+

data/qiniu2upyun.gemspec

@@ -0,0 +1,25 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+require File.expand_path('../lib/qiniu2upyun/version', __FILE__)
+
+Gem::Specification.new do |spec|
+  spec.name          = "qiniu2upyun"
+  spec.version       = Qiniu2Upyun::VERSION
+  spec.authors       = ["zhouguangming"]
+  spec.email         = ["zhouguangming1989@gmail.com"]
+  spec.summary       = %q{Migrate Qiniu to Upyun.}
+  spec.description   = %q{Migrate Qiniu to Upyun.}
+  spec.homepage      = "https://gitcafe.com/zgm/QINIU-to-UPYUN"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.5"
+  spec.add_development_dependency "rake"
+  spec.add_dependency "ruby-progressbar"
+end

metadata

@@ -0,0 +1,105 @@
+--- !ruby/object:Gem::Specification
+name: qiniu2upyun
+version: !ruby/object:Gem::Version
+  version: 0.0.5
+platform: ruby
+authors:
+- zhouguangming
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-04-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: ruby-progressbar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Migrate Qiniu to Upyun.
+email:
+- zhouguangming1989@gmail.com
+executables:
+- qiniu2upyun
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/qiniu2upyun
+- lib/qiniu2upyun.rb
+- lib/qiniu2upyun/cli.rb
+- lib/qiniu2upyun/config.rb
+- lib/qiniu2upyun/data.rb
+- lib/qiniu2upyun/error.rb
+- lib/qiniu2upyun/migrating.rb
+- lib/qiniu2upyun/qiniu.rb
+- lib/qiniu2upyun/record.rb
+- lib/qiniu2upyun/upyun.rb
+- lib/qiniu2upyun/utils.rb
+- lib/qiniu2upyun/version.rb
+- qiniu2upyun.gemspec
+homepage: https://gitcafe.com/zgm/QINIU-to-UPYUN
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Migrate Qiniu to Upyun.
+test_files: []