baidu-netdisk 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: ea47d1e606bc5f1569f6e2e5b2af19cdc15c996cb5d40f3fb4fd2c1190c043f3
+  data.tar.gz: 8d30769f5856ad1844beaa1a72e6784f4f6ec8b83509fd52b8a833fde746a04d
+SHA512:
+  metadata.gz: 6f715c3b60ce547a64b5590413102692247481c4e9d6d273fbc42d2d501c1187dce3ff5452edb13e903f51e8410406a7f706775cfe262a30f4155e4395913c10
+  data.tar.gz: 8165697d3fe2f35ae4e484de731a42678e9930a60b93788827e13baf75b33071220d60a530088b0c47c15cc2d1fa5373262ebd7012f6f7ecc95a19b3d6be050b

data/.env.example

@@ -0,0 +1,5 @@
+APP_ID='10000'
+APP_KEY='app-key'
+SECRET_KEY='screct-key'
+ACCESS_TOKEN='1234.wasd'
+REFRESH_TOKEN = 4321.qte'

data/.rspec

@@ -0,0 +1,4 @@
+--color
+--require spec_helper
+--order rand
+--format doc

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Hegwin Wang
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,121 @@
+# Baidu Netdisk
+
+[![CI](https://github.com/hegwin/baidu-netdisk/actions/workflows/test.yml/badge.svg)](https://github.com/hegwin/baidu-netdisk/actions/workflows/test.yml)
+[![codecov](https://codecov.io/gh/hegwin/baidu-netdisk/branch/main/graph/badge.svg?token=HCUJ4QDMH6)](https://codecov.io/gh/hegwin/baidu-netdisk)
+[![Maintainability](https://api.codeclimate.com/v1/badges/75bf545e0efd8f0b24e1/maintainability)](https://codeclimate.com/github/hegwin/baidu-netdisk/maintainability)
+
+A Ruby client to upload files to Baidu NetDisk (PCS). It'll auto-split big files and upload slices parallelly.
+
+
+## Installation
+
+Install with bundler:
+
+```
+bundle add baidu-netdisk
+```
+
+Or install with ruby gem
+
+```
+gem install baidu-netdisk
+```
+
+## Configuration
+
+```ruby
+BaiduNetDisk.config do |c|
+  # Required configs
+  c.app_id     = 'your_app_id'
+  c.app_key    = 'your_app_key'
+  c.secret_key = 'your_secret_key'
+
+  # Max threads for uploading file slices, default to 1
+  c.max_uploading_threads = 4
+
+  # The following two are optional;
+  # Fill them in if you want to explicitly indicate uploading to someone else's storage space
+  c.access_token  = 'your_access_token'
+  c.refresh_token = 'your_refresh_token'
+end
+```
+
+## Usage
+
+This client is designed for the two following scenarios.
+
+### Scenario 1: Using Baidu NetDisk as a backup file storage
+
+You always save files to your net disk storage. You don't need a callback to receive auth codes or tokens from Baidu; Instead, you can just get them from your Ruby console.
+
+```ruby
+require 'baidu-netdisk'
+
+# It opens a browser to ask for access to Baidu OAuth;
+# you will get your auth code after you log in to your Baidu NetDisk
+BaiduNetDisk::Auth.get_auth_code
+
+# Pass the code as the argument your received from the previous step
+BaiduNetDisk::Auth.get_token(auth_code)
+
+uploader = BaiduNetDisk::Uploader.new(source_path, target_path)
+uploader.execute
+
+#  => {"category"=>6, "ctime"=>1676019860, "from_type"=>1, "fs_id"=>121127634951625, "isdir"=>0, "md5"=>"79835de6btc0b3482f51b49088c8ccfb", "mtime"=>1676019860, "path"=>"<target_path>", "server_filename"=>"<file_name>", "size"=>76267, "errno"=>0, "name"=>"<target_path>"} 
+```
+
+### Scenario 2: Upload files to your clients' Baidu NetDisk 
+
+You should have a web server and need to implement your webhook to receive a callback from Baidu NetDisk after auth.
+
+Say your callback URL is `https://www.example.com/webhook`, you should have the:
+
+```ruby
+BaiduNetDisk::Auth.get_auth_code('https://www.example.com/webhook')
+
+
+# Auth code is fetched in your callbacks
+BaiduNetDisk::Auth.get_token(auth_code, 'https://www.example.com/webhook')
+
+uploader = BaiduNetDisk::Uploader.new(source_path, target_path,
+  { access_token: 'client.access_token', refresh_token: 'client.refresh_token' })
+
+uploader.execute
+```
+
+It provides a hook after token refreshed so you can save tokens after refreshing.
+
+```ruby
+BaiduNetDisk.config do |c|
+  c.after_token_refreshed = -> (access_token, refresh_token) {
+    user.update access_token: access_token, refresh_token: refresh_token
+  }
+end
+```
+
+## Development
+
+### Copy the env file
+
+It reads ENV from .env for testing.
+
+```
+cp .env.example .env
+```
+
+### Run tests
+
+```
+bundle install
+bundle exec rake
+```
+
+### Todo
+
+1. Improve MAINTAINABILITY
+
+2. To use other HTTP clients instead of RestClient:
+
+  1. RestClient can't catch response body when response is `400 BadRequest`.
+
+  2. Currently, we implemented multi-threaded uploading by hands. Consider to use typhoeus with its native hydra to run HTTP requests in parallel.

data/Rakefile

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+# require "rubocop/rake_task"
+# RuboCop::RakeTask.new
+
+task default: %i[spec]

data/lib/baidu-netdisk/auth.rb

@@ -0,0 +1,64 @@
+require 'rest-client'
+require 'json'
+
+class BaiduNetDisk::Auth
+  class << self
+    # This is a debug tool to get auth code for your own Baidu account
+    def get_auth_code(redirect_uri = 'oob')
+      check_required_configs if redirect_uri == 'oob'
+
+      url = "https://openapi.baidu.com/oauth/2.0/authorize?response_type=code&client_id=#{BaiduNetDisk.app_key}&redirect_uri=#{redirect_uri}&scope=basic,netdisk&device_id=#{BaiduNetDisk.app_id}"
+
+      if redirect_uri == 'oob'
+        system('open', url)
+      else
+        RestClient.get url
+      end
+    end
+
+    def get_token(auth_code, redirect_uri = 'oob')
+      response = RestClient.get "https://openapi.baidu.com/oauth/2.0/token?grant_type=authorization_code&code=#{auth_code}&client_id=#{BaiduNetDisk.app_key}&client_secret=#{BaiduNetDisk.secret_key}&redirect_uri=#{redirect_uri}"
+
+      if redirect_uri == 'oob'
+        response_body = JSON.parse response.body
+        BaiduNetDisk.access_token, BaiduNetDisk.refresh_token = response_body.fetch_values('access_token', 'refresh_token')
+      end
+    end
+
+    def refresh_access_token(refresh_token)
+      response = RestClient.get "https://openapi.baidu.com/oauth/2.0/token?grant_type=refresh_token&refresh_token=#{refresh_token}&client_id=#{BaiduNetDisk.app_key}&client_secret=#{BaiduNetDisk.secret_key}"
+
+      response_body = JSON.parse response.body
+
+      access_token, refresh_token = response_body.fetch_values('access_token', 'refresh_token')
+
+      if BaiduNetDisk.after_token_refreshed&.respond_to? :call
+        BaiduNetDisk.after_token_refreshed.call(access_token, refresh_token)
+      end
+
+      [access_token, refresh_token]
+    rescue RestClient::BadRequest
+      $stdout.puts "Refresh token failed."
+      raise BaiduNetDisk::Exception::RefreshTokenFailed
+    end
+
+    private
+
+    def check_required_configs
+      while BaiduNetDisk.app_key.nil? || BaiduNetDisk.app_key.strip.empty?
+        $stdout.print "Please enter your App Key:\n"
+        BaiduNetDisk.app_key = $stdin.gets.chomp
+      end
+
+      while BaiduNetDisk.app_id.nil? || BaiduNetDisk.app_id.strip.empty?
+        $stdout.print "Please enter your App ID:\n"
+        BaiduNetDisk.app_id = $stdin.gets.chomp
+      end
+
+      while BaiduNetDisk.secret_key.nil? || BaiduNetDisk.secret_key.strip.empty?
+        $stdout.print "Please enter your Secret Key:\n"
+        BaiduNetDisk.secret_key = $stdin.gets.chomp
+      end
+    end
+  end
+end

data/lib/baidu-netdisk/exception.rb

@@ -0,0 +1,35 @@
+module BaiduNetDisk
+  module Exception
+    # Exceptions following Baidu error codes
+    # https://openauth.baidu.com/doc/appendix.html#_4-openapi%E9%94%99%E8%AF%AF%E7%A0%81%E5%88%97%E8%A1%A8
+    class PermissionDenied < StandardError; end
+    class SliceMissing < StandardError; end
+    class FileExisted < StandardError; end
+    class SpaceNotEnough < StandardError; end
+    class AuthenticationFailed < StandardError; end
+    class TooManyRequests < StandardError; end
+    class AccessTokenExpired < StandardError; end
+    class UnknownError < StandardError; end
+    class ArgumentError < StandardError; end
+    class UserAuthorizationRequired < StandardError; end
+
+    # Custom exception
+    class UploadDirectoryNotSupported < StandardError; end
+    class RefreshTokenNotProvided < StandardError; end
+    class RefreshTokenFailed < StandardError; end
+
+    MAPPING = {
+      -10 => SpaceNotEnough,
+      -8  => FileExisted,
+      -7  => PermissionDenied,
+      -6  => AuthenticationFailed,
+      1   => UnknownError,
+      2   => ArgumentError,
+      3   => PermissionDenied,
+      6   => UserAuthorizationRequired,
+      111 => AccessTokenExpired,
+      31034 => TooManyRequests,
+      31363 => SliceMissing
+    }
+  end
+end

data/lib/baidu-netdisk/uploader.rb

@@ -0,0 +1,167 @@
+require 'rest-client'
+require 'json'
+
+class BaiduNetDisk::Uploader
+  SLICE_SIZE = 4 * 1024 * 1024
+
+  def initialize(source_path, target_path, options = {})
+    @is_dir = File.directory?(source_path) ? true : false
+    raise BaiduNetDisk::Exception::UploadDirectoryNotSupported, 'Uploading directory is not supported at present.' if @is_dir
+
+    @source_path = source_path
+    @target_path = target_path
+
+    @verbose = options[:verbose]
+    @slice_prefix = ".tmp_#{Random.hex(6)}_"
+
+    @rtype = options[:overwrite] ? 3 : 1
+    @file_size = File.size @source_path
+    @content_md5  = Digest::MD5.hexdigest(File.read @source_path)
+    @upload_id = nil
+    @slices = []
+    @refresh_token = options[:refresh_token] || BaiduNetDisk.refresh_token
+    @access_token = options[:access_token] || BaiduNetDisk.access_token
+  end
+
+  def execute
+    prepare
+    pre_upload
+    upload_by_slices
+    create_file
+  rescue BaiduNetDisk::Exception::PermissionDenied
+    $stderr.puts 'You are not authorised to upload files to your target path.'
+    return false
+  rescue BaiduNetDisk::Exception::AuthenticationFailed, BaiduNetDisk::Exception::AccessTokenExpired => e
+    raise e if @tried_refresh_token
+    retry if refresh_token!
+  ensure
+    clear_up
+  end
+
+  private
+
+  def prepare
+    return if @slices.any?
+
+    if @file_size > SLICE_SIZE
+      $stdout.puts 'Splitting file into slices...' if @verbose
+
+      `split -b #{SLICE_SIZE} "#{@source_path}" #{@slice_prefix}`
+
+      Dir["#{Dir.getwd}/#{@slice_prefix}*"].sort.each do |slice_file_path|
+        @slices << {
+          md5: Digest::MD5.hexdigest(File.read slice_file_path),
+          slice_file_path: slice_file_path,
+          block_id: nil
+        }
+      end
+
+      $stdout.puts "Split file into #{@slices.size} slices. Done." if @verbose
+    else
+      $stdout.puts 'File size is smaller than 4MB, no split.' if @verbose
+
+      @slices << { md5: @content_md5, slice_file_path: @source_path, block_id: 0}
+    end
+  end
+
+  # API doc in Baidu:
+  # https://pan.baidu.com/union/doc/3ksg0s9r7
+  def pre_upload
+    response = RestClient.post "https://pan.baidu.com/rest/2.0/xpan/file?method=precreate&access_token=#{@access_token}", {
+      path: @target_path,
+        size: @file_size,
+        isdir: @is_dir,
+        autoinit: 1,
+        rtype: @rtype,
+        block_list: @slices.map { |slice| slice[:md5] }.to_json,
+        :'content-md5' => @content_md5
+    }, { 'User-Agent' => 'pan.baidu.com' }
+
+    response_body = JSON.parse response.body
+
+    if response_body['errno'].zero?
+      @upload_id = response_body['uploadid']
+      $stdout.print "Got upload ID #{@upload_id}\n" if @verbose
+
+      response_body['block_list'].each.with_index do |block_id, index|
+        @slices[index][:block_id] = block_id
+      end
+    else
+      raise BaiduNetDisk::Exception::MAPPING[response_body['errno']] || StandardError
+    end
+  end
+
+  # API doc in Baidu:
+  # https://pan.baidu.com/union/doc/nksg0s9vi
+  def upload_by_slices
+    queue = @slices.dup
+
+    # TODO Consider `typhoeus` rather than invoking multiple threads manually
+    BaiduNetDisk.max_uploading_threads.times.map do
+      Thread.new do
+        while queue.length > 0
+          slice = queue.pop
+          if slice
+            upload_slice_file(slice[:slice_file_path], slice[:block_id])
+          end
+        end
+      end
+    end.each(&:join)
+
+    $stdout.puts 'Slices upload complete.' if @verbose
+  end
+
+  def upload_slice_file(slice_file_path, block_id = 0)
+    response = RestClient.post "https://d.pcs.baidu.com/rest/2.0/pcs/superfile2?access_token=#{@access_token}&method=upload&type=tmpfile&path=#{@target_path}&uploadid=#{@upload_id}&partseq=#{block_id}", { file: File.new(slice_file_path, 'rb') }
+
+    $stdout.puts "Slice ##{block_id} uploaded!" if @verbose
+
+    response
+  end
+
+  # API doc in Baidu:
+  # https://pan.baidu.com/union/doc/rksg0sa17
+  def create_file
+    response = RestClient.post "https://pan.baidu.com/rest/2.0/xpan/file?method=create&access_token=#{@access_token}", {
+      path: @target_path,
+        size: @file_size,
+        isdir: @is_dir,
+        rtype: @rtype,
+        uploadid: @upload_id,
+        block_list: @slices.map { |slice| slice[:md5] }.to_json
+    }, { 'User-Agent' => 'pan.baidu.com' }
+
+    response_body = JSON.parse response.body
+
+    if response_body['errno'].zero?
+      $stdout.puts "File was successfully created at #{Time.at(response_body['ctime'])}!"
+      response_body
+    else
+      raise BaiduNetDisk::Exception::MAPPING[response_body['errno']] || StandardError, response.body
+    end
+  end
+
+  def clear_up
+    return if @slices.length < 2
+
+    $stdout.puts 'Cleaning tmp slice files...' if @verbose
+    @slices.each do |slice|
+      File.delete(slice[:slice_file_path]) if File.exist?(slice[:slice_file_path])
+    end
+    $stdout.puts 'Cleaning tmp slice files. Done.' if @verbose
+  end
+
+  def refresh_token!
+    @tried_refresh_token = true
+
+    if @refresh_token
+      $stdout.puts 'Access token expired. Trying to refresh...' if @verbose
+      @access_token, @refresh_token = BaiduNetDisk::Auth.refresh_access_token(@refresh_token)
+      $stdout.puts 'Access token refreshed!' if @verbose
+
+      true
+    else
+      raise BaiduNetDisk::Exception::RefreshTokenNotProvided, 'Access token expired. Please provide a refresh token.'
+    end
+  end
+end

data/lib/baidu-netdisk/version.rb

@@ -0,0 +1,3 @@
+module BaiduNetDisk
+  VERSION = '0.1.0'
+end

data/lib/baidu-netdisk.rb

@@ -0,0 +1,20 @@
+module BaiduNetDisk
+  class << self
+    attr_accessor :app_id, :app_key, :secret_key, :access_token, :refresh_token, :after_token_refreshed
+
+    attr_writer :max_uploading_threads
+
+    def config
+      yield self
+    end
+
+    def max_uploading_threads
+      @max_uploading_threads || 1
+    end
+  end
+
+end
+
+require 'baidu-netdisk/exception'
+require 'baidu-netdisk/auth'
+require 'baidu-netdisk/uploader'

metadata

@@ -0,0 +1,181 @@
+--- !ruby/object:Gem::Specification
+name: baidu-netdisk
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Hegwin Wang
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2023-02-10 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+- !ruby/object:Gem::Dependency
+  name: codecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.6.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.6.0
+- !ruby/object:Gem::Dependency
+  name: dotenv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.8.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.8.1
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.14.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.14.2
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 13.0.6
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 13.0.6
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.12.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.12.0
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.21.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.21.2
+- !ruby/object:Gem::Dependency
+  name: vcr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 6.1.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 6.1.0
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.18.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.18.1
+description: 百度网盘文件上传客户端Ruby版
+email:
+- zwt315@163.com
+- hegwin@hegwin.me
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".env.example"
+- ".rspec"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- lib/baidu-netdisk.rb
+- lib/baidu-netdisk/auth.rb
+- lib/baidu-netdisk/exception.rb
+- lib/baidu-netdisk/uploader.rb
+- lib/baidu-netdisk/version.rb
+homepage: https://github.com/hegwin/baidu-netdisk
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.6.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.6
+signing_key:
+specification_version: 4
+summary: A client to upload files to Baidu NetDisk
+test_files: []