rmega 0.1.7 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1de0443e3d865f515533f1110ba81e188b50c82f
-  data.tar.gz: dcfa1879ea580bba7ed15fb6838084fc2ff3fb28
+  metadata.gz: 6a223b72cd508e47946e1420769a7246fbeb0d5d
+  data.tar.gz: 2ee6197d9a398cb44adce16f0c19aead89bbc4f2
 SHA512:
-  metadata.gz: 0d393bdf468cc8f99f58a742be0e82eb9aab70479450fedb4e63f2f1d72aa15dd205860fbf01bbbac1c23e03a427538d9a84533a6c968b5aabe8b686f7eadf05
-  data.tar.gz: 905433bb7362720a3cdaa5753aee2f7a7a34f6c4f8ce6654efa4fcd75502176fbc8008b29b4dea61a89784a85644adc0d4761f32fdfa69ff5c0a856396bdacdd
+  metadata.gz: c40f1ab57edec730541e3fe060f1f5352724a72fcff7f526ae5813ef45e53dffc08fb3d39d8b6ff393fcc233429d10232efae42fbde19c9b8081e061c0789dfc
+  data.tar.gz: 269f261bb5d5d787540e21094c3252dff07edface5a0d899bc9c4c67a6bc9dc52fe830bd75023b456f5cf697872f77f8c0149922f939e2c9df8083ca55ffa78e

data/.travis.yml

@@ -0,0 +1,6 @@
+language: ruby
+rvm:
+  - 1.9.3
+  - 2.0.0
+  - 2.1.2
+  - 2.2.0

data/CHANGELOG.md

@@ -1,3 +1,19 @@
+## 0.2.0
+
+### New Features
+* resumable downloads
+* rmega-dl command (with folder download support)
+* rmega-up command
+* cbc-mac verification (download)
+* handle network errors without interrupting downloads/uploads
+* cache shared keys
+* `Storage#folders` has been removed (use `Storage#shared` that returns only shared folder nodes)
+
+### Changes
+* `RequestError` class was removed
+* Upload now returns the node `handle`
+* `Storage#download` moved to `Rmega#download`
+
 ## 0.1.7
 
 * \#9 Fixed decryption of nodes shared by you to others

data/README.md

@@ -80,7 +80,7 @@ folder.download("~/Downloads/my_folder")
 
 # Download a file by url
 publid_url = 'https://mega.co.nz/#!MAkg2Iab!bc9Y2U6d93IlRRKVYpcC9hLZjS4G278OPdH6nTFPDNQ'
-storage.download(public_url, '~/Downloads')
+Rmega.download(public_url, '~/Downloads')
 ```
 
 ### Upload

data/TODO.md

@@ -1,7 +1,5 @@
 ## TODO
 
-  * Handle timeouts in download/upload
-  * Handle error code -3 response, that is "A temporary congestion or server malfunction prevented your request from being processed.". I noticed that this occurs quite often when creating/deleting a folder node :/
-  * Sometimes uploaded data is corrupted
-  * Search for TODO in the project for other minor tasks
-  * Refactor the pool class
+  * Add an option to skip mac verification and calculation (upon downloads)
+
+  * Show this gif (https://i.imgur.com/VVl55wj.gif) in the readme?

data/bin/rmega-dl

@@ -0,0 +1,47 @@
+#!/usr/bin/env ruby
+
+require 'rmega'
+require 'rmega/cli'
+
+include Rmega::CLI::Helpers
+
+if ARGV.empty?
+  ARGV << '--help'
+else
+  cli_options[:url] = ARGV[0]
+end
+
+OptionParser.new do |opts|
+  opts.banner = "Usage:\n"
+  opts.banner << "\t#{File.basename(__FILE__)} url [options]\n"
+  opts.banner << "Options:"
+
+  opts.on("-o PATH", "--output", "Local destination path") { |path|
+    cli_options[:output] = path
+  }
+
+  apply_opt_parser_options(opts)
+end.parse!
+
+rescue_errors_and_inerrupt do
+  urls = [cli_options[:url]]
+
+  unless mega_url?(cli_options[:url])
+    urls = scan_mega_urls(Session.new.http_get_content(cli_options[:url])).uniq
+    raise("Nothing to download") if urls.empty?
+  end
+
+  urls.each_with_index do |url, index|
+    node = Rmega::Nodes::Factory.build_from_url(url)
+
+    info = if node.type == :folder
+      stats = node.storage.stats
+      "(#{stats[:files]} file#{'s' if stats[:files] > 1}, #{humanize_bytes(stats[:size])})"
+    end
+
+    puts "[#{index+1}/#{urls.count}] #{node.name} #{info}"
+
+    path = cli_options[:output] || Dir.pwd
+    node.download(path)
+  end
+end

data/bin/rmega-up

@@ -0,0 +1,31 @@
+#!/usr/bin/env ruby
+
+require 'rmega'
+require 'rmega/cli'
+
+include Rmega::CLI::Helpers
+
+if ARGV.empty?
+  ARGV << '--help'
+else
+  cli_options[:path] = ARGV[0]
+end
+
+OptionParser.new do |opts|
+  opts.banner = "Usage:\n"
+  opts.banner << "\t#{File.basename(__FILE__)} path [options]\n"
+  opts.banner << "Options:"
+
+  apply_opt_parser_options(opts)
+end.parse!
+
+rescue_errors_and_inerrupt do
+  raise("File not found - #{cli_options[:path]}") unless File.exists?(cli_options[:path])
+
+  user = cli_options[:user] || raise("User email is required")
+  pass = cli_options[:pass] ||= cli_prompt_password
+
+  session = Rmega::Session.new.login(user, pass)
+  root = session.storage.root
+  root.upload(cli_options[:path])
+end

data/lib/rmega.rb

@@ -1,11 +1,43 @@
+require 'thread'
+require 'ostruct'
+require 'logger'
+require 'uri'
+require 'net/http'
+require 'base64'
+require 'openssl'
+require 'digest/md5'
+
 require 'active_support/json'
+require 'active_support/concern'
 require 'active_support/core_ext/module/delegation'
-require 'active_support/core_ext/string/inflections'
-require 'httpclient'
-require 'execjs'
+
 require 'rmega/version'
+require 'rmega/loggable'
 require 'rmega/options'
+require 'rmega/not_inspectable'
+require 'rmega/errors'
+require 'rmega/api_response'
+require 'rmega/utils'
+require 'rmega/net'
+require 'rmega/pool'
+require 'rmega/progress'
+require 'rmega/crypto'
 require 'rmega/session'
+require 'rmega/storage'
+require 'rmega/nodes/factory'
+
+# Used only in specs
+require 'yaml'
+require 'tmpdir'
+require 'fileutils'
 
 module Rmega
+  def self.login(email, password)
+    Session.new.login(email, password).storage
+  end
+
+  def self.download(public_url, path = Dir.pwd)
+    node = Nodes::Factory.build_from_url(public_url)
+    return node.download(path)
+  end
 end

data/lib/rmega/api_response.rb

@@ -0,0 +1,80 @@
+module Rmega
+  class APIResponse
+    attr_reader :body, :code
+
+    # Check out the error codes list at https://mega.co.nz/#doc (section 11)
+    ERRORS = {
+      -1  => 'An internal error has occurred. Please submit a bug report, detailing the exact circumstances in which this error occurred.',
+      -2  => 'You have passed invalid arguments to this command.',
+      -3  => 'A temporary congestion or server malfunction prevented your request from being processed. No data was altered. Retry. Retries must be spaced with exponential backoff.',
+      -4  => 'You have exceeded your command weight per time quota. Please wait a few seconds, then try again (this should never happen in sane real-life applications).',
+      -5  => 'The upload failed. Please restart it from scratch.',
+      -6  => 'Too many concurrent IP addresses are accessing this upload target URL.',
+      -7  => 'The upload file packet is out of range or not starting and ending on a chunk boundary.',
+      -8  => 'The upload target URL you are trying to access has expired. Please request a fresh one.',
+      -9  => 'Object (typically, node or user) not found',
+      -10 => 'Circular linkage attempted',
+      -11 => 'Access violation (e.g., trying to write to a read-only share)',
+      -12 => 'Trying to create an object that already exists',
+      -13 => 'Trying to access an incomplete resource',
+      -14 => 'A decryption operation failed (never returned by the API)',
+      -15 => 'Invalid or expired user session, please relogin',
+      -16 => 'User blocked',
+      -17 => 'Request over quota',
+      -18 => 'Resource temporarily not available, please try again later',
+      -19 => 'Too many connections on this resource',
+      -20 => 'Write failed',
+      -21 => 'Read failed',
+      -22 => 'Invalid application key; request not processed',
+    }.freeze
+
+    def initialize(http_response)
+      @code = http_response.code.to_i
+      @body = http_response.body ? http_response.body : ""
+    end
+
+    def error?
+      unknown_error? or known_error? or temporary_error?
+    end
+
+    def ok?
+      !error?
+    end
+
+    def as_error
+      if unknown_error?
+        return TemporaryServerError.new
+      elsif temporary_error?
+        return TemporaryServerError.new(error_message)
+      else
+        return ServerError.new(error_message)
+      end
+    end
+
+    def as_json
+      @as_body ||= JSON.parse(body).first
+    end
+
+    private
+
+    def as_error_code
+      @error_code ||= body.scan(/\A\[{0,1}(\-\d+)\]{0,1}\z/).flatten.first.to_i
+    end
+
+    def error_message
+      ERRORS[as_error_code]
+    end
+
+    def temporary_error?
+      known_error? and [-3, -6, -18, -19].include?(as_error_code)
+    end
+
+    def unknown_error?
+      code == 500 or body.empty?
+    end
+
+    def known_error?
+      as_error_code < 0
+    end
+  end
+end

data/lib/rmega/cli.rb

@@ -0,0 +1,121 @@
+require 'optparse'
+require 'io/console'
+require 'active_support/core_ext/hash'
+
+module Rmega
+  module CLI
+    module Helpers
+      def cli_options
+        $cli_options ||= {options: {}}
+      end
+
+      def cli_prompt_password
+        print("Enter password: ")
+        password = STDIN.noecho(&:gets)
+        password = password[0..-2] if password.end_with?("\n")
+        puts
+
+        return password
+      end
+
+      def scan_mega_urls(text)
+        text.to_s.scan(Nodes::Factory::URL_REGEXP).flatten.map { |s| "https://mega.co.nz/##{s}" }
+      end
+
+      def mega_url?(url)
+        Nodes::Factory.url?(url)
+      end
+
+      def configuration_filepath
+        File.expand_path('~/.rmega')
+      end
+
+      def write_configuration_file
+        opts = {options: cli_options[:options]}
+        if cli_options[:user]
+          opts[:user] = cli_options[:user]
+          opts[:pass] = cli_options[:pass] || cli_prompt_password
+        end
+        File.open(configuration_filepath, 'wb') { |file| file.write(opts.to_json) }
+        FileUtils.chmod(0600, configuration_filepath)
+        puts "Options saved into #{configuration_filepath}"
+      end
+
+      def read_configuration_file
+        if File.exists?(configuration_filepath)
+          opts = JSON.parse(File.read(configuration_filepath))
+          $cli_options = opts.deep_symbolize_keys.deep_merge(cli_options)
+          puts "Loaded configuration file #{configuration_filepath}" if cli_options[:debug]
+        end
+      rescue Exception => ex
+        raise(ex) if cli_options[:debug]
+      end
+
+      def apply_cli_options
+        Rmega.logger.level = ::Logger::DEBUG if cli_options[:debug]
+
+        cli_options[:options].each do |key, value|
+          Rmega.options.__send__("#{key}=", value)
+        end
+      end
+
+      def apply_opt_parser_options(opts)
+        opts.on("-t NUM", "--thread_pool_size", "Number of threads to use") { |n|
+          cli_options[:options][:thread_pool_size] = n.to_i
+        }
+
+        opts.on("--proxy-addr ADDRESS", "Http proxy address") { |value|
+          cli_options[:options][:http_proxy_address] = value
+        }
+
+        opts.on("--proxy-port PORT", "Http proxy port") { |value|
+          cli_options[:options][:http_proxy_port] = value.to_i
+        }
+
+        opts.on("-u", "--user USER_EMAIL", "User email address") { |value|
+          cli_options[:user] = value
+        }
+
+        opts.on("--pass [USER_PASSWORD]", "User password (if omitted will prompt for it)") { |value|
+          cli_options[:pass] = value
+        }
+
+        opts.on("--write-cfg", "Write a configuration file with the given options") {
+          cli_options[:write_cfg] = true
+        }
+
+        opts.on("--debug", "Debug mode") {
+          cli_options[:debug] = true
+        }
+
+        opts.on("-v", "--version", "Print the version number") {
+          puts Rmega::VERSION
+          puts Rmega::HOMEPAGE
+          exit
+        }
+      end
+
+      def humanize_bytes(*args)
+        Progress.humanize_bytes(*args)
+      end
+
+      def rescue_errors_and_inerrupt(&block)
+        if cli_options[:write_cfg]
+          write_configuration_file
+        else
+          read_configuration_file
+          apply_cli_options
+          yield
+        end
+      rescue Interrupt
+        puts "\nInterrupted"
+      rescue Exception => ex
+        if cli_options[:debug]
+          raise(ex)
+        else
+          puts "\nError: #{ex.message}"
+        end
+      end
+    end
+  end
+end

data/lib/rmega/crypto.rb

@@ -0,0 +1,20 @@
+require 'rmega/crypto/aes_ecb'
+require 'rmega/crypto/aes_cbc'
+require 'rmega/crypto/aes_ctr'
+require 'rmega/crypto/rsa'
+
+module Rmega
+  module Crypto
+    include AesCbc
+    include AesEcb
+    include AesCtr
+    include Rsa
+
+    # Check if all the used ciphers are supported
+    ciphers = OpenSSL::Cipher.ciphers.map(&:upcase)
+    %w[AES-128-CBC AES-128-CTR AES-128-ECB].each do |name|
+      next if ciphers.include?(name)
+      warn "WARNING: Your Ruby is compiled with OpenSSL #{OpenSSL::VERSION} and does not support cipher #{name}."
+    end
+  end
+end

data/lib/rmega/crypto/aes_cbc.rb

@@ -0,0 +1,46 @@
+module Rmega
+  module Crypto
+    module AesCbc
+      def aes_cbc_cipher
+        OpenSSL::Cipher::AES.new(128, :CBC)
+      end
+
+      def aes_cbc_encrypt(key, data)
+        cipher = aes_cbc_cipher
+        cipher.encrypt
+        cipher.padding = 0
+        cipher.key = key
+        return cipher.update(data) + cipher.final
+      end
+
+      def aes_cbc_decrypt(key, data)
+        cipher = aes_cbc_cipher
+        cipher.decrypt
+        cipher.padding = 0
+        cipher.key = key
+        return cipher.update(data) + cipher.final
+      end
+
+      def aes_cbc_mac(key, data, iv)
+        cipher = aes_cbc_cipher
+        cipher.encrypt
+        cipher.padding = 0
+        cipher.iv = iv if iv
+        cipher.key = key
+
+        n = 0
+        mac = nil
+
+        loop do
+          block = data[n..n+15]
+          break if !block or block.empty?
+          block << "\x0"*(16-block.size) if block.size < 16
+          n += 16
+          mac = cipher.update(block)
+        end
+
+        return mac
+      end
+    end
+  end
+end