rmega 0.0.6 → 0.1.0

data/README.md

@@ -1,94 +1,147 @@
 # Rmega
 
-A ruby library for the Mega.co.nz.  
-Tested using ruby 1.9.3+ (OpenSSL 0.9.8r+)  
-This work is the result of a reverse engineering of the Mega's Javascript code.
+A ruby library for MEGA ([https://mega.co.nz/](https://mega.co.nz/))  
+Requirements: Ruby 1.9.3+ and OpenSSL 0.9.8r+
 
-## Installation
-
-  Rmega is distributed via rubygems, so if you have ruby 1.9.3+ installed
-  system wide, just type `gem install rmega`.
 
-## Usage
+This is the result of a reverse engineering of the MEGA javascript code.  
+This is a work in progress, further functionality are coming.
 
-    $ irb -r rmega
 
-### Login and retrive all the files and folders
+Supported features are:
+  * Login
+  * Searching and browsing
+  * Creating folders
+  * Download of files and folders (multi-thread)
+  * Download with public links (multi-thread)
+  * Upload of files (multi-thread)
+  * Deleting and trashing
 
-```ruby
-storage = Rmega.login 'your_email', 'your_password'
 
-# Fetch all the nodes (files, folders, ecc.)
-nodes = storage.nodes
-```
+## Installation
 
+  **Rmega** is distributed via [rubygems.org](https://rubygems.org/).  
+  If you have ruby installed system wide, just type `gem install rmega`.
 
-### Download a file or a folder
+## Usage
 
 ```ruby
-file = storage.nodes_by_name(/document1/i).first
-file.name # => "MyDocument1.pdf"
-file.download '~/Downloads'
-
-folder = storage.nodes_by_name(/photos/i).first
-folder.download '~/Downloads/MyAlbums'
+require 'rmega'
 ```
 
-
-### Download a file using a public url
+### Login
 
 ```ruby
-storage.download 'https://mega.co.nz/#!cER0GYbD!ZCHruEA08Xl4a_bUZYMI', '~/Downloads'
+storage = Rmega.login('your@email.com', 'your_p4ssw0rd')
 ```
 
-
-### Upload a file
+### Browsing
 
 ```ruby
-# Upload a file (to the root folder)
-storage.upload '~/Downloads/my_file.zip'
-
-# Upload a file to a specific folder
-document_folder = storage.nodes_by_name(/photos/i).first
-storage.upload '~/Downloads/my_file.zip', document_folder
+# Print the name of the files in the root folder
+storage.root.files.each { |file| puts file.name }
+
+# Print the number of files in each folder
+storage.root.folders.each do |folder|
+  puts "Folder #{folder.name} contains #{folder.files.size} files."
+end
+
+# Print the name and the size of the files in the recyble bin
+storage.trash.files.each { |file| puts "#{file.name} of #{file.size} bytes" }
+
+# Print the name of all the files (everywhere)
+storage.nodes.each do |node|
+  next unless node.type == :file
+  puts node.name
+end
+
+# Print all the nodes (files, folders, etc.) within a spefic folder
+folder = storage.root.folders[12]
+folder.children.each do |node|
+  puts "Node #{node.name} (#{node.type})"
+end
 ```
 
-### Other operations
+### Searching
 
 ```ruby
-# Trash a file or a folder
-my_node.trash
-
-# Gets the public url (the sharable one) of a file
-my_node.public_url
+# Search for a file within a specific folder
+folder = storage.root.folders[2]
+folder.files.find { |file| file.name == "to_find.txt" }
 
-# See the attributes of a node
-my_node.attributes
+# Search for a file everywhere
+storage.nodes.find { |node| node.type == :file and node.name =~ /my_file/i }
 
-# Create a folder
-parent_folder = storage.nodes_by_name(/photos/i).first
-folder_node = storage.create_folder parent_folder, "london"
+# Note: A node can be of type :file, :folder, :root, :inbox and :trash
 ```
 
-## Todo
+### Download
 
-  * Handle connection errors during upload/download
-
-
-## Installation
+```ruby
+# Download a single file
+file = storage.root.files.first
+file.download("~/Downloads")
+# => Download in progress 15.0MB of 15.0MB (100.0%)
+
+# Download a folder and all its content recursively
+folder = storage.nodes.find do |node|
+  node.type == :folder and node.name == 'my_folder'
+end
+folder.download("~/Downloads/my_folder")
+
+# Download a file by url
+publid_url = 'https://mega.co.nz/#!MAkg2Iab!bc9Y2U6d93IlRRKVYpcC9hLZjS4G278OPdH6nTFPDNQ'
+storage.download(public_url, '~/Downloads')
+```
 
-Add this line to your application's Gemfile:
+### Upload
 
-    gem 'rmega'
+```ruby
+# Upload a file to a specific folder
+folder = storage.root.folders[3]
+folder.upload("~/Downloads/my_file.txt")
 
-And then execute:
+# Upload a file to the root folder
+storage.root.upload("~/Downloads/my_other_file.txt")
+```
 
-    $ bundle
+### Creating a folder
 
-Or install it yourself as:
+```ruby
+# Create a subfolder of the root folder
+new_folder = storage.root.create_folder("my_documents")
+
+# Create a subfolder of an existing folder
+folder = storage.nodes.find do |node|
+  node.type == :folder and node.name == 'my_folder'
+end
+folder.create_folder("my_documents")
+```
 
-    $ gem install rmega
+### Deleting
 
+```ruby
+# Delete a folder
+folder = storage.root.folders[4]
+folder.delete
+
+# Move a folder to the recyle bin
+folder = storage.root.folders[4]
+folder.trash
+
+# Delete a file
+file = storage.root.folders[3].files.find { |f| f.name =~ /document1/ }
+file.delete
+
+# Move a file to the recyle bin
+file = storage.root.files.last
+file.trash
+
+# Empty the trash
+unless storage.trash.empty?
+  storage.trash.empty!
+end
+```
 
 ## Contributing
 

data/lib/rmega/crypto/aes.rb

@@ -1,3 +1,5 @@
+require 'openssl'
+
 module Rmega
   module Crypto
     module Aes
@@ -8,10 +10,10 @@ module Rmega
       end
 
       def cipher
-        @cipher ||= OpenSSL::Cipher::AES.new 128, :CBC
+        @cipher ||= OpenSSL::Cipher::AES.new(128, :CBC)
       end
 
-      def encrypt key, data
+      def encrypt(key, data)
         cipher.reset
         cipher.padding = 0
         cipher.encrypt
@@ -20,7 +22,7 @@ module Rmega
         result.unpack packing
       end
 
-      def decrypt key, data
+      def decrypt(key, data)
         cipher.reset
         cipher.padding = 0
         cipher.decrypt

data/lib/rmega/crypto/aes_ctr.rb

@@ -1,9 +1,12 @@
+require 'rmega/utils'
+require 'rmega/crypto/aes'
+
 module Rmega
   module Crypto
     module AesCtr
       extend self
 
-      def decrypt key, nonce, data
+      def decrypt(key, nonce, data)
         raise "invalid nonce" if nonce.size != 4 or !nonce.respond_to?(:pack)
         raise "invalid key" if key.size != 4 or !key.respond_to?(:pack)
 
@@ -47,7 +50,7 @@ module Rmega
         {data: decrypted_data, mac: mac}
       end
 
-      def encrypt key, nonce, data
+      def encrypt(key, nonce, data)
         raise "invalid nonce" if nonce.size != 4 or !nonce.respond_to?(:pack)
         raise "invalid key" if key.size != 4 or !key.respond_to?(:pack)
 

data/lib/rmega/crypto/crypto.rb

@@ -1,3 +1,8 @@
+require 'rmega/utils'
+require 'rmega/crypto/aes'
+require 'rmega/crypto/aes_ctr'
+require 'rmega/crypto/rsa'
+
 module Rmega
   module Crypto
     extend self
@@ -6,7 +11,7 @@ module Rmega
       Array.new(6).map { rand(0..0xFFFFFFFF) }
     end
 
-    def prepare_key ary
+    def prepare_key(ary)
       pkey = [0x93C467E3,0x7DB0C7A4,0xD1BE3F81,0x0152CB56]
       65536.times do
         0.step(ary.size-1, 4) do |j|
@@ -20,7 +25,7 @@ module Rmega
       pkey
     end
 
-    def decrypt_sid key, csid, privk
+    def decrypt_sid(key, csid, privk)
       # if csid ...
       t = Utils.mpi2b Utils.base64urldecode(csid)
       privk = Utils.a32_to_str decrypt_key(key, Utils.base64_to_a32(privk))
@@ -40,7 +45,7 @@ module Rmega
       Utils.base64urlencode Utils.b2s(decrypted_t)[0..42]
     end
 
-    def encrypt_attributes key, attributes_hash
+    def encrypt_attributes(key, attributes_hash)
       a32key = key.dup
       if a32key.size > 4
         a32key = [a32key[0] ^ a32key[4], a32key[1] ^ a32key[5], a32key[2] ^ a32key[6], a32key[3] ^ a32key[7]]
@@ -50,7 +55,7 @@ module Rmega
       Crypto::Aes.encrypt a32key, Utils.str_to_a32(attributes_str)
     end
 
-    def decrypt_attributes key, attributes_base64
+    def decrypt_attributes(key, attributes_base64)
       a32key = key.dup
       if a32key.size > 4
         a32key = [a32key[0] ^ a32key[4], a32key[1] ^ a32key[5], a32key[2] ^ a32key[6], a32key[3] ^ a32key[7]]
@@ -60,11 +65,11 @@ module Rmega
       JSON.parse attributes.gsub(/^MEGA/, '').rstrip
     end
 
-    def prepare_key_pw password_str
+    def prepare_key_pw(password_str)
       prepare_key Utils.str_to_a32(password_str)
     end
 
-    def stringhash aes_key, string
+    def stringhash(aes_key, string)
       s32 = Utils::str_to_a32 string
       h32 = [0,0,0,0]
 
@@ -74,7 +79,7 @@ module Rmega
       Utils::a32_to_base64 [h32[0],h32[2]]
     end
 
-    def encrypt_key key, data
+    def encrypt_key(key, data)
       return Aes.encrypt(key, data) if data.size == 4
       x = []
       (0..data.size).step(4) do |i|
@@ -85,7 +90,7 @@ module Rmega
       x
     end
 
-    def decrypt_key key, data
+    def decrypt_key(key, data)
       return Aes.decrypt(key, data) if data.size == 4
       x = []
       (0..data.size).step(4) do |i|

data/lib/rmega/crypto/rsa.rb

@@ -1,3 +1,5 @@
+require 'execjs'
+
 module Rmega
   module Crypto
     module Rsa
@@ -11,7 +13,7 @@ module Rmega
         @context ||= ExecJS.compile File.read(script_path)
       end
 
-      def decrypt t, privk
+      def decrypt(t, privk)
         context.call "RSAdecrypt", t, privk[2], privk[0], privk[1], privk[3]
       end
     end

data/lib/rmega/downloader.rb

@@ -1,3 +1,8 @@
+require 'rmega/loggable'
+require 'rmega/utils'
+require 'rmega/pool'
+require 'rmega/progress'
+
 module Rmega
   class Downloader
     include Loggable
@@ -5,7 +10,7 @@ module Rmega
     attr_reader :pool, :base_url, :filesize, :local_path
 
     def initialize(params)
-      @pool = Thread.pool(params[:threads] || 5)
+      @pool = Pool.new(params[:threads])
       @filesize = params[:filesize]
       @base_url = params[:base_url]
       @local_path = params[:local_path]
@@ -17,7 +22,7 @@ module Rmega
       `dd if=/dev/zero of="#{local_path}" bs=1 count=0 seek=#{filesize} > /dev/null 2>&1`
       raise "Unable to create file #{local_path}" if File.size(local_path) != filesize
 
-      File.open(local_path, 'r+b').tap { |f| f.rewind }
+      ::File.open(local_path, 'r+b').tap { |f| f.rewind }
     end
 
     # Downloads a part of the remote file, starting from the start-n byte
@@ -25,38 +30,30 @@ module Rmega
     def download_chunk(start, size)
       stop = start + size - 1
       url = "#{base_url}/#{start}-#{stop}"
-      # logger.debug "#{Thread.current} downloading chunk @ #{start}"
       HTTPClient.new.get_content(url)
     end
 
     # Writes a buffer in the local file, starting from the start-n byte.
     def write_chunk(start, buffer)
-      # logger.debug "#{Thread.current} writing chunk @ #{position}"
       @local_file.seek(start)
       @local_file.write(buffer)
     end
 
-    # Shows the progress bar in console
-    def show_progress(increment)
-      Utils.show_progress(:download, filesize, increment)
-    end
-
     def chunks
-      Storage.chunks(filesize)
+      Utils.chunks(filesize)
     end
 
-    # TODO: checksum check
     def download(&block)
       @local_file = allocate
 
-      show_progress(0)
+      progress = Progress.new(filesize: filesize, verb: 'download')
 
       chunks.each do |start, size|
         pool.defer do
-          buffer = download_chunk(start, size)
-          buffer = yield(start, buffer) if block_given?
-          show_progress(size)
-          pool.synchronize { write_chunk(start, buffer) }
+          encrypted_buffer = download_chunk(start, size)
+          clean_buffer = yield(start, encrypted_buffer)
+          progress.increment(size)
+          pool.synchronize { write_chunk(start, clean_buffer) }
         end
       end
 

data/lib/rmega/loggable.rb

@@ -1,11 +1,18 @@
+require 'logger'
+
 module Rmega
   module Loggable
     def logger
-      Rmega.logger
+      @@logger ||= begin
+        Logger.new($stdout).tap do |l|
+          l.formatter = Proc.new { | severity, time, progname, msg| "#{msg}\n" }
+          l.level = Logger::ERROR
+        end
+      end
     end
 
     def self.included(base)
-      base.send :extend, self
+      base.send(:extend, self)
     end
   end
 end

data/lib/rmega/nodes/deletable.rb

@@ -0,0 +1,16 @@
+require 'rmega/utils'
+require 'rmega/crypto/crypto'
+
+module Rmega
+  module Nodes
+    module Deletable
+      def delete
+        request(a: 'd', n: handle)
+      end
+
+      def trash
+        request(a: 'm', n: handle, t: storage.trash.handle)
+      end
+    end
+  end
+end

data/lib/rmega/nodes/expandable.rb

@@ -0,0 +1,64 @@
+require 'rmega/utils'
+require 'rmega/uploader'
+require 'rmega/crypto/crypto'
+
+module Rmega
+  module Nodes
+    module Expandable
+      def create_folder(name)
+        key = Crypto.random_key
+        encrypted_attributes = Utils.a32_to_base64 Crypto.encrypt_attributes(key[0..3], {n: name.strip})
+        encrypted_key = Utils.a32_to_base64 Crypto.encrypt_key(session.master_key, key)
+        n = [{h: 'xxxxxxxx', t: 1, a: encrypted_attributes, k: encrypted_key}]
+        data = session.request a: 'p', t: parent_handle, n: n
+        Folder.new(session, data['f'][0])
+      end
+
+      def upload_url(filesize)
+        session.request(a: 'u', s: filesize)['p']
+      end
+
+      def upload(local_path)
+        local_path = ::File.expand_path(local_path)
+        filesize = ::File.size(local_path)
+
+        ul_key = Crypto.random_key
+        aes_key = ul_key[0..3]
+        nonce = ul_key[4..5]
+
+        file_mac = [0, 0, 0, 0]
+
+        uploader = Uploader.new(filesize: filesize, base_url: upload_url(filesize), local_path: local_path)
+
+        uploader.upload do |start, clean_buffer|
+          # TODO: should be (chunk_start/0x1000000000) >>> 0, (chunk_start/0x10) >>> 0
+          nonce = [nonce[0], nonce[1], (start/0x1000000000) >> 0, (start/0x10) >> 0]
+
+          encrypted = Crypto::AesCtr.encrypt(aes_key, nonce, clean_buffer)
+          chunk_mac = encrypted[:mac]
+
+          file_mac = [file_mac[0] ^ chunk_mac[0], file_mac[1] ^ chunk_mac[1],
+                      file_mac[2] ^ chunk_mac[2], file_mac[3] ^ chunk_mac[3]]
+          file_mac = Crypto::Aes.encrypt(ul_key[0..3], file_mac)
+
+          encrypted[:data]
+        end
+
+        file_handle = uploader.last_result
+
+        attribs = {n: ::File.basename(local_path)}
+        encrypt_attribs = Utils.a32_to_base64 Crypto.encrypt_attributes(ul_key[0..3], attribs)
+
+        meta_mac = [file_mac[0] ^ file_mac[1], file_mac[2] ^ file_mac[3]]
+
+        key = [ul_key[0] ^ ul_key[4], ul_key[1] ^ ul_key[5], ul_key[2] ^ meta_mac[0],
+               ul_key[3] ^ meta_mac[1], ul_key[4], ul_key[5], meta_mac[0], meta_mac[1]]
+
+        encrypted_key = Utils.a32_to_base64 Crypto.encrypt_key(session.master_key, key)
+        session.request a: 'p', t: handle, n: [{h: file_handle, t: 0, a: encrypt_attribs, k: encrypted_key}]
+
+        attribs[:n]
+      end
+    end
+  end
+end

data/lib/rmega/nodes/factory.rb

@@ -0,0 +1,41 @@
+require 'rmega/nodes/node'
+require 'rmega/nodes/file'
+require 'rmega/nodes/folder'
+require 'rmega/nodes/inbox'
+require 'rmega/nodes/root'
+require 'rmega/nodes/trash'
+
+module Rmega
+  module Nodes
+    module Factory
+      extend self
+
+      def build(session, data)
+        type_name = type(data['t'])
+        node_class = Nodes.const_get("#{type_name.to_s.capitalize}")
+        node_class.new(session, data)
+      end
+
+      # TODO: support other node types than File
+      def build_from_url(session, url)
+        public_handle, key = url.strip.split('!')[1, 2]
+        data = session.request(a: 'g', g: 1, p: public_handle)
+
+        Nodes::File.new(session, data).tap { |n| n.public_url = url }
+      end
+
+      def mega_url?(url)
+        !!(url.to_s =~ /^https:\/\/mega\.co\.nz\/#!.*$/i)
+      end
+
+      def type(number)
+        founded_type = types.find { |k, v| number == v }
+        founded_type.first if founded_type
+      end
+
+      def types
+        {file: 0, folder: 1, root: 2, inbox: 3, trash: 4}
+      end
+    end
+  end
+end

data/lib/rmega/nodes/file.rb

@@ -0,0 +1,39 @@
+require 'rmega/downloader'
+require 'rmega/nodes/node'
+require 'rmega/nodes/deletable'
+
+module Rmega
+  module Nodes
+    class File < Node
+      include Deletable
+
+      def storage_url
+        @storage_url ||= data['g'] || request(a: 'g', g: 1, n: handle)['g']
+      end
+
+      def size
+        data['s']
+      end
+
+      def download(path)
+        path = ::File.expand_path(path)
+        path = Dir.exists?(path) ? ::File.join(path, name) : path
+
+        logger.info "Download #{name} (#{size} bytes) => #{path}"
+
+        k = decrypted_file_key
+        k = [k[0] ^ k[4], k[1] ^ k[5], k[2] ^ k[6], k[3] ^ k[7]]
+        nonce = decrypted_file_key[4..5]
+
+        donwloader = Downloader.new(base_url: storage_url, filesize: size, local_path: path)
+
+        donwloader.download do |start, buffer|
+          nonce = [nonce[0], nonce[1], (start/0x1000000000) >> 0, (start/0x10) >> 0]
+          Crypto::AesCtr.decrypt(k, nonce, buffer)[:data]
+        end
+
+        path
+      end
+    end
+  end
+end

data/lib/rmega/nodes/folder.rb

@@ -0,0 +1,30 @@
+require 'rmega/crypto/crypto'
+require 'rmega/utils'
+require 'rmega/nodes/node'
+require 'rmega/nodes/expandable'
+require 'rmega/nodes/traversable'
+require 'rmega/nodes/deletable'
+
+module Rmega
+  module Nodes
+    class Folder < Node
+      include Expandable
+      include Traversable
+      include Deletable
+
+      def download(path)
+        children.each do |node|
+          if node.type == :file
+            node.download path
+          elsif node.type == :folder
+            subfolder = ::File.expand_path ::File.join(path, node.name)
+            Dir.mkdir(subfolder) unless Dir.exists?(subfolder)
+            node.download subfolder
+          end
+        end
+
+        nil
+      end
+    end
+  end
+end

data/lib/rmega/nodes/inbox.rb

@@ -0,0 +1,8 @@
+require 'rmega/nodes/node'
+
+module Rmega
+  module Nodes
+    class Inbox < Node
+    end
+  end
+end