rmega 0.0.5 → 0.0.6

data/README.md

@@ -1,18 +1,16 @@
-# Rmega [![Build Status](https://travis-ci.org/daniele-m/rmega.png)](https://travis-ci.org/daniele-m/rmega)
+# Rmega
 
+A ruby library for the Mega.co.nz.  
+Tested using ruby 1.9.3+ (OpenSSL 0.9.8r+)  
+This work is the result of a reverse engineering of the Mega's Javascript code.
 
-Ruby library for the Mega.co.nz API.  
-Tested using ruby 1.9.3+ (OpenSSL 0.9.8r+)
-
-<div style="background-color: #000000; border-radius: 8px">
-  <img src="https://eu.static.mega.co.nz/images/mega/logo.png" />
-</div>
-
+## Installation
 
+  Rmega is distributed via rubygems, so if you have ruby 1.9.3+ installed
+  system wide, just type `gem install rmega`.
 
 ## Usage
 
-    $ gem install rmega
     $ irb -r rmega
 
 ### Login and retrive all the files and folders
@@ -66,8 +64,16 @@ my_node.public_url
 
 # See the attributes of a node
 my_node.attributes
+
+# Create a folder
+parent_folder = storage.nodes_by_name(/photos/i).first
+folder_node = storage.create_folder parent_folder, "london"
 ```
 
+## Todo
+
+  * Handle connection errors during upload/download
+
 
 ## Installation
 
@@ -95,7 +101,5 @@ Or install it yourself as:
 
 ## Copyright
 
-This work is the result of a reverse engineering of the Mega's Javascript code.
-
 Copyright (c) 2013 Daniele Molteni  
 MIT License

data/lib/rmega/downloader.rb

@@ -0,0 +1,73 @@
+module Rmega
+  class Downloader
+    include Loggable
+
+    attr_reader :pool, :base_url, :filesize, :local_path
+
+    def initialize(params)
+      @pool = Thread.pool(params[:threads] || 5)
+      @filesize = params[:filesize]
+      @base_url = params[:base_url]
+      @local_path = params[:local_path]
+    end
+
+    # Creates the local file allocating filesize-n bytes (of /dev/zero) for it.
+    # Opens the local file to start writing from the beginning of it.
+    def allocate
+      `dd if=/dev/zero of="#{local_path}" bs=1 count=0 seek=#{filesize} > /dev/null 2>&1`
+      raise "Unable to create file #{local_path}" if File.size(local_path) != filesize
+
+      File.open(local_path, 'r+b').tap { |f| f.rewind }
+    end
+
+    # Downloads a part of the remote file, starting from the start-n byte
+    # and ending after size-n bytes.
+    def download_chunk(start, size)
+      stop = start + size - 1
+      url = "#{base_url}/#{start}-#{stop}"
+      # logger.debug "#{Thread.current} downloading chunk @ #{start}"
+      HTTPClient.new.get_content(url)
+    end
+
+    # Writes a buffer in the local file, starting from the start-n byte.
+    def write_chunk(start, buffer)
+      # logger.debug "#{Thread.current} writing chunk @ #{position}"
+      @local_file.seek(start)
+      @local_file.write(buffer)
+    end
+
+    # Shows the progress bar in console
+    def show_progress(increment)
+      Utils.show_progress(:download, filesize, increment)
+    end
+
+    def chunks
+      Storage.chunks(filesize)
+    end
+
+    # TODO: checksum check
+    def download(&block)
+      @local_file = allocate
+
+      show_progress(0)
+
+      chunks.each do |start, size|
+        pool.defer do
+          buffer = download_chunk(start, size)
+          buffer = yield(start, buffer) if block_given?
+          show_progress(size)
+          pool.synchronize { write_chunk(start, buffer) }
+        end
+      end
+
+      # waits for the last running threads to finish
+      pool.wait_done
+
+      @local_file.flush
+
+      pool.shutdown
+    ensure
+      @local_file.close rescue nil
+    end
+  end
+end

data/lib/rmega/loggable.rb

@@ -0,0 +1,11 @@
+module Rmega
+  module Loggable
+    def logger
+      Rmega.logger
+    end
+
+    def self.included(base)
+      base.send :extend, self
+    end
+  end
+end

data/lib/rmega/nodes/file_node.rb

@@ -1,41 +1,27 @@
+require_relative '../downloader'
+
 module Rmega
   class FileNode < Node
     def storage_url
       @storage_url ||= data['g'] || request(a: 'g', g: 1, n: handle)['g']
     end
 
-    def chunks
-      Storage.chunks filesize
-    end
-
-    def download path
-      path = File.expand_path path
+    def download(path)
+      path = File.expand_path(path)
       path = Dir.exists?(path) ? File.join(path, name) : path
 
-      logger.info "Starting download into #{path}"
-
-      Utils.show_progress :download, filesize
+      logger.info "Download #{name} (#{filesize} bytes) => #{path}"
 
       k = decrypted_file_key
       k = [k[0] ^ k[4], k[1] ^ k[5], k[2] ^ k[6], k[3] ^ k[7]]
       nonce = decrypted_file_key[4..5]
 
-      file = File.open path, 'wb'
-      connection = HTTPClient.new.get_async storage_url
-      message = connection.pop
+      donwloader = Downloader.new(base_url: storage_url, filesize: filesize, local_path: path)
 
-      chunks.each do |chunk_start, chunk_size|
-        buffer = message.content.read chunk_size
-        # TODO: should be (chunk_start/0x1000000000) >>> 0, (chunk_start/0x10) >>> 0
-        nonce = [nonce[0], nonce[1], (chunk_start/0x1000000000) >> 0, (chunk_start/0x10) >> 0]
-        decryption_result = Crypto::AesCtr.decrypt(k, nonce, buffer)
-        file.write(decryption_result[:data])
-        Utils.show_progress :download, filesize, chunk_size
+      donwloader.download do |start, buffer|
+        nonce = [nonce[0], nonce[1], (start/0x1000000000) >> 0, (start/0x10) >> 0]
+        Crypto::AesCtr.decrypt(k, nonce, buffer)[:data]
       end
-
-      nil
-    ensure
-      file.close if file
     end
   end
 end

data/lib/rmega/nodes/node.rb

@@ -4,24 +4,22 @@ module Rmega
 
     def initialize session, data
       @session = session
-      @data = data
+
+      if self.class.mega_url?(data)
+        @data = self.class.public_data(session, data)
+        @public_url = data
+      else
+        @data = data
+      end
     end
 
     def self.fabricate session, data
-      type_name = type_by_number(data['t']).to_s
+      type_name = mega_url?(data) ? :file : type_by_number(data['t'])
       node_class = Rmega.const_get("#{type_name}_node".camelize) rescue nil
       node_class ||= Rmega::Node
       node_class.new session, data
     end
 
-    def self.initialize_by_public_url session, public_url
-      public_handle, key = public_url.split('!')[1, 2]
-
-      node = new session, public_data(session, public_handle)
-      node.instance_variable_set '@public_url', public_url
-      node
-    end
-
     def self.types
       {file: 0, folder: 1, root: 2, inbox: 3, trash: 4}
     end
@@ -31,6 +29,10 @@ module Rmega
       founded_type.first if founded_type
     end
 
+    def self.mega_url? url
+      !!(url.to_s =~ /^https:\/\/mega\.co\.nz\/#!.*$/i)
+    end
+
     def logger
       Rmega.logger
     end
@@ -57,8 +59,9 @@ module Rmega
 
     # Other methods
 
-    def self.public_data session, public_handle
-      request a: 'g', g: 1, p: public_handle
+    def self.public_data session, public_url
+      public_handle, key = public_url.strip.split('!')[1, 2]
+      session.request a: 'g', g: 1, p: public_handle
     end
 
     def public_handle

data/lib/rmega/pool.rb

@@ -0,0 +1,66 @@
+require 'thread'
+
+class Thread
+  # Helper to create a Pool instance.
+  def self.pool(max)
+    Pool.new(max)
+  end
+
+  class Pool
+    def initialize(max)
+      Thread.abort_on_exception = true
+      @mutex = Mutex.new
+      @threads = Array.new(max)
+    end
+
+    # Gets the first position of the pool in which
+    # a thread could be started.
+    def available_slot
+      @threads.each_with_index do |thread, index|
+        return index if thread.nil? or !thread.alive?
+      end
+      nil
+    end
+
+    def synchronize(&block)
+      @mutex.synchronize(&block)
+    end
+
+    # Returns true if all the threads are finished,
+    # false otherwise.
+    def done?
+      @threads.each { |thread| return false if thread and thread.alive? }
+      true
+    end
+
+    # Blocking. Waits until all the threads are finished.
+    def wait_done
+      sleep 0.01 until done?
+    end
+
+    # Blocking. Waits until a pool's slot become available and
+    # returns that position.
+    # TODO: raise an error on wait timeout.
+    def wait_available_slot
+      while true
+        index = available_slot
+        return index if index
+        sleep 0.01
+      end
+    end
+
+    # Sends a KILL signal to all the threads.
+    def shutdown
+      @threads.each { |thread| thread.kill if thread.respond_to?(:kill) }
+      @threads.map! { nil }
+    end
+
+    # Blocking. Starts a new thread with the given block when a pool's slot
+    # become available.
+    def defer(&block)
+      index = wait_available_slot
+      @threads[index].kill if @threads[index].respond_to?(:kill)
+      @threads[index] = Thread.new(&block)
+    end
+  end
+end

data/lib/rmega/storage.rb

@@ -85,7 +85,7 @@ module Rmega
     end
 
     def download public_url, path
-      Node.initialize_by_public_url(session, public_url).download path
+      Node.fabricate(session, public_url).download(path)
     end
 
 

data/lib/rmega/utils.rb

@@ -4,7 +4,10 @@ module Rmega
 
     def show_progress direction, total, increment = 0
       return unless Rmega.options.show_progress
-      @progressbar_progress = 0 if increment.zero?
+      if increment.zero?
+        @progressbar = nil
+        @progressbar_progress = 0
+      end
       @progressbar_progress += increment
       format = "#{direction.to_s.capitalize} in progress #{Utils.format_bytes(@progressbar_progress)} of #{Utils.format_bytes(total)} | %P% | %e        "
       @progressbar ||= ProgressBar.create format: format, total: total

data/lib/rmega/version.rb

@@ -1,3 +1,3 @@
 module Rmega
-  VERSION = "0.0.5"
+  VERSION = "0.0.6"
 end

data/lib/rmega.rb

@@ -13,6 +13,7 @@ require "ruby-progressbar"
 
 # Require all the other files
 require "rmega/version"
+require "rmega/loggable"
 require "rmega/utils"
 require "rmega/crypto/rsa"
 require "rmega/crypto/aes"
@@ -24,6 +25,8 @@ require "rmega/nodes/file_node"
 require "rmega/nodes/folder_node"
 require "rmega/session"
 require "rmega/api_request_error"
+require "rmega/pool"
+require "rmega/downloader"
 
 module Rmega
   def self.logger

data/spec/integration/file_operations_spec.rb

@@ -0,0 +1,49 @@
+require 'integration_spec_helper'
+require 'fileutils'
+
+describe 'File operations' do
+
+  if account_file_exists?
+
+    let(:temp_folder) { "/tmp/.rmega_spec" }
+
+    before { FileUtils.mkdir_p(temp_folder) }
+
+    after { FileUtils.rm_rf(temp_folder) }
+
+    context 'give a public mega url, related to a small file' do
+
+      # A file called testfile.txt containting the string "helloworld!"
+      let(:url) { 'https://mega.co.nz/#!MAkg2Iab!bc9Y2U6d93IlRRKVYpcC9hLZjS4G278OPdH6nTFPDNQ' }
+
+      it 'downloads the related file' do
+        storage.download(url, temp_folder)
+        related_file = File.join(temp_folder, 'testfile.txt')
+        expect(File.read(related_file)).to eq "helloworld!\n"
+      end
+    end
+
+    context 'give a public mega url, related to a big file' do
+
+      # A file called testfile_big_15mb.txt containting the word "topac" repeated 3145728 times (~ 15mb)
+      let(:url) { 'https://mega.co.nz/#!NYVkDaLD!BKyN5SRpOaEtGnTcwiAqcxmJc7p-k0IPWKAW-471KRE' }
+
+      it 'downloads the related file' do
+        storage.download(url, temp_folder)
+        related_file = File.join(temp_folder, 'testfile_big_15mb.txt')
+
+        expect(File.size(related_file)).to eql 15_728_640
+
+        count = 0
+        File.open(related_file, 'rb') do |f|
+          while (word = f.read(3840))
+            break if word != "topac"*768
+            count += 768
+          end
+        end
+
+        expect(count).to eql(15_728_640 / 5)
+      end
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rmega
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-06 00:00:00.000000000 Z
+date: 2013-07-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pry
@@ -142,17 +142,20 @@ files:
 - lib/rmega/crypto/crypto.rb
 - lib/rmega/crypto/rsa.rb
 - lib/rmega/crypto/rsa_mega.js
+- lib/rmega/downloader.rb
+- lib/rmega/loggable.rb
 - lib/rmega/nodes/file_node.rb
 - lib/rmega/nodes/folder_node.rb
 - lib/rmega/nodes/node.rb
+- lib/rmega/pool.rb
 - lib/rmega/session.rb
 - lib/rmega/storage.rb
 - lib/rmega/utils.rb
 - lib/rmega/version.rb
 - rmega.gemspec
+- spec/integration/file_operations_spec.rb
 - spec/integration/folder_operations_spec.rb
 - spec/integration/login_spec.rb
-- spec/integration/rmega_account.yml.example
 - spec/integration_spec_helper.rb
 - spec/rmega/lib/crypto/aes_spec.rb
 - spec/rmega/lib/crypto/crypto_spec.rb
@@ -183,9 +186,9 @@ signing_key:
 specification_version: 3
 summary: mega.co.nz ruby api
 test_files:
+- spec/integration/file_operations_spec.rb
 - spec/integration/folder_operations_spec.rb
 - spec/integration/login_spec.rb
-- spec/integration/rmega_account.yml.example
 - spec/integration_spec_helper.rb
 - spec/rmega/lib/crypto/aes_spec.rb
 - spec/rmega/lib/crypto/crypto_spec.rb

data/spec/integration/rmega_account.yml.example

@@ -1,2 +0,0 @@
-email: your_email@email.com
-password: your_password