rmega 0.0.5 → 0.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/README.md CHANGED
@@ -1,18 +1,16 @@
1
- # Rmega [![Build Status](https://travis-ci.org/daniele-m/rmega.png)](https://travis-ci.org/daniele-m/rmega)
1
+ # Rmega
2
2
 
3
+ A ruby library for the Mega.co.nz.
4
+ Tested using ruby 1.9.3+ (OpenSSL 0.9.8r+)
5
+ This work is the result of a reverse engineering of the Mega's Javascript code.
3
6
 
4
- Ruby library for the Mega.co.nz API.
5
- Tested using ruby 1.9.3+ (OpenSSL 0.9.8r+)
6
-
7
- <div style="background-color: #000000; border-radius: 8px">
8
- <img src="https://eu.static.mega.co.nz/images/mega/logo.png" />
9
- </div>
10
-
7
+ ## Installation
11
8
 
9
+ Rmega is distributed via rubygems, so if you have ruby 1.9.3+ installed
10
+ system wide, just type `gem install rmega`.
12
11
 
13
12
  ## Usage
14
13
 
15
- $ gem install rmega
16
14
  $ irb -r rmega
17
15
 
18
16
  ### Login and retrive all the files and folders
@@ -66,8 +64,16 @@ my_node.public_url
66
64
 
67
65
  # See the attributes of a node
68
66
  my_node.attributes
67
+
68
+ # Create a folder
69
+ parent_folder = storage.nodes_by_name(/photos/i).first
70
+ folder_node = storage.create_folder parent_folder, "london"
69
71
  ```
70
72
 
73
+ ## Todo
74
+
75
+ * Handle connection errors during upload/download
76
+
71
77
 
72
78
  ## Installation
73
79
 
@@ -95,7 +101,5 @@ Or install it yourself as:
95
101
 
96
102
  ## Copyright
97
103
 
98
- This work is the result of a reverse engineering of the Mega's Javascript code.
99
-
100
104
  Copyright (c) 2013 Daniele Molteni
101
105
  MIT License
@@ -0,0 +1,73 @@
1
+ module Rmega
2
+ class Downloader
3
+ include Loggable
4
+
5
+ attr_reader :pool, :base_url, :filesize, :local_path
6
+
7
+ def initialize(params)
8
+ @pool = Thread.pool(params[:threads] || 5)
9
+ @filesize = params[:filesize]
10
+ @base_url = params[:base_url]
11
+ @local_path = params[:local_path]
12
+ end
13
+
14
+ # Creates the local file allocating filesize-n bytes (of /dev/zero) for it.
15
+ # Opens the local file to start writing from the beginning of it.
16
+ def allocate
17
+ `dd if=/dev/zero of="#{local_path}" bs=1 count=0 seek=#{filesize} > /dev/null 2>&1`
18
+ raise "Unable to create file #{local_path}" if File.size(local_path) != filesize
19
+
20
+ File.open(local_path, 'r+b').tap { |f| f.rewind }
21
+ end
22
+
23
+ # Downloads a part of the remote file, starting from the start-n byte
24
+ # and ending after size-n bytes.
25
+ def download_chunk(start, size)
26
+ stop = start + size - 1
27
+ url = "#{base_url}/#{start}-#{stop}"
28
+ # logger.debug "#{Thread.current} downloading chunk @ #{start}"
29
+ HTTPClient.new.get_content(url)
30
+ end
31
+
32
+ # Writes a buffer in the local file, starting from the start-n byte.
33
+ def write_chunk(start, buffer)
34
+ # logger.debug "#{Thread.current} writing chunk @ #{position}"
35
+ @local_file.seek(start)
36
+ @local_file.write(buffer)
37
+ end
38
+
39
+ # Shows the progress bar in console
40
+ def show_progress(increment)
41
+ Utils.show_progress(:download, filesize, increment)
42
+ end
43
+
44
+ def chunks
45
+ Storage.chunks(filesize)
46
+ end
47
+
48
+ # TODO: checksum check
49
+ def download(&block)
50
+ @local_file = allocate
51
+
52
+ show_progress(0)
53
+
54
+ chunks.each do |start, size|
55
+ pool.defer do
56
+ buffer = download_chunk(start, size)
57
+ buffer = yield(start, buffer) if block_given?
58
+ show_progress(size)
59
+ pool.synchronize { write_chunk(start, buffer) }
60
+ end
61
+ end
62
+
63
+ # waits for the last running threads to finish
64
+ pool.wait_done
65
+
66
+ @local_file.flush
67
+
68
+ pool.shutdown
69
+ ensure
70
+ @local_file.close rescue nil
71
+ end
72
+ end
73
+ end
@@ -0,0 +1,11 @@
1
+ module Rmega
2
+ module Loggable
3
+ def logger
4
+ Rmega.logger
5
+ end
6
+
7
+ def self.included(base)
8
+ base.send :extend, self
9
+ end
10
+ end
11
+ end
@@ -1,41 +1,27 @@
1
+ require_relative '../downloader'
2
+
1
3
  module Rmega
2
4
  class FileNode < Node
3
5
  def storage_url
4
6
  @storage_url ||= data['g'] || request(a: 'g', g: 1, n: handle)['g']
5
7
  end
6
8
 
7
- def chunks
8
- Storage.chunks filesize
9
- end
10
-
11
- def download path
12
- path = File.expand_path path
9
+ def download(path)
10
+ path = File.expand_path(path)
13
11
  path = Dir.exists?(path) ? File.join(path, name) : path
14
12
 
15
- logger.info "Starting download into #{path}"
16
-
17
- Utils.show_progress :download, filesize
13
+ logger.info "Download #{name} (#{filesize} bytes) => #{path}"
18
14
 
19
15
  k = decrypted_file_key
20
16
  k = [k[0] ^ k[4], k[1] ^ k[5], k[2] ^ k[6], k[3] ^ k[7]]
21
17
  nonce = decrypted_file_key[4..5]
22
18
 
23
- file = File.open path, 'wb'
24
- connection = HTTPClient.new.get_async storage_url
25
- message = connection.pop
19
+ donwloader = Downloader.new(base_url: storage_url, filesize: filesize, local_path: path)
26
20
 
27
- chunks.each do |chunk_start, chunk_size|
28
- buffer = message.content.read chunk_size
29
- # TODO: should be (chunk_start/0x1000000000) >>> 0, (chunk_start/0x10) >>> 0
30
- nonce = [nonce[0], nonce[1], (chunk_start/0x1000000000) >> 0, (chunk_start/0x10) >> 0]
31
- decryption_result = Crypto::AesCtr.decrypt(k, nonce, buffer)
32
- file.write(decryption_result[:data])
33
- Utils.show_progress :download, filesize, chunk_size
21
+ donwloader.download do |start, buffer|
22
+ nonce = [nonce[0], nonce[1], (start/0x1000000000) >> 0, (start/0x10) >> 0]
23
+ Crypto::AesCtr.decrypt(k, nonce, buffer)[:data]
34
24
  end
35
-
36
- nil
37
- ensure
38
- file.close if file
39
25
  end
40
26
  end
41
27
  end
@@ -4,24 +4,22 @@ module Rmega
4
4
 
5
5
  def initialize session, data
6
6
  @session = session
7
- @data = data
7
+
8
+ if self.class.mega_url?(data)
9
+ @data = self.class.public_data(session, data)
10
+ @public_url = data
11
+ else
12
+ @data = data
13
+ end
8
14
  end
9
15
 
10
16
  def self.fabricate session, data
11
- type_name = type_by_number(data['t']).to_s
17
+ type_name = mega_url?(data) ? :file : type_by_number(data['t'])
12
18
  node_class = Rmega.const_get("#{type_name}_node".camelize) rescue nil
13
19
  node_class ||= Rmega::Node
14
20
  node_class.new session, data
15
21
  end
16
22
 
17
- def self.initialize_by_public_url session, public_url
18
- public_handle, key = public_url.split('!')[1, 2]
19
-
20
- node = new session, public_data(session, public_handle)
21
- node.instance_variable_set '@public_url', public_url
22
- node
23
- end
24
-
25
23
  def self.types
26
24
  {file: 0, folder: 1, root: 2, inbox: 3, trash: 4}
27
25
  end
@@ -31,6 +29,10 @@ module Rmega
31
29
  founded_type.first if founded_type
32
30
  end
33
31
 
32
+ def self.mega_url? url
33
+ !!(url.to_s =~ /^https:\/\/mega\.co\.nz\/#!.*$/i)
34
+ end
35
+
34
36
  def logger
35
37
  Rmega.logger
36
38
  end
@@ -57,8 +59,9 @@ module Rmega
57
59
 
58
60
  # Other methods
59
61
 
60
- def self.public_data session, public_handle
61
- request a: 'g', g: 1, p: public_handle
62
+ def self.public_data session, public_url
63
+ public_handle, key = public_url.strip.split('!')[1, 2]
64
+ session.request a: 'g', g: 1, p: public_handle
62
65
  end
63
66
 
64
67
  def public_handle
data/lib/rmega/pool.rb ADDED
@@ -0,0 +1,66 @@
1
+ require 'thread'
2
+
3
+ class Thread
4
+ # Helper to create a Pool instance.
5
+ def self.pool(max)
6
+ Pool.new(max)
7
+ end
8
+
9
+ class Pool
10
+ def initialize(max)
11
+ Thread.abort_on_exception = true
12
+ @mutex = Mutex.new
13
+ @threads = Array.new(max)
14
+ end
15
+
16
+ # Gets the first position of the pool in which
17
+ # a thread could be started.
18
+ def available_slot
19
+ @threads.each_with_index do |thread, index|
20
+ return index if thread.nil? or !thread.alive?
21
+ end
22
+ nil
23
+ end
24
+
25
+ def synchronize(&block)
26
+ @mutex.synchronize(&block)
27
+ end
28
+
29
+ # Returns true if all the threads are finished,
30
+ # false otherwise.
31
+ def done?
32
+ @threads.each { |thread| return false if thread and thread.alive? }
33
+ true
34
+ end
35
+
36
+ # Blocking. Waits until all the threads are finished.
37
+ def wait_done
38
+ sleep 0.01 until done?
39
+ end
40
+
41
+ # Blocking. Waits until a pool's slot become available and
42
+ # returns that position.
43
+ # TODO: raise an error on wait timeout.
44
+ def wait_available_slot
45
+ while true
46
+ index = available_slot
47
+ return index if index
48
+ sleep 0.01
49
+ end
50
+ end
51
+
52
+ # Sends a KILL signal to all the threads.
53
+ def shutdown
54
+ @threads.each { |thread| thread.kill if thread.respond_to?(:kill) }
55
+ @threads.map! { nil }
56
+ end
57
+
58
+ # Blocking. Starts a new thread with the given block when a pool's slot
59
+ # become available.
60
+ def defer(&block)
61
+ index = wait_available_slot
62
+ @threads[index].kill if @threads[index].respond_to?(:kill)
63
+ @threads[index] = Thread.new(&block)
64
+ end
65
+ end
66
+ end
data/lib/rmega/storage.rb CHANGED
@@ -85,7 +85,7 @@ module Rmega
85
85
  end
86
86
 
87
87
  def download public_url, path
88
- Node.initialize_by_public_url(session, public_url).download path
88
+ Node.fabricate(session, public_url).download(path)
89
89
  end
90
90
 
91
91
 
data/lib/rmega/utils.rb CHANGED
@@ -4,7 +4,10 @@ module Rmega
4
4
 
5
5
  def show_progress direction, total, increment = 0
6
6
  return unless Rmega.options.show_progress
7
- @progressbar_progress = 0 if increment.zero?
7
+ if increment.zero?
8
+ @progressbar = nil
9
+ @progressbar_progress = 0
10
+ end
8
11
  @progressbar_progress += increment
9
12
  format = "#{direction.to_s.capitalize} in progress #{Utils.format_bytes(@progressbar_progress)} of #{Utils.format_bytes(total)} | %P% | %e "
10
13
  @progressbar ||= ProgressBar.create format: format, total: total
data/lib/rmega/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module Rmega
2
- VERSION = "0.0.5"
2
+ VERSION = "0.0.6"
3
3
  end
data/lib/rmega.rb CHANGED
@@ -13,6 +13,7 @@ require "ruby-progressbar"
13
13
 
14
14
  # Require all the other files
15
15
  require "rmega/version"
16
+ require "rmega/loggable"
16
17
  require "rmega/utils"
17
18
  require "rmega/crypto/rsa"
18
19
  require "rmega/crypto/aes"
@@ -24,6 +25,8 @@ require "rmega/nodes/file_node"
24
25
  require "rmega/nodes/folder_node"
25
26
  require "rmega/session"
26
27
  require "rmega/api_request_error"
28
+ require "rmega/pool"
29
+ require "rmega/downloader"
27
30
 
28
31
  module Rmega
29
32
  def self.logger
@@ -0,0 +1,49 @@
1
+ require 'integration_spec_helper'
2
+ require 'fileutils'
3
+
4
+ describe 'File operations' do
5
+
6
+ if account_file_exists?
7
+
8
+ let(:temp_folder) { "/tmp/.rmega_spec" }
9
+
10
+ before { FileUtils.mkdir_p(temp_folder) }
11
+
12
+ after { FileUtils.rm_rf(temp_folder) }
13
+
14
+ context 'give a public mega url, related to a small file' do
15
+
16
+ # A file called testfile.txt containting the string "helloworld!"
17
+ let(:url) { 'https://mega.co.nz/#!MAkg2Iab!bc9Y2U6d93IlRRKVYpcC9hLZjS4G278OPdH6nTFPDNQ' }
18
+
19
+ it 'downloads the related file' do
20
+ storage.download(url, temp_folder)
21
+ related_file = File.join(temp_folder, 'testfile.txt')
22
+ expect(File.read(related_file)).to eq "helloworld!\n"
23
+ end
24
+ end
25
+
26
+ context 'give a public mega url, related to a big file' do
27
+
28
+ # A file called testfile_big_15mb.txt containting the word "topac" repeated 3145728 times (~ 15mb)
29
+ let(:url) { 'https://mega.co.nz/#!NYVkDaLD!BKyN5SRpOaEtGnTcwiAqcxmJc7p-k0IPWKAW-471KRE' }
30
+
31
+ it 'downloads the related file' do
32
+ storage.download(url, temp_folder)
33
+ related_file = File.join(temp_folder, 'testfile_big_15mb.txt')
34
+
35
+ expect(File.size(related_file)).to eql 15_728_640
36
+
37
+ count = 0
38
+ File.open(related_file, 'rb') do |f|
39
+ while (word = f.read(3840))
40
+ break if word != "topac"*768
41
+ count += 768
42
+ end
43
+ end
44
+
45
+ expect(count).to eql(15_728_640 / 5)
46
+ end
47
+ end
48
+ end
49
+ end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: rmega
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.5
4
+ version: 0.0.6
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2013-03-06 00:00:00.000000000 Z
12
+ date: 2013-07-10 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: pry
@@ -142,17 +142,20 @@ files:
142
142
  - lib/rmega/crypto/crypto.rb
143
143
  - lib/rmega/crypto/rsa.rb
144
144
  - lib/rmega/crypto/rsa_mega.js
145
+ - lib/rmega/downloader.rb
146
+ - lib/rmega/loggable.rb
145
147
  - lib/rmega/nodes/file_node.rb
146
148
  - lib/rmega/nodes/folder_node.rb
147
149
  - lib/rmega/nodes/node.rb
150
+ - lib/rmega/pool.rb
148
151
  - lib/rmega/session.rb
149
152
  - lib/rmega/storage.rb
150
153
  - lib/rmega/utils.rb
151
154
  - lib/rmega/version.rb
152
155
  - rmega.gemspec
156
+ - spec/integration/file_operations_spec.rb
153
157
  - spec/integration/folder_operations_spec.rb
154
158
  - spec/integration/login_spec.rb
155
- - spec/integration/rmega_account.yml.example
156
159
  - spec/integration_spec_helper.rb
157
160
  - spec/rmega/lib/crypto/aes_spec.rb
158
161
  - spec/rmega/lib/crypto/crypto_spec.rb
@@ -183,9 +186,9 @@ signing_key:
183
186
  specification_version: 3
184
187
  summary: mega.co.nz ruby api
185
188
  test_files:
189
+ - spec/integration/file_operations_spec.rb
186
190
  - spec/integration/folder_operations_spec.rb
187
191
  - spec/integration/login_spec.rb
188
- - spec/integration/rmega_account.yml.example
189
192
  - spec/integration_spec_helper.rb
190
193
  - spec/rmega/lib/crypto/aes_spec.rb
191
194
  - spec/rmega/lib/crypto/crypto_spec.rb
@@ -1,2 +0,0 @@
1
- email: your_email@email.com
2
- password: your_password