rmega 0.0.5 → 0.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.md +15 -11
- data/lib/rmega/downloader.rb +73 -0
- data/lib/rmega/loggable.rb +11 -0
- data/lib/rmega/nodes/file_node.rb +9 -23
- data/lib/rmega/nodes/node.rb +15 -12
- data/lib/rmega/pool.rb +66 -0
- data/lib/rmega/storage.rb +1 -1
- data/lib/rmega/utils.rb +4 -1
- data/lib/rmega/version.rb +1 -1
- data/lib/rmega.rb +3 -0
- data/spec/integration/file_operations_spec.rb +49 -0
- metadata +7 -4
- data/spec/integration/rmega_account.yml.example +0 -2
data/README.md
CHANGED
@@ -1,18 +1,16 @@
|
|
1
|
-
# Rmega
|
1
|
+
# Rmega
|
2
2
|
|
3
|
+
A ruby library for the Mega.co.nz.
|
4
|
+
Tested using ruby 1.9.3+ (OpenSSL 0.9.8r+)
|
5
|
+
This work is the result of a reverse engineering of the Mega's Javascript code.
|
3
6
|
|
4
|
-
|
5
|
-
Tested using ruby 1.9.3+ (OpenSSL 0.9.8r+)
|
6
|
-
|
7
|
-
<div style="background-color: #000000; border-radius: 8px">
|
8
|
-
<img src="https://eu.static.mega.co.nz/images/mega/logo.png" />
|
9
|
-
</div>
|
10
|
-
|
7
|
+
## Installation
|
11
8
|
|
9
|
+
Rmega is distributed via rubygems, so if you have ruby 1.9.3+ installed
|
10
|
+
system wide, just type `gem install rmega`.
|
12
11
|
|
13
12
|
## Usage
|
14
13
|
|
15
|
-
$ gem install rmega
|
16
14
|
$ irb -r rmega
|
17
15
|
|
18
16
|
### Login and retrive all the files and folders
|
@@ -66,8 +64,16 @@ my_node.public_url
|
|
66
64
|
|
67
65
|
# See the attributes of a node
|
68
66
|
my_node.attributes
|
67
|
+
|
68
|
+
# Create a folder
|
69
|
+
parent_folder = storage.nodes_by_name(/photos/i).first
|
70
|
+
folder_node = storage.create_folder parent_folder, "london"
|
69
71
|
```
|
70
72
|
|
73
|
+
## Todo
|
74
|
+
|
75
|
+
* Handle connection errors during upload/download
|
76
|
+
|
71
77
|
|
72
78
|
## Installation
|
73
79
|
|
@@ -95,7 +101,5 @@ Or install it yourself as:
|
|
95
101
|
|
96
102
|
## Copyright
|
97
103
|
|
98
|
-
This work is the result of a reverse engineering of the Mega's Javascript code.
|
99
|
-
|
100
104
|
Copyright (c) 2013 Daniele Molteni
|
101
105
|
MIT License
|
@@ -0,0 +1,73 @@
|
|
1
|
+
module Rmega
|
2
|
+
class Downloader
|
3
|
+
include Loggable
|
4
|
+
|
5
|
+
attr_reader :pool, :base_url, :filesize, :local_path
|
6
|
+
|
7
|
+
def initialize(params)
|
8
|
+
@pool = Thread.pool(params[:threads] || 5)
|
9
|
+
@filesize = params[:filesize]
|
10
|
+
@base_url = params[:base_url]
|
11
|
+
@local_path = params[:local_path]
|
12
|
+
end
|
13
|
+
|
14
|
+
# Creates the local file allocating filesize-n bytes (of /dev/zero) for it.
|
15
|
+
# Opens the local file to start writing from the beginning of it.
|
16
|
+
def allocate
|
17
|
+
`dd if=/dev/zero of="#{local_path}" bs=1 count=0 seek=#{filesize} > /dev/null 2>&1`
|
18
|
+
raise "Unable to create file #{local_path}" if File.size(local_path) != filesize
|
19
|
+
|
20
|
+
File.open(local_path, 'r+b').tap { |f| f.rewind }
|
21
|
+
end
|
22
|
+
|
23
|
+
# Downloads a part of the remote file, starting from the start-n byte
|
24
|
+
# and ending after size-n bytes.
|
25
|
+
def download_chunk(start, size)
|
26
|
+
stop = start + size - 1
|
27
|
+
url = "#{base_url}/#{start}-#{stop}"
|
28
|
+
# logger.debug "#{Thread.current} downloading chunk @ #{start}"
|
29
|
+
HTTPClient.new.get_content(url)
|
30
|
+
end
|
31
|
+
|
32
|
+
# Writes a buffer in the local file, starting from the start-n byte.
|
33
|
+
def write_chunk(start, buffer)
|
34
|
+
# logger.debug "#{Thread.current} writing chunk @ #{position}"
|
35
|
+
@local_file.seek(start)
|
36
|
+
@local_file.write(buffer)
|
37
|
+
end
|
38
|
+
|
39
|
+
# Shows the progress bar in console
|
40
|
+
def show_progress(increment)
|
41
|
+
Utils.show_progress(:download, filesize, increment)
|
42
|
+
end
|
43
|
+
|
44
|
+
def chunks
|
45
|
+
Storage.chunks(filesize)
|
46
|
+
end
|
47
|
+
|
48
|
+
# TODO: checksum check
|
49
|
+
def download(&block)
|
50
|
+
@local_file = allocate
|
51
|
+
|
52
|
+
show_progress(0)
|
53
|
+
|
54
|
+
chunks.each do |start, size|
|
55
|
+
pool.defer do
|
56
|
+
buffer = download_chunk(start, size)
|
57
|
+
buffer = yield(start, buffer) if block_given?
|
58
|
+
show_progress(size)
|
59
|
+
pool.synchronize { write_chunk(start, buffer) }
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
# waits for the last running threads to finish
|
64
|
+
pool.wait_done
|
65
|
+
|
66
|
+
@local_file.flush
|
67
|
+
|
68
|
+
pool.shutdown
|
69
|
+
ensure
|
70
|
+
@local_file.close rescue nil
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
@@ -1,41 +1,27 @@
|
|
1
|
+
require_relative '../downloader'
|
2
|
+
|
1
3
|
module Rmega
|
2
4
|
class FileNode < Node
|
3
5
|
def storage_url
|
4
6
|
@storage_url ||= data['g'] || request(a: 'g', g: 1, n: handle)['g']
|
5
7
|
end
|
6
8
|
|
7
|
-
def
|
8
|
-
|
9
|
-
end
|
10
|
-
|
11
|
-
def download path
|
12
|
-
path = File.expand_path path
|
9
|
+
def download(path)
|
10
|
+
path = File.expand_path(path)
|
13
11
|
path = Dir.exists?(path) ? File.join(path, name) : path
|
14
12
|
|
15
|
-
logger.info "
|
16
|
-
|
17
|
-
Utils.show_progress :download, filesize
|
13
|
+
logger.info "Download #{name} (#{filesize} bytes) => #{path}"
|
18
14
|
|
19
15
|
k = decrypted_file_key
|
20
16
|
k = [k[0] ^ k[4], k[1] ^ k[5], k[2] ^ k[6], k[3] ^ k[7]]
|
21
17
|
nonce = decrypted_file_key[4..5]
|
22
18
|
|
23
|
-
|
24
|
-
connection = HTTPClient.new.get_async storage_url
|
25
|
-
message = connection.pop
|
19
|
+
donwloader = Downloader.new(base_url: storage_url, filesize: filesize, local_path: path)
|
26
20
|
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
nonce = [nonce[0], nonce[1], (chunk_start/0x1000000000) >> 0, (chunk_start/0x10) >> 0]
|
31
|
-
decryption_result = Crypto::AesCtr.decrypt(k, nonce, buffer)
|
32
|
-
file.write(decryption_result[:data])
|
33
|
-
Utils.show_progress :download, filesize, chunk_size
|
21
|
+
donwloader.download do |start, buffer|
|
22
|
+
nonce = [nonce[0], nonce[1], (start/0x1000000000) >> 0, (start/0x10) >> 0]
|
23
|
+
Crypto::AesCtr.decrypt(k, nonce, buffer)[:data]
|
34
24
|
end
|
35
|
-
|
36
|
-
nil
|
37
|
-
ensure
|
38
|
-
file.close if file
|
39
25
|
end
|
40
26
|
end
|
41
27
|
end
|
data/lib/rmega/nodes/node.rb
CHANGED
@@ -4,24 +4,22 @@ module Rmega
|
|
4
4
|
|
5
5
|
def initialize session, data
|
6
6
|
@session = session
|
7
|
-
|
7
|
+
|
8
|
+
if self.class.mega_url?(data)
|
9
|
+
@data = self.class.public_data(session, data)
|
10
|
+
@public_url = data
|
11
|
+
else
|
12
|
+
@data = data
|
13
|
+
end
|
8
14
|
end
|
9
15
|
|
10
16
|
def self.fabricate session, data
|
11
|
-
type_name = type_by_number(data['t'])
|
17
|
+
type_name = mega_url?(data) ? :file : type_by_number(data['t'])
|
12
18
|
node_class = Rmega.const_get("#{type_name}_node".camelize) rescue nil
|
13
19
|
node_class ||= Rmega::Node
|
14
20
|
node_class.new session, data
|
15
21
|
end
|
16
22
|
|
17
|
-
def self.initialize_by_public_url session, public_url
|
18
|
-
public_handle, key = public_url.split('!')[1, 2]
|
19
|
-
|
20
|
-
node = new session, public_data(session, public_handle)
|
21
|
-
node.instance_variable_set '@public_url', public_url
|
22
|
-
node
|
23
|
-
end
|
24
|
-
|
25
23
|
def self.types
|
26
24
|
{file: 0, folder: 1, root: 2, inbox: 3, trash: 4}
|
27
25
|
end
|
@@ -31,6 +29,10 @@ module Rmega
|
|
31
29
|
founded_type.first if founded_type
|
32
30
|
end
|
33
31
|
|
32
|
+
def self.mega_url? url
|
33
|
+
!!(url.to_s =~ /^https:\/\/mega\.co\.nz\/#!.*$/i)
|
34
|
+
end
|
35
|
+
|
34
36
|
def logger
|
35
37
|
Rmega.logger
|
36
38
|
end
|
@@ -57,8 +59,9 @@ module Rmega
|
|
57
59
|
|
58
60
|
# Other methods
|
59
61
|
|
60
|
-
def self.public_data session,
|
61
|
-
|
62
|
+
def self.public_data session, public_url
|
63
|
+
public_handle, key = public_url.strip.split('!')[1, 2]
|
64
|
+
session.request a: 'g', g: 1, p: public_handle
|
62
65
|
end
|
63
66
|
|
64
67
|
def public_handle
|
data/lib/rmega/pool.rb
ADDED
@@ -0,0 +1,66 @@
|
|
1
|
+
require 'thread'
|
2
|
+
|
3
|
+
class Thread
|
4
|
+
# Helper to create a Pool instance.
|
5
|
+
def self.pool(max)
|
6
|
+
Pool.new(max)
|
7
|
+
end
|
8
|
+
|
9
|
+
class Pool
|
10
|
+
def initialize(max)
|
11
|
+
Thread.abort_on_exception = true
|
12
|
+
@mutex = Mutex.new
|
13
|
+
@threads = Array.new(max)
|
14
|
+
end
|
15
|
+
|
16
|
+
# Gets the first position of the pool in which
|
17
|
+
# a thread could be started.
|
18
|
+
def available_slot
|
19
|
+
@threads.each_with_index do |thread, index|
|
20
|
+
return index if thread.nil? or !thread.alive?
|
21
|
+
end
|
22
|
+
nil
|
23
|
+
end
|
24
|
+
|
25
|
+
def synchronize(&block)
|
26
|
+
@mutex.synchronize(&block)
|
27
|
+
end
|
28
|
+
|
29
|
+
# Returns true if all the threads are finished,
|
30
|
+
# false otherwise.
|
31
|
+
def done?
|
32
|
+
@threads.each { |thread| return false if thread and thread.alive? }
|
33
|
+
true
|
34
|
+
end
|
35
|
+
|
36
|
+
# Blocking. Waits until all the threads are finished.
|
37
|
+
def wait_done
|
38
|
+
sleep 0.01 until done?
|
39
|
+
end
|
40
|
+
|
41
|
+
# Blocking. Waits until a pool's slot become available and
|
42
|
+
# returns that position.
|
43
|
+
# TODO: raise an error on wait timeout.
|
44
|
+
def wait_available_slot
|
45
|
+
while true
|
46
|
+
index = available_slot
|
47
|
+
return index if index
|
48
|
+
sleep 0.01
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
# Sends a KILL signal to all the threads.
|
53
|
+
def shutdown
|
54
|
+
@threads.each { |thread| thread.kill if thread.respond_to?(:kill) }
|
55
|
+
@threads.map! { nil }
|
56
|
+
end
|
57
|
+
|
58
|
+
# Blocking. Starts a new thread with the given block when a pool's slot
|
59
|
+
# become available.
|
60
|
+
def defer(&block)
|
61
|
+
index = wait_available_slot
|
62
|
+
@threads[index].kill if @threads[index].respond_to?(:kill)
|
63
|
+
@threads[index] = Thread.new(&block)
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
data/lib/rmega/storage.rb
CHANGED
data/lib/rmega/utils.rb
CHANGED
@@ -4,7 +4,10 @@ module Rmega
|
|
4
4
|
|
5
5
|
def show_progress direction, total, increment = 0
|
6
6
|
return unless Rmega.options.show_progress
|
7
|
-
|
7
|
+
if increment.zero?
|
8
|
+
@progressbar = nil
|
9
|
+
@progressbar_progress = 0
|
10
|
+
end
|
8
11
|
@progressbar_progress += increment
|
9
12
|
format = "#{direction.to_s.capitalize} in progress #{Utils.format_bytes(@progressbar_progress)} of #{Utils.format_bytes(total)} | %P% | %e "
|
10
13
|
@progressbar ||= ProgressBar.create format: format, total: total
|
data/lib/rmega/version.rb
CHANGED
data/lib/rmega.rb
CHANGED
@@ -13,6 +13,7 @@ require "ruby-progressbar"
|
|
13
13
|
|
14
14
|
# Require all the other files
|
15
15
|
require "rmega/version"
|
16
|
+
require "rmega/loggable"
|
16
17
|
require "rmega/utils"
|
17
18
|
require "rmega/crypto/rsa"
|
18
19
|
require "rmega/crypto/aes"
|
@@ -24,6 +25,8 @@ require "rmega/nodes/file_node"
|
|
24
25
|
require "rmega/nodes/folder_node"
|
25
26
|
require "rmega/session"
|
26
27
|
require "rmega/api_request_error"
|
28
|
+
require "rmega/pool"
|
29
|
+
require "rmega/downloader"
|
27
30
|
|
28
31
|
module Rmega
|
29
32
|
def self.logger
|
@@ -0,0 +1,49 @@
|
|
1
|
+
require 'integration_spec_helper'
|
2
|
+
require 'fileutils'
|
3
|
+
|
4
|
+
describe 'File operations' do
|
5
|
+
|
6
|
+
if account_file_exists?
|
7
|
+
|
8
|
+
let(:temp_folder) { "/tmp/.rmega_spec" }
|
9
|
+
|
10
|
+
before { FileUtils.mkdir_p(temp_folder) }
|
11
|
+
|
12
|
+
after { FileUtils.rm_rf(temp_folder) }
|
13
|
+
|
14
|
+
context 'give a public mega url, related to a small file' do
|
15
|
+
|
16
|
+
# A file called testfile.txt containting the string "helloworld!"
|
17
|
+
let(:url) { 'https://mega.co.nz/#!MAkg2Iab!bc9Y2U6d93IlRRKVYpcC9hLZjS4G278OPdH6nTFPDNQ' }
|
18
|
+
|
19
|
+
it 'downloads the related file' do
|
20
|
+
storage.download(url, temp_folder)
|
21
|
+
related_file = File.join(temp_folder, 'testfile.txt')
|
22
|
+
expect(File.read(related_file)).to eq "helloworld!\n"
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
context 'give a public mega url, related to a big file' do
|
27
|
+
|
28
|
+
# A file called testfile_big_15mb.txt containting the word "topac" repeated 3145728 times (~ 15mb)
|
29
|
+
let(:url) { 'https://mega.co.nz/#!NYVkDaLD!BKyN5SRpOaEtGnTcwiAqcxmJc7p-k0IPWKAW-471KRE' }
|
30
|
+
|
31
|
+
it 'downloads the related file' do
|
32
|
+
storage.download(url, temp_folder)
|
33
|
+
related_file = File.join(temp_folder, 'testfile_big_15mb.txt')
|
34
|
+
|
35
|
+
expect(File.size(related_file)).to eql 15_728_640
|
36
|
+
|
37
|
+
count = 0
|
38
|
+
File.open(related_file, 'rb') do |f|
|
39
|
+
while (word = f.read(3840))
|
40
|
+
break if word != "topac"*768
|
41
|
+
count += 768
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
expect(count).to eql(15_728_640 / 5)
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rmega
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.6
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-
|
12
|
+
date: 2013-07-10 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: pry
|
@@ -142,17 +142,20 @@ files:
|
|
142
142
|
- lib/rmega/crypto/crypto.rb
|
143
143
|
- lib/rmega/crypto/rsa.rb
|
144
144
|
- lib/rmega/crypto/rsa_mega.js
|
145
|
+
- lib/rmega/downloader.rb
|
146
|
+
- lib/rmega/loggable.rb
|
145
147
|
- lib/rmega/nodes/file_node.rb
|
146
148
|
- lib/rmega/nodes/folder_node.rb
|
147
149
|
- lib/rmega/nodes/node.rb
|
150
|
+
- lib/rmega/pool.rb
|
148
151
|
- lib/rmega/session.rb
|
149
152
|
- lib/rmega/storage.rb
|
150
153
|
- lib/rmega/utils.rb
|
151
154
|
- lib/rmega/version.rb
|
152
155
|
- rmega.gemspec
|
156
|
+
- spec/integration/file_operations_spec.rb
|
153
157
|
- spec/integration/folder_operations_spec.rb
|
154
158
|
- spec/integration/login_spec.rb
|
155
|
-
- spec/integration/rmega_account.yml.example
|
156
159
|
- spec/integration_spec_helper.rb
|
157
160
|
- spec/rmega/lib/crypto/aes_spec.rb
|
158
161
|
- spec/rmega/lib/crypto/crypto_spec.rb
|
@@ -183,9 +186,9 @@ signing_key:
|
|
183
186
|
specification_version: 3
|
184
187
|
summary: mega.co.nz ruby api
|
185
188
|
test_files:
|
189
|
+
- spec/integration/file_operations_spec.rb
|
186
190
|
- spec/integration/folder_operations_spec.rb
|
187
191
|
- spec/integration/login_spec.rb
|
188
|
-
- spec/integration/rmega_account.yml.example
|
189
192
|
- spec/integration_spec_helper.rb
|
190
193
|
- spec/rmega/lib/crypto/aes_spec.rb
|
191
194
|
- spec/rmega/lib/crypto/crypto_spec.rb
|