rmega 0.1.7 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +6 -0
- data/CHANGELOG.md +16 -0
- data/README.md +1 -1
- data/TODO.md +3 -5
- data/bin/rmega-dl +47 -0
- data/bin/rmega-up +31 -0
- data/lib/rmega.rb +35 -3
- data/lib/rmega/api_response.rb +80 -0
- data/lib/rmega/cli.rb +121 -0
- data/lib/rmega/crypto.rb +20 -0
- data/lib/rmega/crypto/aes_cbc.rb +46 -0
- data/lib/rmega/crypto/aes_ctr.rb +15 -84
- data/lib/rmega/crypto/aes_ecb.rb +25 -0
- data/lib/rmega/crypto/rsa.rb +21 -12
- data/lib/rmega/errors.rb +3 -51
- data/lib/rmega/loggable.rb +0 -3
- data/lib/rmega/net.rb +56 -0
- data/lib/rmega/nodes/deletable.rb +0 -3
- data/lib/rmega/nodes/downloadable.rb +73 -30
- data/lib/rmega/nodes/expandable.rb +14 -10
- data/lib/rmega/nodes/factory.rb +30 -17
- data/lib/rmega/nodes/file.rb +0 -4
- data/lib/rmega/nodes/folder.rb +4 -14
- data/lib/rmega/nodes/inbox.rb +0 -2
- data/lib/rmega/nodes/node.rb +48 -25
- data/lib/rmega/nodes/node_key.rb +44 -0
- data/lib/rmega/nodes/root.rb +0 -4
- data/lib/rmega/nodes/trash.rb +0 -3
- data/lib/rmega/nodes/uploadable.rb +42 -33
- data/lib/rmega/not_inspectable.rb +10 -0
- data/lib/rmega/options.rb +22 -5
- data/lib/rmega/pool.rb +18 -7
- data/lib/rmega/progress.rb +53 -13
- data/lib/rmega/session.rb +125 -52
- data/lib/rmega/storage.rb +25 -21
- data/lib/rmega/utils.rb +23 -183
- data/lib/rmega/version.rb +2 -1
- data/rmega.gemspec +3 -5
- data/spec/integration/file_download_spec.rb +14 -32
- data/spec/integration/file_integrity_spec.rb +41 -0
- data/spec/integration/file_upload_spec.rb +11 -57
- data/spec/integration/folder_download_spec.rb +17 -0
- data/spec/integration/folder_operations_spec.rb +30 -30
- data/spec/integration/login_spec.rb +3 -3
- data/spec/integration/resume_download_spec.rb +53 -0
- data/spec/integration_spec_helper.rb +9 -4
- data/spec/rmega/lib/cli_spec.rb +12 -0
- data/spec/rmega/lib/session_spec.rb +31 -0
- data/spec/rmega/lib/storage_spec.rb +27 -0
- data/spec/rmega/lib/utils_spec.rb +16 -78
- data/spec/spec_helper.rb +1 -4
- metadata +30 -40
- data/lib/rmega/crypto/aes.rb +0 -35
- data/lib/rmega/crypto/crypto.rb +0 -107
- data/lib/rmega/crypto/rsa_mega.js +0 -455
- data/spec/rmega/lib/crypto/aes_spec.rb +0 -12
- data/spec/rmega/lib/crypto/crypto_spec.rb +0 -27
data/lib/rmega/options.rb
CHANGED
@@ -1,12 +1,15 @@
|
|
1
|
-
require 'ostruct'
|
2
|
-
|
3
1
|
module Rmega
|
4
2
|
def self.default_options
|
5
3
|
{
|
6
|
-
|
4
|
+
thread_pool_size: 4,
|
7
5
|
max_retries: 10,
|
8
|
-
retry_interval:
|
9
|
-
|
6
|
+
retry_interval: 3,
|
7
|
+
http_open_timeout: 180,
|
8
|
+
http_read_timeout: 180,
|
9
|
+
# http_proxy_address: '127.0.0.1',
|
10
|
+
# http_proxy_port: 8080,
|
11
|
+
show_progress: true,
|
12
|
+
file_integrity_check: true,
|
10
13
|
api_url: 'https://eu.api.mega.co.nz/cs'
|
11
14
|
}
|
12
15
|
end
|
@@ -14,4 +17,18 @@ module Rmega
|
|
14
17
|
def self.options
|
15
18
|
@options ||= OpenStruct.new(default_options)
|
16
19
|
end
|
20
|
+
|
21
|
+
module Options
|
22
|
+
extend ActiveSupport::Concern
|
23
|
+
|
24
|
+
def options
|
25
|
+
Rmega.options
|
26
|
+
end
|
27
|
+
|
28
|
+
module ClassMethods
|
29
|
+
def options
|
30
|
+
Rmega.options
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
17
34
|
end
|
data/lib/rmega/pool.rb
CHANGED
@@ -1,29 +1,36 @@
|
|
1
|
-
require 'thread'
|
2
|
-
|
3
1
|
module Rmega
|
4
2
|
class Pool
|
5
|
-
|
3
|
+
include Options
|
6
4
|
|
7
|
-
def initialize
|
8
|
-
|
5
|
+
def initialize
|
6
|
+
threads_raises_exceptions
|
9
7
|
|
10
8
|
@mutex = Mutex.new
|
11
9
|
@resource = ConditionVariable.new
|
12
|
-
@max =
|
10
|
+
@max = options.thread_pool_size
|
13
11
|
|
14
12
|
@running = []
|
15
13
|
@queue = []
|
16
14
|
end
|
17
15
|
|
16
|
+
def threads_raises_exceptions
|
17
|
+
Thread.abort_on_exception = true
|
18
|
+
end
|
19
|
+
|
18
20
|
def defer(&block)
|
19
21
|
synchronize { @queue << block }
|
20
22
|
process_queue
|
21
23
|
end
|
22
24
|
|
25
|
+
alias :process :defer
|
26
|
+
|
23
27
|
def wait_done
|
28
|
+
return if done?
|
24
29
|
synchronize { @resource.wait(@mutex) }
|
25
30
|
end
|
26
31
|
|
32
|
+
alias :shutdown :wait_done
|
33
|
+
|
27
34
|
private
|
28
35
|
|
29
36
|
def synchronize(&block)
|
@@ -47,10 +54,14 @@ module Rmega
|
|
47
54
|
synchronize { @resource.signal }
|
48
55
|
end
|
49
56
|
|
57
|
+
def thread_terminated
|
58
|
+
synchronize { @running.reject! { |thread| thread == Thread.current } }
|
59
|
+
end
|
60
|
+
|
50
61
|
def thread_proc(&block)
|
51
62
|
Proc.new do
|
52
63
|
block.call
|
53
|
-
|
64
|
+
thread_terminated
|
54
65
|
process_queue
|
55
66
|
signal_done if done?
|
56
67
|
end
|
data/lib/rmega/progress.rb
CHANGED
@@ -1,28 +1,61 @@
|
|
1
1
|
module Rmega
|
2
2
|
class Progress
|
3
|
+
include Options
|
3
4
|
|
4
|
-
def initialize(
|
5
|
-
@total =
|
6
|
-
@caption =
|
5
|
+
def initialize(total, options = {})
|
6
|
+
@total = total
|
7
|
+
@caption = options[:caption]
|
7
8
|
@bytes = 0
|
9
|
+
@real_bytes = 0
|
10
|
+
@mutex = Mutex.new
|
8
11
|
@start_time = Time.now
|
9
12
|
|
10
13
|
show
|
11
14
|
end
|
12
15
|
|
16
|
+
def show?
|
17
|
+
options.show_progress
|
18
|
+
end
|
19
|
+
|
13
20
|
def show
|
14
|
-
|
21
|
+
return unless show?
|
15
22
|
|
16
|
-
message = "[#{@caption}]
|
23
|
+
message = @caption ? "[#{@caption}] " : ""
|
24
|
+
message << "#{humanize_bytes(@bytes)} of #{humanize_bytes(@total)}"
|
17
25
|
|
18
26
|
if ended?
|
19
27
|
message << ". Completed in #{elapsed_time} sec.\n"
|
20
28
|
else
|
21
|
-
message << "
|
29
|
+
message << ", #{percentage}% @ #{humanize_bytes(speed, 1)}/s, #{options.thread_pool_size} threads"
|
30
|
+
end
|
31
|
+
|
32
|
+
print_r(message)
|
33
|
+
end
|
34
|
+
|
35
|
+
def stty_size_columns
|
36
|
+
return @stty_size_columns unless @stty_size_columns.nil?
|
37
|
+
@stty_size_columns ||= (`stty size`.split[1].to_i rescue false)
|
38
|
+
end
|
39
|
+
|
40
|
+
def columns
|
41
|
+
stty_size_columns || 80
|
42
|
+
end
|
43
|
+
|
44
|
+
def print_r(message)
|
45
|
+
if message.size + 10 > columns
|
46
|
+
puts message
|
47
|
+
else
|
48
|
+
blank_line = ' ' * (message.size + 10)
|
49
|
+
print "\r#{blank_line}\r#{message}"
|
22
50
|
end
|
51
|
+
end
|
52
|
+
|
53
|
+
def percentage
|
54
|
+
(100.0 * @bytes / @total).round(2)
|
55
|
+
end
|
23
56
|
|
24
|
-
|
25
|
-
|
57
|
+
def speed
|
58
|
+
@real_bytes.to_f / (Time.now - @start_time).to_f
|
26
59
|
end
|
27
60
|
|
28
61
|
def elapsed_time
|
@@ -33,17 +66,24 @@ module Rmega
|
|
33
66
|
@total == @bytes
|
34
67
|
end
|
35
68
|
|
36
|
-
def increment(bytes)
|
37
|
-
@
|
38
|
-
|
69
|
+
def increment(bytes, options = {})
|
70
|
+
@mutex.synchronize do
|
71
|
+
@bytes += bytes
|
72
|
+
@real_bytes += bytes unless options[:real] == false
|
73
|
+
show
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
def humanize_bytes(*args)
|
78
|
+
self.class.humanize_bytes(*args)
|
39
79
|
end
|
40
80
|
|
41
|
-
def
|
81
|
+
def self.humanize_bytes(bytes, round = 2)
|
42
82
|
units = ['bytes', 'kb', 'MB', 'GB', 'TB', 'PB']
|
43
83
|
e = (bytes == 0 ? 0 : Math.log(bytes)) / Math.log(1024)
|
44
84
|
value = bytes.to_f / (1024 ** e.floor)
|
45
85
|
|
46
|
-
"#{value.round(round)} #{units[e]}"
|
86
|
+
return "#{value.round(round)} #{units[e]}"
|
47
87
|
end
|
48
88
|
end
|
49
89
|
end
|
data/lib/rmega/session.rb
CHANGED
@@ -1,82 +1,155 @@
|
|
1
|
-
require 'rmega/storage'
|
2
|
-
require 'rmega/errors'
|
3
|
-
require 'rmega/crypto/crypto'
|
4
|
-
require 'rmega/utils'
|
5
|
-
|
6
1
|
module Rmega
|
7
|
-
def self.login(email, password)
|
8
|
-
Session.new(email, password).storage
|
9
|
-
end
|
10
|
-
|
11
2
|
class Session
|
3
|
+
include NotInspectable
|
12
4
|
include Loggable
|
5
|
+
include Net
|
6
|
+
include Options
|
7
|
+
include Crypto
|
8
|
+
extend Crypto
|
13
9
|
|
14
|
-
attr_reader :
|
10
|
+
attr_reader :request_id, :sid, :shared_keys, :rsa_privk
|
11
|
+
attr_accessor :master_key
|
15
12
|
|
16
|
-
def initialize
|
17
|
-
@email = email
|
13
|
+
def initialize
|
18
14
|
@request_id = random_request_id
|
19
15
|
@shared_keys = {}
|
16
|
+
end
|
20
17
|
|
21
|
-
|
18
|
+
def storage
|
19
|
+
@storage ||= Storage.new(self)
|
22
20
|
end
|
23
21
|
|
24
|
-
def
|
25
|
-
|
22
|
+
def decrypt_rsa_private_key(encrypted_privk)
|
23
|
+
privk = aes_ecb_decrypt(@master_key, Utils.base64urldecode(encrypted_privk))
|
24
|
+
|
25
|
+
# Decompose private key
|
26
|
+
decomposed_key = []
|
27
|
+
|
28
|
+
4.times do
|
29
|
+
len = ((privk[0].ord * 256 + privk[1].ord + 7) >> 3) + 2
|
30
|
+
privk_part = privk[0, len]
|
31
|
+
decomposed_key << Utils.string_to_bignum(privk[0..len-1][2..-1])
|
32
|
+
privk = privk[len..-1]
|
33
|
+
end
|
34
|
+
|
35
|
+
return decomposed_key
|
26
36
|
end
|
27
37
|
|
28
|
-
|
29
|
-
|
38
|
+
def hash_password(password)
|
39
|
+
self.class.hash_password(password)
|
40
|
+
end
|
30
41
|
|
31
|
-
def
|
32
|
-
|
42
|
+
def self.hash_password(password)
|
43
|
+
pwd = password.dup.force_encoding('BINARY')
|
44
|
+
pkey = "\x93\xc4\x67\xe3\x7d\xb0\xc7\xa4\xd1\xbe\x3f\x81\x1\x52\xcb\x56".force_encoding('BINARY')
|
45
|
+
null_byte = "\x0".force_encoding('BINARY').freeze
|
46
|
+
blank = (null_byte*16).force_encoding('BINARY').freeze
|
47
|
+
keys = {}
|
48
|
+
|
49
|
+
65536.times do
|
50
|
+
(0..pwd.size-1).step(16) do |j|
|
51
|
+
|
52
|
+
keys[j] ||= begin
|
53
|
+
key = blank.dup
|
54
|
+
16.times { |i| key[i] = pwd[i+j] || null_byte if i+j < pwd.size }
|
55
|
+
key
|
56
|
+
end
|
57
|
+
|
58
|
+
pkey = aes_ecb_encrypt(keys[j], pkey)
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
return pkey
|
63
|
+
end
|
64
|
+
|
65
|
+
def decrypt_session_id(csid)
|
66
|
+
csid = Utils.base64_mpi_to_bn(csid)
|
67
|
+
csid = rsa_decrypt(csid, @rsa_privk)
|
68
|
+
csid = csid.to_s(16)
|
69
|
+
csid = '0' + csid if csid.length % 2 > 0
|
70
|
+
csid = Utils.hexstr_to_bstr(csid)[0,43]
|
71
|
+
csid = Utils.base64urlencode(csid)
|
72
|
+
return csid
|
73
|
+
end
|
74
|
+
|
75
|
+
def user_hash(aes_key, email)
|
76
|
+
s_bytes = email.bytes.to_a
|
77
|
+
hash = Array.new(16, 0)
|
78
|
+
s_bytes.size.times { |n| hash[n & 15] = hash[n & 15] ^ s_bytes[n] }
|
79
|
+
hash = hash.pack('c*')
|
80
|
+
16384.times { hash = aes_ecb_encrypt(aes_key, hash) }
|
81
|
+
hash = hash[0..4-1] + hash[8..12-1]
|
82
|
+
return Utils.base64urlencode(hash)
|
33
83
|
end
|
34
84
|
|
35
|
-
|
36
|
-
|
37
|
-
|
85
|
+
# If the user_hash is found on the server it returns:
|
86
|
+
# * The user master_key (128 bit for AES) encrypted with the password_hash
|
87
|
+
# * The RSA private key ecrypted with the master_key
|
88
|
+
# * A brand new session_id encrypted with the RSA private key
|
89
|
+
def login(email, password)
|
90
|
+
# Derive an hash from the user password
|
91
|
+
password_hash = hash_password(password)
|
92
|
+
u_hash = user_hash(password_hash, email.strip.downcase)
|
38
93
|
|
39
|
-
|
40
|
-
|
41
|
-
@master_key =
|
94
|
+
resp = request(a: 'us', user: email.strip, uh: u_hash)
|
95
|
+
|
96
|
+
@master_key = aes_cbc_decrypt(password_hash, Utils.base64urldecode(resp['k']))
|
97
|
+
@rsa_privk = decrypt_rsa_private_key(resp['privk'])
|
98
|
+
@sid = decrypt_session_id(resp['csid'])
|
99
|
+
@shared_keys = {}
|
42
100
|
|
43
|
-
|
44
|
-
@rsa_privk = Crypto.decrypt_rsa_privk(@master_key, resp['privk'])
|
45
|
-
@sid = Crypto.decrypt_sid(@rsa_privk, resp['csid'])
|
101
|
+
return self
|
46
102
|
end
|
47
103
|
|
48
|
-
def
|
49
|
-
|
104
|
+
def ephemeral_login(user_handle, password)
|
105
|
+
resp = request(a: 'us', user: user_handle)
|
106
|
+
|
107
|
+
password_hash = hash_password(password)
|
108
|
+
|
109
|
+
@master_key = aes_cbc_decrypt(password_hash, Utils.base64urldecode(resp['k']))
|
110
|
+
@sid = resp['tsid']
|
111
|
+
@rsa_privk = nil
|
112
|
+
@shared_keys = {}
|
113
|
+
|
114
|
+
return self
|
50
115
|
end
|
51
116
|
|
52
|
-
def
|
53
|
-
|
54
|
-
|
55
|
-
|
117
|
+
def self.ephemeral
|
118
|
+
master_key = OpenSSL::Random.random_bytes(16)
|
119
|
+
password = OpenSSL::Random.random_bytes(16)
|
120
|
+
password_hash = hash_password(password)
|
121
|
+
challenge = OpenSSL::Random.random_bytes(16)
|
122
|
+
|
123
|
+
session = new
|
124
|
+
|
125
|
+
user_handle = session.request(a: 'up', k: Utils.base64urlencode(aes_ecb_encrypt(password_hash, master_key)),
|
126
|
+
ts: Utils.base64urlencode(challenge + aes_ecb_encrypt(master_key, challenge)))
|
127
|
+
|
128
|
+
return session.ephemeral_login(user_handle, password)
|
56
129
|
end
|
57
130
|
|
58
|
-
def
|
59
|
-
|
60
|
-
|
131
|
+
def random_request_id
|
132
|
+
rand(1E7..1E9).to_i
|
133
|
+
end
|
61
134
|
|
62
|
-
|
63
|
-
|
135
|
+
def request_url(params = {})
|
136
|
+
params = params.merge(sid: @sid) if @sid
|
137
|
+
params = params.to_a.map { |a| a.join("=") }.join("&")
|
138
|
+
params = "&#{params}" unless params.empty?
|
64
139
|
|
65
|
-
|
140
|
+
return "#{options.api_url}?id=#{@request_id}#{params}"
|
141
|
+
end
|
66
142
|
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
143
|
+
def request(body, query_params = {})
|
144
|
+
survive do
|
145
|
+
@request_id += 1
|
146
|
+
api_response = APIResponse.new(http_post(request_url(query_params), [body].to_json))
|
147
|
+
if api_response.ok?
|
148
|
+
return(api_response.as_json)
|
149
|
+
else
|
150
|
+
raise(api_response.as_error)
|
151
|
+
end
|
73
152
|
end
|
74
|
-
rescue SocketError, Errors::ServerError => error
|
75
|
-
raise(error) if retries < 0
|
76
|
-
raise(error) if error.respond_to?(:temporary?) && !error.temporary?
|
77
|
-
retries -= 1
|
78
|
-
sleep(retry_interval)
|
79
|
-
retry
|
80
153
|
end
|
81
154
|
end
|
82
155
|
end
|
data/lib/rmega/storage.rb
CHANGED
@@ -1,8 +1,8 @@
|
|
1
|
-
require 'rmega/nodes/factory'
|
2
|
-
|
3
1
|
module Rmega
|
4
2
|
class Storage
|
3
|
+
include NotInspectable
|
5
4
|
include Loggable
|
5
|
+
include Crypto
|
6
6
|
|
7
7
|
attr_reader :session
|
8
8
|
|
@@ -20,14 +20,34 @@ module Rmega
|
|
20
20
|
quota['mstrg']
|
21
21
|
end
|
22
22
|
|
23
|
+
def stats
|
24
|
+
stats_hash = {files: 0, size: 0}
|
25
|
+
|
26
|
+
nodes.each do |n|
|
27
|
+
next unless n.type == :file
|
28
|
+
stats_hash[:files] += 1
|
29
|
+
stats_hash[:size] += n.filesize
|
30
|
+
end
|
31
|
+
|
32
|
+
return stats_hash
|
33
|
+
end
|
34
|
+
|
23
35
|
def quota
|
24
36
|
session.request(a: 'uq', strg: 1)
|
25
37
|
end
|
26
38
|
|
39
|
+
def nodes=(list)
|
40
|
+
@nodes = list
|
41
|
+
end
|
42
|
+
|
27
43
|
def nodes
|
44
|
+
return @nodes if @nodes
|
45
|
+
|
28
46
|
results = session.request(a: 'f', c: 1)
|
29
47
|
|
30
|
-
|
48
|
+
(results['ok'] || []).each do |hash|
|
49
|
+
shared_keys[hash['h']] ||= aes_ecb_decrypt(master_key, Utils.base64urldecode(hash['k']))
|
50
|
+
end
|
31
51
|
|
32
52
|
(results['f'] || []).map do |node_data|
|
33
53
|
node = Nodes::Factory.build(session, node_data)
|
@@ -36,12 +56,8 @@ module Rmega
|
|
36
56
|
end
|
37
57
|
end
|
38
58
|
|
39
|
-
def
|
40
|
-
|
41
|
-
root_handle = list.find { |node| node.type == :root }.handle
|
42
|
-
list.select do |node|
|
43
|
-
node.shared_root? || (node.type == :folder && node.parent_handle == root_handle)
|
44
|
-
end
|
59
|
+
def shared
|
60
|
+
nodes.select { |node| node.shared_root? }
|
45
61
|
end
|
46
62
|
|
47
63
|
def trash
|
@@ -51,17 +67,5 @@ module Rmega
|
|
51
67
|
def root
|
52
68
|
@root ||= nodes.find { |n| n.type == :root }
|
53
69
|
end
|
54
|
-
|
55
|
-
def download(public_url, path)
|
56
|
-
Nodes::Factory.build_from_url(session, public_url).download(path)
|
57
|
-
end
|
58
|
-
|
59
|
-
private
|
60
|
-
|
61
|
-
def store_shared_keys(list)
|
62
|
-
list.each do |hash|
|
63
|
-
shared_keys[hash['h']] = Crypto.decrypt_key(master_key, Utils.base64_to_a32(hash['k']))
|
64
|
-
end
|
65
|
-
end
|
66
70
|
end
|
67
71
|
end
|