rmega 0.1.7 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +6 -0
  3. data/CHANGELOG.md +16 -0
  4. data/README.md +1 -1
  5. data/TODO.md +3 -5
  6. data/bin/rmega-dl +47 -0
  7. data/bin/rmega-up +31 -0
  8. data/lib/rmega.rb +35 -3
  9. data/lib/rmega/api_response.rb +80 -0
  10. data/lib/rmega/cli.rb +121 -0
  11. data/lib/rmega/crypto.rb +20 -0
  12. data/lib/rmega/crypto/aes_cbc.rb +46 -0
  13. data/lib/rmega/crypto/aes_ctr.rb +15 -84
  14. data/lib/rmega/crypto/aes_ecb.rb +25 -0
  15. data/lib/rmega/crypto/rsa.rb +21 -12
  16. data/lib/rmega/errors.rb +3 -51
  17. data/lib/rmega/loggable.rb +0 -3
  18. data/lib/rmega/net.rb +56 -0
  19. data/lib/rmega/nodes/deletable.rb +0 -3
  20. data/lib/rmega/nodes/downloadable.rb +73 -30
  21. data/lib/rmega/nodes/expandable.rb +14 -10
  22. data/lib/rmega/nodes/factory.rb +30 -17
  23. data/lib/rmega/nodes/file.rb +0 -4
  24. data/lib/rmega/nodes/folder.rb +4 -14
  25. data/lib/rmega/nodes/inbox.rb +0 -2
  26. data/lib/rmega/nodes/node.rb +48 -25
  27. data/lib/rmega/nodes/node_key.rb +44 -0
  28. data/lib/rmega/nodes/root.rb +0 -4
  29. data/lib/rmega/nodes/trash.rb +0 -3
  30. data/lib/rmega/nodes/uploadable.rb +42 -33
  31. data/lib/rmega/not_inspectable.rb +10 -0
  32. data/lib/rmega/options.rb +22 -5
  33. data/lib/rmega/pool.rb +18 -7
  34. data/lib/rmega/progress.rb +53 -13
  35. data/lib/rmega/session.rb +125 -52
  36. data/lib/rmega/storage.rb +25 -21
  37. data/lib/rmega/utils.rb +23 -183
  38. data/lib/rmega/version.rb +2 -1
  39. data/rmega.gemspec +3 -5
  40. data/spec/integration/file_download_spec.rb +14 -32
  41. data/spec/integration/file_integrity_spec.rb +41 -0
  42. data/spec/integration/file_upload_spec.rb +11 -57
  43. data/spec/integration/folder_download_spec.rb +17 -0
  44. data/spec/integration/folder_operations_spec.rb +30 -30
  45. data/spec/integration/login_spec.rb +3 -3
  46. data/spec/integration/resume_download_spec.rb +53 -0
  47. data/spec/integration_spec_helper.rb +9 -4
  48. data/spec/rmega/lib/cli_spec.rb +12 -0
  49. data/spec/rmega/lib/session_spec.rb +31 -0
  50. data/spec/rmega/lib/storage_spec.rb +27 -0
  51. data/spec/rmega/lib/utils_spec.rb +16 -78
  52. data/spec/spec_helper.rb +1 -4
  53. metadata +30 -40
  54. data/lib/rmega/crypto/aes.rb +0 -35
  55. data/lib/rmega/crypto/crypto.rb +0 -107
  56. data/lib/rmega/crypto/rsa_mega.js +0 -455
  57. data/spec/rmega/lib/crypto/aes_spec.rb +0 -12
  58. data/spec/rmega/lib/crypto/crypto_spec.rb +0 -27
@@ -0,0 +1,10 @@
1
+ module Rmega
2
+ module NotInspectable
3
+ def inspect(attributes = {})
4
+ memaddr = (__send__(:object_id) << 1).to_s(16)
5
+ string = "#<#{self.class.name}:#{memaddr}"
6
+ attributes.each { |k, v| string << " #{k}=#{v}" }
7
+ string << ">"
8
+ end
9
+ end
10
+ end
data/lib/rmega/options.rb CHANGED
@@ -1,12 +1,15 @@
1
- require 'ostruct'
2
-
3
1
  module Rmega
4
2
  def self.default_options
5
3
  {
6
- upload_timeout: 120,
4
+ thread_pool_size: 4,
7
5
  max_retries: 10,
8
- retry_interval: 1,
9
- api_request_timeout: 20,
6
+ retry_interval: 3,
7
+ http_open_timeout: 180,
8
+ http_read_timeout: 180,
9
+ # http_proxy_address: '127.0.0.1',
10
+ # http_proxy_port: 8080,
11
+ show_progress: true,
12
+ file_integrity_check: true,
10
13
  api_url: 'https://eu.api.mega.co.nz/cs'
11
14
  }
12
15
  end
@@ -14,4 +17,18 @@ module Rmega
14
17
  def self.options
15
18
  @options ||= OpenStruct.new(default_options)
16
19
  end
20
+
21
+ module Options
22
+ extend ActiveSupport::Concern
23
+
24
+ def options
25
+ Rmega.options
26
+ end
27
+
28
+ module ClassMethods
29
+ def options
30
+ Rmega.options
31
+ end
32
+ end
33
+ end
17
34
  end
data/lib/rmega/pool.rb CHANGED
@@ -1,29 +1,36 @@
1
- require 'thread'
2
-
3
1
  module Rmega
4
2
  class Pool
5
- MAX = 4
3
+ include Options
6
4
 
7
- def initialize(max = MAX)
8
- Thread.abort_on_exception = true
5
+ def initialize
6
+ threads_raises_exceptions
9
7
 
10
8
  @mutex = Mutex.new
11
9
  @resource = ConditionVariable.new
12
- @max = max || MAX
10
+ @max = options.thread_pool_size
13
11
 
14
12
  @running = []
15
13
  @queue = []
16
14
  end
17
15
 
16
+ def threads_raises_exceptions
17
+ Thread.abort_on_exception = true
18
+ end
19
+
18
20
  def defer(&block)
19
21
  synchronize { @queue << block }
20
22
  process_queue
21
23
  end
22
24
 
25
+ alias :process :defer
26
+
23
27
  def wait_done
28
+ return if done?
24
29
  synchronize { @resource.wait(@mutex) }
25
30
  end
26
31
 
32
+ alias :shutdown :wait_done
33
+
27
34
  private
28
35
 
29
36
  def synchronize(&block)
@@ -47,10 +54,14 @@ module Rmega
47
54
  synchronize { @resource.signal }
48
55
  end
49
56
 
57
+ def thread_terminated
58
+ synchronize { @running.reject! { |thread| thread == Thread.current } }
59
+ end
60
+
50
61
  def thread_proc(&block)
51
62
  Proc.new do
52
63
  block.call
53
- @running.reject! { |thread| thread == Thread.current }
64
+ thread_terminated
54
65
  process_queue
55
66
  signal_done if done?
56
67
  end
@@ -1,28 +1,61 @@
1
1
  module Rmega
2
2
  class Progress
3
+ include Options
3
4
 
4
- def initialize(params)
5
- @total = params[:total]
6
- @caption = params[:caption]
5
+ def initialize(total, options = {})
6
+ @total = total
7
+ @caption = options[:caption]
7
8
  @bytes = 0
9
+ @real_bytes = 0
10
+ @mutex = Mutex.new
8
11
  @start_time = Time.now
9
12
 
10
13
  show
11
14
  end
12
15
 
16
+ def show?
17
+ options.show_progress
18
+ end
19
+
13
20
  def show
14
- percentage = (100.0 * @bytes / @total).round(2)
21
+ return unless show?
15
22
 
16
- message = "[#{@caption}] #{humanize(@bytes)} of #{humanize(@total)}"
23
+ message = @caption ? "[#{@caption}] " : ""
24
+ message << "#{humanize_bytes(@bytes)} of #{humanize_bytes(@total)}"
17
25
 
18
26
  if ended?
19
27
  message << ". Completed in #{elapsed_time} sec.\n"
20
28
  else
21
- message << " (#{percentage}%)"
29
+ message << ", #{percentage}% @ #{humanize_bytes(speed, 1)}/s, #{options.thread_pool_size} threads"
30
+ end
31
+
32
+ print_r(message)
33
+ end
34
+
35
+ def stty_size_columns
36
+ return @stty_size_columns unless @stty_size_columns.nil?
37
+ @stty_size_columns ||= (`stty size`.split[1].to_i rescue false)
38
+ end
39
+
40
+ def columns
41
+ stty_size_columns || 80
42
+ end
43
+
44
+ def print_r(message)
45
+ if message.size + 10 > columns
46
+ puts message
47
+ else
48
+ blank_line = ' ' * (message.size + 10)
49
+ print "\r#{blank_line}\r#{message}"
22
50
  end
51
+ end
52
+
53
+ def percentage
54
+ (100.0 * @bytes / @total).round(2)
55
+ end
23
56
 
24
- blank_line = ' ' * (message.size + 15)
25
- print "\r#{blank_line}\r#{message}"
57
+ def speed
58
+ @real_bytes.to_f / (Time.now - @start_time).to_f
26
59
  end
27
60
 
28
61
  def elapsed_time
@@ -33,17 +66,24 @@ module Rmega
33
66
  @total == @bytes
34
67
  end
35
68
 
36
- def increment(bytes)
37
- @bytes += bytes
38
- show
69
+ def increment(bytes, options = {})
70
+ @mutex.synchronize do
71
+ @bytes += bytes
72
+ @real_bytes += bytes unless options[:real] == false
73
+ show
74
+ end
75
+ end
76
+
77
+ def humanize_bytes(*args)
78
+ self.class.humanize_bytes(*args)
39
79
  end
40
80
 
41
- def humanize(bytes, round = 2)
81
+ def self.humanize_bytes(bytes, round = 2)
42
82
  units = ['bytes', 'kb', 'MB', 'GB', 'TB', 'PB']
43
83
  e = (bytes == 0 ? 0 : Math.log(bytes)) / Math.log(1024)
44
84
  value = bytes.to_f / (1024 ** e.floor)
45
85
 
46
- "#{value.round(round)} #{units[e]}"
86
+ return "#{value.round(round)} #{units[e]}"
47
87
  end
48
88
  end
49
89
  end
data/lib/rmega/session.rb CHANGED
@@ -1,82 +1,155 @@
1
- require 'rmega/storage'
2
- require 'rmega/errors'
3
- require 'rmega/crypto/crypto'
4
- require 'rmega/utils'
5
-
6
1
  module Rmega
7
- def self.login(email, password)
8
- Session.new(email, password).storage
9
- end
10
-
11
2
  class Session
3
+ include NotInspectable
12
4
  include Loggable
5
+ include Net
6
+ include Options
7
+ include Crypto
8
+ extend Crypto
13
9
 
14
- attr_reader :email, :request_id, :sid, :master_key, :shared_keys, :rsa_privk
10
+ attr_reader :request_id, :sid, :shared_keys, :rsa_privk
11
+ attr_accessor :master_key
15
12
 
16
- def initialize(email, password)
17
- @email = email
13
+ def initialize
18
14
  @request_id = random_request_id
19
15
  @shared_keys = {}
16
+ end
20
17
 
21
- login(password)
18
+ def storage
19
+ @storage ||= Storage.new(self)
22
20
  end
23
21
 
24
- def options
25
- Rmega.options
22
+ def decrypt_rsa_private_key(encrypted_privk)
23
+ privk = aes_ecb_decrypt(@master_key, Utils.base64urldecode(encrypted_privk))
24
+
25
+ # Decompose private key
26
+ decomposed_key = []
27
+
28
+ 4.times do
29
+ len = ((privk[0].ord * 256 + privk[1].ord + 7) >> 3) + 2
30
+ privk_part = privk[0, len]
31
+ decomposed_key << Utils.string_to_bignum(privk[0..len-1][2..-1])
32
+ privk = privk[len..-1]
33
+ end
34
+
35
+ return decomposed_key
26
36
  end
27
37
 
28
- delegate :api_url, :api_request_timeout, to: :options
29
- delegate :max_retries, :retry_interval, to: :options
38
+ def hash_password(password)
39
+ self.class.hash_password(password)
40
+ end
30
41
 
31
- def storage
32
- @storage ||= Storage.new(self)
42
+ def self.hash_password(password)
43
+ pwd = password.dup.force_encoding('BINARY')
44
+ pkey = "\x93\xc4\x67\xe3\x7d\xb0\xc7\xa4\xd1\xbe\x3f\x81\x1\x52\xcb\x56".force_encoding('BINARY')
45
+ null_byte = "\x0".force_encoding('BINARY').freeze
46
+ blank = (null_byte*16).force_encoding('BINARY').freeze
47
+ keys = {}
48
+
49
+ 65536.times do
50
+ (0..pwd.size-1).step(16) do |j|
51
+
52
+ keys[j] ||= begin
53
+ key = blank.dup
54
+ 16.times { |i| key[i] = pwd[i+j] || null_byte if i+j < pwd.size }
55
+ key
56
+ end
57
+
58
+ pkey = aes_ecb_encrypt(keys[j], pkey)
59
+ end
60
+ end
61
+
62
+ return pkey
63
+ end
64
+
65
+ def decrypt_session_id(csid)
66
+ csid = Utils.base64_mpi_to_bn(csid)
67
+ csid = rsa_decrypt(csid, @rsa_privk)
68
+ csid = csid.to_s(16)
69
+ csid = '0' + csid if csid.length % 2 > 0
70
+ csid = Utils.hexstr_to_bstr(csid)[0,43]
71
+ csid = Utils.base64urlencode(csid)
72
+ return csid
73
+ end
74
+
75
+ def user_hash(aes_key, email)
76
+ s_bytes = email.bytes.to_a
77
+ hash = Array.new(16, 0)
78
+ s_bytes.size.times { |n| hash[n & 15] = hash[n & 15] ^ s_bytes[n] }
79
+ hash = hash.pack('c*')
80
+ 16384.times { hash = aes_ecb_encrypt(aes_key, hash) }
81
+ hash = hash[0..4-1] + hash[8..12-1]
82
+ return Utils.base64urlencode(hash)
33
83
  end
34
84
 
35
- def login(password)
36
- uh = Crypto.stringhash Crypto.prepare_key_pw(password), email.downcase
37
- resp = request(a: 'us', user: email, uh: uh)
85
+ # If the user_hash is found on the server it returns:
86
+ # * The user master_key (128 bit for AES) encrypted with the password_hash
87
+ # * The RSA private key ecrypted with the master_key
88
+ # * A brand new session_id encrypted with the RSA private key
89
+ def login(email, password)
90
+ # Derive an hash from the user password
91
+ password_hash = hash_password(password)
92
+ u_hash = user_hash(password_hash, email.strip.downcase)
38
93
 
39
- # Decrypts the master key
40
- encrypted_key = Crypto.prepare_key_pw(password)
41
- @master_key = Crypto.decrypt_key(encrypted_key, Utils.base64_to_a32(resp['k']))
94
+ resp = request(a: 'us', user: email.strip, uh: u_hash)
95
+
96
+ @master_key = aes_cbc_decrypt(password_hash, Utils.base64urldecode(resp['k']))
97
+ @rsa_privk = decrypt_rsa_private_key(resp['privk'])
98
+ @sid = decrypt_session_id(resp['csid'])
99
+ @shared_keys = {}
42
100
 
43
- # Generates the session id
44
- @rsa_privk = Crypto.decrypt_rsa_privk(@master_key, resp['privk'])
45
- @sid = Crypto.decrypt_sid(@rsa_privk, resp['csid'])
101
+ return self
46
102
  end
47
103
 
48
- def random_request_id
49
- rand(1E7..1E9).to_i
104
+ def ephemeral_login(user_handle, password)
105
+ resp = request(a: 'us', user: user_handle)
106
+
107
+ password_hash = hash_password(password)
108
+
109
+ @master_key = aes_cbc_decrypt(password_hash, Utils.base64urldecode(resp['k']))
110
+ @sid = resp['tsid']
111
+ @rsa_privk = nil
112
+ @shared_keys = {}
113
+
114
+ return self
50
115
  end
51
116
 
52
- def request_url
53
- "#{api_url}?id=#{@request_id}".tap do |url|
54
- url << "&sid=#{@sid}" if @sid
55
- end
117
+ def self.ephemeral
118
+ master_key = OpenSSL::Random.random_bytes(16)
119
+ password = OpenSSL::Random.random_bytes(16)
120
+ password_hash = hash_password(password)
121
+ challenge = OpenSSL::Random.random_bytes(16)
122
+
123
+ session = new
124
+
125
+ user_handle = session.request(a: 'up', k: Utils.base64urlencode(aes_ecb_encrypt(password_hash, master_key)),
126
+ ts: Utils.base64urlencode(challenge + aes_ecb_encrypt(master_key, challenge)))
127
+
128
+ return session.ephemeral_login(user_handle, password)
56
129
  end
57
130
 
58
- def request(content, retries = max_retries)
59
- @request_id += 1
60
- logger.debug "POST #{request_url} #{content.inspect}"
131
+ def random_request_id
132
+ rand(1E7..1E9).to_i
133
+ end
61
134
 
62
- response = HTTPClient.new.post(request_url, [content].to_json, timeout: api_request_timeout)
63
- code, body = response.code.to_i, response.body
135
+ def request_url(params = {})
136
+ params = params.merge(sid: @sid) if @sid
137
+ params = params.to_a.map { |a| a.join("=") }.join("&")
138
+ params = "&#{params}" unless params.empty?
64
139
 
65
- logger.debug("#{code} #{body}")
140
+ return "#{options.api_url}?id=#{@request_id}#{params}"
141
+ end
66
142
 
67
- if code == 500 && body.to_s.empty?
68
- raise Errors::ServerError.new("Server too busy", temporary: true)
69
- else
70
- json = JSON.parse(body).first
71
- raise Errors::ServerError.new(json) if json.to_s =~ /\A\-\d+\z/
72
- json
143
+ def request(body, query_params = {})
144
+ survive do
145
+ @request_id += 1
146
+ api_response = APIResponse.new(http_post(request_url(query_params), [body].to_json))
147
+ if api_response.ok?
148
+ return(api_response.as_json)
149
+ else
150
+ raise(api_response.as_error)
151
+ end
73
152
  end
74
- rescue SocketError, Errors::ServerError => error
75
- raise(error) if retries < 0
76
- raise(error) if error.respond_to?(:temporary?) && !error.temporary?
77
- retries -= 1
78
- sleep(retry_interval)
79
- retry
80
153
  end
81
154
  end
82
155
  end
data/lib/rmega/storage.rb CHANGED
@@ -1,8 +1,8 @@
1
- require 'rmega/nodes/factory'
2
-
3
1
  module Rmega
4
2
  class Storage
3
+ include NotInspectable
5
4
  include Loggable
5
+ include Crypto
6
6
 
7
7
  attr_reader :session
8
8
 
@@ -20,14 +20,34 @@ module Rmega
20
20
  quota['mstrg']
21
21
  end
22
22
 
23
+ def stats
24
+ stats_hash = {files: 0, size: 0}
25
+
26
+ nodes.each do |n|
27
+ next unless n.type == :file
28
+ stats_hash[:files] += 1
29
+ stats_hash[:size] += n.filesize
30
+ end
31
+
32
+ return stats_hash
33
+ end
34
+
23
35
  def quota
24
36
  session.request(a: 'uq', strg: 1)
25
37
  end
26
38
 
39
+ def nodes=(list)
40
+ @nodes = list
41
+ end
42
+
27
43
  def nodes
44
+ return @nodes if @nodes
45
+
28
46
  results = session.request(a: 'f', c: 1)
29
47
 
30
- store_shared_keys(results['ok'] || [])
48
+ (results['ok'] || []).each do |hash|
49
+ shared_keys[hash['h']] ||= aes_ecb_decrypt(master_key, Utils.base64urldecode(hash['k']))
50
+ end
31
51
 
32
52
  (results['f'] || []).map do |node_data|
33
53
  node = Nodes::Factory.build(session, node_data)
@@ -36,12 +56,8 @@ module Rmega
36
56
  end
37
57
  end
38
58
 
39
- def folders
40
- list = nodes
41
- root_handle = list.find { |node| node.type == :root }.handle
42
- list.select do |node|
43
- node.shared_root? || (node.type == :folder && node.parent_handle == root_handle)
44
- end
59
+ def shared
60
+ nodes.select { |node| node.shared_root? }
45
61
  end
46
62
 
47
63
  def trash
@@ -51,17 +67,5 @@ module Rmega
51
67
  def root
52
68
  @root ||= nodes.find { |n| n.type == :root }
53
69
  end
54
-
55
- def download(public_url, path)
56
- Nodes::Factory.build_from_url(session, public_url).download(path)
57
- end
58
-
59
- private
60
-
61
- def store_shared_keys(list)
62
- list.each do |hash|
63
- shared_keys[hash['h']] = Crypto.decrypt_key(master_key, Utils.base64_to_a32(hash['k']))
64
- end
65
- end
66
70
  end
67
71
  end