rmega 0.2.7 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 48d1cd2037a5511fdc9adc693fbd1b6e74331327a3e46a93e2326a3c9aef7f42
-  data.tar.gz: 159cd701d79d5fdb8d903a562f01fff6a439fedb2d6efff926acf8df772182f0
+  metadata.gz: ee612b39a7d211f89deb0c30044ec6d75ae231da61314e73432389f96e7f7a0b
+  data.tar.gz: 1742781c48533cea360740dc14f9b3d67bec0933e671605e3df4e0d4d2608a68
 SHA512:
-  metadata.gz: 74c4ac1a15aeafeea21aed3dce4f17e8bdc7f82bf50608d776bc3c9df27e57ece398e8b8cf1b77d6dcd34114a020b7b1bcbcf3bdeca398da80b0ab80c3e72f39
-  data.tar.gz: 48a51f07da3be0c88619068afa1fc93a42d4f77dc008ddf6a5504a66aa536584e46af9e3c799fcf8f70fec44c6d31c5aabdaf74bd13257fa4910df9418c793b9
+  metadata.gz: 5cf7d33fe73f7626160d4045f27e48b546943fc492dca38c8f528f7ec0b3c07ed341527a8c17305b8448c08efd2aea3d44ecf517a1aaf9684f172830b6586c4a
+  data.tar.gz: f919c67bd100e4e94c7f3dd4c319d0df784173a3a7f90d45b24914b4b8fa474196b4a858f89e414abfaa3301eac2e9c0bf3602d1f3d870010ec1c8c8a1d1511f

data/.circleci/config.yml

@@ -0,0 +1,52 @@
+# Use the latest 2.1 version of CircleCI pipeline process engine.
+# See: https://circleci.com/docs/2.0/configuration-reference
+version: 2.1
+
+# Orbs are reusable packages of CircleCI configuration that you may share across projects, enabling you to create encapsulated, parameterized commands, jobs, and executors that can be used across multiple projects.
+# See: https://circleci.com/docs/2.0/orb-intro/
+orbs:
+  ruby: circleci/ruby@0.1.2
+
+# Define a job to be invoked later in a workflow.
+# See: https://circleci.com/docs/2.0/configuration-reference/#jobs
+jobs:
+  test_on_ruby_latest:
+    docker:
+      - image: circleci/ruby
+    executor: ruby/default
+    steps:
+      - checkout
+      - run: 'ruby -v'
+      - run: 'bundle install'
+      - run: 'bundle exec rspec'
+  test_on_ruby25:
+    docker:
+      - image: 'circleci/ruby:2.5'
+    executor: ruby/default
+    steps:
+      - checkout
+      - run: 'ruby -v'
+      - run: 'bundle install'
+      - run: 'bundle exec rspec'
+  test_on_ruby21:
+    docker:
+      - image: 'circleci/ruby:2.1'
+    executor: ruby/default
+    steps:
+      - checkout
+      - run: 'ruby -v'
+      - run: 'bundle install'
+      - run: 'bundle exec rspec'
+
+# Invoke jobs via workflows
+# See: https://circleci.com/docs/2.0/configuration-reference/#workflows
+workflows:
+  test: # This is the name of the workflow, feel free to change it to better match your workflow.
+    # Inside the workflow, you define the jobs you want to run.
+    jobs:
+      - test_on_ruby21:
+          context: rmega_secrets
+      - test_on_ruby25:
+          context: rmega_secrets
+      - test_on_ruby_latest:
+          context: rmega_secrets

data/README.md

@@ -1,10 +1,11 @@
 [![Gem Version](https://badge.fury.io/rb/rmega.svg)](https://badge.fury.io/rb/rmega)
-[![Build Status](https://travis-ci.org/topac/rmega.svg?branch=master)](https://travis-ci.org/topac/rmega)
+[![rmega](https://circleci.com/gh/topac/rmega.svg?style=svg)](https://circleci.com/gh/topac/rmega)
+
 
 # rmega
 
 Pure ruby library for <img src="https://mega.co.nz/favicon.ico" alt=""/> **MEGA** [https://mega.nz/](https://mega.nz/).  
-Works on Linux and OSX with Ruby 1.9.3+.
+Works on Linux and OSX with Ruby 2.1+.
 
 ## Installation
 
@@ -93,7 +94,7 @@ public_url = 'https://mega.nz/file/MAkg2Iab#bc9Y2U6d93IlRRKVYpcC9hLZjS4G278OPdH6
 Rmega.download(public_url, '~/Downloads')
 ```
 
-### Upload
+### Upload a file
 
 ```ruby
 # Upload a file to a specific folder
@@ -104,6 +105,17 @@ folder.upload("~/Downloads/my_file.txt")
 storage.root.upload("~/Downloads/my_other_file.txt")
 ```
 
+### Upload a directory
+
+```ruby
+# Upload a directory to a specific folder
+folder = storage.root.folders[3]
+folder.upload_dir("~/Downloads/my_directory")
+
+# Upload a directory to the root folder
+storage.root.upload_dir("~/Downloads/my_other_directory")
+```
+
 ### Creating a folder
 
 ```ruby

data/lib/rmega/conn_pool.rb

@@ -0,0 +1,98 @@
+module Rmega
+  module ConnPool
+    include Loggable
+    include Options
+    extend MonitorMixin
+    
+    # url: URI / String
+    # options: any options that Net::HTTP.new accepts
+    def self.get(url, options = {})
+      uri = url.is_a?(URI) ? url : URI(url)
+      connection_manager.get_client(uri, options)
+    end
+    
+    def self.connection_manager
+      synchronize do
+        @connection_managers ||= []
+
+        # we first clear all old ones
+        removed = @connection_managers.reject!(&:stale?)
+        (removed || []).each(&:close_connections!)
+        
+        # get the manager
+        Thread.current[:http_connection_manager] ||= self.synchronize do
+          manager = ConnectionManager.new
+          @connection_managers << manager
+          manager
+        end
+      end
+    end
+  
+    # @see https://www.dmitry-ishkov.com/2021/05/turbocharge-http-requests-in-ruby.html
+    class ConnectionManager
+      include MonitorMixin
+      # if a client wasn't used within this time range
+      # it gets removed from the cache and the connection closed.
+      # This helps to make sure there are no memory leaks.
+      STALE_AFTER = 180
+  
+      # Seconds to reuse the connection of the previous request. 
+      # If the idle time is less than this Keep-Alive Timeout, Net::HTTP reuses the TCP/IP socket used by the previous communication. Source: Ruby docs
+      KEEP_ALIVE_TIMEOUT = 300
+  
+      # KEEP_ALIVE_TIMEOUT vs STALE_AFTER
+      # STALE_AFTER - how long an Net::HTTP client object is cached in ruby
+      # KEEP_ALIVE_TIMEOUT - how long that client keeps TCP/IP socket open.
+  
+      attr_accessor :clients_store, :last_used
+  
+      def initialize(*args)
+        super
+        self.clients_store = {}
+        self.last_used = Time.now
+      end
+  
+      def get_client(uri, options)
+        synchronize do
+          # refresh the last time a client was used,
+          # this prevents the client from becoming stale
+          self.last_used = Time.now
+  
+          # we use params as a cache key for clients.
+          # 2 connections to the same host but with different
+          # options are going to use different HTTP clients
+          params = [uri.host, uri.port, options]
+          client = clients_store[params]
+  
+          return client if client
+  
+          client = ::Net::HTTP.new(uri.host, uri.port)
+          client.keep_alive_timeout = KEEP_ALIVE_TIMEOUT
+
+          # set SSL to true if a scheme is https
+          client.use_ssl = uri.scheme == "https"
+  
+          # open connection
+          client.start
+  
+          # cache the client
+          clients_store[params] = client
+  
+          client
+        end
+      end
+  
+      # close connections for each client
+      def close_connections!
+        synchronize do
+          clients_store.values.each(&:finish)
+          self.clients_store = {}
+        end
+      end
+
+      def stale?
+        Time.now - last_used > STALE_AFTER
+      end
+    end
+  end
+end

data/lib/rmega/net.rb

@@ -33,9 +33,6 @@ module Rmega
       req.body = data
       logger.debug("REQ POST #{url} #{cut_string(data)}")
 
-      # if you don't use Net::Http#start it will not keep the socket open even if you set
-      # the connection header BUT setting the connection header to 'keep-alive' its enough
-      # to fool MEGA servers and don't let them reset your connection!
       req['Connection'] = 'keep-alive'
 
       response = net_http(uri).request(req)
@@ -46,8 +43,7 @@ module Rmega
     private
 
     def net_http(uri)
-      http = ::Net::HTTP.new(uri.host, uri.port)
-      http.use_ssl = true if uri.scheme == 'https'
+      http = Rmega::ConnPool.get(uri)
 
       # apply common http options
       http.proxy_from_env = false if options.http_proxy_address

data/lib/rmega/nodes/downloadable.rb

@@ -95,7 +95,7 @@ module Rmega
         end
 
         # waits for the last running threads to finish
-        pool.shutdown
+        pool.wait_done
 
         if options.file_integrity_check
           file_mac = aes_cbc_mac(@node_key.aes_key, chunk_macs.sort.map(&:last).join, "\x0"*16)

data/lib/rmega/nodes/uploadable.rb

@@ -66,7 +66,7 @@ module Rmega
           end
         end
 
-        pool.shutdown
+        pool.wait_done
 
         # encrypt attributes
         _attr = serialize_attributes(:n => Utils.utf8(::File.basename(path)))
@@ -85,6 +85,20 @@ module Rmega
       ensure
         file.close if file
       end
+
+      def upload_dir(dir)
+        return create_folder(::File.basename(dir)).upload_dir(dir) unless name == ::File.basename(dir)
+
+        ::Dir["#{dir}/*"].each do |path|
+          name = ::File.basename(path)
+
+          if ::File.directory?(path)
+            create_folder(name).upload_dir(path)
+          elsif ::File.size(path) > 0
+            upload(path)
+          end
+        end
+      end
     end
   end
 end

data/lib/rmega/pool.rb

@@ -1,70 +1,58 @@
 module Rmega
   class Pool
     include Options
-
+    
     def initialize
       threads_raises_exceptions
 
-      @mutex = Mutex.new
-      @resource = ConditionVariable.new
-      @max = options.thread_pool_size
+      @queue = Queue.new
+      @queue_closed = false
+      @threads = []
+      @cv = ConditionVariable.new
+      @working_threads = 0
+      
+      options.thread_pool_size.times do
+        @threads << Thread.new do
+          while proc = @queue.pop
+            mutex.synchronize do
+              @working_threads += 1
+            end
+            
+            proc.call
+
+            mutex.synchronize do
+              @working_threads -= 1
+              
+              if @queue_closed and @queue.empty? and @working_threads == 0
+                @cv.signal
+              end
+            end
+          end
+        end
+      end
+    end
 
-      @running = []
-      @queue = []
+    def mutex
+      @mutex ||= Mutex.new
     end
 
     def threads_raises_exceptions
       Thread.abort_on_exception = true
     end
 
-    def defer(&block)
-      synchronize { @queue << block }
-      process_queue
+    def process(&block)
+      @queue << block
     end
-
-    alias :process :defer
-
+    
     def wait_done
-      return if done?
-      synchronize { @resource.wait(@mutex) }
-    end
-
-    alias :shutdown :wait_done
-
-    private
-
-    def synchronize(&block)
-      @mutex.synchronize(&block)
-    end
+      @queue.close if @queue.respond_to?(:close)
+      @queue_closed = true
 
-    def process_queue
-      synchronize do
-        if @running.size < @max
-          proc = @queue.shift
-          @running << Thread.new(&thread_proc(&proc)) if proc
-        end
+      mutex.synchronize do
+        @cv.wait(mutex)
       end
-    end
-
-    def done?
-      synchronize { @queue.empty? && @running.empty? }
-    end
 
-    def signal_done
-      synchronize { @resource.signal }
-    end
-
-    def thread_terminated
-      synchronize { @running.reject! { |thread| thread == Thread.current } }
-    end
-
-    def thread_proc(&block)
-      Proc.new do
-        block.call
-        thread_terminated
-        process_queue
-        signal_done if done?
-      end
+      @threads.each(&:kill)
     end
   end
 end

data/lib/rmega/version.rb

@@ -1,4 +1,4 @@
 module Rmega
-  VERSION = "0.2.7"
+  VERSION = "0.3.1"
   HOMEPAGE = "https://github.com/topac/rmega"
 end

data/lib/rmega.rb

@@ -22,6 +22,7 @@ require 'rmega/not_inspectable'
 require 'rmega/errors'
 require 'rmega/api_response'
 require 'rmega/utils'
+require 'rmega/conn_pool'
 require 'rmega/net'
 require 'rmega/pool'
 require 'rmega/progress'

data/spec/integration/file_download_spec.rb

@@ -4,7 +4,7 @@ describe 'File download' do
 
   context 'given a public mega url (a small file)' do
 
-    let(:url) { 'https://mega.nz/#!QQhADCbL!vUY_phwxvkC004t5NKx7vynL16SvFfHYFkiX5vUlgjQ' }
+    let(:url) { 'https://mega.nz/file/muAVRRbb#zp9dvPvoVck8-4IwTazqsUqol6yiUK7kwLWOwrD8Jqo' }
 
     it 'downloads the related file' do
       Rmega.download(url, temp_folder)
@@ -15,13 +15,13 @@ describe 'File download' do
 
   context 'given a public mega url (a big file)' do
 
-    let(:url) { 'https://mega.nz/#!oAhCnBKR!CPeG8X92nBjvFsBF9EprZNW_TqIUwItHMkF9G2IZEIo' }
+    let(:url) { 'https://mega.nz/file/3zpE1ToL#B1L4o8POE4tER4h1tyVoGNxaXFhbjwfxhe3Eyp9nrN8' }
 
     it 'downloads the related file' do
       Rmega.download(url, temp_folder)
-      related_file = File.join(temp_folder, 'testfile_big_15mb.txt')
+      related_file = File.join(temp_folder, 'testfile_big_15mb.binary')
       md5 = Digest::MD5.file(related_file).hexdigest
-      expect(md5).to eq("0451dc82ac003dbef703342e40a1b8f6")
+      expect(md5).to eq("a92ec9994911866e3ea31aa1d914ac23")
     end
   end
 end

data/spec/integration/folder_download_spec.rb

@@ -4,14 +4,14 @@ describe 'Folder download' do
 
   context 'given a public mega url (folder)' do
 
-    let(:url) { 'https://mega.nz/#F!oQYEUaBD!QtYCjQDbBzefFeIM994FIg' }
+    let(:url) { 'https://mega.nz/folder/GvgkUIIK#v2hd_5GSvciGKazNeWSa6A' }
 
     it 'downloads the related file' do
       Rmega.download(url, temp_folder)
-      list = Dir["#{temp_folder}/test_folder/**/*"].map do |p|
-        p.gsub("#{temp_folder}/test_folder/", "")
+      list = Dir["#{temp_folder}/another_test_folder/**/*"].map do |p|
+        p.gsub("#{temp_folder}/another_test_folder/", "")
       end
-      expect(list.sort).to eq(["a.txt", "b.txt", "c", "c/c.txt"])
+      expect(list.sort).to eq(["b.txt", "c", "c/c.txt"])
     end
   end
 end

data/spec/integration/folder_upload_spec.rb

@@ -0,0 +1,48 @@
+require 'integration_spec_helper'
+
+describe 'Folder upload' do
+
+  if account?
+
+    before(:all) do
+      @storage = login
+    end
+
+    context "when a folder is uploaded" do
+
+      let(:folder) { SecureRandom.hex(6) }
+      let(:subfolder_empty) { SecureRandom.hex(6) }
+      let(:subfolder_with_content) { SecureRandom.hex(6) }
+      let(:file1) { SecureRandom.hex(6) }
+      let(:file2) { SecureRandom.hex(6) }
+
+      it 'all its content is found' do
+        Dir.mkdir("#{temp_folder}/#{folder}")
+        Dir.mkdir("#{temp_folder}/#{folder}/#{subfolder_empty}")
+        Dir.mkdir("#{temp_folder}/#{folder}/#{subfolder_with_content}")
+        File.write("#{temp_folder}/#{folder}/#{subfolder_with_content}/#{file1}", SecureRandom.random_bytes(1000))
+        File.write("#{temp_folder}/#{folder}/#{subfolder_with_content}/#{file2}", SecureRandom.random_bytes(2000))
+
+        @storage.root.upload_dir("#{temp_folder}/#{folder}")
+
+        uploaded_folder = @storage.root.folders.find { |f| f.name == folder }
+
+        expect(uploaded_folder.folders.size).to eq 2
+        folder1 = uploaded_folder.folders.find {|f| f.name == subfolder_empty}
+        folder2 = uploaded_folder.folders.find {|f| f.name == subfolder_with_content}
+        
+        expect(folder1).not_to be_nil
+        expect(folder2).not_to be_nil
+        
+        expect(folder1.folders).to be_empty
+        expect(folder1.files).to be_empty
+        
+        expect(folder2.folders).to be_empty
+        expect(folder2.files.find { |f| f.name == file1 }).not_to be_nil
+        expect(folder2.files.find { |f| f.name == file2 }).not_to be_nil
+
+        uploaded_folder.delete
+      end
+    end
+  end
+end

data/spec/integration/resume_download_spec.rb

@@ -3,7 +3,7 @@ require 'integration_spec_helper'
 module Rmega
   describe 'Resumable download' do
 
-    let(:download_url) { 'https://mega.nz/#!oAhCnBKR!CPeG8X92nBjvFsBF9EprZNW_TqIUwItHMkF9G2IZEIo' }
+    let(:download_url) { 'https://mega.nz/file/3zpE1ToL#B1L4o8POE4tER4h1tyVoGNxaXFhbjwfxhe3Eyp9nrN8' }
 
     let(:destination_file) { "#{temp_folder}/temp.txt" }
 
@@ -15,18 +15,17 @@ module Rmega
 
     it 'resume a download of a file' do
       node = Nodes::Factory.build_from_url(download_url)
-      content = nil
-
+      
       thread = Thread.new do
         node.download(destination_file)
       end
-
+      
       loop do
         next unless File.exists?(destination_file)
+        content = nil
         node.file_io_synchronize { content = File.read(destination_file) }
-        content.strip!
-        break if content.size > 5_000_000
-        sleep(0.5)
+        break if content.force_encoding("BINARY").strip.size > 2_000_000
+        sleep(0.1)
       end
 
       thread.kill
@@ -36,7 +35,7 @@ module Rmega
       node.download(destination_file)
 
       md5 = Digest::MD5.file(destination_file).hexdigest
-      expect(md5).to eq("0451dc82ac003dbef703342e40a1b8f6")
+      expect(md5).to eq("a92ec9994911866e3ea31aa1d914ac23")
     end
   end
 end

data/spec/integration/rmega-dl_spec.rb

@@ -2,7 +2,7 @@ require 'integration_spec_helper'
 
 describe "rmega-dl" do
 
-  let(:url) { 'https://mega.nz/#!QQhADCbL!vUY_phwxvkC004t5NKx7vynL16SvFfHYFkiX5vUlgjQ' }
+  let(:url) { 'https://mega.nz/file/muAVRRbb#zp9dvPvoVck8-4IwTazqsUqol6yiUK7kwLWOwrD8Jqo' }
 
   def call(*args)
     `bundle exec ./bin/rmega-dl #{args.join(' ')}`
@@ -26,9 +26,9 @@ describe "rmega-dl" do
   if account?
     context "given an account and a path" do
       it "downloads a file" do
-        call("/test_folder/a.txt -u #{account['email']} --pass #{account['password']} -o #{temp_folder}")
-        downloaded_file = "#{temp_folder}/a.txt"
-        expect(File.read(downloaded_file)).to eq "hello\n"
+        call("/test_folder/b.txt -u #{account['email']} --pass '#{account['password']}' -o #{temp_folder}")
+        downloaded_file = "#{temp_folder}/b.txt"
+        expect(File.read(downloaded_file)).to eq "foo\n"
       end
     end
   end

data/spec/integration/rmega-up_spec.rb

@@ -38,14 +38,14 @@ describe "rmega-up" do
   if account?
     context "when the remote path is missing" do
       it "fails" do
-        resp = call("#{filepath} -u #{account['email']} --pass #{account['password']} -r /foobar")
+        resp = call("#{filepath} -u #{account['email']} --pass '#{account['password']}' -r /foobar")
         expect(resp).to match(/error/i)
       end
     end
 
     context "without specifying a remote folder" do
       it "uploads a file to the root node" do
-        call("#{filepath} -u #{account['email']} --pass #{account['password']}")
+        call("#{filepath} -u #{account['email']} --pass '#{account['password']}'")
         storage = login
         node = storage.root.files.find { |f| f.name == filename }
         node.delete if node
@@ -55,7 +55,7 @@ describe "rmega-up" do
 
     context "when specifying a remote folder" do
       it "uploads a file into that folder" do
-        call("#{filepath} -u #{account['email']} --pass #{account['password']} -r test_folder2")
+        call("#{filepath} -u #{account['email']} --pass '#{account['password']}' -r test_folder2")
         storage = login
         node = storage.root.folders.find { |f| f.name == "test_folder2" }
         node = node.files.find { |f| f.name == filename }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rmega
 version: !ruby/object:Gem::Version
-  version: 0.2.7
+  version: 0.3.1
 platform: ruby
 authors:
 - topac
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-06-23 00:00:00.000000000 Z
+date: 2023-02-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pbkdf2-ruby
@@ -75,8 +75,8 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".circleci/config.yml"
 - ".gitignore"
-- ".travis.yml"
 - CHANGELOG.md
 - Dockerfile
 - Gemfile
@@ -89,6 +89,7 @@ files:
 - lib/rmega.rb
 - lib/rmega/api_response.rb
 - lib/rmega/cli.rb
+- lib/rmega/conn_pool.rb
 - lib/rmega/crypto.rb
 - lib/rmega/crypto/aes_cbc.rb
 - lib/rmega/crypto/aes_ctr.rb
@@ -124,6 +125,7 @@ files:
 - spec/integration/file_upload_spec.rb
 - spec/integration/folder_download_spec.rb
 - spec/integration/folder_operations_spec.rb
+- spec/integration/folder_upload_spec.rb
 - spec/integration/login_spec.rb
 - spec/integration/resume_download_spec.rb
 - spec/integration/rmega-dl_spec.rb
@@ -163,6 +165,7 @@ test_files:
 - spec/integration/file_upload_spec.rb
 - spec/integration/folder_download_spec.rb
 - spec/integration/folder_operations_spec.rb
+- spec/integration/folder_upload_spec.rb
 - spec/integration/login_spec.rb
 - spec/integration/resume_download_spec.rb
 - spec/integration/rmega-dl_spec.rb

data/.travis.yml

@@ -1,11 +0,0 @@
-language: ruby
-sudo: false
-cache: bundler
-rvm:
-  - 1.9.3
-  - 2.0
-  - 2.1
-  - 2.2
-  - 2.4
-  - 2.5
-  - ruby-head