cnvrg 1.4.9.4 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 57cfbe5e8781c875703c195139bb558edd39db3c70d35920efc48152412f63d2
-  data.tar.gz: 35c51593cfef1e3a54e33a95f801e5b09114df80ce5be03f2770408cc33bb0f5
+  metadata.gz: 7f9ea846489f30df4e7a770ef453e640e3dcaff0176ebbc6a4ae000bfbcba0e8
+  data.tar.gz: 7f5bb7c85aaa16c43834bf705325470bacf8edd2bd342e8d5cf39bd853843136
 SHA512:
-  metadata.gz: b60b3f6689af7a7d7446a52cf01c34043af9811ad3e132a5788372b51bbc83cd429ffe94d899ca53121018d3328028b14d82046bf1d0be22fefa0118fb0a17e3
-  data.tar.gz: 3c083bf5f3dc563a4863b34ba0db6a82e53f134c324b2d37e20ad08dbfc9ad9e6809e3778df197a897a43df2538a18f571293ae2e26bc62a484d738f91c56d22
+  metadata.gz: 30cc30763ae8758f8526eb8cac1ef559f3e39c649d521112c7f22908226719da771340907c5928f9ab86f3672fc5f37c1b3a2bd6dd09b8d2b1ac10fe809ad9ed
+  data.tar.gz: f1f20925bd2b889676303efbcd657ba063f7bc28274ab074048ccc91a904ebb1046883d8a4854bf77f2c951b88c6b4f75cf468e5221a203a5ac3d0b4692d42c7

data/cnvrg.gemspec

@@ -44,5 +44,4 @@ Gem::Specification.new do |spec|
     spec.add_runtime_dependency 'ruby-progressbar'
     spec.add_runtime_dependency 'net-ssh'
     spec.add_runtime_dependency 'down'
-    spec.add_runtime_dependency 'thread'
 end

data/lib/cnvrg/cli.rb

@@ -167,7 +167,7 @@ module Cnvrg
         super
       end
     end
-    desc "data", "upload and manage datasets", :hide => false
+    desc "data [COMMAND]", "Upload and manage datasets", :hide => false
     subcommand "data", Data
 
     desc "job", "manage running jobs", :hide => false
@@ -242,13 +242,16 @@ module Cnvrg
 
     map %w(-v --version) => :version
 
-    desc 'api', 'set api url, e.g cnvrg --api "https://cnvrg.io/api"'
+    desc 'api CNVRG_APPLICATION_URL', 'Set api url, e.g cnvrg --api "https://cnvrg.io/api"'
     method_option :verify_ssl, :type => :boolean, :aliases => ["-s", "--verify_ssl"], :default => true
 
     def set_api_url(url)
       log_handler()
       log_start(__method__, args, options)
       home_dir = File.expand_path('~')
+      if url.end_with? "/"
+        url = url.chomp("/")
+      end
       if !url.end_with? "/api"
         url = url + "/api"
       end
@@ -356,9 +359,9 @@ module Cnvrg
       end
     end
 
-    map %w(-api --api) => :set_api_url
+    map %w(api -api --api) => :set_api_url
 
-    desc 'set_default_owner', 'set default owner'
+    desc 'set_default_owner', 'Set default owner'
 
     def set_default_owner
       begin
@@ -423,7 +426,7 @@ module Cnvrg
       end
     end
 
-    desc 'set_compression_path', 'set compression path'
+    desc 'set_compression_path', 'Set compression path'
     method_option :reset, :type => :boolean, :aliases => ["-r", "--reset"], :default => false
 
     def set_compression_path(*compression_path)
@@ -765,7 +768,7 @@ module Cnvrg
       end
     end
 
-    desc 'data init', 'Init dataset directory'
+    desc 'data init', 'Init dataset directory', :hide => true
     method_option :public, :type => :boolean, :aliases => ["-p", "--public"], :default => false
 
     def init_data(public, bucket: nil)
@@ -804,7 +807,7 @@ module Cnvrg
         exit(1)
       end
     end
-    desc 'data delete', 'delete dataset'
+    desc 'data delete', 'delete dataset', :hide => true
     def delete_data(dataset_slug)
       begin
         verify_logged_in(false)
@@ -829,7 +832,7 @@ module Cnvrg
       end
     end
 
-    desc 'data verify', 'Verify datasets'
+    desc 'data verify', 'Verify datasets', :hide => true
     method_option :timeout, :type => :numeric, :aliases => ["-t", "--timeout"], :desc => "Time to wait before returning final answer", :default => 15
 
     def verify_datasets(dataset_titles, timeout=0)
@@ -848,7 +851,7 @@ module Cnvrg
       end
     end
 
-    desc 'data clone', 'Clone dataset'
+    desc 'data clone', 'Clone dataset', :hide => true
     method_option :commit, :type => :string, :aliases => ["-c", "--commit"], :default => ""
     method_option :only_tree, :type => :boolean, :aliases => ["-t", "--tree"], :default => false
     method_option :query, :type => :string, :aliases => ["-q", "--query"], :default => nil
@@ -884,7 +887,10 @@ module Cnvrg
           @files = Cnvrg::Datafiles.new(owner, slug, dataset: @dataset)
           log_message("Downloading files", Thor::Shell::Color::BLUE)
           if @dataset.softlinked?
-            return @dataset.download_softlink
+            @files.cp_ds
+            Cnvrg::CLI.log_message("Clone finished successfully", Thor::Shell::Color::GREEN)
+            @dataset.write_success
+            return
           end
 
           if only_tree
@@ -903,6 +909,7 @@ module Cnvrg
                                            :total => files_count,
                                            :autofinish => true)
 
+          
           while files['keys'].length > 0
             Cnvrg::Logger.log_info("download multiple files, #{downloaded_files.size} files downloaded")
             @files.download_multiple_files_s3(files, @dataset.local_path, progressbar: progressbar, read_only: read)
@@ -933,7 +940,7 @@ module Cnvrg
       end
     end
 
-    desc 'data clone_query', 'Clone dataset _query'
+    desc 'data clone_query', 'Clone dataset _query', :hide => true
     method_option :query, :type => :string, :aliases => ["-q", "--query"], :default => ""
     def clone_data_query(dataset_url,query=nil)
       begin
@@ -1074,7 +1081,7 @@ module Cnvrg
       end
     end
 
-    desc 'data download', 'pull data'
+    desc 'data download', 'pull data', :hide => true
     method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
     method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
 
@@ -1432,7 +1439,7 @@ module Cnvrg
     end
 
 
-    desc 'data upload', 'push data'
+    desc 'data upload', 'Upload files from local dataset directory to remote server', :hide => true
     method_option :ignore, :type => :array, :aliases => ["-i", "--i"], :desc => "ignore following files"
     method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
     method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
@@ -1630,7 +1637,7 @@ module Cnvrg
 
     end
 
-    desc 'data list', 'List all dataset you currently have'
+    desc 'data list', 'List all dataset you currently have', :hide => true
 
     def list_dataset
       verify_logged_in(false)
@@ -1652,7 +1659,7 @@ module Cnvrg
 
     end
 
-    desc 'data queries', 'List all data search queries you currently have'
+    desc 'data queries', 'List all data search queries you currently have', :hide => true
 
     def queries
       verify_logged_in(false)
@@ -1663,7 +1670,7 @@ module Cnvrg
       print_table(result)
     end
 
-    desc 'data download_tags_yaml', 'Download dataset tags yml file in current directory'
+    desc 'data download_tags_yaml', 'Download dataset tags yml files in current directory', :hide => true
 
     def download_tags_yaml
       verify_logged_in(false)
@@ -1678,7 +1685,7 @@ module Cnvrg
       end
     end
 
-    desc 'data query_files', 'List all data search queries you currently have'
+    desc 'data query_files', 'List all data search queries you currently have', :hide => true
     def query_files(query)
       verify_logged_in(false)
       log_start(__method__, args, options)
@@ -1689,7 +1696,7 @@ module Cnvrg
       print_table(result)
     end
 
-    desc 'data commits', 'List all commits for a specific dataset'
+    desc 'data commits', 'List all commits for a specific dataset', :hide => true
 
     def list_dataset_commits()
       verify_logged_in(true)
@@ -1704,7 +1711,7 @@ module Cnvrg
 
     end
 
-    desc 'commits', 'List all commits for a specific dataset'
+    desc 'commits', 'List all commits for a specific Project'
 
     def list_commits()
       verify_logged_in(true)
@@ -1774,7 +1781,7 @@ module Cnvrg
 
     end
 
-    desc 'clone', 'Clone project'
+    desc 'clone PROJECT_URL', 'Clone project'
     method_option :remote, :type => :boolean, :aliases => ["-r", "--r"], :default => false
     method_option :commit, :type => :string, :aliases => ["-c", "--c"], :default => nil
 
@@ -1956,7 +1963,7 @@ module Cnvrg
              :direct=>false, :force =>force, :sync =>true, :tags =>tags, :parallel => parallel
 
     end
-    desc 'upload_data_new', 'upload_data_new'
+    desc 'upload_data_new', 'upload_data_new', :hide => true
     method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
     method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
     method_option :force, :type => :boolean, :aliases => ["-f","--force"], :default => false
@@ -2005,7 +2012,7 @@ module Cnvrg
       end
     end
 
-    desc 'start_commit', 'start data commit'
+    desc 'start_commit', 'start data commit', :hide => true
     method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
     method_option :direct, :type => :boolean, :aliases => ["-d", "--direct"], :desc => "was called directed", :default => true
     method_option :force, :type => :boolean, :aliases => ["-f", "--force"], :default => false
@@ -2039,7 +2046,7 @@ module Cnvrg
     end
 
 
-    desc 'end_commit', 'start data commit'
+    desc 'end_commit', 'start data commit', :hide => true
     method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
     method_option :force, :type => :boolean, :aliases => ["-f","--force"], :default => false
 
@@ -2066,7 +2073,7 @@ module Cnvrg
         end
       end
     end
-    desc 'list_files', 'list files in dataset'
+    desc 'list_files', 'list files in dataset', :hide => true
     method_option :json, :type => :boolean, :aliases => ["-j","--json"],:default => true,   :desc => "response as json"
     method_option :commit, :type => :string, :aliases => ["-c","--commit"], :default => nil
 
@@ -2100,7 +2107,7 @@ module Cnvrg
       end
 
     end
-    desc 'upload_data', 'Upload updated files'
+    desc 'upload_data', 'Upload updated files', :hide => true
     method_option :ignore, :type => :string, :aliases => ["-i"], :desc => "ignore following files", :default => ""
     method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
     method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
@@ -2347,7 +2354,7 @@ module Cnvrg
       end
     end
 
-    desc 'download_file_data', 'Download one data files'
+    desc 'download_file_data', 'Download one data files', :hide => true
     method_option :verbose, :type => :boolean, :aliases => ["-v", "--verbose"], :default => false
     method_option :commit, :type => :string, :aliases => ["-c", "--commit"], :desc => "download specified commit", :default => nil
     method_option :link, :type => :boolean, :aliases => ["-l","--link"], :desc => "download specified commit", :default => false
@@ -2409,7 +2416,7 @@ module Cnvrg
     end
 
 
-    desc 'download_data_new', 'Download updated files'
+    desc 'download_data_new', 'Download updated files', :hide => true
     method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
     method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
     method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
@@ -2642,7 +2649,7 @@ module Cnvrg
 
 
 
-    desc 'jump', 'jump to specific commit'
+    desc 'jump', 'Jump to specific commit'
     def jump(commit_sha1)
       begin
         verify_logged_in()
@@ -3071,8 +3078,8 @@ module Cnvrg
                     stats = remote ? usage_metrics_in_docker(docker_id) : Helpers.ubuntu? ? {memory: memory_usage, cpu: cpu_usage} : {}
                     if is_on_gpu
                       gu = gpu_util
-                      stats['gpu'] = gu[0]
-                      stats['gpu_util'] = gu[1]
+                      stats['gpu_util'] = gu[0]
+                      stats['gpu'] = gu[1]
                     end
                     @exp.send_machine_stats [stats] unless stats.empty?
                   rescue => e
@@ -3508,7 +3515,7 @@ module Cnvrg
     method_option :dataset_only_tree, :type => :boolean, :aliases => [ "--dataset_only_tree"], :default => false
     method_option :data_query, :type => :string, :aliases => ["-q", "--query"], :default => ""
 
-    desc 'notebook', 'starts a notebook session remotely or locally'
+    desc 'notebook', 'Starts a notebook session remotely or locally'
 
     def notebook
       verify_logged_in(true)
@@ -3636,7 +3643,7 @@ module Cnvrg
       end
     end
 
-    desc 'remote_notebook', 'run notebook server on remote server'
+    desc 'remote_notebook', 'Run notebook server on remote server'
     method_option :machine_type, :type => :string, :default => ""
     method_option :notebook_type, :type => :string, :aliases => ["-n", "--notebook_type"], :default => ""
     method_option :data, :type => :string, :aliases => ["-d", "--data"], :default => ""
@@ -4439,7 +4446,7 @@ module Cnvrg
 
     end
 
-    desc 'push', 'push image to cnvrg repository'
+    desc 'push', 'push image to cnvrg repository', :hide=> true
 
     def push(*name)
       verify_logged_in(true)
@@ -4846,7 +4853,7 @@ module Cnvrg
 
     end
 
-    desc 'upload_image', 'Upload new docker image to cnvrg'
+    desc 'upload_image', 'Upload new docker image to cnvrg', :hide => true
     method_option :workdir, :type => :string, :aliases => ["-w","--workdir"], :desc => "workdir of docker image", :default => "/root"
     method_option :description, :type => :string, :aliases => ["-d", "--description"], :desc => "description for docker image", :default => ""
     method_option :user, :type => :string, :aliases => ["-u","--user"], :default => "root"
@@ -4999,7 +5006,7 @@ module Cnvrg
       end
     end
 
-    desc 'list_images', 'lists all custom images you can pull'
+    desc 'list_images', 'lists all custom images you can pull', :hide => true
 
     def list_images
       begin
@@ -5026,7 +5033,7 @@ module Cnvrg
 
     end
 
-    desc 'list_machines', 'lists all machines belong to your organization'
+    desc 'list_machines', 'Lists all machines belong to your organization'
 
     def list_machines
       begin
@@ -5051,7 +5058,7 @@ module Cnvrg
 
     end
 
-    desc 'experiments', 'list project experiments'
+    desc 'experiments', 'List project experiments'
     method_option :id, :type => :string, :aliases => ["-id"], :desc => "Get info for specific experiments", :default => ""
     method_option :tag, :type => :string, :aliases => ["-t"], :desc => "Get info for specific experiment tag", :default => ""
 
@@ -5134,7 +5141,7 @@ module Cnvrg
     end
 
 
-    desc 'pull_image', 'downloads and loads an image'
+    desc 'pull_image', 'downloads and loads an image', :hide => true
 
     def pull_image(image_name)
       begin
@@ -5217,7 +5224,7 @@ module Cnvrg
 
     end
 
-    desc 'set_image', 'set image to a porject'
+    desc 'set_image', 'set image to a porject', :hide => true
 
     def set_image(docker_image)
       verify_logged_in(true)

data/lib/cnvrg/data.rb

@@ -1,6 +1,5 @@
-require 'thread/pool'
-require 'thread/future'
-
+require 'cnvrg/cli'
+require 'thor'
 class SubCommandBase < Thor
  def self.banner(command, namespace = nil, subcommand = false)
   "#{basename}  #{command.usage}"
@@ -14,7 +13,7 @@ end
 module Cnvrg
   class Data < SubCommandBase
     class_option :no_compression, :type => :boolean, :aliases => ["-nc", "--no_compression"], :default => false
-    desc "data init", "init data folder"
+    desc "data init", "Set current directory as dataset directory"
     method_option :public, :type => :boolean, :aliases => ["-p", "--public"], :default => false
     method_option :bucket, :type => :string, :aliases => ["-b", "--bucket"], :default => ""
     def init
@@ -23,7 +22,7 @@ module Cnvrg
       bucket = options["bucket"]
       cli.init_data(public, bucket: bucket)
     end
-    desc "data upload", "upload data folder"
+    desc "data upload", "Upload files from local dataset directory to remote server"
     method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
     method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
     method_option :force, :type => :boolean, :aliases => ["-f","--force"], :default => false
@@ -40,7 +39,7 @@ module Cnvrg
       tags = options["tags"]
       cli.upload_data_new(new_branch,  verbose,sync,force, tags, chunk_size)
     end
-    desc 'sync', 'sync_data_new'
+    desc 'data sync', 'Synchronise local dataset directory with remote server'
     method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
     method_option :force, :type => :boolean, :aliases => ["-f","--force"], :default => false
     method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
@@ -64,7 +63,7 @@ module Cnvrg
       init = options["init"]
       cli.sync_data_new(new_branch, force, verbose,commit,all_files, tags,parallel, chunk_size, init)
     end
-    desc 'data download', 'pull data'
+    desc 'data download', 'Download files from remote server'
     method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits", :default => false
     method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
     method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
@@ -80,7 +79,7 @@ module Cnvrg
       all_files = options["all_files"]
       cli.download_data_new(verbose,sync,new_branch, commit,all_files)
     end
-    desc 'data clone', 'clone datset'
+    desc 'data clone DATASET_URL', 'Clone dataset'
     method_option :only_tree, :type => :boolean, :aliases => ["-t", "--tree"], :default => false
     method_option :commit, :type => :string, :aliases => ["-c", "--commit"], :default => nil
     method_option :query, :type => :string, :aliases => ["-q", "--query"], :default => nil
@@ -97,7 +96,7 @@ module Cnvrg
       cli.clone_data(dataset_url, only_tree=only_tree,commit=commit, query=query, read=read, remote=remote)
     end
 
-    desc 'data verify', 'verify datasets', :hide => true
+    desc 'data verify DATASETS_TITLES', 'verify datasets', :hide => true
     method_option :timeout, :type => :numeric, :aliases => ["-t", "--timeout"], :desc => "Time to wait before returning final answer", :default => 15
 
     def verify(*dataset_titles)
@@ -106,7 +105,7 @@ module Cnvrg
       cli.verify_datasets(dataset_titles, timeout)
     end
 
-    desc 'data set --url=dataset_url', 'Set dataset url to other url'
+    desc 'data set --url=DATASET_URL', 'Set dataset url to other url'
     method_option :url, :type => :string, :aliases => ["--url"], :default => ''
     def set
       cli = Cnvrg::CLI.new
@@ -117,7 +116,7 @@ module Cnvrg
       end
     end
 
-    desc 'data put dataset_url', 'clone datset'
+    desc 'data put DATASET_URL FILES_PREFIX', 'Upload selected files from local dataset directory to remote server'
     method_option :dir, :type => :string, :aliases => ["-d", "--dir"], :default => ''
     def put(dataset_url, *files)
       cli = Cnvrg::CLI.new()
@@ -125,7 +124,7 @@ module Cnvrg
       cli.data_put(dataset_url, files: files, dir: dir)
     end
 
-    desc 'data clone_query', 'clone query datset'
+    desc 'data clone_query --query=QUERY_SLUG DATASET_URL', 'Clone dataset with specific query'
     method_option :query, :type => :string, :aliases => ["-q", "--query"], :default => nil
     def clone_query(dataset_url)
       cli = Cnvrg::CLI.new()
@@ -133,20 +132,20 @@ module Cnvrg
       cli.clone_data_query(dataset_url,query=query)
     end
 
-    desc 'data delete', 'delete datset'
+    desc 'data delete DATASET_SLUG', 'Delete dataset'
     def delete(dataset_slug)
       cli = Cnvrg::CLI.new()
       cli.delete_data(dataset_slug)
 
     end
-    desc 'data list', 'list of datasets'
+    desc 'data list', 'Show list of all datasets'
     def list()
       cli = Cnvrg::CLI.new()
 
       cli.list_dataset()
 
     end
-    desc 'data commits', 'pull data'
+    desc 'data commits', 'List all commits for a current dataset'
 
     def commits()
       cli = Cnvrg::CLI.new()
@@ -154,7 +153,7 @@ module Cnvrg
 
     end
 
-    desc 'data files', 'list dataset files'
+    desc 'data files DATASET_URL', 'Show list of dataset files'
     method_option :offset, :type => :numeric, :aliases => ["-o", "--offset"], :default => 0
     method_option :limit, :type => :numeric, :aliases => ["-l", "--limit"], :default => 1000
     method_option :expires, :type => :numeric, :aliases => ["-ex", "--expires"], :default => 3600
@@ -172,19 +171,19 @@ module Cnvrg
       cli.log_message(files)
     end
 
-    desc 'data queries', 'list of all queries'
+    desc 'data queries', 'List all dataset queries related to current dataset'
     def queries()
       cli = Cnvrg::CLI.new()
       cli.queries()
     end
 
-    desc 'data query_files', 'list of all queries'
+    desc 'data query_files QUERY_NAME', 'Show list of all files in specific query'
     def query_files(query)
       cli = Cnvrg::CLI.new()
       cli.query_files(query)
     end
 
-    desc 'data download_tags_yaml', 'Download dataset tags yml file in current directory'
+    desc 'data download_tags_yaml', 'Download dataset tags yml files in current dataset directory'
     def download_tags_yaml
       cli = Cnvrg::CLI.new()
       cli.download_tags_yaml()

data/lib/cnvrg/datafiles.rb

@@ -70,6 +70,35 @@ module Cnvrg
       end
     end
 
+    def cp_ds
+      prefix = @dataset.get_dataset["bucket_prefix"]
+      current_batch = @downloader.fetch_files(prefix: prefix, limit: 500)
+      pbar = ProgressBar.create(:title => "Download Progress",
+                                        :progress_mark => '=',
+                                        :format => "%b%i| %c Files downloaded",
+                                        :starting_at => 0,
+                                        :total => nil,
+                                        :autofinish => true)
+      parallel_options = {
+          in_threads: ParallelThreads,
+          in_processes: Cnvrg::CLI::ParallelProcesses,
+          isolation: true,
+          finish: ->(*args) {pbar.progress += 1}
+      }
+      while current_batch.size > 0
+        Parallel.map(current_batch, parallel_options) do |file|
+          #current_batch.map do |file|
+          next if file.end_with? "/"
+          cutted_key = Cnvrg::Downloader::Clients::S3Client.cut_prefix(prefix, file)
+          dest_path = File.join(@dataset.local_path, cutted_key)
+          @downloader.download(file, dest_path, decrypt: false)
+          file
+        end
+        marker = current_batch.last
+        current_batch = @downloader.fetch_files(prefix: prefix, marker: marker, limit: 500)
+      end
+    end
+
 
     def put_commit(commit_sha1)
       response = Cnvrg::API.request("#{@base_resource}/commit/latest", 'PUT', {commit_sha1: commit_sha1})

data/lib/cnvrg/dataset.rb

@@ -63,21 +63,6 @@ module Cnvrg
     end
 
 
-    def download_softlink
-      @storage = self.get_storage_client
-      @progressbar = ProgressBar.create(:title => "Download Progress",
-                                        :progress_mark => '=',
-                                        :format => "%b%i| %c Files downloaded",
-                                        :starting_at => 0,
-                                        :total => nil,
-                                        :autofinish => true)
-      @storage.cp(get_dataset["bucket_prefix"], @local_path, progress: @progressbar)
-      @progressbar.finish
-      Cnvrg::CLI.log_message("Clone finished successfully", Thor::Shell::Color::GREEN)
-      write_success
-    end
-
-
     def get_storage_client
       response = Cnvrg::API.request("users/#{@owner}/datasets/#{@slug}/client", 'GET')
       if Cnvrg::CLI.is_response_success(response, false)
@@ -262,6 +247,9 @@ module Cnvrg
       end
 
       ignore_list = []
+      if not File.exists? "#{self.local_path}/.cnvrgignore"
+        return ignore_list
+      end
       File.open(self.local_path + "/.cnvrgignore", "r").each_line do |line|
         line = line.strip
         if line.start_with? "#" or ignore_list.include? line or line.empty?

data/lib/cnvrg/downloader/clients/s3_client.rb

@@ -26,8 +26,10 @@ module Cnvrg
                                     })
           end
 
-          @client = Aws::S3::Client.new(options)
-          @bucket = Aws::S3::Resource.new(client: @client).bucket(@bucket_name)
+          @options = options
+
+          #@client = Aws::S3::Client.new(options)
+          #@bucket = Aws::S3::Resource.new(client: @client).bucket(@bucket_name)
           @upload_options = {:use_accelerate_endpoint => storage == 's3'}
           if encryption.present?
             @upload_options[:server_side_encryption] = encryption
@@ -39,54 +41,39 @@ module Cnvrg
           storage_path = Cnvrg::Helpers.decrypt(@key, @iv, storage_path) if decrypt
           resp = nil
           File.open(local_path, 'w+') do |file|
-            resp = @client.get_object({bucket: @bucket_name, key: storage_path}, target: file)
+            resp = aws_client.get_object({bucket: @bucket_name, key: storage_path}, target: file)
           end
           resp
         rescue => e
-          p e.message
-          p e.backtrace
+          Cnvrg::Logger.log_error(e)
+          raise e
         end
 
         def upload(storage_path, local_path)
           ### storage path is the path inside s3 (after the bucket)
           # local path is fullpath for the file /home/ubuntu/user.../hazilim.py
-          o = @bucket.object(storage_path)
+          o = aws_bucket.object(storage_path)
           o.upload_file(local_path, @upload_options)
         end
 
-
-        def cp(prefix, local_path, progress: nil)
-          files = []
-          files_thread = Thread.new do
-            current_marker = nil
-            while true
-              batch_files = @bucket.objects(prefix: prefix, marker: current_marker).first(500)
-              batch_files = batch_files.to_a
-              break if batch_files.size === 0
-              files += batch_files
-              current_marker = batch_files.last.key
-            end
-          end
-          thread_pool = Thread.pool(Cnvrg::Helpers.parallel_threads)
-          while true
-            while files.present?
-              Thread.future thread_pool do
-                f = files.pop
-                cutted_key = cut_prefix(prefix, f.key)
-                dest_path = File.join(local_path, cutted_key)
-                download(f.key, dest_path, decrypt: false)
-                progress.increment
-              end
-            end
-            break if files.size == 0 and files_thread.status == false
-            sleep(1)
-          end
-          thread_pool.wait(:done)
+        def fetch_files(prefix: nil, marker: nil, limit: 1000)
+          batch_files = aws_bucket.objects(prefix: prefix, marker: marker).first(limit)
+          batch_files.to_a.map(&:key)
         end
 
-        def cut_prefix(prefix, file)
+        def self.cut_prefix(prefix, file)
           file.gsub(prefix, '').gsub(/^\/*/, '')
         end
+
+
+        private
+        def aws_client
+          Aws::S3::Client.new(@options)
+        end
+
+        def aws_bucket
+          Aws::S3::Resource.new(client: aws_client).bucket(@bucket_name)
+        end
       end
     end
   end

data/lib/cnvrg/helpers.rb

@@ -12,6 +12,14 @@ module Cnvrg
       ### set max threads to be 100k
       [threads, 100000].min
     end
+
+    def self.parallel_options
+      {
+          in_processes: Cnvrg::CLI::ParallelProcesses,
+          in_thread: Cnvrg::CLI::ParallelThreads,
+          isolation: true
+      }
+    end
     def checkmark
       checkmark = "\u2713"
       return checkmark.encode('utf-8')

data/lib/cnvrg/version.rb

@@ -1,4 +1,4 @@
 module Cnvrg
-  VERSION = '1.4.9.4'
+  VERSION = '1.5.0'
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 1.4.9.4
+  version: 1.5.0
 platform: ruby
 authors:
 - Yochay Ettun
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-06-25 00:00:00.000000000 Z
+date: 2019-07-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -372,20 +372,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: thread
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 description: A CLI tool for interacting with cnvrg.io.
 email:
 - info@cnvrg.io
@@ -446,7 +432,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubyforge_project: 
+rubygems_version: 2.7.9
 signing_key: 
 specification_version: 4
 summary: A CLI tool for interacting with cnvrg.io.