cnvrg 1.10.19 → 1.11.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 057f3283dffe548f882ed6cbdcfbc176e12fa75291f8a4867c8eee74fefd701f
-  data.tar.gz: fe30a40d86418cca0c98156ed89212ccfb159ead16dd248388a05f9d5bb3a05f
+  metadata.gz: 5b5e5904880441ee1b4610eb69e2d3d5c74e0e58f4f7d1dbd0e1547de0c47e9c
+  data.tar.gz: 37b7b8c6f82b00ca93ccc8a323c84653e2c83757008c57c04d5a98c32bcad2de
 SHA512:
-  metadata.gz: 5b5ccc6f603a06affd160bc928c8ee99acf045eccf3905dcc1c23ceca8b4efdb185c3f78716bb111973df0420a5356dbf3565c52a9fe298f4fccdc886f3b07d7
-  data.tar.gz: 1cc6cbaa4737fb0151bed559352292908479ffce6e2bb1ba65812aef19d91b935265ec80f8a2a0f52111c02b3944aecb79d89e8d72518f4db094c75766834228
+  metadata.gz: 8a5d6540f9da9ab77321435d5f4ece63010faf6d0ce05a76ce938236eaa6cd4c1853aa004f732313d69b644b66df50da505b9c55a4fc165a6e353f7e48c414c9
+  data.tar.gz: 4f9f90f9f465db06e62a2f25fe84161b0a0994afb68d9d7ff4604959f3627bdbea20eead733fa50942e5a93191d445b010f7b152211c9948b746e69e8c4e839f

data/lib/cnvrg/cli.rb

@@ -896,10 +896,12 @@ module Cnvrg
             return
           end
 
+
           if only_tree
-            success = Dataset.clone_tree(commit: commit, dataset_home: dataset_home)
-            return if success
+            Dataset.clone_tree(commit: commit, dataset_home: dataset_home)
+            return
           end
+
           commit = response["result"]["commit"]
           files_count = response["result"]["file_count"]
           files = @files.get_clone_chunk(commit: commit)
@@ -911,6 +913,7 @@ module Cnvrg
                                            :total => files_count,
                                            :autofinish => true)
 
+          Dataset.clone_tree(commit: commit, dataset_home: dataset_home, progressbar: progressbar)
 
           while files['keys'].length > 0
             Cnvrg::Logger.log_info("download multiple files, #{downloaded_files.size} files downloaded")
@@ -964,10 +967,9 @@ module Cnvrg
         Cnvrg::CLI.is_response_success(response,true)
         dataset_name = response["results"]["name"]
         dataset_slug = response["results"]["slug"]
-        dataset_home = Dir.pwd+"/"+dataset_slug
+        dataset_home = Dir.pwd + "/" + dataset_slug
         Dataset.stop_if_dataset_present(dataset_home, dataset_name) if soft
 
-        # dataset_home = Dir.pwd
         if Dataset.blank_clone(owner, dataset_name, dataset_slug)
           dataset = Dataset.new(dataset_home)
           log_message("Cloning #{dataset_name}", Thor::Shell::Color::BLUE)
@@ -986,12 +988,13 @@ module Cnvrg
           begin
             log_message("Downloading files", Thor::Shell::Color::BLUE)
             Parallel.map((response["results"]["query_files"]), parallel_options) do |f|
-              relative_path = f["fullpath"].gsub(/^#{dataset_home}/, "")
+              relative_path = f["fullpath"].gsub(/^#{dataset_home}/, "").gsub(/^#{slug}/, "")
               relative_path_dir = relative_path.split("/")
               file_name = relative_path_dir.pop()
               relative_path_dir = relative_path_dir.join("/")
               abs_path = dataset_home + "/" + relative_path_dir
               abs_path = dataset_home if flatten
+
               begin
                 FileUtils.mkdir_p(abs_path) unless File.exist? (abs_path + "/" + file_name)
               rescue
@@ -1194,6 +1197,7 @@ module Cnvrg
     desc '', '', :hide => true
     def data_put(dataset_url, files: [], dir: '', commit: '', chunk_size: 1000, force: false, override: false, threads: 15, message: nil)
       begin
+        verify_logged_in(false)
         log_start(__method__, args, options)
 
         owner, slug = get_owner_slug(dataset_url)
@@ -1217,7 +1221,7 @@ module Cnvrg
           Cnvrg::Logger.info("Put files in latest commit")
           response = @datafiles.last_valid_commit()
           unless response #means we failed in the start commit.
-            raise SignalException.new(1, "Cant put files into commit:#{commit}, check the dataset id and commit")
+            raise SignalException.new(1, "Cant put files into commit:#{commit}, check the dataset id and commitc")
           end
           @commit = response['result']['sha1']
         else
@@ -1780,8 +1784,7 @@ module Cnvrg
       print_table(list)
     end
 
-    desc 'commits', 'List all commits for a specific Project'
-
+    desc 'commits', 'List all commits for a specific Project', :hide => true
     def list_commits()
       verify_logged_in(true)
       log_start(__method__, args, options)
@@ -1795,7 +1798,7 @@ module Cnvrg
     end
 
 
-    desc 'unlink', 'Unlink a project from current directory'
+    desc 'unlink', 'Unlink a project from current directory', :hide => true
 
     def unlink
       verify_logged_in(false)
@@ -2892,7 +2895,7 @@ module Cnvrg
       end
     end
 
-    desc 'show', 'Show specific file from a specific commit'
+    desc 'show', 'Show specific file from a specific commit', :hide => true
     method_option :path, :type => :string, :aliases => ["-p"], :desc => "File path", :default => ""
     method_option :commit, :type => :string, :aliases => ["-c"], :desc => "Commit sha1", :default => nil
 
@@ -4805,8 +4808,7 @@ module Cnvrg
 
     end
 
-    desc 'list_machines', 'Lists all machines belong to your organization'
-
+    desc 'list_machines', 'Lists all machines belong to your organization', :hide => true
     def list_machines
       begin
         verify_logged_in(false)

data/lib/cnvrg/datafiles.rb

@@ -52,9 +52,6 @@ module Cnvrg
         end
         raise SignalException.new(1, "Cant find file #{file}") unless File.exists? "#{Dir.pwd}/#{file}"
       end
-      paths.each_slice(1000) do |chunk|
-        chunk.each { |p| p.gsub!(/^\.\//, '') }
-      end
       paths
     end
 
@@ -432,9 +429,10 @@ module Cnvrg
 
         file_chunks = files.each_slice(chunk_size).to_a
         # Fetch the required files from the server:
-        Parallel.map((file_chunks), in_threads: 10) do |files_chunk|
+        Parallel.map((file_chunks), in_threads: threads) do |chunk|
+          files_chunk = chunk.map{|p| p.gsub(/^\.\//, '')}
           Cnvrg::Logger.info("Generating chunk idx")
-          tree = @dataset.generate_chunked_idx(files_chunk, prefix: prefix, threads: threads)
+          tree = @dataset.generate_chunked_idx(files_chunk, prefix: prefix, threads: threads, cli: cli)
           Cnvrg::Logger.info("Getting files info from server")
           results = request_upload_files(commit_sha1, tree, override, new_branch, partial_commit)
           next unless results
@@ -1202,7 +1200,9 @@ module Cnvrg
     end
 
     def end_commit(commit_sha1, force, success: true, uploaded_files: 0, commit_type: nil)
+      counter = 0
       begin
+        counter += 1
         response = Cnvrg::API.request(
             "#{base_resource}/commit/end",
             'POST',
@@ -1214,9 +1214,11 @@ module Cnvrg
                 commit_type: commit_type
             }
         )
-        Cnvrg::CLI.is_response_success(response, true)
+        is_success = Cnvrg::CLI.is_response_success(response, false)
+        raise Exception.new("Invalid response #{response}") unless is_success
         return response
       rescue => e
+        retry if counter <= 20
         return false
       end
     end

data/lib/cnvrg/dataset.rb

@@ -471,22 +471,32 @@ module Cnvrg
       response.to_json
     end
 
-    def self.clone_tree(commit: 'latest', dataset_home: nil)
+    def self.clone_tree(commit: 'latest', dataset_home: nil, progressbar: nil)
       @dataset = Cnvrg::Dataset.new(dataset_home)
       @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug, dataset: @dataset)
       trees = @files.get_trees(commit: commit)
       return false if trees.nil?
-      pb = ProgressBar.create(:title => "Download Progress",
-                              :progress_mark => '=',
-                              :format => "%b>>%i| %p%% %t",
-                              :starting_at => 0,
-                              :total => trees.size,
-                              :autofinish => true)
+
+      if progressbar
+        pb = progressbar
+        pb.total += trees.size
+      else
+        pb = ProgressBar.create(
+          :title => "Download Progress",
+          :progress_mark => '=',
+          :format => "%b>>%i| %p%% %t",
+          :starting_at => 0,
+          :total => trees.size,
+          :autofinish => true
+        )
+      end
+
       trees.each do |tree|
         pb.progress += 1
         @files.download_dir(dataset_home, tree)
       end
-      pb.finish
+
+      pb.finish unless progressbar
       @dataset.write_success
       true
     end
@@ -541,7 +551,7 @@ module Cnvrg
       "#{url}/#{self.owner}/datasets/#{self.slug}"
     end
 
-    def generate_chunked_idx(list_files = [], threads: 15, prefix: '')
+    def generate_chunked_idx(list_files = [], threads: 15, prefix: '', cli: nil)
       tree = {}
       Parallel.map(list_files, in_threads: threads) do |file|
 
@@ -552,7 +562,11 @@ module Cnvrg
         label = safe_path.gsub(self.local_path + "/", "")
         label = "#{prefix}/#{label}" if prefix.present?
         if not Cnvrg::Files.valid_file_name?(label)
-          raise StandardError.new("#{label} is not a valid file name.")
+          if cli
+            cli.log_message("#{label} is not a valid file name, skipping it", Thor::Shell::Color::RED)
+          else
+            puts "#{label} is not a valid file name, , skipping it"
+          end
         end
         if File.directory? file
           tree[label + "/"] = nil

data/lib/cnvrg/downloader/client.rb

@@ -14,8 +14,15 @@ module Cnvrg
       end
 
       def extract_key_iv(sts_path)
-        sts = open(sts_path, {ssl_verify_mode: 0}).read rescue nil
-        raise StandardError.new("Cant open sts") if sts.blank?
+        count = 20
+        begin
+          count += 1
+          sts = open(sts_path, {ssl_verify_mode: 0}).read rescue nil
+        rescue => e
+          Cnvrg::Logger.log_error(e)
+          retry if count <= 20
+          raise StandardError.new("Cant access storage: #{e.message}")
+        end
         sts.split("\n")
       end
 

data/lib/cnvrg/files.rb

@@ -115,7 +115,7 @@ module Cnvrg
         #upload files
         blob_id_chunk = Parallel.map(files.keys, in_threads: ParallelThreads) do |file|
           begin
-            Cnvrg::Helpers.try_until_success{self.upload_single_file(files[file].merge(parsed_chunk_of_files[file]))}
+            upload_single_file(files[file].merge(parsed_chunk_of_files[file]))
           rescue => e
             Cnvrg::CLI.log_message("Failed to upload #{file}: #{e.message}", 'red') unless suppress_exceptions
             Cnvrg::Logger.log_error(e)
@@ -188,7 +188,7 @@ module Cnvrg
     def upload_single_file(file)
       path = file['path']
       absolute_path = file[:absolute_path]
-      @client.upload(path, absolute_path)
+      @client.safe_upload(path, absolute_path)
     end
 
     def parse_file(file)

data/lib/cnvrg/version.rb

@@ -1,3 +1,3 @@
 module Cnvrg
-  VERSION = '1.10.19'
+  VERSION = '1.11.3'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 1.10.19
+  version: 1.11.3
 platform: ruby
 authors:
 - Yochay Ettun
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-09-22 00:00:00.000000000 Z
+date: 2020-11-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -453,7 +453,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.0.4
 signing_key:
 specification_version: 4
 summary: A CLI tool for interacting with cnvrg.io.