cnvrg 1.10.18 → 1.11.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8009827a102d9ea4bda3e07d95ad52a23fe7202b2670850952fb345acac7b906
-  data.tar.gz: aa91b22c83120d50f0631945d06f4d878e1cea4d2f9d950f6e29fbba17305f21
+  metadata.gz: 70906f4870c9aa2b3457673212893d09e6b5f22e15a5dc4c3b0cc2115e74d8d9
+  data.tar.gz: ff9de0565cf2c81102c68a21154af1f97d843e65d51fc8eb69f3a490b64bb9a3
 SHA512:
-  metadata.gz: 2159ce3f1e3b55390b599a72b80c1b6de559554380d5b294922ddd001cff2c944cf6614bb582f184c62be6299826f4a1860257a5426c6534a254ea195bb9910c
-  data.tar.gz: ccb989d604602dbcfcee6f40725ae8a45875744f222b7aa717ddfe2760483361d03a16f6e016179acf82c0c493d9a3518a548d18dbb850a7cf0fc0b99aef0120
+  metadata.gz: d839249d813d8004c9d00553f81273e890d7beb3bfab4bb65b64453f459bd6e4c366c59b35cee2773716e2c26f6f5b30223937837e088fe964e98a60b242c6e3
+  data.tar.gz: f7cac4b24a2f7fc275e28dd03b2fad4b7374dd52c648dd84f36652d6e7bd61f24e834aa89b1f826ac39e7680aa7cbfc6ca1564bc3611fdfe53dfd650368d7319

data/lib/cnvrg/cli.rb

@@ -964,10 +964,9 @@ module Cnvrg
         Cnvrg::CLI.is_response_success(response,true)
         dataset_name = response["results"]["name"]
         dataset_slug = response["results"]["slug"]
-        dataset_home = Dir.pwd+"/"+dataset_slug
+        dataset_home = Dir.pwd + "/" + dataset_slug
         Dataset.stop_if_dataset_present(dataset_home, dataset_name) if soft
 
-        # dataset_home = Dir.pwd
         if Dataset.blank_clone(owner, dataset_name, dataset_slug)
           dataset = Dataset.new(dataset_home)
           log_message("Cloning #{dataset_name}", Thor::Shell::Color::BLUE)
@@ -986,12 +985,13 @@ module Cnvrg
           begin
             log_message("Downloading files", Thor::Shell::Color::BLUE)
             Parallel.map((response["results"]["query_files"]), parallel_options) do |f|
-              relative_path = f["fullpath"].gsub(/^#{dataset_home}/, "")
+              relative_path = f["fullpath"].gsub(/^#{dataset_home}/, "").gsub(/^#{slug}/, "")
               relative_path_dir = relative_path.split("/")
               file_name = relative_path_dir.pop()
               relative_path_dir = relative_path_dir.join("/")
               abs_path = dataset_home + "/" + relative_path_dir
               abs_path = dataset_home if flatten
+
               begin
                 FileUtils.mkdir_p(abs_path) unless File.exist? (abs_path + "/" + file_name)
               rescue
@@ -1781,8 +1781,7 @@ module Cnvrg
       print_table(list)
     end
 
-    desc 'commits', 'List all commits for a specific Project'
-
+    desc 'commits', 'List all commits for a specific Project', :hide => true
     def list_commits()
       verify_logged_in(true)
       log_start(__method__, args, options)
@@ -1796,7 +1795,7 @@ module Cnvrg
     end
 
 
-    desc 'unlink', 'Unlink a project from current directory'
+    desc 'unlink', 'Unlink a project from current directory', :hide => true
 
     def unlink
       verify_logged_in(false)
@@ -1906,7 +1905,7 @@ module Cnvrg
               log_message("Remove dir in order to clone #{project_name}", Thor::Shell::Color::RED)
               exit(1)
             end
-
+  
           end
           clone_resp = Project.clone_dir(slug, owner, project_name,git)
           project_home = Dir.pwd + "/" + project_name
@@ -2893,7 +2892,7 @@ module Cnvrg
       end
     end
 
-    desc 'show', 'Show specific file from a specific commit'
+    desc 'show', 'Show specific file from a specific commit', :hide => true
     method_option :path, :type => :string, :aliases => ["-p"], :desc => "File path", :default => ""
     method_option :commit, :type => :string, :aliases => ["-c"], :desc => "Commit sha1", :default => nil
 
@@ -4806,8 +4805,7 @@ module Cnvrg
 
     end
 
-    desc 'list_machines', 'Lists all machines belong to your organization'
-
+    desc 'list_machines', 'Lists all machines belong to your organization', :hide => true
     def list_machines
       begin
         verify_logged_in(false)

data/lib/cnvrg/datafiles.rb

@@ -52,7 +52,7 @@ module Cnvrg
         end
         raise SignalException.new(1, "Cant find file #{file}") unless File.exists? "#{Dir.pwd}/#{file}"
       end
-      paths.map{|p| p.gsub(/^\.\//, '')}
+      paths
     end
 
     def get_files_and_folders(paths)
@@ -429,9 +429,10 @@ module Cnvrg
 
         file_chunks = files.each_slice(chunk_size).to_a
         # Fetch the required files from the server:
-        Parallel.map((file_chunks), in_threads: 10) do |files_chunk|
+        Parallel.map((file_chunks), in_threads: threads) do |chunk|
+          files_chunk = chunk.map{|p| p.gsub(/^\.\//, '')}
           Cnvrg::Logger.info("Generating chunk idx")
-          tree = @dataset.generate_chunked_idx(files_chunk, prefix: prefix, threads: threads)
+          tree = @dataset.generate_chunked_idx(files_chunk, prefix: prefix, threads: threads, cli: cli)
           Cnvrg::Logger.info("Getting files info from server")
           results = request_upload_files(commit_sha1, tree, override, new_branch, partial_commit)
           next unless results
@@ -1199,7 +1200,9 @@ module Cnvrg
     end
 
     def end_commit(commit_sha1, force, success: true, uploaded_files: 0, commit_type: nil)
+      counter = 0
       begin
+        counter += 1
         response = Cnvrg::API.request(
             "#{base_resource}/commit/end",
             'POST',
@@ -1211,9 +1214,11 @@ module Cnvrg
                 commit_type: commit_type
             }
         )
-        Cnvrg::CLI.is_response_success(response, true)
+        is_success = Cnvrg::CLI.is_response_success(response, false)
+        raise Exception.new("Invalid response #{response}") unless is_success
         return response
       rescue => e
+        retry if counter <= 20
         return false
       end
     end

data/lib/cnvrg/dataset.rb

@@ -541,7 +541,7 @@ module Cnvrg
       "#{url}/#{self.owner}/datasets/#{self.slug}"
     end
 
-    def generate_chunked_idx(list_files = [], threads: 15, prefix: '')
+    def generate_chunked_idx(list_files = [], threads: 15, prefix: '', cli: nil)
       tree = {}
       Parallel.map(list_files, in_threads: threads) do |file|
 
@@ -552,7 +552,11 @@ module Cnvrg
         label = safe_path.gsub(self.local_path + "/", "")
         label = "#{prefix}/#{label}" if prefix.present?
         if not Cnvrg::Files.valid_file_name?(label)
-          raise StandardError.new("#{label} is not a valid file name.")
+          if cli
+            cli.log_message("#{label} is not a valid file name, skipping it", Thor::Shell::Color::RED)
+          else
+            puts "#{label} is not a valid file name, , skipping it"
+          end
         end
         if File.directory? file
           tree[label + "/"] = nil

data/lib/cnvrg/downloader/client.rb

@@ -14,8 +14,15 @@ module Cnvrg
       end
 
       def extract_key_iv(sts_path)
-        sts = open(sts_path).read rescue nil
-        raise StandardError.new("Cant open sts") if sts.blank?
+        count = 20
+        begin
+          count += 1
+          sts = open(sts_path, {ssl_verify_mode: 0}).read rescue nil
+        rescue => e
+          Cnvrg::Logger.log_error(e)
+          retry if count <= 20
+          raise StandardError.new("Cant access storage: #{e.message}")
+        end
         sts.split("\n")
       end
 

data/lib/cnvrg/files.rb

@@ -115,7 +115,7 @@ module Cnvrg
         #upload files
         blob_id_chunk = Parallel.map(files.keys, in_threads: ParallelThreads) do |file|
           begin
-            Cnvrg::Helpers.try_until_success{self.upload_single_file(files[file].merge(parsed_chunk_of_files[file]))}
+            upload_single_file(files[file].merge(parsed_chunk_of_files[file]))
           rescue => e
             Cnvrg::CLI.log_message("Failed to upload #{file}: #{e.message}", 'red') unless suppress_exceptions
             Cnvrg::Logger.log_error(e)
@@ -188,7 +188,7 @@ module Cnvrg
     def upload_single_file(file)
       path = file['path']
       absolute_path = file[:absolute_path]
-      @client.upload(path, absolute_path)
+      @client.safe_upload(path, absolute_path)
     end
 
     def parse_file(file)

data/lib/cnvrg/helpers.rb

@@ -3,7 +3,7 @@ module Cnvrg
 
     extend self
     def parallel_threads
-      15
+      5
     end
 
     def self.parallel_options

data/lib/cnvrg/version.rb

@@ -1,3 +1,3 @@
 module Cnvrg
-  VERSION = '1.10.18'
+  VERSION = '1.11.2'
 end

metadata

@@ -1,16 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 1.10.18
+  version: 1.11.2
 platform: ruby
 authors:
 - Yochay Ettun
 - Leah Kolben
 - Omer Shacham
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-09-15 00:00:00.000000000 Z
+date: 2020-11-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -438,7 +438,7 @@ files:
 homepage: https://cnvrg.io
 licenses: []
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -453,8 +453,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key: 
+rubygems_version: 3.0.4
+signing_key:
 specification_version: 4
 summary: A CLI tool for interacting with cnvrg.io.
 test_files: []