cnvrg 1.10.17 → 1.11.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 29b4ab96b94066d5e799cbd5e648eaae0871a634ac6383a66ca3c82a3af9a038
-  data.tar.gz: aca3f52a3959cf6db6614bed5b7669c41e9607f0abf25ec83e870783c997f484
+  metadata.gz: aa004590a2756120371db2d6322a564e364076453f9d7b858e83c94e2068b4b9
+  data.tar.gz: d23ee94fbbd6bfb6b09b303d71f4fa5c646d6fc76cb6d049e96f7d3dddfff348
 SHA512:
-  metadata.gz: 96fbd5de39c5aa44d93e24c529b5e149cbd8f689c56329e7c5b4a6f9990d19ca8a7d1fdb1f5f00b8abc05c8dccc061a74de65ac8c81524c27f65b0e349dfdc23
-  data.tar.gz: 2412c51b6926725be234c022faca6191110caf45de2c83381b171e411f7d7bee78e74d519b91f42e01d07b69413df5df009747f35bd34e480d5733718ccd824c
+  metadata.gz: 0f35da8bf64a49b8ddfa3baea029ad08aa29665632763799b550886cc2647aa81933f967ec6bb5b2b41ab536d1784b265b9d139c0151ee92cbee02dcc33e47a0
+  data.tar.gz: e355bf1a4081462895a278998010b8ce587d9d4e065a63df3d8cab65948abc09468fb84511e665f29354782dcf94bc9298d701a03592d5daf5503dbb6228afc9

data/lib/cnvrg/cli.rb

@@ -964,10 +964,9 @@ module Cnvrg
         Cnvrg::CLI.is_response_success(response,true)
         dataset_name = response["results"]["name"]
         dataset_slug = response["results"]["slug"]
-        dataset_home = Dir.pwd+"/"+dataset_slug
+        dataset_home = Dir.pwd + "/" + dataset_slug
         Dataset.stop_if_dataset_present(dataset_home, dataset_name) if soft
 
-        # dataset_home = Dir.pwd
         if Dataset.blank_clone(owner, dataset_name, dataset_slug)
           dataset = Dataset.new(dataset_home)
           log_message("Cloning #{dataset_name}", Thor::Shell::Color::BLUE)
@@ -986,12 +985,13 @@ module Cnvrg
           begin
             log_message("Downloading files", Thor::Shell::Color::BLUE)
             Parallel.map((response["results"]["query_files"]), parallel_options) do |f|
-              relative_path = f["fullpath"].gsub(/^#{dataset_home}/, "")
+              relative_path = f["fullpath"].gsub(/^#{dataset_home}/, "").gsub(/^#{slug}/, "")
               relative_path_dir = relative_path.split("/")
               file_name = relative_path_dir.pop()
               relative_path_dir = relative_path_dir.join("/")
               abs_path = dataset_home + "/" + relative_path_dir
               abs_path = dataset_home if flatten
+
               begin
                 FileUtils.mkdir_p(abs_path) unless File.exist? (abs_path + "/" + file_name)
               rescue
@@ -1781,8 +1781,7 @@ module Cnvrg
       print_table(list)
     end
 
-    desc 'commits', 'List all commits for a specific Project'
-
+    desc 'commits', 'List all commits for a specific Project', :hide => true
     def list_commits()
       verify_logged_in(true)
       log_start(__method__, args, options)
@@ -1796,7 +1795,7 @@ module Cnvrg
     end
 
 
-    desc 'unlink', 'Unlink a project from current directory'
+    desc 'unlink', 'Unlink a project from current directory', :hide => true
 
     def unlink
       verify_logged_in(false)
@@ -2893,7 +2892,7 @@ module Cnvrg
       end
     end
 
-    desc 'show', 'Show specific file from a specific commit'
+    desc 'show', 'Show specific file from a specific commit', :hide => true
     method_option :path, :type => :string, :aliases => ["-p"], :desc => "File path", :default => ""
     method_option :commit, :type => :string, :aliases => ["-c"], :desc => "Commit sha1", :default => nil
 
@@ -4806,8 +4805,7 @@ module Cnvrg
 
     end
 
-    desc 'list_machines', 'Lists all machines belong to your organization'
-
+    desc 'list_machines', 'Lists all machines belong to your organization', :hide => true
     def list_machines
       begin
         verify_logged_in(false)

data/lib/cnvrg/datafiles.rb

@@ -52,7 +52,7 @@ module Cnvrg
         end
         raise SignalException.new(1, "Cant find file #{file}") unless File.exists? "#{Dir.pwd}/#{file}"
       end
-      paths.map{|p| p.gsub(/^\.\//, '')}
+      paths
     end
 
     def get_files_and_folders(paths)
@@ -316,21 +316,21 @@ module Cnvrg
           #Cnvrg::Logger.log_method(bind: binding)
           retry_count += 1
 
-          puts "Failed request upload files: #{Time.current}"
-          puts upload_resp
+          Cnvrg::Logger.log_info("Failed request upload files: #{Time.current}, retry: #{retry_count}")
 
-          if retry_count > 5
-            raise Exception.new("Got an error message from server, #{upload_resp.try(:fetch, "message")}")
+          if retry_count > 20
+            puts "Failed to upload files: #{Time.current}, trying next chunk"
+            return nil
           end
           sleep 5
           next
         end
         return upload_resp['result'].with_indifferent_access
       end
-
     end
 
     def upload_multiple_files_optimized(files, commit_sha1, threads: 15, chunk_size: 1000, override: false, new_branch: false, prefix: '', partial_commit: nil)
+      Thread.report_on_exception = false
       cli = CLI.new
       cli.log_message("Using #{threads} threads with chunk size of #{chunk_size}.", Thor::Shell::Color::GREEN)
 
@@ -342,6 +342,7 @@ module Cnvrg
       file_queue = Queue.new
       progress_queue = Queue.new
       worker_threads = []
+      progress_threads = []
 
       # Vars to keep track of uploaded files and directories
       dirs = []
@@ -359,56 +360,82 @@ module Cnvrg
                 cli.log_message("Error while uploading file: #{file[:absolute_path]}", Thor::Shell::Color::RED)
                 Cnvrg::Logger.log_error_message("Error while upload single file #{file["path"]}")
               end
+              while progress_queue.size > 15000
+                sleep(0.1)
+              end
               progress_queue << file
             end
           end
         end
 
         # init the thread that handles the file upload progress and saving them in the server
-        progress_thread = Thread.new do
-          loop do
-            file = progress_queue.deq(non_block: true) rescue nil # to prevent deadlocks
-            unless file.nil?
-              progress_mutex.synchronize {
-                progressbar.progress += 1
-                uploaded_files.append(file) if file[:success]
-              }
-
-              if uploaded_files.size == chunk_size or progressbar.finished?
-                refresh_storage_token
-                Cnvrg::Logger.info("Finished upload chunk of #{chunk_size} files")
-                Cnvrg::Logger.info("Sending Upload files save")
-                blob_ids = uploaded_files.map {|f| f['bv_id']}
-                Cnvrg::Logger.info("Sending chunk to server")
-                upload_resp = Cnvrg::API.request(@base_resource + "upload_files_save", "POST", {commit: commit_sha1, blob_ids: blob_ids, dirs: dirs})
-                unless Cnvrg::CLI.is_response_success(upload_resp, false)
-                  Cnvrg::Logger.log_method(bind: binding)
-                  raise Exception.new("Got an error message from server, #{upload_resp.try(:fetch, "message")}")
+        threads.times do |i|
+          progress_threads[i] = Thread.new do
+            loop do
+              file = progress_queue.deq(non_block: true) rescue nil # to prevent deadlocks
+              unless file.nil?
+                blob_ids = []
+                dirs_to_upload = []
+
+                progress_mutex.synchronize {
+                  progressbar.progress += 1
+                  uploaded_files.append(file) if file[:success]
+
+                  if uploaded_files.size >= chunk_size or progressbar.finished?
+                    blob_ids = uploaded_files.map {|f| f['bv_id']}
+                    dirs_to_upload = dirs.clone
+                    uploaded_files = []
+                    dirs = []
+                  end
+                }
+
+                if blob_ids.present?
+                  refresh_storage_token
+                  Cnvrg::Logger.info("Finished upload chunk of #{chunk_size} files, Sending Upload files save")
+
+
+                  retry_count = 0
+                  loop do
+                    upload_resp = Cnvrg::API.request(@base_resource + "upload_files_save", "POST", {commit: commit_sha1, blob_ids: blob_ids, dirs: dirs_to_upload})
+
+                    if not (Cnvrg::CLI.is_response_success(upload_resp, false))
+                      retry_count += 1
+                      Cnvrg::Logger.log_error_message("Failed request save files: #{Time.current}, retry: #{retry_count}")
+                      Cnvrg::Logger.info("Got an error message from server, #{upload_resp.try(:fetch, "message")}")
+                      if retry_count > 20
+                        puts "Failed to save files: #{Time.current}, trying next chunk"
+                        break
+                      end
+                      sleep 5
+                      next
+                    end
+                    Cnvrg::Logger.info("Chunk saved on server")
+                    break
+                  end
                 end
-                Cnvrg::Logger.info("Chunk saved on server")
-                uploaded_files = []
-                dirs = []
+              else
+                sleep(0.1)
               end
-            else
-              sleep(0.1)
-            end
 
-            if progressbar.finished?
-              Cnvrg::Logger.info("Progress bar finished closing queues")
-              file_queue.close()
-              progress_queue.close()
-              Thread.exit
+              if progressbar.finished?
+                Cnvrg::Logger.info("Progress bar finished closing queues")
+                file_queue.close()
+                progress_queue.close()
+                Thread.exit
+              end
             end
           end
         end
 
         file_chunks = files.each_slice(chunk_size).to_a
         # Fetch the required files from the server:
-        Parallel.map((file_chunks), in_threads: 10) do |files_chunk|
+        Parallel.map((file_chunks), in_threads: threads) do |chunk|
+          files_chunk = chunk.map{|p| p.gsub(/^\.\//, '')}
           Cnvrg::Logger.info("Generating chunk idx")
-          tree = @dataset.generate_chunked_idx(files_chunk, prefix: prefix, threads: threads)
+          tree = @dataset.generate_chunked_idx(files_chunk, prefix: prefix, threads: threads, cli: cli)
           Cnvrg::Logger.info("Getting files info from server")
           results = request_upload_files(commit_sha1, tree, override, new_branch, partial_commit)
+          next unless results
 
           if results['files'].blank?
             progress_mutex.synchronize { progressbar.progress += tree.keys.length }
@@ -431,12 +458,11 @@ module Cnvrg
             file_queue.push tree[key].merge(files_to_upload[key])
           end
         end
-        Cnvrg::Logger.info("Finishing sub processes of datasets' upload")
-        progress_thread.join()
+        Cnvrg::Logger.info("Waiting to progress and workers to finish")
+        progress_threads.each(&:join)
         worker_threads.each(&:join)
-
+        Thread.report_on_exception = true
       rescue => e
-        puts e
         Cnvrg::Logger.log_method(bind: binding)
         Cnvrg::Logger.log_error(e)
         raise e
@@ -500,7 +526,7 @@ module Cnvrg
         upload_resp = Cnvrg::API.request(@base_resource + "upload_file", 'POST_FILE', {absolute_path: absolute_path, relative_path: relative_path,
                                                                                        commit_sha1: commit_sha1, file_name: file_name,
                                                                                        file_size: file_size, file_content_type: content_type, sha1: sha1})
-        puts upload_resp
+
 
       end
 
@@ -547,7 +573,7 @@ module Cnvrg
           return false
         end
       rescue => e
-        puts e.message
+        #puts e.message
         return false
       end
     end
@@ -792,7 +818,7 @@ module Cnvrg
         return resp
 
       rescue => e
-        puts e.message
+        #puts e.message
         return false
 
       end
@@ -1174,7 +1200,9 @@ module Cnvrg
     end
 
     def end_commit(commit_sha1, force, success: true, uploaded_files: 0, commit_type: nil)
+      counter = 0
       begin
+        counter += 1
         response = Cnvrg::API.request(
             "#{base_resource}/commit/end",
             'POST',
@@ -1186,9 +1214,11 @@ module Cnvrg
                 commit_type: commit_type
             }
         )
-        Cnvrg::CLI.is_response_success(response, true)
+        is_success = Cnvrg::CLI.is_response_success(response, false)
+        raise Exception.new("Invalid response #{response}") unless is_success
         return response
       rescue => e
+        retry if counter <= 20
         return false
       end
     end

data/lib/cnvrg/dataset.rb

@@ -541,7 +541,7 @@ module Cnvrg
       "#{url}/#{self.owner}/datasets/#{self.slug}"
     end
 
-    def generate_chunked_idx(list_files = [], threads: 15, prefix: '')
+    def generate_chunked_idx(list_files = [], threads: 15, prefix: '', cli: nil)
       tree = {}
       Parallel.map(list_files, in_threads: threads) do |file|
 
@@ -552,12 +552,21 @@ module Cnvrg
         label = safe_path.gsub(self.local_path + "/", "")
         label = "#{prefix}/#{label}" if prefix.present?
         if not Cnvrg::Files.valid_file_name?(label)
-          raise StandardError.new("#{label} is not a valid file name.")
+          if cli
+            cli.log_message("#{label} is not a valid file name, skipping it", Thor::Shell::Color::RED)
+          else
+            puts "#{label} is not a valid file name, , skipping it"
+          end
         end
         if File.directory? file
           tree[label + "/"] = nil
         else
-          sha1 = OpenSSL::Digest::SHA1.file(file).hexdigest
+          begin
+            sha1 = OpenSSL::Digest::SHA1.file(file).hexdigest
+          rescue => e
+            #puts "Could'nt calculate sha1 for: #{file}, Error: #{e.message}"
+            next
+          end
           file_name = File.basename file
           file_size = File.size(file).to_f
           mime_type = MimeMagic.by_path(file)

data/lib/cnvrg/downloader/client.rb

@@ -14,8 +14,15 @@ module Cnvrg
       end
 
       def extract_key_iv(sts_path)
-        sts = open(sts_path).read rescue nil
-        raise StandardError.new("Cant open sts") if sts.blank?
+        count = 20
+        begin
+          count += 1
+          sts = open(sts_path, {ssl_verify_mode: 0}).read rescue nil
+        rescue => e
+          Cnvrg::Logger.log_error(e)
+          retry if count <= 20
+          raise StandardError.new("Cant access storage: #{e.message}")
+        end
         sts.split("\n")
       end
 

data/lib/cnvrg/version.rb

@@ -1,3 +1,3 @@
 module Cnvrg
-  VERSION = '1.10.17'
+  VERSION = '1.11.1'
 end

metadata

@@ -1,16 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 1.10.17
+  version: 1.11.1
 platform: ruby
 authors:
 - Yochay Ettun
 - Leah Kolben
 - Omer Shacham
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-09-15 00:00:00.000000000 Z
+date: 2020-10-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -438,7 +438,7 @@ files:
 homepage: https://cnvrg.io
 licenses: []
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -454,7 +454,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.1.2
-signing_key: 
+signing_key:
 specification_version: 4
 summary: A CLI tool for interacting with cnvrg.io.
 test_files: []