cnvrg 1.11.15 → 1.11.24

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8798ab84db1e0742dc733a0f74b5830e2c5396099fc52bdb4a04ef108a43408e
-  data.tar.gz: 795988ec46ba506091136240bda258f923469e4b3000d58e84371df09f65840d
+  metadata.gz: e8a5d09f38ee4ce25caecdaf73b77e37fadff3b745454030cb15863210c1b319
+  data.tar.gz: 1f3e27bc4fa778ff048e7257f2ce753009ceaf0801c4ca3f4facfdc3011ad940
 SHA512:
-  metadata.gz: '0497d165396b52d2975c3d3ab8c097596c173f04c63711519cf2c122fb9dc5f90ecc493aab7ba2cba0d7064dc14b52a7e1d7f2f6f5e68d3af020d4b3f1f5b35c'
-  data.tar.gz: 72c65364f4c9ad8d5acf68546f0a428c113dd9697613674ce83b52a3ce7fcec8064a7517d9b5b5ae7f21a8690ba935d028622688d72f87eb3e1927d7fe5ea9e2
+  metadata.gz: 14d79f14723eff1d8df81907183fabbe97270fd49610d4aa456fa820e4c19994c3d2bd1b7c64160f4d66a0636ff385a32b86f6fed2bedf9c0ed98c7968267c23
+  data.tar.gz: 80b4644fc6f560e54280ab726983f4e5d89978255991b29c510452b5514019025a3dcae45e4ad5f9a36e97823909eb6d1f2ee892cc63ef5090fa78ecacf595fe

data/cnvrg.gemspec

@@ -23,8 +23,8 @@ Gem::Specification.new do |spec|
     spec.add_development_dependency 'rspec', '~> 3.0'
     spec.add_development_dependency 'vcr', '~> 3.0'
     spec.add_development_dependency 'aruba'
-    spec.add_development_dependency 'pry'	
-    
+    spec.add_development_dependency 'pry'
+
     spec.add_runtime_dependency 'mimemagic', '~> 0.3.1','>=0.3.2'
     spec.add_runtime_dependency 'faraday', '~> 0.15.2'
     spec.add_runtime_dependency 'netrc', '~> 0.11.0'

data/lib/cnvrg/api.rb

@@ -31,6 +31,10 @@ module Cnvrg
         end
         def self.request(resource, method = 'GET', data = {}, parse_request = true)
             resource = URI::encode resource
+
+            # We need to remoe all double slashes from the url to work with the proxy
+            resource = resource.gsub(/[\/]{2,}/, "/").gsub("https:/", "https://").gsub("http:/", "http://")
+
             begin
                 n = Netrc.read
             rescue => e

data/lib/cnvrg/cli.rb

@@ -1210,6 +1210,7 @@ module Cnvrg
         @dataset = Dataset.new(dataset_info: {:owner =>  owner, :slug => slug})
         @datafiles = Cnvrg::Datafiles.new(owner, slug, dataset: @dataset)
         @files = @datafiles.verify_files_exists(files)
+        @files = @files.uniq { |t| t.gsub('./', '')}
 
         if @files.blank?
           raise SignalException.new(1, "Cant find files to upload, exiting.")
@@ -2346,7 +2347,6 @@ module Cnvrg
             log_message("#{check} Project is up to date", Thor::Shell::Color::GREEN, (((options["sync"] or sync) and !direct) ? false : true))
             return true
           end
-          force = true
         end
 
         if ignore.nil? or ignore.empty?
@@ -3323,7 +3323,7 @@ module Cnvrg
                                real: real_time
                     }
                     if print_log
-                      puts line
+                      puts({log: line, timestamp: Time.now, exp_logs: true}.to_json)
                     end
                     log << cur_log
                     if log.size >= 10

data/lib/cnvrg/datafiles.rb

@@ -43,8 +43,11 @@ module Cnvrg
     def verify_files_exists(files)
       paths = []
       files.each do |file|
+        # dir shouldnt have ending slash.
+        file = file[0..-2] if file.end_with? '/'
         if File.exists? file
           if File.directory? file
+            paths << file unless file == '.'
             paths += Dir.glob("#{file}/**/*")
           else
             paths << file
@@ -349,11 +352,11 @@ module Cnvrg
       progress_mutex = Mutex.new
       file_queue = Queue.new
       progress_queue = Queue.new
+      dirs_queue = Queue.new
       worker_threads = []
       progress_threads = []
 
       # Vars to keep track of uploaded files and directories
-      dirs = []
       uploaded_files = []
 
       begin
@@ -376,6 +379,29 @@ module Cnvrg
           end
         end
 
+        dir_thread = Thread.new do
+          dirs_to_create = []
+          loop do
+            progress_mutex.synchronize {
+              dir = dirs_queue.deq(non_block: true) rescue nil
+              dirs_to_create << dir unless dir.nil?
+            }
+            if dirs_to_create.size >= 1000 || progressbar.finished?
+              resp = Cnvrg::API.request(@base_resource + "create_dirs", "POST", {dirs: dirs_to_create, commit_sha1: commit_sha1})
+
+              break if resp == false # if resp is false it means 404 which is old server
+              unless Cnvrg::CLI.is_response_success(resp, false)
+                time = Time.current
+                Cnvrg::Logger.log_error_message("Failed to create dirs: #{time}, #{resp.try(:fetch, "message")}")
+                dirs_to_create = []
+                next
+              end
+              dirs_to_create = []
+            end
+            break if progressbar.finished? && dirs_queue.empty? && dirs_to_create.empty?
+          end
+        end
+
         # init the thread that handles the file upload progress and saving them in the server
         threads.times do |i|
           progress_threads[i] = Thread.new do
@@ -383,7 +409,6 @@ module Cnvrg
               file = progress_queue.deq(non_block: true) rescue nil # to prevent deadlocks
               unless file.nil?
                 blob_ids = []
-                dirs_to_upload = []
 
                 progress_mutex.synchronize {
                   progressbar.progress += 1
@@ -391,9 +416,7 @@ module Cnvrg
 
                   if uploaded_files.size >= chunk_size or progressbar.finished?
                     blob_ids = uploaded_files.map {|f| f['bv_id']}
-                    dirs_to_upload = dirs.clone
                     uploaded_files = []
-                    dirs = []
                   end
                 }
 
@@ -401,10 +424,9 @@ module Cnvrg
                   refresh_storage_token
                   Cnvrg::Logger.info("Finished upload chunk of #{chunk_size} files, Sending Upload files save")
 
-
                   retry_count = 0
                   loop do
-                    upload_resp = Cnvrg::API.request(@base_resource + "upload_files_save", "POST", {commit: commit_sha1, blob_ids: blob_ids, dirs: dirs_to_upload})
+                    upload_resp = Cnvrg::API.request(@base_resource + "upload_files_save", "POST", {commit: commit_sha1, blob_ids: blob_ids})
 
                     if not (Cnvrg::CLI.is_response_success(upload_resp, false))
                       retry_count += 1
@@ -427,8 +449,8 @@ module Cnvrg
 
               if progressbar.finished?
                 Cnvrg::Logger.info("Progress bar finished closing queues")
-                file_queue.close()
-                progress_queue.close()
+                file_queue.close
+                progress_queue.close
                 Thread.exit
               end
             end
@@ -441,7 +463,21 @@ module Cnvrg
           files_chunk = chunk.map{|p| p.gsub(/^\.\//, '')}
           Cnvrg::Logger.info("Generating chunk idx")
           tree = @dataset.generate_chunked_idx(files_chunk, prefix: prefix, threads: threads, cli: cli)
+
+          progress_mutex.synchronize {
+            # Handle directories:
+            new_dirs = tree.keys.select { |k| tree[k].nil? }
+
+            if new_dirs.blank?
+              ## we need to send 1 file so we will inflated dirs from in case when we dont have folders in the tree
+              file = tree.keys.find { |k| tree[k] != nil }
+              dirs_queue.push file
+            end
+
+            new_dirs.each { |dir| dirs_queue.push dir }
+          }
           Cnvrg::Logger.info("Getting files info from server")
+
           results = request_upload_files(commit_sha1, tree, override, new_branch, partial_commit)
           next unless results
 
@@ -450,11 +486,8 @@ module Cnvrg
             next
           end
 
-          # Handle directories:
-          new_dirs = tree.keys.select {|k| tree[k].nil?}
-          dirs += new_dirs
-
           files_to_upload = results['files']
+
           progress_mutex.synchronize {
             progressbar.progress += tree.keys.length - files_to_upload.length
           }
@@ -466,7 +499,10 @@ module Cnvrg
             file_queue.push tree[key].merge(files_to_upload[key])
           end
         end
+
         Cnvrg::Logger.info("Waiting to progress and workers to finish")
+        dir_thread.join
+        dirs_queue.close
         progress_threads.each(&:join)
         worker_threads.each(&:join)
         Thread.report_on_exception = true

data/lib/cnvrg/dataset.rb

@@ -564,7 +564,8 @@ module Cnvrg
         safe_path = file
         safe_path = file[1..-1] if file.start_with? "/"
 
-        label = safe_path.gsub(self.local_path + "/", "")
+        dataset_local_path = self.local_path + "/"
+        label = safe_path.start_with?(dataset_local_path) ? safe_path.sub(dataset_local_path, "") : safe_path
         label = "#{prefix}/#{label}" if prefix.present?
         if not Cnvrg::Files.valid_file_name?(label)
           if cli
@@ -598,6 +599,7 @@ module Cnvrg
           }
         end
       end
+
       if prefix.present? #add the prefix as dirs to the files
         #lets say the prefix is a/b/c so we want that a/, a/b/, a/b/c/ will be in our files_list
         dirs = prefix.split('/')

data/lib/cnvrg/downloader/client.rb

@@ -17,7 +17,9 @@ module Cnvrg
         count = 0
         begin
           count += 1
-          sts = open(sts_path, {ssl_verify_mode: 0}).read rescue nil
+          sts_file = open(sts_path, {ssl_verify_mode: 0})
+          sts = sts_file.read
+          sts.split("\n")
         rescue => e
           backoff_time_seconds = backoff_time(count)
           sleep backoff_time_seconds
@@ -25,7 +27,6 @@ module Cnvrg
           retry if count <= 20
           raise StandardError.new("Cant access storage: #{e.message}")
         end
-        sts.split("\n")
       end
 
       def cut_prefix(prefix, file)

data/lib/cnvrg/files.rb

@@ -787,6 +787,7 @@ module Cnvrg
             end
           else
             file_path += postfix
+
             # blob
             begin
               if not File.exists?(project_home+"/"+File.dirname(file_path))
@@ -795,12 +796,19 @@ module Cnvrg
               local_path = project_home+"/"+file_path
               storage_path = f["path"]
               @client.safe_download(storage_path, local_path)
-              progress.progress += 1 if progress.present?
-              download_succ_count += 1
             rescue => e
               cli.log_message("Could not download file #{file_path}. error: #{e.message}", Thor::Shell::Color::RED)
               raise e
             end
+
+            # progressbar can throw an exception so we no longer trust it!
+            begin
+              progress.progress += 1 if progress.present?
+            rescue
+              nil
+            ensure
+              download_succ_count += 1
+            end
           end
         end
         if download_succ_count == files["keys"].size

data/lib/cnvrg/flow.rb

@@ -62,7 +62,7 @@ module Cnvrg
       resp = Cnvrg::API.request(url, 'POST', {flow_version: recipe.to_json}) || {}
       if resp["status"] == 200
         return [Flows.new(resp["flow_version"]["flow_id"], project: project), resp["flow_version"]["id"]]
-      elsif resp["status"] == 400
+      elsif resp["status"].between?(400,499)
         raise StandardError.new(resp["message"])
       end
       raise StandardError.new("Can't create new flow")

data/lib/cnvrg/project.rb

@@ -448,8 +448,8 @@ module Cnvrg
           next
         end
         if File.directory? e
-
-          tree_idx[label + "/"] = nil
+          dir_name = (label.ends_with? "/") ? label : (label + "/")
+          tree_idx[dir_name] = nil
         else
           file_in_idx = old_idx[:tree][label] rescue nil
           last_modified = File.mtime(e).to_f
@@ -513,10 +513,11 @@ module Cnvrg
         #upload
         local_idx = self.generate_idx(deploy: deploy, files: specific_files)
       end
+
       commit = local_idx[:commit]
       tree = local_idx[:tree]
       ignore_list = self.send_ignore_list()
-      if force or specific_files.present?
+      if force
         added = []
         if tree.present?
           added += local_idx[:tree].keys

data/lib/cnvrg/version.rb

@@ -1,3 +1,3 @@
 module Cnvrg
-  VERSION = '1.11.15'
-end
+  VERSION = '1.11.24'
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 1.11.15
+  version: 1.11.24
 platform: ruby
 authors:
 - Yochay Ettun
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-01-25 00:00:00.000000000 Z
+date: 2021-02-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler