cnvrg 1.6.14 → 1.6.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 464db61203d8184ba21a0b17054b08eac5c55f63cfead2484f4ef0b00d902f94
-  data.tar.gz: f56302c84bc02289d876699dd8ac4df31dba148f15beabd92533d8fca824bfdc
+  metadata.gz: da2bf544fbb1248b902085a73ea2bb9ab008afd8b635b7be82dd3f6c6678ec72
+  data.tar.gz: 29829d584b729b86bc0637647063924927182c6a99d08f3757649516cd6dbefb
 SHA512:
-  metadata.gz: 4373ec0d34b3049a5a0da66dfaa71df71569770c5c6581cc49306f4041420d1c5e043502a89f139eb43d75b2585170a82990f6d13ab5e8acf8614284d904fd69
-  data.tar.gz: 4228f9fd6547d7d649771b10a13c72f2d1aa24ca6903b8ae828c1837cd3fa92080b41c0f927de34789d62ef89427998eb699787662bff4b38f94cb675b8b5104
+  metadata.gz: 12901887e4d8e2a39107ec863f0b520c70dde335fbbab3626fa5fdc8f14830a3fd00b26445fea305ddc2fa943bd4f66d38eb234f10c6f107c414be6723c70f58
+  data.tar.gz: 9ea2f33d969fd555401a87556f765532e1fcaa9e385da8fa309a9a269b7dce2d506409013574e0f0b198c71a1d9e329063f85d32afac1f8daa69ec1aa4930143

data/cnvrg.gemspec

@@ -31,7 +31,7 @@ Gem::Specification.new do |spec|
     spec.add_runtime_dependency 'open4', '~> 1.3', '>= 1.3.4'
     spec.add_runtime_dependency 'highline', '~> 1.7', '>= 1.7.8'
     spec.add_runtime_dependency 'thor', '~> 0.19.0','>=0.19.1'
-    spec.add_runtime_dependency 'aws-sdk', '~> 2'
+    spec.add_runtime_dependency 'aws-sdk', '~> 2.11.417'
     spec.add_runtime_dependency 'signet', '~> 0.11.0'
     spec.add_runtime_dependency 'google-cloud-env', '~> 1.2.1'
     spec.add_runtime_dependency 'google-cloud-core', '~> 1.3.2'

data/lib/cnvrg/cli.rb

@@ -845,7 +845,6 @@ module Cnvrg
       begin
         verify_logged_in(false)
         log_start(__method__, args, options)
-        return if check_pod_restart[1] ## It means that all datasets downloaded successfully
         commit = options["commit"] || commit
         only_tree = options["only_tree"] || only_tree
         read = options["read"] || read || false
@@ -1781,7 +1780,6 @@ module Cnvrg
       begin
         verify_logged_in(false)
         log_start(__method__, args, options)
-        return if check_pod_restart[0] ## It means that project downloaded successfully
         url_parts = project_url.split("/")
         project_index = Cnvrg::Helpers.look_for_in_path(project_url, "projects")
         slug = url_parts[project_index + 1]
@@ -2155,6 +2153,7 @@ module Cnvrg
                                                    new_branch: new_branch,
                                                    progressbar: progressbar,
                                                    partial_commit: partial_commit)
+
         files_uploaded += upload_resp
         temp_tree.each do |k, v|
           new_tree[k] = (v.present?) ? {sha1: v.try(:fetch, :sha1, nil), commit_time: nil} : nil
@@ -2293,7 +2292,9 @@ module Cnvrg
                                          :total => (to_upload.size + deleted.size),
                                          :autofinish => true)
         @files.upload_multiple_files(to_upload, commit_sha1, progress: progressbar)
+
         @files.delete_files_from_server(deleted, commit_sha1)
+
         progressbar.finish
         res = @files.end_commit(commit_sha1, force: force, message: commit_msg)
         unless Cnvrg::CLI.is_response_success(res, false)

data/lib/cnvrg/datafiles.rb

@@ -116,6 +116,7 @@ module Cnvrg
 
     def upload_multiple_files(commit_sha1, tree, threads: ParallelThreads, force: false, new_branch: false, progressbar: nil, prefix: '', partial_commit: nil)
       begin
+        cli=CLI.new
         Cnvrg::Logger.log_info("Sending Upload Files request")
         upload_resp = Cnvrg::API.request(@base_resource + "upload_files", 'POST_JSON', {commit_sha1: commit_sha1, tree: tree, force: force, is_branch: new_branch, partial_commit: partial_commit})
         unless Cnvrg::CLI.is_response_success(upload_resp, false)
@@ -132,16 +133,24 @@ module Cnvrg
         progressbar.progress += tree.keys.length - files.length if progressbar.present?
         Parallel.map((files.keys), {in_threads: threads}) do |k|
           o = tree[k].merge(files[k])
-          upload_single_file(o)
+          success = upload_single_file(o)
+          if not success
+            files.except!(k)
+            tree.except!(k)
+            cli.log_message("Error while uploading file: #{o[:absolute_path]}", Thor::Shell::Color::RED)
+            Cnvrg::Logger.log_error_message("Error while upload single file #{o["path"]}")
+          end
           progressbar.progress += 1 if progressbar.present?
         end
         blob_ids = files.values.map {|f| f['bv_id']}
-        dirs = tree.keys.select {|k| tree[k].nil?} || []
-        Cnvrg::Logger.info("Sending Upload files save")
-        upload_resp = Cnvrg::API.request(@base_resource + "upload_files_save", "POST", {commit: commit_sha1, blob_ids: blob_ids, dirs: dirs})
-        unless Cnvrg::CLI.is_response_success(upload_resp, false)
-          Cnvrg::Logger.log_method(bind: binding)
-          raise Exception.new("Got an error message from server, #{upload_resp.try(:fetch, "message")}")
+        if blob_ids.present?
+          dirs = tree.keys.select {|k| tree[k].nil?} || []
+          Cnvrg::Logger.info("Sending Upload files save")
+          upload_resp = Cnvrg::API.request(@base_resource + "upload_files_save", "POST", {commit: commit_sha1, blob_ids: blob_ids, dirs: dirs})
+          unless Cnvrg::CLI.is_response_success(upload_resp, false)
+            Cnvrg::Logger.log_method(bind: binding)
+            raise Exception.new("Got an error message from server, #{upload_resp.try(:fetch, "message")}")
+          end
         end
         Cnvrg::Logger.log_info("Upload Success")
         return files.keys.length
@@ -155,13 +164,17 @@ module Cnvrg
 
     def upload_single_file(file)
       begin
-      file = file.as_json
-      Cnvrg::Logger.log_info("Uploading #{file["absolute_path"]}")
-      @downloader.upload(file["path"], file["absolute_path"])
-      Cnvrg::Logger.log_info("#{file["absolute_path"]} uploaded.")
+        cli = CLI.new
+        file = file.as_json
+        Cnvrg::Logger.log_info("Uploading #{file["absolute_path"]}")
+        success = @downloader.upload(file["path"], file["absolute_path"])
+        return false if not success
+        Cnvrg::Logger.log_info("#{file["absolute_path"]} uploaded.")
       rescue => e
+        cli.log_message(e.message, Thor::Shell::Color::RED)
         Cnvrg::Logger.log_error_message("Error while upload single file #{file["path"]}")
         Cnvrg::Logger.log_error(e)
+        return false
       end
     end
 

data/lib/cnvrg/downloader/clients/azure_client.rb

@@ -20,7 +20,26 @@ module Cnvrg
         end
 
         def upload(storage_path, local_path)
-          client.create_block_blob(@container, storage_path, File.open(local_path, "rb"))
+          begin
+            try_upload_file(storage_path, local_path)
+          rescue => e
+            raise e
+          end
+        end
+
+        def try_upload_file(storage_path, local_path)
+          c = 20
+          error = nil
+          while c > 0
+            begin
+              client.create_block_blob(@container, storage_path, File.open(local_path, "rb"))
+              break
+            rescue => e
+              c -= 1
+              error = e
+            end
+          end
+          raise error if error.present?
         end
 
         def fetch_files(prefix: nil, marker: nil, limit: 10000)

data/lib/cnvrg/downloader/clients/gcp_client.rb

@@ -11,7 +11,7 @@ module Cnvrg
           @tempfile = nil
           @bucket_name = Cnvrg::Helpers.decrypt(@key, @iv, bucket_name)
           init_gcp_credentials
-          @storage = Google::Cloud::Storage.new(project_id: @project_id, credentials: @credentials)
+          @storage = Google::Cloud::Storage.new(project_id: @project_id, credentials: @credentials, retries: 20, timeout: 10)
           @bucket = @storage.bucket(@bucket_name)
           @bucket.name
         rescue => e
@@ -38,7 +38,11 @@ module Cnvrg
         end
 
         def upload(storage_path, local_path)
-          @bucket.create_file(local_path, storage_path)
+          begin
+            @bucket.create_file(local_path, storage_path)
+          rescue => e
+            raise e
+          end
         end
       end
     end

data/lib/cnvrg/downloader/clients/s3_client.rb

@@ -54,6 +54,8 @@ module Cnvrg
           # local path is fullpath for the file /home/ubuntu/user.../hazilim.py
           o = aws_bucket.object(storage_path)
           o.upload_file(local_path, @upload_options)
+        rescue => e
+          raise e
         end
 
         def fetch_files(prefix: nil, marker: nil, limit: 1000)

data/lib/cnvrg/files.rb

@@ -85,6 +85,7 @@ module Cnvrg
         Cnvrg::Logger.log_info("Upload files to older server..")
         return self.upload_files_old(files_list, commit_sha1, progress: progress)
       end
+
       files_list = files_list.map{|x| [x,self.parse_file(x)]}.to_h
       resp = Cnvrg::API.request(@base_resource + "upload_files", 'POST', {files: files_list, commit: commit_sha1})
       unless Cnvrg::CLI.is_response_success(resp, false)
@@ -99,6 +100,7 @@ module Cnvrg
         begin
           Cnvrg::Helpers.try_until_success{self.upload_single_file(files[file].merge(files_list[file]))}
         rescue => e
+
           Cnvrg::CLI.log_message("Failed to upload #{file}: #{e.message}", 'red')
           Cnvrg::Logger.log_error(e)
           Cnvrg::Logger.log_method(bind: binding)

data/lib/cnvrg/project.rb

@@ -329,10 +329,13 @@ module Cnvrg
     def get_storage_client
       response = Cnvrg::API.request("users/#{@owner}/projects/#{@slug}/client", 'GET')
       if Cnvrg::CLI.is_response_success(response, false)
+
         client_params = response['client']
       else
+
         client_params = get_storage_client_fallback
       end
+
       Cnvrg::Downloader::Client.factory(client_params)
     end
 

data/lib/cnvrg/version.rb

@@ -1,3 +1,3 @@
 module Cnvrg
-  VERSION = '1.6.14'
+  VERSION = '1.6.15'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 1.6.14
+  version: 1.6.15
 platform: ruby
 authors:
 - Yochay Ettun
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-01-21 00:00:00.000000000 Z
+date: 2020-01-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -210,14 +210,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2'
+        version: 2.11.417
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2'
+        version: 2.11.417
 - !ruby/object:Gem::Dependency
   name: signet
   requirement: !ruby/object:Gem::Requirement
@@ -477,7 +477,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.0.4
 signing_key: 
 specification_version: 4
 summary: A CLI tool for interacting with cnvrg.io.