cnvrg 1.11.5 → 1.11.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b48bf1399bcafb013474322d63fb0d06ad9c484866f4d59dce963eddcd2fb67a
-  data.tar.gz: bc4ed0f3b02a869babc60beec4b090542492d5b53db50810ca7cbe5eaa7901f7
+  metadata.gz: 54dfcb511e29a518772481a3d99a6b30107234c4a218655babc69ddc36496d6d
+  data.tar.gz: 42ea59d4ef4fedef1afd2ba5fb8dcc7e86e439a2e9cc2dfa084409769dc7daf1
 SHA512:
-  metadata.gz: b685ded1e59dfdb12a766ccd05e682b9e053a3069adf377ddc09fdae7d94be0ede6bd3f2ef67dbefdc0c6dc4d8d9a0c4e0ca9754a5cc939aa93340f33cbc295b
-  data.tar.gz: 20a00dbc9a98c4f96785ea1bb684b9f217aa97ad39a583f1638c317a392ccb821f58de01c939bccde883c8aa02cf124b60757b0f4f1eaade669d87784aa394b5
+  metadata.gz: 5c82e60bebc3777880873744d66a366cd2a8893443c62b82606f2986afaa5812a6cf7816f0f2fd9ba35f28819baede432e65c89440c7c594e94d767b31f102c5
+  data.tar.gz: e345d41daab3f9109f47f0d56fb0fd0c1d6d0991bb43b01b5718f1b1f35c182af35989eef36030f384cbbc758524656fa1e08c75d5bbc2abccbeae5be26114a9

data/lib/cnvrg/cli.rb

@@ -972,6 +972,7 @@ module Cnvrg
 
         if Dataset.blank_clone(owner, dataset_name, dataset_slug)
           dataset = Dataset.new(dataset_home)
+          downloader = dataset.get_storage_client
           log_message("Cloning #{dataset_name}", Thor::Shell::Color::BLUE)
           parallel_options = {
               :progress => {
@@ -994,15 +995,20 @@ module Cnvrg
               relative_path_dir = relative_path_dir.join("/")
               abs_path = dataset_home + "/" + relative_path_dir
               abs_path = dataset_home if flatten
+              fullpath = abs_path + "/" + file_name
 
               begin
-                FileUtils.mkdir_p(abs_path) unless File.exist? (abs_path + "/" + file_name)
+                FileUtils.mkdir_p(abs_path) unless File.exist? (fullpath)
               rescue
                 log_message("Could not create directory: #{abs_path}", Thor::Shell::Color::RED)
                 exit(1)
               end
               begin
-                File.write "#{abs_path}/#{file_name}", open(f["url"]).read unless File.exist? (abs_path + "/" + file_name)
+                unless File.exist?(fullpath)
+                  downloader.safe_operation("#{abs_path}/#{file_name}") do
+                    File.open(fullpath, "w") { |file| file.write open(f["url"]).read }
+                  end
+                end
               rescue => e
                 log_message("Could not download file: #{f["fullpath"]}", Thor::Shell::Color::RED)
                 exit(1)
@@ -3062,6 +3068,7 @@ module Cnvrg
     method_option :notify_on_success, :type => :boolean, :aliases => ["-nos", "--notify_on_success"], :default => nil
     method_option :emails, :type => :string, :aliases => ["-es", "--emails"], :default => "", :desc => "additional emails to notify on success / or error, comma separated"
     method_option :wait, :type => :boolean, :aliases => ["-w", "--wait"], :default => false, :desc => "keep command session open until experiment finished to return exit status"
+    method_option :debug, :type => :boolean, :aliases => ["--debug"], :default => true
 
     def run(*cmd)
       verify_logged_in(true)
@@ -3070,6 +3077,7 @@ module Cnvrg
       sync_before = options["sync_before"]
       sync_after = options["sync_after"]
       log = options["log"]
+      debug = options["debug"]
       title = options["title"]
       commit = options["commit"] || nil
       email_notification = options["email_notification"]
@@ -3103,6 +3111,7 @@ module Cnvrg
         wait = false
       end
 
+
       if !data.present? and data_query.present?
         log_message("Please provide data with data_query", Thor::Shell::Color::RED)
         exit(1)
@@ -3149,7 +3158,7 @@ module Cnvrg
                :image => image, :grid => grid, :data => data, :data_commit => data_commit, :ignore => ignore, :force => force, :sync_before_terminate => sync_before_terminate,
                :max_time => max_time,
                :periodic_sync => periodic_sync, :dataset_only_tree=> dataset_only_tree,
-               :output_dir=>output_dir, :data_query=>data_query, :git_commit =>git_commit, :git_branch=> git_branch,
+               :output_dir=>output_dir, :data_query=>data_query, :git_commit =>git_commit, :git_branch=> git_branch, :debug => debug,
                :restart_if_stuck =>restart_if_stuck, :local_folders => local_folders, :datasets => datasets, :prerun => prerun, :requirements => requirements,
                :email_notification_error => email_notification_error, :email_notification_success => email_notification_success, :emails => emails, :wait => wait
 
@@ -3475,6 +3484,7 @@ module Cnvrg
     method_option :email_notification_success, :type => :boolean, :aliases => ["-nos", "--email_notification_success"], :default => true
     method_option :emails, :type => :string, :aliases => ["-es", "--emails"], :default => "", :desc => "additional emails to notify on success / or error"
     method_option :wait, :type => :boolean, :aliases => ["-w", "--wait"], :default => false, :desc => "keep command session open until experiment finished to return exit status"
+    method_option :debug, :type => :boolean, :aliases => ["--debug"], :default => true
 
     def exec_remote(*cmd)
 
@@ -3492,6 +3502,7 @@ module Cnvrg
         data_query = options["data_query"] || nil
         sync_before = options["sync_before"]
         force = options["force"]
+        debug = options["debug"]
         prerun = options["prerun"]
         requirements = options["requirements"]
         email_notification_error = options["email_notification_error"]
@@ -3535,8 +3546,8 @@ module Cnvrg
         local_folders_options = options["local_folders"]
         options_hash.except!("schedule", "recurring", "machine_type", "image", "upload_output", "grid", "data", "data_commit", "title",
                              "local", "small", "medium", "large", "gpu", "gpuxl", "gpuxxl","max_time","dataset_only_tree",
-                             "data_query", "git_commit","git_branch", "restart_if_stuck","local_folders","output_dir", "commit", "datasets",
-                             "requirements", "prerun", "email_notification_error", "email_notification_success", "emails", "wait")
+                             "data_query", "git_commit","git_branch","restart_if_stuck","local_folders","output_dir", "commit", "datasets",
+                             "requirements", "prerun", "email_notification_error", "email_notification_success", "emails", "wait","debug")
         exec_options = options_hash.map {|x| "--#{x[0]}=#{x[1]}"}.flatten.join(" ")
         command = "#{exec_options} #{remote}  #{upload_output_option} #{cmd.flatten.join(" ")}"
         commit_to_run = options["commit"] || nil
@@ -3579,20 +3590,6 @@ module Cnvrg
           end
         end
 
-        if command.include? "'"
-          oc = command.to_enum(:scan, /'/).map {Regexp.last_match}
-          pairs = oc.enum_for(:each_slice, 2).to_a
-          pairs.each_with_index do |p, i|
-            add = 0
-            if i != 0
-              add = 2 * i
-            end
-            total_loc = command[p[0].offset(0)[0] + add..p[1].offset(0)[0] + add]
-            command[p[0].offset(0)[0] + add..p[1].offset(0)[0] + add] = "\"#{total_loc}\""
-          end
-
-
-        end
         log_message("Running remote experiment", Thor::Shell::Color::BLUE)
         exp = Experiment.new(project.owner, project.slug)
         if forced_commit and (commit_to_run.nil? or commit_to_run.empty?)
@@ -3603,7 +3600,7 @@ module Cnvrg
 
         res = exp.exec_remote(command, commit_to_run, instance_type, image, schedule, local_timestamp, grid, path_to_cmd, data, data_commit,
                               periodic_sync, sync_before_terminate, max_time, ds_sync_options,output_dir,
-                              data_query, git_commit, git_branch, restart_if_stuck,local_folders_options, title, datasets, prerun: prerun, requirements: requirements, recurring: recurring,
+                              data_query, git_commit, git_branch,debug, restart_if_stuck,local_folders_options, title, datasets, prerun: prerun, requirements: requirements, recurring: recurring,
                               email_notification_error: email_notification_error, email_notification_success: email_notification_success, emails_to_notify: emails)
         if Cnvrg::CLI.is_response_success(res)
           check = Helpers.checkmark()
@@ -3655,6 +3652,7 @@ module Cnvrg
                 sleep 3
                 tries += 1
                 retry if tries <= 5
+                exit(1)
               end
             end
           end

data/lib/cnvrg/datafiles.rb

@@ -1338,7 +1338,7 @@ module Cnvrg
             #   Cnvrg::Logger.log_info("Trying to download #{local_path} but its already exists, skipping..")
             #   next
             # end
-            resp = @downloader.download(storage_path, local_path)
+            resp = @downloader.safe_download(storage_path, local_path)
             Cnvrg::Logger.log_info("Download #{local_path} success resp: #{resp}")
           rescue => e
             Cnvrg::Logger.log_error(e)

data/lib/cnvrg/downloader/client.rb

@@ -30,10 +30,14 @@ module Cnvrg
         file.gsub(prefix, '').gsub(/^\/*/, '')
       end
 
-      def download(storage_path, local_path)
+      def download(storage_path, local_path, decrypt: true)
         ### need to be implemented..
       end
 
+      def safe_download(storage_path, local_path, decrypt: true)
+        safe_operation(local_path) { self.download(storage_path, local_path, decrypt: decrypt) }
+      end
+
       def upload(storage_path, local_path)
         ### need to be implemented..
       end
@@ -51,17 +55,34 @@ module Cnvrg
       end
 
       def safe_upload(storage_path, local_path)
+        safe_operation(local_path) { self.upload(storage_path, local_path) }
+      end
+
+      def self.factory(params)
+        params = params.as_json
+        case params["storage"]
+        when 's3', 'minio'
+          return Cnvrg::Downloader::Clients::S3Client.new(sts_path: params["path_sts"], access_key: params["sts_a"], secret: params["sts_s"], session_token: params["sts_st"], region: params["region"], bucket: params["bucket"], encryption: params["encryption"], endpoint: params["endpoint"], storage: params["storage"])
+        when 'azure'
+          azure_params = params.symbolize_keys.slice(*[:storage_account_name, :storage_access_key, :container, :sts])
+          return Cnvrg::Downloader::Clients::AzureClient.new(**azure_params)
+        when 'gcp'
+          return Cnvrg::Downloader::Clients::GcpClient.new(project_id: params["project_id"], credentials: params["credentials"], bucket_name: params["bucket_name"], sts: params["sts"])
+        end
+      end
+
+      def safe_operation(local_path)
         n = 1
         error = nil
         while n <= RETRIES
           begin
-            self.upload(storage_path, local_path)
+            yield
             error = nil
             break
           rescue => e
             backoff_time_seconds = backoff_time(n)
 
-            message = "Got error: #{e.class.name} with message: #{e.message} while uploading a single file: #{local_path}, retry: #{n} of: #{RETRIES}"
+            message = "Got error: #{e.class.name} with message: #{e.message} while uploading / downloading a single file: #{local_path}, retry: #{n} of: #{RETRIES}"
             if n < RETRIES
               message += ", next retry in: #{backoff_time_seconds} seconds"
             else
@@ -79,19 +100,6 @@ module Cnvrg
         true
       end
 
-      def self.factory(params)
-        params = params.as_json
-        case params["storage"]
-        when 's3', 'minio'
-          return Cnvrg::Downloader::Clients::S3Client.new(sts_path: params["path_sts"], access_key: params["sts_a"], secret: params["sts_s"], session_token: params["sts_st"], region: params["region"], bucket: params["bucket"], encryption: params["encryption"], endpoint: params["endpoint"], storage: params["storage"])
-        when 'azure'
-          azure_params = params.symbolize_keys.slice(*[:storage_account_name, :storage_access_key, :container, :sts])
-          return Cnvrg::Downloader::Clients::AzureClient.new(**azure_params)
-        when 'gcp'
-          return Cnvrg::Downloader::Clients::GcpClient.new(project_id: params["project_id"], credentials: params["credentials"], bucket_name: params["bucket_name"], sts: params["sts"])
-        end
-      end
-
       private
 
       def random_number_milliseconds

data/lib/cnvrg/downloader/clients/gcp_client.rb

@@ -31,7 +31,7 @@ module Cnvrg
           @tempfile = t
         end
 
-        def download(storage_path, local_path)
+        def download(storage_path, local_path, decrypt: true)
           prepare_download(local_path)
           file = @bucket.file(decrypt(storage_path))
           file.download local_path

data/lib/cnvrg/experiment.rb

@@ -110,7 +110,7 @@ module Cnvrg
 
     def exec_remote(command, commit_to_run, instance_type, image_slug,schedule,local_timestamp, grid,path_to_cmd,data, data_commit,periodic_sync,
                     sync_before_terminate, max_time, ds_sync_options=0,output_dir=nil,data_query=nil,
-                    git_commit=nil, git_branch=nil, restart_if_stuck=nil, local_folders=nil,title=nil, datasets=nil, prerun: true, requirements: true, recurring: nil,
+                    git_commit=nil, git_branch=nil,debug=true, restart_if_stuck=nil, local_folders=nil,title=nil, datasets=nil, prerun: true, requirements: true, recurring: nil,
                     email_notification_error: false, email_notification_success: false, emails_to_notify: nil)
       response = Cnvrg::API.request("users/#{@owner}/projects/#{@project_slug}/experiment/remote", 'POST', {command: command, image_slug: image_slug,
                                                                                                             commit_sha1: commit_to_run,
@@ -119,6 +119,7 @@ module Cnvrg
                                                                                                             local_timestamp:local_timestamp,
                                                                                                             datasets: datasets,
                                                                                                             grid: grid,
+                                                                                                            debug:debug,
                                                                                                             path_to_cmd:path_to_cmd,dataset_slug:data,
                                                                                                             dataset_commit: data_commit,max_time:max_time,
                                                                                                             periodic_sync:periodic_sync, sync_before_terminate:sync_before_terminate,

data/lib/cnvrg/files.rb

@@ -789,7 +789,7 @@ module Cnvrg
               end
               local_path = project_home+"/"+file_path
               storage_path = f["path"]
-              @client.download(storage_path, local_path)
+              @client.safe_download(storage_path, local_path)
               progress.progress += 1 if progress.present?
               download_succ_count += 1
             rescue => e
@@ -962,7 +962,7 @@ module Cnvrg
 
     def download_file(file_path: '', key: '', iv: '', bucket: '', path: '', client: nil)
       local_path = @project_home+"/"+file_path
-      @client.download(path, local_path)
+      @client.safe_download(path, local_path)
     end
 
     def delete(file)

data/lib/cnvrg/storage.rb

@@ -34,7 +34,7 @@ module Cnvrg
         @element.get_clone_chunk(commit: commit, chunk_size: params[:limit], offset: params[:offset])
       end
       action = Proc.new do |storage, local|
-        @client.download(storage, local)
+        @client.safe_download(storage, local)
       end
 
       @stats = @element.get_stats

data/lib/cnvrg/version.rb

@@ -1,3 +1,3 @@
 module Cnvrg
-  VERSION = '1.11.5'
+  VERSION = '1.11.10'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 1.11.5
+  version: 1.11.10
 platform: ruby
 authors:
 - Yochay Ettun
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-11-12 00:00:00.000000000 Z
+date: 2020-12-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -453,7 +453,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.0.4
 signing_key:
 specification_version: 4
 summary: A CLI tool for interacting with cnvrg.io.