cnvrg 1.11.7 → 1.11.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1402d0759d20d1c1d9182cebc35fe5539c14a8a1bb2c4eeddf791f43ff768082
-  data.tar.gz: 4146973b1a98ea3d26af0efb96ee481093e29fe9fce6fb88c54a539b34e5c0cf
+  metadata.gz: 74c9b6b821c1d0fadae87a6b17579fa43a445fb0166a973bb14f686d778109e5
+  data.tar.gz: 525cae9497069129defa5c02995d04548adf4155e70e2ccf5ffe4f23bd883ed2
 SHA512:
-  metadata.gz: 53951eb4b7f82322640e75fdeae5ece07bb7189cc5b81257a9a78fd1398ab04c43ef0ed2784c1963f8d53dd983f874f2ed1ccc0ac4953ffced37deb75953d679
-  data.tar.gz: 8a046e229d0d14a2d36cb896c467aff3599e98b0e1cfb93e96fb72e80342751a47df4e1a90a57ff62e09ed92f1720afb783fe354dd34a5e06767e98c5bd8bdb6
+  metadata.gz: 736ae2e6aa54cdd18ff40d0a196cf6a10c764eaca2cc651fa4c2816b671b1d84bdd5d2329dea01082ab4175a95223797a3904cfa712aca51a07b6c13f63b95ab
+  data.tar.gz: bf58276b9169bc539f8d505b7482b5e9c40959e2773d1b08a323b86fd5acf9aa729f34390f56e1c3e854e9deb5d252cb4ff7f242fedba59f4ca4152c2d4f7cf3

data/lib/cnvrg/api.rb

@@ -65,6 +65,16 @@ module Cnvrg
                     response = conn.get "#{resource}", data
                     success = true
                     Cnvrg::API.parse_version(response)
+                    if response.to_hash[:status].to_i != 200
+                      Cnvrg::Logger.log_info("Got back bad status #{response.to_hash[:status]}")
+                    end
+                    if [503, 502, 429].include?(response.to_hash[:status].to_i)
+                      Cnvrg::Logger.log_info("Got back status #{response.to_hash[:status]}, will retry in #{5 * retries} seconds")
+                      success = false
+                      sleep(5 * retries)
+                      retries +=1
+                      next
+                    end
                   rescue => e
                     Cnvrg::Logger.log_error(e)
                     sleep(5)
@@ -95,11 +105,20 @@ module Cnvrg
                         response = conn.put "#{resource}", data.to_json if method.eql? 'PUT'
                         success = true
                         Cnvrg::API.parse_version(response)
-
+                        if response.to_hash[:status].to_i != 200
+                          Cnvrg::Logger.log_info("Got back bad status #{response.to_hash[:status]}")
+                        end
+                        if [503, 502, 429].include?(response.to_hash[:status].to_i)
+                          Cnvrg::Logger.log_info("Got back status #{response.to_hash[:status]}, will retry in #{5 * retries} seconds")
+                          success = false
+                          sleep(5 * retries)
+                          retries +=1
+                          next
+                        end
                     rescue => e
                       Cnvrg::Logger.log_error(e)
-                        sleep(5)
-                        retries +=1
+                      sleep(5)
+                      retries +=1
                     end
                 end
                 if !success

data/lib/cnvrg/cli.rb

@@ -1006,7 +1006,7 @@ module Cnvrg
               begin
                 unless File.exist?(fullpath)
                   downloader.safe_operation("#{abs_path}/#{file_name}") do
-                    File.open(fullpath, "w") { |f| f.write open(f["url"]).read }
+                    File.open(fullpath, "w") { |file| file.write open(f["url"]).read }
                   end
                 end
               rescue => e
@@ -3068,6 +3068,7 @@ module Cnvrg
     method_option :notify_on_success, :type => :boolean, :aliases => ["-nos", "--notify_on_success"], :default => nil
     method_option :emails, :type => :string, :aliases => ["-es", "--emails"], :default => "", :desc => "additional emails to notify on success / or error, comma separated"
     method_option :wait, :type => :boolean, :aliases => ["-w", "--wait"], :default => false, :desc => "keep command session open until experiment finished to return exit status"
+    method_option :debug, :type => :boolean, :aliases => ["--debug"], :default => true
 
     def run(*cmd)
       verify_logged_in(true)
@@ -3076,6 +3077,7 @@ module Cnvrg
       sync_before = options["sync_before"]
       sync_after = options["sync_after"]
       log = options["log"]
+      debug = options["debug"]
       title = options["title"]
       commit = options["commit"] || nil
       email_notification = options["email_notification"]
@@ -3109,6 +3111,7 @@ module Cnvrg
         wait = false
       end
 
+
       if !data.present? and data_query.present?
         log_message("Please provide data with data_query", Thor::Shell::Color::RED)
         exit(1)
@@ -3155,7 +3158,7 @@ module Cnvrg
                :image => image, :grid => grid, :data => data, :data_commit => data_commit, :ignore => ignore, :force => force, :sync_before_terminate => sync_before_terminate,
                :max_time => max_time,
                :periodic_sync => periodic_sync, :dataset_only_tree=> dataset_only_tree,
-               :output_dir=>output_dir, :data_query=>data_query, :git_commit =>git_commit, :git_branch=> git_branch,
+               :output_dir=>output_dir, :data_query=>data_query, :git_commit =>git_commit, :git_branch=> git_branch, :debug => debug,
                :restart_if_stuck =>restart_if_stuck, :local_folders => local_folders, :datasets => datasets, :prerun => prerun, :requirements => requirements,
                :email_notification_error => email_notification_error, :email_notification_success => email_notification_success, :emails => emails, :wait => wait
 
@@ -3379,10 +3382,10 @@ module Cnvrg
                 if @project.is_git
                   output_dir = output_dir || @exp.output_dir
                   if output_dir.present?
-                    upload(false, false, true, ignore, true, true, output_dir, "Experiment", @exp.slug, true )
+                    upload(false, false, true, ignore, true, false, output_dir, "Experiment", @exp.slug, true )
                   end
                 else
-                  upload(false, false, true, ignore, true, true, nil, "Experiment", @exp.slug, true )
+                  upload(false, false, true, ignore, true, false, nil, "Experiment", @exp.slug, true )
                 end
               end
 
@@ -3481,6 +3484,7 @@ module Cnvrg
     method_option :email_notification_success, :type => :boolean, :aliases => ["-nos", "--email_notification_success"], :default => true
     method_option :emails, :type => :string, :aliases => ["-es", "--emails"], :default => "", :desc => "additional emails to notify on success / or error"
     method_option :wait, :type => :boolean, :aliases => ["-w", "--wait"], :default => false, :desc => "keep command session open until experiment finished to return exit status"
+    method_option :debug, :type => :boolean, :aliases => ["--debug"], :default => true
 
     def exec_remote(*cmd)
 
@@ -3498,6 +3502,7 @@ module Cnvrg
         data_query = options["data_query"] || nil
         sync_before = options["sync_before"]
         force = options["force"]
+        debug = options["debug"]
         prerun = options["prerun"]
         requirements = options["requirements"]
         email_notification_error = options["email_notification_error"]
@@ -3541,8 +3546,8 @@ module Cnvrg
         local_folders_options = options["local_folders"]
         options_hash.except!("schedule", "recurring", "machine_type", "image", "upload_output", "grid", "data", "data_commit", "title",
                              "local", "small", "medium", "large", "gpu", "gpuxl", "gpuxxl","max_time","dataset_only_tree",
-                             "data_query", "git_commit","git_branch", "restart_if_stuck","local_folders","output_dir", "commit", "datasets",
-                             "requirements", "prerun", "email_notification_error", "email_notification_success", "emails", "wait")
+                             "data_query", "git_commit","git_branch","restart_if_stuck","local_folders","output_dir", "commit", "datasets",
+                             "requirements", "prerun", "email_notification_error", "email_notification_success", "emails", "wait","debug")
         exec_options = options_hash.map {|x| "--#{x[0]}=#{x[1]}"}.flatten.join(" ")
         command = "#{exec_options} #{remote}  #{upload_output_option} #{cmd.flatten.join(" ")}"
         commit_to_run = options["commit"] || nil
@@ -3585,20 +3590,6 @@ module Cnvrg
           end
         end
 
-        if command.include? "'"
-          oc = command.to_enum(:scan, /'/).map {Regexp.last_match}
-          pairs = oc.enum_for(:each_slice, 2).to_a
-          pairs.each_with_index do |p, i|
-            add = 0
-            if i != 0
-              add = 2 * i
-            end
-            total_loc = command[p[0].offset(0)[0] + add..p[1].offset(0)[0] + add]
-            command[p[0].offset(0)[0] + add..p[1].offset(0)[0] + add] = "\"#{total_loc}\""
-          end
-
-
-        end
         log_message("Running remote experiment", Thor::Shell::Color::BLUE)
         exp = Experiment.new(project.owner, project.slug)
         if forced_commit and (commit_to_run.nil? or commit_to_run.empty?)
@@ -3609,7 +3600,7 @@ module Cnvrg
 
         res = exp.exec_remote(command, commit_to_run, instance_type, image, schedule, local_timestamp, grid, path_to_cmd, data, data_commit,
                               periodic_sync, sync_before_terminate, max_time, ds_sync_options,output_dir,
-                              data_query, git_commit, git_branch, restart_if_stuck,local_folders_options, title, datasets, prerun: prerun, requirements: requirements, recurring: recurring,
+                              data_query, git_commit, git_branch,debug, restart_if_stuck,local_folders_options, title, datasets, prerun: prerun, requirements: requirements, recurring: recurring,
                               email_notification_error: email_notification_error, email_notification_success: email_notification_success, emails_to_notify: emails)
         if Cnvrg::CLI.is_response_success(res)
           check = Helpers.checkmark()
@@ -3661,6 +3652,7 @@ module Cnvrg
                 sleep 3
                 tries += 1
                 retry if tries <= 5
+                exit(1)
               end
             end
           end

data/lib/cnvrg/datafiles.rb

@@ -11,6 +11,7 @@ module Cnvrg
 
     LARGE_FILE=1024*1024*5
     MULTIPART_SPLIT=10000000
+    RETRIES = ENV['UPLOAD_FILE_RETRIES'].try(:to_i) || 10
 
     attr_reader :base_resource
 
@@ -256,6 +257,7 @@ module Cnvrg
     end
 
     def delete_file_chunk(commit_sha1, regex_list, chunk_size, offset)
+      retry_count = 0
       begin
         resp = Cnvrg::API.request(
             @base_resource + "delete_files_by_chunk",
@@ -268,13 +270,19 @@ module Cnvrg
             }
         )
         unless Cnvrg::CLI.is_response_success(resp, false)
-          Cnvrg::Logger.log_method(bind: binding)
           raise Exception.new("Got an error message from server, #{resp.try(:fetch, "message")}")
         end
         return resp["total_changes"]
       rescue => e
         Cnvrg::Logger.log_method(bind: binding)
         Cnvrg::Logger.log_error(e)
+
+        if retry_count < RETRIES
+          sleep(2**retry_count) # Exponential backoff
+          retry_count += 1
+          retry
+        end
+
         raise e
       end
     end

data/lib/cnvrg/downloader/client.rb

@@ -14,11 +14,13 @@ module Cnvrg
       end
 
       def extract_key_iv(sts_path)
-        count = 20
+        count = 0
         begin
           count += 1
           sts = open(sts_path, {ssl_verify_mode: 0}).read rescue nil
         rescue => e
+          backoff_time_seconds = backoff_time(count)
+          sleep backoff_time_seconds
           Cnvrg::Logger.log_error(e)
           retry if count <= 20
           raise StandardError.new("Cant access storage: #{e.message}")

data/lib/cnvrg/experiment.rb

@@ -110,7 +110,7 @@ module Cnvrg
 
     def exec_remote(command, commit_to_run, instance_type, image_slug,schedule,local_timestamp, grid,path_to_cmd,data, data_commit,periodic_sync,
                     sync_before_terminate, max_time, ds_sync_options=0,output_dir=nil,data_query=nil,
-                    git_commit=nil, git_branch=nil, restart_if_stuck=nil, local_folders=nil,title=nil, datasets=nil, prerun: true, requirements: true, recurring: nil,
+                    git_commit=nil, git_branch=nil,debug=true, restart_if_stuck=nil, local_folders=nil,title=nil, datasets=nil, prerun: true, requirements: true, recurring: nil,
                     email_notification_error: false, email_notification_success: false, emails_to_notify: nil)
       response = Cnvrg::API.request("users/#{@owner}/projects/#{@project_slug}/experiment/remote", 'POST', {command: command, image_slug: image_slug,
                                                                                                             commit_sha1: commit_to_run,
@@ -119,6 +119,7 @@ module Cnvrg
                                                                                                             local_timestamp:local_timestamp,
                                                                                                             datasets: datasets,
                                                                                                             grid: grid,
+                                                                                                            debug:debug,
                                                                                                             path_to_cmd:path_to_cmd,dataset_slug:data,
                                                                                                             dataset_commit: data_commit,max_time:max_time,
                                                                                                             periodic_sync:periodic_sync, sync_before_terminate:sync_before_terminate,

data/lib/cnvrg/helpers/agent.rb

@@ -75,9 +75,11 @@ class Cnvrg::Helpers::Agent
 
   def exec!
     log_internal("Command: #{@command} with slug: #{@slug} started!")
-    if should_run?
+    if @command.blank?
+      @exit_status = 0
+    elsif should_run?
       send_logs(status: Status::STARTED)
-      periodic_thread
+      periodic_thread_handle = periodic_thread
       execute_command
     else
       @exit_status = 127
@@ -86,6 +88,9 @@ class Cnvrg::Helpers::Agent
     finish_log += " after #{@real_execution_retries} retries" if @real_execution_retries > 0
     log_internal(finish_log)
     send_logs(exit_status: @exit_status, status: Status::FINISHED)
+    if periodic_thread_handle.present?
+      periodic_thread_handle.join
+    end
   end
 
   def get_logs_to_send

data/lib/cnvrg/helpers/executer.rb

@@ -120,17 +120,31 @@ class Cnvrg::Helpers::Executer
   end
 
   def init
-    resp = Cnvrg::API.request(activity_url, "PUT", {stats: executer_stats})
-    machine_activity = resp["machine_activity"]
-    Cnvrg::Logger.log_info("Got back machine activity #{machine_activity}")
-    if machine_activity.present? and @machine_activity != machine_activity
-      Cnvrg::Logger.log_info("Changing to machine activity #{machine_activity}")
-      machine_activity_yml = {slug: machine_activity}
-      File.open("/conf/.machine_activity.yml", "w+") {|f| f.write machine_activity_yml.to_yaml}
-      @machine_activity = machine_activity
+    retries = 0
+    success = false
+    puts("Agent started, connecting to #{Cnvrg::API.get_api}")
+    STDOUT.flush
+    while !success and retries < 100
+      begin
+        resp = Cnvrg::API.request(activity_url, "PUT", {stats: executer_stats})
+        machine_activity = resp["machine_activity"]
+        success = true
+        puts("Connected to server")
+        STDOUT.flush
+        Cnvrg::Logger.log_info("Got back machine activity #{machine_activity}")
+        if machine_activity.present? and @machine_activity != machine_activity
+          Cnvrg::Logger.log_info("Changing to machine activity #{machine_activity}")
+          machine_activity_yml = {slug: machine_activity}
+          File.open("/conf/.machine_activity.yml", "w+") {|f| f.write machine_activity_yml.to_yaml}
+          @machine_activity = machine_activity
+        end
+      rescue => e
+        Cnvrg::Logger.log_error(e)
+        Cnvrg::Logger.info("Sleeping for #{5 * retries}")
+        sleep(5 * retries)
+        retries +=1
+      end
     end
-  rescue => e
-    Cnvrg::Logger.log_error(e)
   end
 
   def polling_thread
@@ -156,7 +170,7 @@ class Cnvrg::Helpers::Executer
         next
       end
       cmd = @commands_q.pop.symbolize_keys
-      command_json = Cnvrg::API.request([activity_url, "commands", cmd[:slug]].join('/'), "GET")
+      comman  d_json = Cnvrg::API.request([activity_url, "commands", cmd[:slug]].join('/'), "GET")
 
       cmd_status = command_json["status"] rescue ""
 

data/lib/cnvrg/version.rb

@@ -1,3 +1,3 @@
 module Cnvrg
-  VERSION = '1.11.7'
+  VERSION = '1.11.13'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 1.11.7
+  version: 1.11.13
 platform: ruby
 authors:
 - Yochay Ettun
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-11-18 00:00:00.000000000 Z
+date: 2021-01-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -453,7 +453,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.4
+rubygems_version: 3.1.2
 signing_key:
 specification_version: 4
 summary: A CLI tool for interacting with cnvrg.io.