cnvrg 0.4.4 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6533e6de1028236f163d504243a2afef14da2f2b
-  data.tar.gz: 5327a071845d376bff81c76197a0b6844377d36e
+  metadata.gz: e44751ed52140fdaee81469a183a0f06f86968e1
+  data.tar.gz: 7b95fb6b3720a17d2edfe7a12da9c21ef0d3034f
 SHA512:
-  metadata.gz: d959e3931999dfbf64b45105b9f5e1e048219ee0b6e4f695d87193540229e998093347d3aa81896faa6f6e4d72e6cdfb629aca448558c3e19b7b88d8c2b397ce
-  data.tar.gz: b0ba98342dd6675c19e34376f6bd4f3157431e562c9d3850f988ec3b74a5d60008592b9a21cd3498c3a53de389e1f82460bcd930b09d736edd070ca8c3dba16d
+  metadata.gz: 2766aba6a508f5b073b4b12e987ee1d4568f84c47c632e6fefa6bf6fc0416a3737727cae88dfa7b97873c3f6461c5821d15a4ca9c7c54e32e9ca763b3e8b79d2
+  data.tar.gz: 8d2985f8c0ff5ea5d70131e993d896ce3ddd20a1f855dcf7ba7cad4a862a09fa13424bcdb3c5ebccb2a4e2ec9746f00eff2d7be49b7ee810826dc94d41a05a35

data/lib/cnvrg/cli.rb

@@ -3095,9 +3095,10 @@ module Cnvrg
     method_option :small, :type => :boolean, :aliases => ["-sm", "--small"], :default => false
     method_option :medium, :type => :boolean, :aliases => ["-md", "--medium"], :default => false
     method_option :large, :type => :boolean, :aliases => ["-lg", "--large"], :default => false
-    method_option :gpu_1, :type => :boolean, :aliases => ["--gpu"], :default => false
-    method_option :gpu_2, :type => :boolean, :aliases => ["--gpuxl"], :default => false
-    method_option :gpu_3, :type => :boolean, :aliases => ["--gpuxxl"], :default => false
+    method_option :gpu, :type => :boolean, :aliases => ["--gpu"], :default => false
+    method_option :gpuxl, :type => :boolean, :aliases => ["--gpuxl"], :default => false
+    method_option :gpuxxl, :type => :boolean, :aliases => ["--gpuxxl"], :default => false
+    method_option :machine, :type => :string, :aliases => ["-m", "--machine"], :default => nil
     method_option :sync_before, :type => :boolean, :aliases => ["-sb", "--sync_before"], :default => true
     method_option :sync_after, :type => :boolean, :aliases => ["-sa", "--sync_after"], :default => true
     method_option :title, :type => :string, :aliases => ["-t", "--title"], :default => ""
@@ -3139,12 +3140,9 @@ module Cnvrg
       force = options["force"]
       max_time = options["max_time"]
       dataset_only_tree = options["dataset_only_tree"]
+      custom_machine = options["machine"]
 
       options_hash = Hash[options]
-      real_options = []
-      options_hash.each do |o|
-        real_options << o if (!o[1].eql? "" and !["small", "medium", "large", "gpu_1", "gpu_2", "gpu_3"].include? o[0])
-      end
       if local
         invoke :exec, [cmd], :sync_before => sync_before, :sync_after => sync_after, :title => title,
                :log => log, :email_notification => email_notification, :upload_output => upload_output,
@@ -3163,15 +3161,10 @@ module Cnvrg
 
           end
         end
-        real_options.delete(["local", false])
         instances = {"small" => options["small"], "medium" => options["medium"], "large" => options["large"],
-                      "gpu_1" => options["gpu_2"], "gpu_3" => options["gpuxxl"]}
+                      "gpu" => options["gpu"], "gpuxl" => options["gpuxl"], "gpuxxl" => options["gpuxxl"],
+                     options["machine"] => !options["machine"].blank?}
         instance_type = get_instance_type(instances)
-        if !instance_type.nil? and !instance_type.empty?
-          real_options << ["machine_type", instance_type]
-        end
-        exec_options = real_options.map {|x| "--#{x[0]}=#{x[1]}"}.flatten.join(" ")
-        cmd_to_exec = "#{exec_options} #{cmd.join(" ")}"
         invoke :exec_remote, [cmd], :sync_before => sync_before, :sync_after => sync_after, :title => title, :machine_type => instance_type,
                :schedule => schedule, :log => log, :email_notification => email_notification, :upload_output => upload_output, :commit => commit,
                :image => image, :grid => grid, :data => data, :data_commit => data_commit, :ignore => ignore, :force => force, :sync_before_terminate => sync_before_terminate,
@@ -3227,7 +3220,6 @@ module Cnvrg
       time_to_upload = calc_output_time(upload_output)
       project_home = get_project_home
       @project = Project.new(project_home)
-
       is_new_branch = @project.compare_commit(commit)
       begin
         if !commit.nil? and !commit.empty?
@@ -3244,8 +3236,6 @@ module Cnvrg
         end
         if !indocker
           image_proj = is_project_with_docker(working_dir)
-
-
           if image_proj and image_proj.is_docker
             container = image_proj.get_container
             if !container
@@ -3292,7 +3282,7 @@ module Cnvrg
             stdout, stderr = '', ''
             begin
 
-              # if remote
+              if remote
                 if @exp.sync_before_terminate
                   spot_status_thread = Thread.new do
                     begin
@@ -3312,24 +3302,11 @@ module Cnvrg
                     end
                   end
                 end
-                if @exp.sync_delay_time
-                  regular_sync = Thread.new do
-                    begin
-                      loop do
-                        # sync
-                        sleep(@exp.sync_delay_time)
-
-                        download(sync = true, ignore_list = [])
-                        upload(link = false, sync = true, direct = false, ignore_list = [])
-                      end
-                    rescue => e
-                      log_error(e)
-                    end
-                  end
-                end
-              # end
+              end
+              process_running = true
               stats_thread = Thread.new do
-                loop do
+                while process_running do
+                  sleep 30
                   begin
                     stats = remote ? usage_metrics_in_docker(docker_id) : Helpers.linux? ? {memory: memory_usage, cpu: cpu_usage} : {}
                     if is_on_gpu
@@ -3337,85 +3314,90 @@ module Cnvrg
                       stats['gpu'] = gu[0]
                       stats['gpu_util'] = gu[1]
                     end
-                    @exp.send_machine_stats [stats]
+                    @exp.send_machine_stats [stats] unless stats.empty?
                   rescue => e
                     log_error(e)
                     log_message("Failed to upload ongoing stats, continuing with experiment", Thor::Shell::Color::YELLOW)
-                  ensure
-                    sleep(30)
                   end
                 end
               end
-
-
               log_thread = Thread.new do
-                loop do
-                  log_count = log.count
-                  time_to_upload = calc_output_log_time(log_count)
+                while process_running or !log.empty? do
                   begin
-                    if log_count != 0
-                      if time_to_upload <= Time.now - start_loop
-                        @exp.upload_temp_log(log)
-                        log = []
-                        start_loop = Time.now
-                      end
+                    temp_log = log
+                    if !temp_log.empty?
+                      @exp.upload_temp_log(temp_log) unless temp_log.empty?
+
                     end
+                    log -= temp_log
                   rescue => e
                     log_message("Failed to upload ongoing results, continuing with experiment", Thor::Shell::Color::YELLOW)
                     log_error(e)
+                  ensure
+                    sleep 10
                   end
-                  sleep(1)
                 end
               end
-
               PTY.spawn(cmd) do |stdout, stdin, pid, stderr|
                 begin
                   stdout.each do |line|
                     cur_time = Time.now
                     real_time = Time.now - real
-
                     cur_log = {time: cur_time,
                                message: line,
                                type: "stdout",
                                real: real_time
-
                     }
-
-
+                    $LOG.info(cur_log)
                     if print_log
                       puts cur_log
                     end
                     log << cur_log
+                    # if log.size >= 10
+                    #   @exp.upload_temp_log(log)
+                    #   log = []
+                    # end
                   end
-
-
                   if stderr
-
                     stderr.each do |err|
-
                       log << {time: Time.now, message: err, type: "stderr"}
                     end
                   end
-
                 rescue Errno::EIO => e
                   log_error(e)
-                rescue Errno::ENOENT => e
+                  if !log.empty?
 
+                    temp_log = log
+                    @exp.upload_temp_log(temp_log) unless temp_log.empty?
+
+                    log -= temp_log
+                  end
+                rescue Errno::ENOENT => e
                   exp_success = false
                   log_message("command \"#{cmd}\" couldn't be executed, verify command is valid", Thor::Shell::Color::RED)
                   log_error(e)
-                rescue Open4::ChildExited
-                  exp_success = false
-                  log_message("The process exited!", Thor::Shell::Color::RED)
+                # rescue Open4::ChildExited
+                #   exp_success = false
+                #   log_message("The process exited!", Thor::Shell::Color::RED)
                 rescue => e
                   sleep(20) # end cycle
-                  res = @exp.end(log, 1, start_commit, cpu_average, memory_average)
+                  res = @exp.end(log, 1, start_commit, 0, 0)
                   log_message("Error occurred,aborting", Thor::Shell::Color::RED)
                   log_error(e)
                   exit(0)
                 end
                 ::Process.wait pid
               end
+              process_running = false
+
+              if !log.empty?
+
+                temp_log = log
+                  @exp.upload_temp_log(temp_log) unless temp_log.empty?
+
+                log -= temp_log
+              end
+
               cpu_average = cpu_total.inject(0) {|sum, el| sum + el}.to_f / cpu_total.size
               memory_average = memory_total.inject(0) {|sum, el| sum + el}.to_f / memory_total.size
               exit_status = $?.exitstatus
@@ -3438,8 +3420,10 @@ module Cnvrg
                 end
                 end_commit = @project.last_local_commit
 
-                sleep(30) # end cycle
-                res = @exp.end(log, exit_status, end_commit, cpu_average, memory_average)
+                log_thread.join
+                stats_thread.join
+
+              res = @exp.end(log, exit_status, end_commit, cpu_average, memory_average)
 
 
                 if !exp_success
@@ -3465,13 +3449,15 @@ module Cnvrg
 
                 }
                 log << cur_log
-                sleep(20) # end cycle
+                process_running = false
+                log_thread.join
+                stats_thread.join
                 res = @exp.end(log, "-1", end_commit, cpu_average, memory_average)
 
               end
               log_error(e)
-              Thread.kill(log_thread)
-              Thread.kill(stats_thread)
+              # Thread.kill(log_thread)
+              # Thread.kill(stats_thread)
 
               exit(1)
             end
@@ -3483,7 +3469,9 @@ module Cnvrg
       rescue SignalException
         exit_status = -1
         end_commit = @project.last_local_commit
-        sleep(20) # end cycle
+        process_running = false
+        log_thread.join
+        stats_thread.join
 
         res = @exp.end(log, exit_status, end_commit, cpu_average, memory_average)
         if container
@@ -5519,7 +5507,12 @@ module Cnvrg
           if config and !config.nil? and !config.empty? and !config.to_h[:compression_path].nil?
             compression_path = config.to_h[:compression_path]
           end
-          config = {owner: owner, username: username, version_last_check: get_start_day(), api: url, compression_path: compression_path}
+          verify_ssl = false
+
+          if config and !config.nil? and !config.empty? and !config.to_h[:verfiy_ssl].nil?
+            verify_ssl = config.to_h[:verify_ssl]
+          end
+          config = {owner: owner, username: username, version_last_check: get_start_day(), api: url, compression_path: compression_path, verfiy_ssl:verify_ssl}
 
           File.open(home_dir + "/.cnvrg/config.yml", "w+") {|f| f.write config.to_yaml}
           return true
@@ -6145,27 +6138,30 @@ module Cnvrg
       end
 
       def gpu_util
-        gpu = [0.0, 0.0]
+        stats = [[],[]]
         begin
-          gpu_stats = `nvidia-smi --query-gpu=utilization.gpu,utilization.memory --format=csv |tail -1`
+          gpu_stats = `nvidia-smi --query-gpu=utilization.gpu,utilization.memory --format=csv`
 
           if !gpu_stats.nil?
-            gpu = gpu_stats.strip
-            gpu = gpu_stats.gsub!("%", "").split(",")
-            gpu[0] = gpu[0].to_f
-            gpu[1] = gpu[1].to_f
-            return gpu
+            gpu_stats = gpu_stats.split("\n")[1..-1]
+            stats = [[],[]]
+            gpu_stats.each do |stat|
+              gpu = stat.strip.gsub!("%", "").split(",")
+              stats[0] << gpu[0].to_f
+              stats[1] << gpu[1].to_f
+            end
+            return stats
           end
 
         rescue
-          return gpu
+          return stats
         end
 
 
       end
 
       def usage_metrics_in_docker(docker_id)
-        res = {cpu: 0.0, memory: 0.0}
+        res = {cpu: 0.0, memory: 0.0, block_io: {input: 0, output: 0.0}}
         begin
           if docker_id.nil?
             docker_id = `cat /etc/hostname`
@@ -6174,15 +6170,46 @@ module Cnvrg
           if !stats.nil?
             conv = stats.split(",")
             cpu = conv[0].gsub!("%", "").to_f
+            res[:cpu] = cpu
             memory = conv[1].gsub!("%", "").to_f
-            res = {cpu: cpu, memory: memory}
+            res[:memory] = memory
+            block_io = parse_io conv[2]
+            res = {cpu: cpu, memory: memory, block_io: block_io}
             return res
           end
         rescue
           return res
         end
+      end
+
 
 
+      def parse_io(block_io_str)
+        block_io = block_io_str.gsub!(" ", "").split('/')
+        input = block_io[0]
+        output = block_io[1]
+        r = Regexp.new('(\d+(\.\d+)?)([A-Za-z]+)')
+        input_match = r.match(input)
+        input = input_match[1].to_f * size_to_bytes(input_match[3])
+        output_match = r.match(output)
+        output = output_match[1].to_f * size_to_bytes(output_match[3])
+        {input: input, output: output}
+      end
+
+
+      def size_to_bytes size
+        case size.try(:downcase)
+        when 'b'
+          1
+        when 'kb'
+          2**10
+        when 'mb'
+          2**20
+        when 'gb'
+          2**30
+        else
+          1
+        end
       end
 
     end

data/lib/cnvrg/project.rb

@@ -381,7 +381,7 @@ module Cnvrg
         end
         response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/status", 'POST', {idx: local_idx, new_branch: new_branch,
                                                                                                    current_commit: commit,ignore:ignore_list, force:force,in_exp:in_exp})
-        CLI.is_response_success(response,false)
+        CLI.is_response_success(response,true)
         return response
       end
     def jump_idx(new_branch, commit=last_local_commit)

data/lib/cnvrg/version.rb

@@ -1,4 +1,4 @@
 module Cnvrg
-  VERSION = '0.4.4'
+  VERSION = '0.5.0'
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 0.4.4
+  version: 0.5.0
 platform: ruby
 authors:
 - Yochay Ettun
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-05-17 00:00:00.000000000 Z
+date: 2018-06-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler