cnvrg 0.0.14254 → 0.0.14256

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c569282c58ef4a5a88ba3f62190a2640bdf4e00f
-  data.tar.gz: a9f9f19980dec4fa966f79a2116646f35571683d
+  metadata.gz: e16a2aac3b10ceece9436b1d704e614bf03fbec0
+  data.tar.gz: fcdb72eb0cd3da08bef8b8baa84e15ae74137a0e
 SHA512:
-  metadata.gz: cfe491fa37c5830b7930afc840127ba45b223298fda50c975180422b3b41916004a87d600c7efde36bdd574ea3edafee7690240775c75074df4f87cf95ea30c3
-  data.tar.gz: 70ef2d66fcee9c0f718b347b651ba7a066191910ddbf9f0028042949ee98d528630b005ca58919de2995baa079c07e8de0ceb171ce5ed7b0b6f271e255465977
+  metadata.gz: 71d7e5bfe772ff2265f84c25063323656e50e37c3357d531a9b0372ad3af05d4f6a3045f5d909d052f69dcc412957f3becddf83ec7d4acc5ff12693c29100108
+  data.tar.gz: 23cf4218ff64a183ca9da4c2e51c94c0cc0f3b17ebfd274c9663d286c03c6a1035b3d6eb5f2757ed5d6ad4624462a1a21fa30ec1e698de01ceefc2a100f801ae

data/lib/cnvrg/cli.rb

@@ -35,7 +35,6 @@ require 'thor'
 require 'pathname'
 require 'enumerator'
 
-
 class Thor
   module Base
     def initialize(args = [], local_options = {}, config = {})
@@ -2194,6 +2193,8 @@ module Cnvrg
     method_option :data, :type => :string, :aliases => ["-d", "--data"], :default => ""
     method_option :data_commit, :type => :string, :aliases => ["-dc", "--data_commit"], :default => ""
     method_option :ignore, :type => :string, :aliases => ["-i", "--ignore"], :desc => "ignore following files", :default => ""
+    method_option :remote, :type => :boolean, :aliases => ["--remote"], :default => false
+    method_option :gpu, :type => :boolean, :aliases => ["--gpu"], :default => false
     def exec(*cmd)
 
       log = []
@@ -2260,6 +2261,12 @@ module Cnvrg
             exit(0)
           end
         end
+        remote = options["remote"]
+        if remote
+          docker_id = `cat /etc/hostname`
+          docker_id = docker_id.strip()
+        end
+        is_on_gpu = options["gpu"]
         start_commit = @project.last_local_commit
         cmd = cmd.join("\s")
 
@@ -2283,19 +2290,34 @@ module Cnvrg
                 begin
                   stdout.each do |line|
                     cur_time = Time.now
-                    monitor = %x{ps aux|awk  '{print $2,$3,$4}'|grep #{pid} }
-                    monitor_by = monitor.split(" ")
-                    memory = monitor_by[2]
-                    cpu = monitor_by[1]
+                    if remote
+                      stats = usage_metrics_in_docker(docker_id)
+                      cpu = stats[0]
+                      memory = stats[1]
+                    else
+                      memory = memory_usage()
+                      cpu = cpu_usage()
+                    end
+
                     memory_total << memory.to_f
                     cpu_total << cpu.to_f
                     real_time= Time.now-real
 
-                    cur_log = {time: cur_time,
+                    cur_log = { time: cur_time,
                                message: line,
                                type: "stdout",
-                               real: real_time}
-                    if print_log
+                               real: real_time,
+                                cpu: cpu,
+                                memory: memory
+                    }
+                    if is_on_gpu
+                      gpu_stats = gpu_util
+                      gpu_utilization = gpu_stats[0]
+                      gpu_memory_util = gpu_stats[1]
+                      cur_log.merge!(gpu_util:gpu_utilization,gpu_memory_util:gpu_memory_util)
+                    end
+
+                      if print_log
                       puts cur_log
                     end
                     log << cur_log
@@ -2304,8 +2326,8 @@ module Cnvrg
                       if time_to_upload !=0
                         if time_to_upload <= Time.now - start_loop
                           #upload current log
-                          cpu_average = cpu_total.inject(0) { |sum, el| sum + el }.to_f / cpu_total.size
-                          memory_average = memory_total.inject(0) { |sum, el| sum + el }.to_f / memory_total.size
+                          # cpu_average = cpu_total.inject(0) { |sum, el| sum + el }.to_f / cpu_total.size
+                          # memory_average = memory_total.inject(0) { |sum, el| sum + el }.to_f / memory_total.size
 
                           @exp.upload_temp_log(log, cpu_average, memory_average)
                           log = []
@@ -2621,8 +2643,7 @@ module Cnvrg
     method_option :data_commit, :type => :string, :aliases => ["--data_commit"], :default => ""
     method_option :ignore, :type => :string, :aliases => ["-i", "--ignore"], :desc => "ignore following files", :default => ""
 
-    def
-    exec_remote(*cmd)
+    def exec_remote(*cmd)
       verify_logged_in(true)
       log_start(__method__, args, options)
       working_dir = is_cnvrg_dir
@@ -2661,10 +2682,15 @@ module Cnvrg
         else
           upload_output_option = "--upload_output=#{upload_output}"
         end
+        remote = "--remote=true"
+        if !instance_type.nil? and instance_type.include? "gpu"
+          remote= "#{remote} --gpu=true"
+        end
+
         options_hash = Hash[options]
         options_hash.except!("schedule", "machine_type", "image", "upload_output", "grid", "data", "data_commit", "local", "small", "medium", "large", "gpu", "gpuxl", "gpuxxl")
         exec_options = options_hash.map { |x| "--#{x[0]}=#{x[1]}" }.flatten.join(" ")
-        command = "#{exec_options}  #{upload_output_option} #{cmd.flatten.join(" ")}"
+        command = "#{exec_options} #{remote}  #{upload_output_option} #{cmd.flatten.join(" ")}"
         commit_to_run = options["commit"] || nil
         if !schedule.nil? and !schedule.empty?
 
@@ -3853,6 +3879,7 @@ module Cnvrg
                 '8888/tcp' => {},
             },
             'HostConfig' => {
+                'Binds'=> ["/var/run/docker.sock:/var/run/docker.sock","/usr/bin/docker:/usr/bin/docker" ],
                 'PortBindings' => {
                     '8888/tcp' => [
                         {'HostPort' => "#{port}", 'HostIp' => 'localhost'}
@@ -3877,6 +3904,7 @@ module Cnvrg
         container.exec(command, tty: true)
         say "#{container.id}:#{port}##{tensport}"
       rescue => e
+        puts e
         if e.message.include? "is not running"
           return config_remote(image_name, port-1, tensport-1)
         end
@@ -3947,7 +3975,7 @@ module Cnvrg
         #     },
         # }
 
-        container_id = `nvidia-docker run -itd -p #{port}:8888 -p #{tensport}:6006 -w #{app_dir} -v /usr/bin/nvidia-smi:/usr/bin/nvidia-smi #{image_name}:latest #{cmd} `
+        container_id = `nvidia-docker run -itd -p #{port}:8888 -p #{tensport}:6006 -w #{app_dir} -v /usr/bin/nvidia-smi:/usr/bin/nvidia-smi  -v /var/run/docker.sock:/var/run/docker.sock -v /usr/bin/docker:/usr/bin/docker #{image_name}:latest #{cmd} `
         container_id = container_id.gsub("\n", "")
         container = Docker::Container.get(container_id)
         # container.start()
@@ -4445,11 +4473,10 @@ module Cnvrg
             url = "https://cnvrg.io/api"
           end
           config = YAML.load_file(home_dir+"/.cnvrg/config.yml")
-
           compression_path =  "#{home_dir}/.cnvrg/tmp"
-            if config and !config.nil? and !config.empty? and !config.to_h[:compression_path].nil?
-              compression_path = config.to_h[:compression_path]
-            end
+          if config and !config.nil? and !config.empty? and !config.to_h[:compression_path].nil?
+            compression_path = config.to_h[:compression_path]
+          end
           config = {owner: owner, username: username, version_last_check: get_start_day(), api: url,compression_path:compression_path}
 
           File.open(home_dir+"/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
@@ -4504,7 +4531,7 @@ module Cnvrg
 
       def log_end(exit_status=0, error=nil)
         begin
-          if exit_status==1
+          if exit_status != 0
             $LOG.error exit_status: exit_status, error: error
           else
             $LOG.info exit_status: exit_status
@@ -4932,6 +4959,76 @@ module Cnvrg
       def extarct_tar(file_path, dir_path)
         `tar -xvf #{file_path} -C #{dir_path} > /dev/null 2>&1`
         return $?.success?
+      end
+      def cpu_usage
+        cpu_usage = 0.0
+        begin
+          cpu = `top b -n 2 -d 2 |grep %Cpu |tail -1 |awk '{print $2+$3}'`
+        if !cpu.nil?
+          cpu_usage = cpu.to_f
+        end
+        rescue
+          cpu_usage = 0.0
+        end
+
+        return cpu_usage
+      end
+      def memory_usage
+        prec = 0.0
+        begin
+        used = `free -mt |grep Mem: |awk '{print $3}'`
+        total = `free -mt |grep Mem: |awk '{print $2}'`
+
+        used_f = used.to_f if !used.nil?
+        total_f = total.to_f if !total.nil?
+          prec = (used_f / total_f)*100
+          prec = prec.round(2)
+        rescue
+          prec = 0.0
+        end
+        return prec
+
+
+      end
+      def gpu_util
+        gpu = [0.0, 0.0]
+        begin
+        gpu_stats = `nvidia-smi --query-gpu=utilization.gpu,utilization.memory --format=csv |tail -1`
+
+        if !gpu_stats.nil?
+          gpu = gpu_stats.strip
+          gpu = gpu_stats.gsub!("%","").split(",")
+          gpu[0] = gpu[0].to_f
+          gpu[1] = gpu[1].to_f
+          return gpu
+        end
+
+        rescue
+          return gpu
+        end
+
+
+      end
+      def usage_metrics_in_docker(docker_id)
+        begin
+          if docker_id.nil?
+            docker_id = `cat /etc/hostname`
+          end
+          stats = `sudo docker stats #{docker_id} --no-stream |tail -1 | awk '{print $2,$8}'`
+          if !stats.nil?
+            conv = stats.gsub!("%","").split(" ")
+            cpu = conv[0].to_f
+            memory = conv[1].to_f
+            return [cpu,memory]
+          end
+
+
+        rescue
+          return [0.0,0.0]
+        end
+
+
+
       end
 
       def with_progress_bar

data/lib/cnvrg/experiment.rb

@@ -92,8 +92,7 @@ module Cnvrg
 
     def upload_temp_log(temp_log, cpu_average, memory_average)
       response = Cnvrg::API.request(@base_resource + "experiment/upload_temp_log", 'POST', {output: temp_log,
-                                                                                            exp_slug: @slug, cpu_average: cpu_average,
-                                                                                            memory_average: memory_average})
+                                                                                            exp_slug: @slug})
       Cnvrg::CLI.is_response_success(response,false)
     end
 

data/lib/cnvrg/version.rb

@@ -1,4 +1,5 @@
 module Cnvrg
-    VERSION = '0.0.14254'
+    VERSION = '0.0.14256'
+
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 0.0.14254
+  version: 0.0.14256
 platform: ruby
 authors:
 - Yochay Ettun
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-07-03 00:00:00.000000000 Z
+date: 2017-07-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -373,9 +373,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.6.12
 signing_key: 
 specification_version: 4
 summary: A CLI tool for interacting with cnvrg.io.
 test_files: []
-has_rdoc: