cnvrg 1.6.0.1 → 1.6.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 689273a5f98807ad5a58f7bd13c64afe7524c57bf7cd5a53041cb072dff1976a
-  data.tar.gz: f404ae18536e6f7e43b4d0b9b342ed7e7e64250bf427efa0017488bfb08677c7
+  metadata.gz: 7d86069ef477a0e9e5d05eb754caa0cd025f97c4eeb43640b89494e5ff966d95
+  data.tar.gz: e2c315c26dbc155ed538d1992cd0833d01f6cdb0f714d4c4dbba0bd7482446d3
 SHA512:
-  metadata.gz: c2114fb5d5de328e8ea7df2892a40be729e86707d507a4c462ea560acdc6eb6b552bf720a3d48f1b6dad341dbca66bc3988b0ae5030bdaabc0e69d16ea785ac1
-  data.tar.gz: 99b6e69acc9c4399720a18c1b561050828e8f4e47b6ced70ae6c10434ecade93f858b7a4ee4df09c8668f0eb6893d9ed6e3bc883c4d3c28512a13a2ad561ddaf
+  metadata.gz: 2baa36b55fc4cea10cdde2a69c2144d13331301efce03477563fb900b59fd1b4a6a5c4ade0bef5176515537243291b01cda2a6b073e507e2a98fe3feb27a94f1
+  data.tar.gz: 91b217fe795cae96230bc40e3b1a107368ed1c00a67492ea195c237ae50d1f6c8700922357fb399c7dac6eeb1188d8a3150c65ab0a2c2c6640b79dd2928d5b23

data/lib/cnvrg/cli.rb

@@ -2976,7 +2976,8 @@ module Cnvrg
         else
           invoke :exec, [cmd], :sync_before => sync_before, :sync_after => sync_after, :title => title,
                  :log => log, :email_notification => email_notification, :upload_output => upload_output,
-                 :commit => commit, :image => image, :data => data, :data_commit => data_commit, :ignore => ignore, :force => force, :output_dir=>output_dir, :data_query=>data_query
+                 :commit => commit, :image => image, :data => data, :data_commit => data_commit,
+                 :ignore => ignore, :force => force, :output_dir=>output_dir, :data_query=>data_query
           return
         end
       else
@@ -5108,6 +5109,61 @@ module Cnvrg
 
     end
 
+
+    desc 'compare-experiments', 'compare experiments using tensorboard', :hide => true
+    method_option :slugs, :type => :string, :aliases => ["-ids"], :desc => "List of experiments slugs to compare"
+    method_option :namespace, :type => :string, :aliases => ["-n"], :desc => "experiments pods namespace", :default => "cnvrg"
+    method_option :project_slug, :type => :string, :aliases => ["-s"], :desc => "project slug"
+    method_option :project_owner, :type => :string, :aliases => ["-o"], :desc => "project slug"
+    method_option :frequency, :type => :numeric, :aliases => ["-f"], :desc => "poll frequency"
+
+    def compare_experiments
+      verify_logged_in(true)
+      log_start(__method__, args, options)
+      exps_map = {}
+
+      if options[:slugs].blank?
+        log_message("No experiments slugs given", Thor::Shell::Color::RED)
+        return false
+      end
+      slugs = options[:slugs].split(",")
+      if slugs.blank?
+        log_message("No experiments slugs given", Thor::Shell::Color::RED)
+        return false
+      end
+      frequency = options[:frequency] || 5
+      namespace = options[:namespace]
+      project_dir = is_cnvrg_dir(Dir.pwd)
+      @project = Project.new(project_home=project_dir, slug: options[:project_slug], owner: options[:project_owner])
+
+      while true
+        slugs.each do |exp_slug|
+
+          begin
+            if exps_map[exp_slug].blank?
+              exp = @project.get_experiment(exp_slug)["experiment"]
+            else
+              exp = exps_map[exp_slug]
+              log_message("#{exp["title"]} has ended skipping it", Thor::Shell::Color::BLUE)
+              next
+            end
+            exp_name = exp["title"]
+            if exp["end_commit"].present? and exp["status"] != "Ongoing"
+              log_message("#{exp_name} has ended, getting files from end commit", Thor::Shell::Color::BLUE)
+              Cnvrg::Helpers.get_experiment_events_log_from_server(exp, @project)
+              exps_map[exp_slug] = exp
+            elsif exp["machine_activity"].present?
+              log_message("#{exp_name} is running should get logs", Thor::Shell::Color::BLUE)
+              Cnvrg::Helpers.get_experiment_events_log_via_kubectl(exp, namespace)
+            end
+          rescue => e
+            Cnvrg::Logger.log_error(e)
+          end
+        end
+        sleep frequency
+      end
+    end
+
     desc 'experiments', 'List project experiments'
     method_option :id, :type => :string, :aliases => ["--id"], :desc => "Get info for specific experiments", :default => ""
     method_option :tag, :type => :string, :aliases => ["-t"], :desc => "Get info for specific experiment tag", :default => ""

data/lib/cnvrg/helpers.rb

@@ -359,5 +359,56 @@ parameters:
       return {client: client, key: key, iv: iv, bucket: bucket, upload_options: upload_options}
     end
 
+    def get_experiment_events_log_from_server(exp, project)
+      dest_dir = exp["slug"]
+      commit = exp["end_commit"]
+      response = project.clone(0, commit)
+      Cnvrg::CLI.is_response_success(response, should_exit=false)
+      commit_sha1 = response["result"]["commit"]
+      files = response["result"]["tree"].keys
+      files = files.select do |f| f.include?("tfevents") end
+      @files = Cnvrg::Files.new(project.owner, project.slug, project_home: "", project: project)
+      @files.download_files(files, commit_sha1, progress: nil)
+      FileUtils.rm_rf("#{dest_dir}")
+      FileUtils.mkdir_p(dest_dir)
+      files.each do |f|
+        FileUtils.mv(f, "#{dest_dir}/#{File.basename(f)}")
+      end
+    end
+
+    def get_experiment_events_log_via_kubectl(exp, namespace)
+      dest_dir = exp["slug"]
+      result = `kubectl -n #{namespace} get pods | grep #{exp["machine_activity"]}`
+      pod_name = result.split(" ")[0]
+      if pod_name.present?
+        FileUtils.mkdir_p(dest_dir)
+        working_dir = `kubectl -n #{namespace} exec #{pod_name} -- pwd`
+        working_dir.strip!
+        res = `kubectl -n #{namespace} exec #{pod_name} -- /bin/bash -c "ls -R #{working_dir}"`
+        files_and_folders = res.split("\n\n")
+        all_files = []
+
+        files_and_folders.each do |file_and_folder|
+          files = file_and_folder.split("\n")
+          if files.first.include?(":")
+            folder = files.first.gsub(":", "")
+            files = files.drop(1)
+          end
+          files.each do |file|
+            if file.include?("tfevents")
+              all_files << "#{folder}/#{file}"
+            end
+          end
+        end
+
+        all_files.each do |file|
+          res = `kubectl -n #{namespace} cp #{pod_name}:#{file} #{dest_dir}/#{File.basename(file)}`
+        end
+
+      end
+    rescue => e
+      Cnvrg::Logger.log_error(e)
+    end
   end
+
 end

data/lib/cnvrg/image.rb

@@ -79,7 +79,7 @@ module Cnvrg
       end
       post_build_update(true)
     rescue => e
-      @cli.log_message("Image Build failed: #{e.message}")
+      @cli.log_message("Image Build failed")
       post_build_update(false, e.message)
     end
 

data/lib/cnvrg/version.rb

@@ -1,4 +1,4 @@
 module Cnvrg
-  VERSION = '1.6.0.1'
+  VERSION = '1.6.0.2'
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 1.6.0.1
+  version: 1.6.0.2
 platform: ruby
 authors:
 - Yochay Ettun
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-10-28 00:00:00.000000000 Z
+date: 2019-11-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler