cnvrg 1.9.9.9.7 → 1.10.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cb866c9448de7ea2062759a1f9cd9bf0f9db23b3fae06c8b61d178dbd1a227c9
-  data.tar.gz: dfc63cef384e0a8df9a367e03f544462638f5d0b43913dfe40b5dd59429ea2f8
+  metadata.gz: d9e8e76336bf9862fca95649740a7b5a5148565469ece725f4760041db28ffc4
+  data.tar.gz: f65852f97fb1294258c3c6b4b6941fb62248eeb1691d432e47bb4e2acb60a313
 SHA512:
-  metadata.gz: 760d90fb1e7516a3e2f9ae72a7662ff02ae0e6324a883a17471ad5edacccc252f3b127a5bddf7ec0afcff6d42fe3b3f5596a6697fa058f5740ff0799894934b0
-  data.tar.gz: ae6196d1a37f4bc68534aa728199d93500ee8299c99a896452158d22cf1e73578cf277ba0742f3e302ef9e677c466a335e9dac214c608e15c8ed088d870ddb60
+  metadata.gz: c5be8e6c872dd256560805d8146b3e2d7672a72844b932081dddc94c62e3fde4f9d7f66dd18d4f882c005c9e6e2183a91de689254b6dce07c265932058a4a1b0
+  data.tar.gz: 345eee06f18b24a30826665c7803b1275bdbc33d94779e5e37812cce7f95738bb819c5e2064871618145a27e72d133373e2493fb57f63b1c85ce8072c6057a54

data/lib/cnvrg/api.rb

@@ -212,7 +212,6 @@ module Cnvrg
                return nil
             rescue SignalException
                 return false
-
             end
 
         end

data/lib/cnvrg/api_v2.rb

@@ -2,13 +2,76 @@ module Cnvrg
   class API_V2 < API
     ENDPOINT_VERSION = 'v2'
 
+    class CnvrgAPIError < StandardError; end
+
     def self.endpoint_uri
       api = get_api()
       return "#{api}/#{Cnvrg::API_V2::ENDPOINT_VERSION}"
     end
 
+    def self.request(resource, method = 'GET', data = {}, parse_request = true)
+      resource = URI::encode resource
+      n = Netrc.read
+      if n['cnvrg.io'].nil?
+        puts 'You\'re not logged in'
+        puts 'Please log in via `cnvrg login`'
+        return
+      end
+
+      _, pass = n[Cnvrg::Helpers.netrc_domain]
+
+      conn = Faraday.new endpoint_uri, :ssl => {:verify => !!Helpers.is_verify_ssl}
+      conn.headers['Auth-Token'] = pass
+      conn.headers['User-Agent'] = Cnvrg::API::USER_AGENT
+      conn.headers['Content-Type'] = "application/json"
+      conn.options.timeout = 420
+      conn.options.open_timeout = 180
+
+      20.times do
+        begin
+          response = send_request conn, resource, method, data
+          Cnvrg::API.parse_version response
+          is_response_success response
+          if parse_request
+            return JSON.parse(response.body)
+          else
+            return response.body
+          end
+        rescue CnvrgAPIError => e
+          raise e
+        rescue => e
+          Cnvrg::Logger.log_error e
+          sleep 0.5
+          retry
+        end
+      end
+    rescue CnvrgAPIError => e
+      Cnvrg::Logger.log_error e
+      raise e
+    rescue SignalException
+      return false
+    rescue => e
+      Cnvrg::Logger.log_error e
+      return nil
+    end
+
+    private
+
+    def self.send_request(conn, resource, method, data)
+      case method
+      when 'GET'
+        conn.get resource, data
+      when 'POST'
+        conn.post resource, data.to_json
+      when 'PUT'
+        conn.put resource, data.to_json
+      when 'DELETE'
+        conn.delete resource, data
+      end
+    end
+
     def self.is_response_success(response)
-      raise Exception.new("Bad status in response #{response.status}") if response.status != 200
+      raise CnvrgAPIError.new(JSON(response.body)['errors']) if response.status != 200
     end
   end
-end
+end

data/lib/cnvrg/cli.rb

@@ -49,6 +49,7 @@ require 'cnvrg/job_cli'
 require 'cnvrg/job_ssh'
 require 'cnvrg/connect_job_ssh'
 require 'cnvrg/api_v2'
+require 'rubygems/package'
 
 class Thor
   module Base
@@ -857,7 +858,7 @@ module Cnvrg
     method_option :read, :type => :boolean, :aliases => ["-r", "--read"], :default => false
     method_option :remote, :type => :boolean, :aliases => ["-h", "--remote"], :default => false
     method_option :soft, :type => :boolean, :aliases => ["-s", "--soft"], :default => false, :hide => true
-    def clone_data(dataset_url, only_tree=false, commit=nil, query=nil, read=false, remote=false, flatten: false, relative: false, soft: false)
+    def clone_data(dataset_url, only_tree=false, commit=nil, query=nil, read=false, remote=false, flatten: false, relative: false, soft: false, threads: 15)
       begin
         verify_logged_in(false)
         log_start(__method__, args, options)
@@ -913,7 +914,7 @@ module Cnvrg
 
           while files['keys'].length > 0
             Cnvrg::Logger.log_info("download multiple files, #{downloaded_files.size} files downloaded")
-            @files.download_multiple_files_s3(files, @dataset.local_path, progressbar: progressbar, read_only: read, flatten: flatten)
+            @files.download_multiple_files_s3(files, @dataset.local_path, progressbar: progressbar, read_only: read, flatten: flatten, threads: threads)
 
             downloaded_files += files['keys'].length
             files = @files.get_clone_chunk(commit: commit, latest_id: files['latest'])
@@ -1267,7 +1268,7 @@ module Cnvrg
         @datafiles = Cnvrg::Datafiles.new(owner, slug, dataset: @dataset)
 
         # Init a new commit
-        response = @datafiles.start_commit(false, true, chunks: 1, message: message )
+        response = @datafiles.start_commit(false, false, chunks: 1, message: message )
         unless response #means we failed in the start commit.
           raise SignalException.new(1, "Cant put files into dataset, check the dataset id")
         end
@@ -1846,6 +1847,7 @@ module Cnvrg
     method_option :remote, :type => :boolean, :aliases => ["-r", "--r"], :default => false
     method_option :commit, :type => :string, :aliases => ["-c", "--c"], :default => nil
     method_option :soft, :type => :boolean, :aliases => ["-s", "--soft"], :default => false, :hide => true
+    method_option :threads, :type => :numeric, :aliases => ["--threads"], :default => 15
     def clone(project_url)
       begin
         verify_logged_in(false)
@@ -1856,7 +1858,7 @@ module Cnvrg
         owner = url_parts[project_index - 1]
         remote = options["remote"] || false
         soft = options["soft"] || false
-
+        threads = options[:threads] || Cnvrg::Helpers.parallel_threads
 
         response = Cnvrg::API.request("users/#{owner}/projects/#{slug}/get_project", 'GET')
         Cnvrg::CLI.is_response_success(response)
@@ -1907,9 +1909,10 @@ module Cnvrg
           idx = {commit: response["result"]["commit"], tree: response["result"]["tree"]}
           log_message("Downloading files", Thor::Shell::Color::BLUE)
           progressbar = @files.create_progressbar(files.size, "Clone Progress")
-          @files.download_files(files, commit_sha1, progress: progressbar)
+          @files.download_files(files, commit_sha1, progress: progressbar, threads: threads)
           progressbar.finish
           Project.verify_cnvrgignore_exist(project_name, remote)
+          Cnvrg::Logger.log_info("Generating idx")
           @project.set_idx(idx)
           log_message("Done")
           log_message("Downloaded #{files.size} files")
@@ -2274,9 +2277,9 @@ module Cnvrg
     method_option :job_type, :type => :string, :aliases => [ "--job_type"], :default => nil, :hide=>true
     method_option :suppress_exceptions, :type => :boolean, :aliases => ["--suppress-exceptions"], :default => true
     method_option :debug_mode, :type => :boolean, :aliases => ["--debug-mode"], :default => false
-    method_option :chunk_size, :type => :numeric, :aliases => ["--chunk"], :default => 100
+    method_option :chunk_size, :type => :numeric, :aliases => ["--chunk"], :default => 1000
 
-    def upload(link = false, sync = false, direct = false, ignore_list = "", in_exp = false, force = false, output_dir = "output", job_type = nil, job_slug = nil, suppress_exceptions = true,chunk_size=100)
+    def upload(link = false, sync = false, direct = false, ignore_list = "", in_exp = false, force = false, output_dir = "output", job_type = nil, job_slug = nil, suppress_exceptions = true,chunk_size=1000)
       begin
         # we are passing "force" twice.. doesnt really make sense :\\
         verify_logged_in(true)
@@ -2973,7 +2976,7 @@ module Cnvrg
     method_option :git_diff, :type => :boolean, :aliases => ["--git_diff"], :default => false
     method_option :suppress_exceptions, :type => :boolean, :aliases => ["--suppress-exceptions"], :default => true
     method_option :debug_mode, :type => :boolean, :aliases => ["--debug-mode"], :default => false
-    method_option :chunk_size, :type => :numeric, :aliases => ["--chunk"], :default => 100
+    method_option :chunk_size, :type => :numeric, :aliases => ["--chunk"], :default => 1000
 
     def sync(direct = true)
       verify_logged_in(true) if direct
@@ -3043,6 +3046,7 @@ module Cnvrg
     method_option :notify_on_error, :type => :boolean, :aliases => ["-noe", "--notify_on_error"], :default => nil
     method_option :notify_on_success, :type => :boolean, :aliases => ["-nos", "--notify_on_success"], :default => nil
     method_option :emails, :type => :string, :aliases => ["-es", "--emails"], :default => "", :desc => "additional emails to notify on success / or error, comma separated"
+    method_option :wait, :type => :boolean, :aliases => ["-w", "--wait"], :default => false, :desc => "keep command session open until experiment finished to return exit status"
 
     def run(*cmd)
       verify_logged_in(true)
@@ -3077,6 +3081,7 @@ module Cnvrg
       email_notification_error = options["notify_on_error"]
       email_notification_success = options["notify_on_success"]
       emails = options["emails"]
+      wait = options["wait"]
 
       if !data.present? and data_query.present?
         log_message("Please provide data with data_query", Thor::Shell::Color::RED)
@@ -3122,14 +3127,14 @@ module Cnvrg
         invoke :exec_remote, [cmd], :sync_before => sync_before, :sync_after => sync_after, :title => title, :machine_type => instance_type,
                :schedule => schedule, :recurring => recurring, :log => log, :email_notification => email_notification, :upload_output => upload_output, :commit => commit,
                :image => image, :grid => grid, :data => data, :data_commit => data_commit, :ignore => ignore, :force => force, :sync_before_terminate => sync_before_terminate,
-                :max_time => max_time,
+               :max_time => max_time,
                :periodic_sync => periodic_sync, :dataset_only_tree=> dataset_only_tree,
                :output_dir=>output_dir, :data_query=>data_query, :git_commit =>git_commit, :git_branch=> git_branch,
                :restart_if_stuck =>restart_if_stuck, :local_folders => local_folders, :datasets => datasets, :prerun => prerun, :requirements => requirements,
-               :email_notification_error => email_notification_error, :email_notification_success => email_notification_success, :emails => emails
+               :email_notification_error => email_notification_error, :email_notification_success => email_notification_success, :emails => emails, :wait => wait
+
         return
       end
-
     end
 
     desc '', '', :hide => true
@@ -3449,6 +3454,7 @@ module Cnvrg
     method_option :email_notification_error, :type => :boolean, :aliases => ["-noe", "--email_notification_error"], :default => true
     method_option :email_notification_success, :type => :boolean, :aliases => ["-nos", "--email_notification_success"], :default => true
     method_option :emails, :type => :string, :aliases => ["-es", "--emails"], :default => "", :desc => "additional emails to notify on success / or error"
+    method_option :wait, :type => :boolean, :aliases => ["-w", "--wait"], :default => false, :desc => "keep command session open until experiment finished to return exit status"
 
     def exec_remote(*cmd)
 
@@ -3510,7 +3516,7 @@ module Cnvrg
         options_hash.except!("schedule", "recurring", "machine_type", "image", "upload_output", "grid", "data", "data_commit", "title",
                              "local", "small", "medium", "large", "gpu", "gpuxl", "gpuxxl","max_time","dataset_only_tree",
                              "data_query", "git_commit","git_branch", "restart_if_stuck","local_folders","output_dir", "commit", "datasets",
-                             "requirements", "prerun", "email_notification_error", "email_notification_success", "emails")
+                             "requirements", "prerun", "email_notification_error", "email_notification_success", "emails", "wait")
         exec_options = options_hash.map {|x| "--#{x[0]}=#{x[1]}"}.flatten.join(" ")
         command = "#{exec_options} #{remote}  #{upload_output_option} #{cmd.flatten.join(" ")}"
         commit_to_run = options["commit"] || nil
@@ -3588,11 +3594,44 @@ module Cnvrg
           end
 
           log_message(str, Thor::Shell::Color::GREEN)
+          
+          exit_status = 0
+          
+          if options['wait']
+            while true
+              tries = 0
+              begin
+                result = Cnvrg::API_V2.request(
+                    "#{project.owner}/projects/#{project.slug}/experiments/#{res["result"]["exp_url"]}/exit_status",
+                    'GET',
+                    { grid: res["result"]["grid"]}
+                )
+
+                exit_statuses = result.values
+                if exit_statuses.include? nil
+                  system("clear") || system("cls")
+                  msg = "#{Time.current}: waiting for experiment to finish"
+                  msg = "#{Time.current}: waiting for all experiments to finish" if res["result"]["grid"]
+                  puts msg
+                  sleep 3
+                else
+                  result.each do |slug, exit_status|
+                    puts "Experiment #{slug} was exited with status #{exit_status}"
+                  end
+                  break
+                end
+              rescue => e
+                log_error(e)
+                log_message("Error occurred, retrying", Thor::Shell::Color::RED)
+                sleep 3
+                tries += 1
+                retry if tries <= 5
+              end
+            end
+          end
 
-          exit(0)
-          # end
+          exit(exit_status.to_i)
         end
-
       rescue => e
         log_message("Error occurred, Aborting", Thor::Shell::Color::RED)
         log_error(e)
@@ -3608,6 +3647,7 @@ module Cnvrg
         exit(1)
       end
     end
+
     desc 'deploy', 'Deploys model to production'
     method_option :small, :type => :boolean, :aliases => ["-s", "--small"], :default => false
     method_option :medium, :type => :boolean, :aliases => ["-m", "--medium"], :default => false
@@ -4816,17 +4856,19 @@ module Cnvrg
             exp_name = exp["title"]
             if exp["end_commit"].present? and exp["status"] != "Ongoing"
               log_message("#{exp_name} has ended, getting files from end commit", Thor::Shell::Color::BLUE)
-              Cnvrg::Helpers.get_experiment_events_log_from_server(exp, @project)
+              num_of_new_files = Cnvrg::Helpers.get_experiment_events_log_from_server(exp, @project)
               exps_map[exp_slug] = exp
             else
               log_message("#{exp_name} is running should get logs", Thor::Shell::Color::BLUE)
-              success = Cnvrg::Helpers.get_experiment_events_log_via_kubectl(exp, namespace)
+              success, num_of_new_files = Cnvrg::Helpers.get_experiment_events_log_via_kubectl(exp, namespace)
               if !success and exp["last_successful_commit"].present? and !copied_commits.include?(exp["last_successful_commit"])
                 log_message("Failed to get kube files, using last commit", Thor::Shell::Color::BLUE)
-                Cnvrg::Helpers.get_experiment_events_log_from_server(exp, @project, commit: exp["last_successful_commit"])
+                num_of_new_files = Cnvrg::Helpers.get_experiment_events_log_from_server(exp, @project, commit: exp["last_successful_commit"])
                 copied_commits << exp["last_successful_commit"]
               end
             end
+
+            log_message("New tf files copied", Thor::Shell::Color::BLUE) if num_of_new_files > 0
           rescue => e
             Cnvrg::Logger.log_error(e)
           end

data/lib/cnvrg/data.rb

@@ -154,6 +154,7 @@ module Cnvrg
     method_option :relative, :type => :boolean, :aliases => ["-rel", "--relative"], :default => false
     method_option :flatten, :type => :boolean, :aliases => ["-f", "--flatten"], :default => false
     method_option :soft, :type => :boolean, :aliases => ["-s", "--soft"], :default => false, :hide => true
+    method_option :threads, :type => :numeric, :aliases => ["--threads"], :default => 15
     def clone(dataset_url)
       cli = Cnvrg::CLI.new()
       only_tree =options[:only_tree]
@@ -163,6 +164,7 @@ module Cnvrg
       remote = options[:remote]
       soft = options[:soft]
       flatten = options[:flatten]
+      threads = options[:threads]
       cli.clone_data(
           dataset_url,
           only_tree=only_tree,
@@ -172,7 +174,8 @@ module Cnvrg
           remote=remote,
           flatten: flatten,
           relative: options[:relative],
-          soft: soft
+          soft: soft,
+          threads: threads
       )
     end
 

data/lib/cnvrg/datafiles.rb

@@ -409,8 +409,6 @@ module Cnvrg
           tree = @dataset.generate_chunked_idx(files_chunk, prefix: prefix, threads: threads)
           results = request_upload_files(commit_sha1, tree, override, new_branch, partial_commit)
 
-          # puts "Got #{results['files'].size} files to upload from #{files_chunk.size} files"
-
           if results['files'].blank?
             progress_mutex.synchronize { progressbar.progress += tree.keys.length }
             next
@@ -1284,11 +1282,11 @@ module Cnvrg
       end
     end
 
-    def download_multiple_files_s3(files, project_home, conflict: false, progressbar: nil, read_only:false, flatten: false)
+    def download_multiple_files_s3(files, project_home, conflict: false, progressbar: nil, read_only:false, flatten: false, threads: 15)
       begin
         refresh_storage_token
         parallel_options = {
-            in_threads: ParallelThreads,
+            in_threads: threads,
             isolation: true
         }
         Parallel.map(files["keys"], parallel_options) do |f|

data/lib/cnvrg/dataset.rb

@@ -544,8 +544,12 @@ module Cnvrg
     def generate_chunked_idx(list_files = [], threads: 15, prefix: '')
       tree = {}
       Parallel.map(list_files, in_threads: threads) do |file|
-        #check if prefix exists do prefix/path otherwise path
-        label = file.gsub(self.local_path + "/", "")
+
+        # Fix for root path issue
+        safe_path = file
+        safe_path = file[1..-1] if file.start_with? "/"
+
+        label = safe_path.gsub(self.local_path + "/", "")
         label = "#{prefix}/#{label}" if prefix.present?
         if not Cnvrg::Files.valid_file_name?(label)
           raise StandardError.new("#{label} is not a valid file name.")
@@ -558,9 +562,16 @@ module Cnvrg
           file_size = File.size(file).to_f
           mime_type = MimeMagic.by_path(file)
           content_type = !(mime_type.nil? or mime_type.text?) ? mime_type.type : "text/plain"
-          relative_path = file.gsub(/^#{@local_path + "/"}/, "")
+          relative_path = safe_path.gsub(/^#{@local_path + "/"}/, "")
           relative_path = "#{prefix}/#{relative_path}" if prefix.present?
-          tree[label] = {sha1: sha1, file_name: file_name, file_size: file_size, content_type: content_type, absolute_path: file, relative_path: relative_path}
+          tree[label] = {
+            sha1: sha1,
+            file_name: file_name,
+            file_size: file_size,
+            content_type: content_type,
+            absolute_path: file,
+            relative_path: relative_path
+          }
         end
       end
       if prefix.present? #add the prefix as dirs to the files

data/lib/cnvrg/files.rb

@@ -723,7 +723,7 @@ module Cnvrg
       @progressbar
     end
 
-    def download_files(files, commit, postfix: '', progress: nil)
+    def download_files(files, commit, postfix: '', progress: nil, threads: 15)
       return if files.blank?
       if Cnvrg::Helpers.server_version < 1
         Cnvrg::Logger.log_info("Download files from older server.")
@@ -733,7 +733,7 @@ module Cnvrg
       unless Cnvrg::CLI.is_response_success(res, false)
         raise SignalException.new("Cant download files from the server.")
       end
-      self.download_multpile_files_s3(res['result'], @project_home, postfix: postfix, progress: progress)
+      self.download_multpile_files_s3(res['result'], @project_home, postfix: postfix, progress: progress, threads: threads)
     end
 
 
@@ -750,7 +750,7 @@ module Cnvrg
       conflicted.each{|file| self.delete_conflict(file); progress.progress += 1 if progress.present?}
     end
 
-    def download_multpile_files_s3(files, project_home, postfix: '', progress: nil)
+    def download_multpile_files_s3(files, project_home, postfix: '', progress: nil, threads: 15)
       begin
         props = {}
         client = props[:client]
@@ -759,7 +759,7 @@ module Cnvrg
         bucket = props[:bucket]
         download_succ_count = 0
         parallel_options = {
-            in_threads: Cnvrg::Helpers.parallel_threads,
+            in_threads: threads,
             isolation: true
         }
 

data/lib/cnvrg/helpers.rb

@@ -2,15 +2,8 @@ module Cnvrg
   module Helpers
 
     extend self
-    def parallel_threads()
-      threads = ENV["CNVRG_PARALLEL_THREADS"].to_i
-
-      
-      ### if empty, default will be 15 threads
-      threads = threads > 0 ? threads : 15
-
-      ### set max threads to be 100k
-      [threads, 100000].min
+    def parallel_threads
+      15
     end
 
     def self.parallel_options
@@ -371,11 +364,14 @@ parameters:
       @files.download_files(files, commit_sha1, progress: nil)
       FileUtils.rm_rf("#{dest_dir}")
       FileUtils.mkdir_p(dest_dir)
+      num_of_new_files = 0
       files.each do |f|
         file_dir = "#{dest_dir}/#{File.dirname(f)}"
         FileUtils.mkdir_p(file_dir)
+        num_of_new_files += 1 unless File.exist?("#{dest_dir}/#{f}")
         FileUtils.mv(f, "#{dest_dir}/#{f}")
       end
+      return num_of_new_files
     end
 
     def get_experiment_events_log_via_kubectl(exp, namespace)
@@ -383,7 +379,7 @@ parameters:
       result = `kubectl -n #{namespace} get pods | grep #{exp["slug"]}`
 
       pod_name = result.split(" ")[0]
-      return false if pod_name.blank?
+      return false, 0 if pod_name.blank?
       FileUtils.mkdir_p(dest_dir)
       working_dir = `kubectl -n #{namespace} exec #{pod_name} -c agent -- pwd`
       working_dir.strip!
@@ -406,16 +402,18 @@ parameters:
         end
       end
 
+      num_of_new_files = 0
       all_files.each do |file|
         file_dir = "#{dest_dir}/#{File.dirname(file)}"
         FileUtils.mkdir_p(file_dir)
+        num_of_new_files += 1 unless File.exist?("#{dest_dir}/#{file}")
         res = `kubectl -n #{namespace} cp #{pod_name}:#{file} -c agent #{dest_dir}/#{file}`
       end
 
-      return true
+      return true, num_of_new_files
     rescue => e
       Cnvrg::Logger.log_error(e)
-      return false
+      return false, 0
     end
   end
 

data/lib/cnvrg/project.rb

@@ -421,7 +421,7 @@ module Cnvrg
 
     def generate_idx(deploy: false, files: [])
       if File.exists? "#{self.local_path}/.cnvrg/idx.yml"
-        old_idx = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
+        old_idx = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml") rescue {:tree => {}, :commit => nil}
       else
         old_idx = {:tree => {}, :commit => nil}
       end

data/lib/cnvrg/version.rb

@@ -1,3 +1,3 @@
 module Cnvrg
-  VERSION = '1.9.9.9.7'
+  VERSION = '1.10.8'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 1.9.9.9.7
+  version: 1.10.8
 platform: ruby
 authors:
 - Yochay Ettun
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-07-21 00:00:00.000000000 Z
+date: 2020-08-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler