cnvrg 1.6.27 → 1.6.36

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1cd284862886ec5b09fd54d38793623e428fec3dbfac5fc6c46667c2e820a302
-  data.tar.gz: 7f3cce766ef0935b737ce0e95dcce47bf827bc4ab45579887fa51b9ec4a02b55
+  metadata.gz: c5e60c1cc7c0af133390dd106ee666db3787b6c995b54d5ef97edff3d073ff14
+  data.tar.gz: 3d37f71642874898b131812fdeb1ca7be3b7eebddc3a8aa521245c31eaa47a0b
 SHA512:
-  metadata.gz: 32562f3442ed727ab26ee6de3af75df36bd56d6c1515d653e697936c03624efa03d741610b7e960b67a7179b3f961b7d07efb71af0e3201ff15b08939aca3719
-  data.tar.gz: 8fbaad6c29f0f7e0ae504912505f36be91825263c3cb72b2ad73df5cc7944115e605eaf3e6711b526c1cbdf9d52572c488728d62a2ce86c5ee40aa9e1a5e99b3
+  metadata.gz: dc82247f7d78186aa0b35084d83b0d4ae72608f0c765727ee57bebcec76d6bc9c84fad2795c7c7ed7ba6ac64921df49c9c45cd3536128b11b8a5b8203e8e0040
+  data.tar.gz: 896d84a1615dd3a698c5e78ce92d7422fca9e6f21e1c7e9ca6011ab315437b39b2f3f5fb0790d683397e2ba6bdaaad4185156255e615e298471b50949201f9d2

data/lib/cnvrg/cli.rb

@@ -1184,7 +1184,7 @@ module Cnvrg
     end
 
     desc '', '', :hide => true
-    def data_put(dataset_url, files: [], dir: '', commit: '', chunk_size: 1000)
+    def data_put(dataset_url, files: [], dir: '', commit: '', chunk_size: 1000, message: nil)
       begin
         verify_logged_in(false)
         log_start(__method__, args, options)
@@ -1204,7 +1204,7 @@ module Cnvrg
         log_message("Uploading #{@files.size} files", Thor::Shell::Color::GREEN)
         number_of_chunks = (@files.size.to_f / chunk_size).ceil
         if commit.blank?
-          response = @datafiles.start_commit(false, true, chunks: number_of_chunks)
+          response = @datafiles.start_commit(false, true, chunks: number_of_chunks, message: message )
           unless response #means we failed in the start commit.
             raise SignalException.new(1, "Cant put files into dataset, check the dataset id")
           end
@@ -1837,8 +1837,6 @@ module Cnvrg
           end
           clone_resp = Project.clone_dir(slug, owner, project_name,git)
           project_home = Dir.pwd + "/" + project_name
-
-
         end
 
         if clone_resp
@@ -1986,31 +1984,7 @@ module Cnvrg
         files_to_upload, upload_errors = invoke :upload_data_files,[commit, files_list: files_list],:new_branch=>new_branch, :verbose =>verbose, :force =>force, :sync =>sync, :chunk_size => chunk_size
 
         upload_size = files_to_upload + upload_errors.try(:size) rescue 0
-
-        if files_to_upload <= 0
-          check = Helpers.checkmark
-          log_message("#{check} Data sync finished", Thor::Shell::Color::GREEN)
-          if total_deleted > 0
-            log_message("#{total_deleted} files deleted successfully.", Thor::Shell::Color::GREEN)
-          end
-
-          if total_downloaded > 0
-            log_message("#{total_downloaded} files downloaded successfully.", Thor::Shell::Color::GREEN)
-          end
-
-          if upload_errors.try(:size) > 0
-            log_message("#{upload_errors.try(:size)}/#{upload_size} files didn't upload:", Thor::Shell::Color::RED)
-            upload_errors.each do |file_hash|
-              log_message("#{file_hash[:absolute_path]}", Thor::Shell::Color::RED)
-            end
-          end
-
-          if upload_errors.blank? and total_downloaded == 0 and total_deleted == 0
-            Cnvrg::CLI.log_message("You are up to date", Thor::Shell::Color::GREEN)
-          end
-          return
-        end
-        invoke :end_commit_data,[commit, success: true, uploaded_files: files_to_upload], :new_branch=>new_branch, :force =>force
+        invoke :end_commit_data,[commit, success: true, uploaded_files: files_to_upload, sync: sync], :new_branch=>new_branch, :force =>force
         if tags
           log_message('Uploading Tags', Thor::Shell::Color::BLUE)
           dataset_dir = is_cnvrg_dir(Dir.pwd)
@@ -2035,8 +2009,10 @@ module Cnvrg
         if total_downloaded > 0
           log_message("#{total_downloaded} files downloaded successfully.", Thor::Shell::Color::GREEN)
         end
+        if upload_size > 0
+          log_message("#{files_to_upload}/#{upload_size} files uploaded successfully.", Thor::Shell::Color::GREEN)
+        end
 
-        log_message("#{files_to_upload}/#{upload_size} files uploaded successfully.", Thor::Shell::Color::GREEN)
         if upload_errors.try(:size) > 0
           log_message("#{upload_errors.try(:size)}/#{upload_size} files didn't upload:", Thor::Shell::Color::RED)
           upload_errors.each do |file_hash|
@@ -2100,7 +2076,7 @@ module Cnvrg
     method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
     method_option :force, :type => :boolean, :aliases => ["-f","--force"], :default => false
 
-    def end_commit_data(commit, success: true, uploaded_files: 0)
+    def end_commit_data(commit, success: true, uploaded_files: 0, sync: false)
       begin
         verify_logged_in(true)
         log_start(__method__, args, options)
@@ -2115,7 +2091,12 @@ module Cnvrg
             @dataset.revert_next_commit #removes the next commit
             log_message("#{check} Dataset is up to date", Thor::Shell::Color::GREEN)
           else
-            log_message("#{check} Data sync finished", Thor::Shell::Color::GREEN)
+            if sync
+              message = "#{check} Data sync finished"
+            else
+              message = "#{check} Data upload finished"
+            end
+            log_message(message, Thor::Shell::Color::GREEN)
             @dataset.remove_next_commit #takes the next commit and put it as current commit
             @dataset.set_partial_commit(nil)
             @dataset.backup_idx
@@ -2911,7 +2892,7 @@ module Cnvrg
     method_option :job_type, :type => :string, :aliases => ["-jt", "--job_type"], :default => nil
     method_option :files, :type => :string, :aliases => ["--files"], :default => nil
     method_option :output_dir, :type => :string, :aliases => ["--output_dir"], :default => nil
-    def  sync(direct = true)
+    def sync(direct = true)
       verify_logged_in(true) if direct
       @project = Project.new(get_project_home)
       log_start(__method__, args, options)
@@ -2973,6 +2954,7 @@ module Cnvrg
     method_option :requirements, :type => :boolean, :aliases => ["-r", "--requirements"], :default => true
     method_option :notify_on_error, :type => :boolean, :aliases => ["-noe", "--notify_on_error"], :default => nil
     method_option :notify_on_success, :type => :boolean, :aliases => ["-nos", "--notify_on_success"], :default => nil
+    method_option :emails, :type => :string, :aliases => ["-es", "--emails"], :default => "", :desc => "additional emails to notify on success / or error, comma separated"
 
     def run(*cmd)
       verify_logged_in(true)
@@ -3006,6 +2988,7 @@ module Cnvrg
       requirements = options["requirements"]
       email_notification_error = options["notify_on_error"]
       email_notification_success = options["notify_on_success"]
+      emails = options["emails"]
 
       if !data.present? and data_query.present?
         log_message("Please provide data with data_query", Thor::Shell::Color::RED)
@@ -3020,7 +3003,6 @@ module Cnvrg
       restart_if_stuck = options["restart_if_stuck"]
 
       options_hash = Hash[options]
-
       if local
         if Cnvrg::Helpers.windows?
             say "Windows is currently not supported for running experiments locally"
@@ -3056,7 +3038,7 @@ module Cnvrg
                :periodic_sync => periodic_sync, :dataset_only_tree=> dataset_only_tree,
                :output_dir=>output_dir, :data_query=>data_query, :git_commit =>git_commit, :git_branch=> git_branch,
                :restart_if_stuck =>restart_if_stuck, :local_folders => local_folders, :datasets => datasets, :prerun => prerun, :requirements => requirements,
-               :email_notification_error => email_notification_error, :email_notification_success => email_notification_success
+               :email_notification_error => email_notification_error, :email_notification_success => email_notification_success, :emails => emails
         return
       end
 
@@ -3362,8 +3344,9 @@ module Cnvrg
     method_option :datasets, :type => :string, :aliases => ["--datasets"], :default => nil
     method_option :prerun, :type => :boolean, :aliases => ["-p", "--prerun"], :default => true
     method_option :requirements, :type => :boolean, :aliases => ["-r", "--requirements"], :default => true
-    method_option :email_notification_error, :type => :boolean, :aliases => ["--email_notification_error"], :default => true
-    method_option :email_notification_success, :type => :boolean, :aliases => ["--email_notification_success"], :default => true
+    method_option :email_notification_error, :type => :boolean, :aliases => ["-noe", "--email_notification_error"], :default => true
+    method_option :email_notification_success, :type => :boolean, :aliases => ["-nos", "--email_notification_success"], :default => true
+    method_option :emails, :type => :string, :aliases => ["-es", "--emails"], :default => "", :desc => "additional emails to notify on success / or error"
 
     def exec_remote(*cmd)
 
@@ -3385,6 +3368,7 @@ module Cnvrg
         requirements = options["requirements"]
         email_notification_error = options["email_notification_error"]
         email_notification_success = options["email_notification_success"]
+        emails = options["emails"]
         max_time = options["max_time"]
         if !max_time.nil? and !max_time.empty?
           max_time = max_time.to_i
@@ -3424,7 +3408,7 @@ module Cnvrg
         options_hash.except!("schedule", "recurring", "machine_type", "image", "upload_output", "grid", "data", "data_commit", "title",
                              "local", "small", "medium", "large", "gpu", "gpuxl", "gpuxxl","max_time","dataset_only_tree",
                              "data_query", "git_commit","git_branch", "restart_if_stuck","local_folders","output_dir", "commit", "datasets", "requirements", "prerun",
-                             "email_notification_error", "email_notification_success")
+                             "email_notification_error", "email_notification_success", "emails")
         exec_options = options_hash.map {|x| "--#{x[0]}=#{x[1]}"}.flatten.join(" ")
         command = "#{exec_options} #{remote}  #{upload_output_option} #{cmd.flatten.join(" ")}"
         commit_to_run = options["commit"] || nil
@@ -3492,7 +3476,7 @@ module Cnvrg
         res = exp.exec_remote(command, commit_to_run, instance_type, image, schedule, local_timestamp, grid, path_to_cmd, data, data_commit,
                               periodic_sync, sync_before_terminate, max_time, ds_sync_options,output_dir,
                               data_query, git_commit, git_branch, restart_if_stuck,local_folders_options, title, datasets, prerun: prerun, requirements: requirements, recurring: recurring,
-                              email_notification_error: email_notification_error, email_notification_success: email_notification_success)
+                              email_notification_error: email_notification_error, email_notification_success: email_notification_success, emails_to_notify: emails)
         if Cnvrg::CLI.is_response_success(res)
           check = Helpers.checkmark()
           str = "#{check} Experiment's is on: #{Cnvrg::Helpers.remote_url}/#{project.owner}/projects/#{project.slug}/experiments/#{res["result"]["exp_url"]}"

data/lib/cnvrg/data.rb

@@ -186,11 +186,14 @@ module Cnvrg
     desc 'data put DATASET_URL FILES_PREFIX', 'Upload selected files from local dataset directory to remote server'
     method_option :dir, :type => :string, :aliases => ["-d", "--dir"], :default => ''
     method_option :commit, :type => :string, :aliases => ["-c", "--commit"], :default => ''
+    method_option :message, :type => :string, :aliases => ["--message"], :desc => "create commit with message", :default => nil
     def put(dataset_url, *files)
       cli = Cnvrg::CLI.new()
       dir = options[:dir]
       commit = options[:commit]
-      cli.data_put(dataset_url, files: files, dir: dir, commit: commit)
+      message = options[:message]
+
+      cli.data_put(dataset_url, files: files, dir: dir, commit: commit, message: message)
     end
 
     desc 'data clone_query --query=QUERY_SLUG DATASET_URL', 'Clone dataset with specific query'

data/lib/cnvrg/datafiles.rb

@@ -20,6 +20,15 @@ module Cnvrg
       @dataset = dataset
       @base_resource = "users/#{owner}/datasets/#{dataset_slug}/"
       @downloader = @dataset.get_storage_client
+      @token_issue_time = Time.current
+    end
+
+    def refresh_storage_token
+      current_time = Time.current
+      if current_time - @token_issue_time > 3.hours
+        @downloader = @dataset.get_storage_client
+        @token_issue_time = Time.current
+      end
     end
 
     def check_file_sha1(filename, org_sha1, tag: 'conflict')
@@ -117,6 +126,7 @@ module Cnvrg
     def upload_multiple_files(commit_sha1, tree, threads: ParallelThreads, force: false, new_branch: false, prefix: '', partial_commit: nil, total: nil)
       begin
         Cnvrg::Logger.log_info("Sending Upload Files request")
+        refresh_storage_token
         error = nil
         upload_resp = nil
         10.times do
@@ -1007,6 +1017,7 @@ module Cnvrg
 
     def download_multiple_files_s3(files, project_home, conflict: false, progressbar: nil, read_only:false)
       begin
+        refresh_storage_token
         parallel_options = {
             in_threads: ParallelThreads,
             isolation: true

data/lib/cnvrg/experiment.rb

@@ -108,7 +108,7 @@ module Cnvrg
     def exec_remote(command, commit_to_run, instance_type, image_slug,schedule,local_timestamp, grid,path_to_cmd,data, data_commit,periodic_sync,
                     sync_before_terminate, max_time, ds_sync_options=0,output_dir=nil,data_query=nil,
                     git_commit=nil, git_branch=nil, restart_if_stuck=nil, local_folders=nil,title=nil, datasets=nil, prerun: true, requirements: true, recurring: nil,
-                    email_notification_error: false, email_notification_success: false)
+                    email_notification_error: false, email_notification_success: false, emails_to_notify: nil)
       response = Cnvrg::API.request("users/#{@owner}/projects/#{@project_slug}/experiment/remote", 'POST', {command: command, image_slug: image_slug,
                                                                                                             commit_sha1: commit_to_run,
                                                                                                             instance_type: instance_type,
@@ -124,7 +124,8 @@ module Cnvrg
                                                                                                             restart_if_stuck:restart_if_stuck, local_folders: local_folders, title:title,
                                                                                                             prerun: prerun, requirements: requirements, recurring: recurring,
                                                                                                             email_notification_error: email_notification_error,
-                                                                                                            email_notification_success: email_notification_success})
+                                                                                                            email_notification_success: email_notification_success,
+                                                                                                            emails_to_notify: emails_to_notify})
 
       return response
     end

data/lib/cnvrg/files.rb

@@ -7,7 +7,7 @@ require 'cnvrg/result'
 module Cnvrg
   class Files
     ParallelThreads = Cnvrg::Helpers.parallel_threads
-    VALID_FILE_NAME = /[\x00\\:\*\?\"<>\|]/
+    VALID_FILE_NAME = /[\x00\\\*\?\"<>\|]/
     LARGE_FILE=1024*1024*5
     MULTIPART_SPLIT=10000000
 
@@ -27,8 +27,16 @@ module Cnvrg
       @custom_progess = false
       @cli = cli
       @options = options
+      @token_issue_time = Time.current
     end
 
+    def refresh_storage_token
+      current_time = Time.current
+      if current_time - @token_issue_time > 3.hours
+        @client = @project.get_storage_client
+        @token_issue_time = Time.current
+      end
+    end
 
     def self.valid_file_name?(fullpath)
       VALID_FILE_NAME.match(fullpath).blank?
@@ -97,7 +105,13 @@ module Cnvrg
       files = res['files']
 
       #upload files
+      token_mutex = Mutex.new
       blob_ids = Parallel.map(files.keys, in_threads: ParallelThreads) do |file|
+
+        token_mutex.synchronize {
+          refresh_storage_token
+        }
+
         begin
           Cnvrg::Helpers.try_until_success{self.upload_single_file(files[file].merge(files_list[file]))}
         rescue => e
@@ -721,8 +735,15 @@ module Cnvrg
             in_threads: Cnvrg::Helpers.parallel_threads,
             isolation: true
         }
+
+        token_mutex = Mutex.new
+
         Parallel.map(files["keys"], parallel_options) do |f|
 
+          token_mutex.synchronize {
+            refresh_storage_token
+          }
+
           file_path = f["name"]
           if file_path.end_with? "/"
             # dir

data/lib/cnvrg/version.rb

@@ -1,3 +1,3 @@
 module Cnvrg
-  VERSION = '1.6.27'
+  VERSION = '1.6.36'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 1.6.27
+  version: 1.6.36
 platform: ruby
 authors:
 - Yochay Ettun
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-02-24 00:00:00.000000000 Z
+date: 2020-06-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler