cnvrg 1.6.26 → 1.6.35

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b90d814761168667e43372f1966e30a2d5a821568bcf28d913dffcbda6b033c7
-  data.tar.gz: c72d65d79f183cad5002bd2c14f940e6d51a9ef7a869bac12172a5bcc4ddbc76
+  metadata.gz: 5f90f59c9a970e96f5ac718fa295cc2ce4f81ff143d8d9f8848b039a4ecdd004
+  data.tar.gz: 66beb8643ccbe900863b674b20f6138d69380cd0719deba77b15ce836c14d3dd
 SHA512:
-  metadata.gz: d093bd678b362dc9297e69a3b1454811e036c2eaf476aab4083df5da5e18492d6951fda662f02818b52776c43c6b6881c624db205066b5cf0ee314ec6a187588
-  data.tar.gz: 06f86968182f724358ffe7d241afaa8dfcc9c6b834cabae3c75b82e2326b286abc547dd9458b9a3c1a984d846abc7611b54379dadea934ca0c5d767d61b721d0
+  metadata.gz: 771c2fc62849c1d33363d49aa555ffd7408ef3f821dc3b3106c5c94004ab1076f4d96574b12af92b80f5a7fe8ca38270875cd06801edd26145d6c7eaef249266
+  data.tar.gz: 53fd6f5f3653ed67777022e9c50f9b042f2a4b35739e23a28ff4777c5480c62e227b62633cd7b34ff44bf19a8d1ed2268555194a5650f2c103578960f2d0bf5f

data/lib/cnvrg/cli.rb

@@ -1184,7 +1184,7 @@ module Cnvrg
     end
 
     desc '', '', :hide => true
-    def data_put(dataset_url, files: [], dir: '', commit: '', chunk_size: 1000)
+    def data_put(dataset_url, files: [], dir: '', commit: '', chunk_size: 1000, message: nil)
       begin
         verify_logged_in(false)
         log_start(__method__, args, options)
@@ -1204,7 +1204,7 @@ module Cnvrg
         log_message("Uploading #{@files.size} files", Thor::Shell::Color::GREEN)
         number_of_chunks = (@files.size.to_f / chunk_size).ceil
         if commit.blank?
-          response = @datafiles.start_commit(false, true, chunks: number_of_chunks)
+          response = @datafiles.start_commit(false, true, chunks: number_of_chunks, message: message )
           unless response #means we failed in the start commit.
             raise SignalException.new(1, "Cant put files into dataset, check the dataset id")
           end
@@ -1229,14 +1229,16 @@ module Cnvrg
         end
         if commit.blank?
           res = @datafiles.put_commit(@commit)
-          #unless res.is_success?
-          #  raise SignalException.new(1, res.msg)
-          #end
+          unless res.is_success?
+            raise SignalException.new(1, res.msg)
+          end
         else
           res = @datafiles.end_commit(@commit,false, success: true )
-          #unless res.is_success?
-          #  raise SignalException.new(1, res.msg)
-          #end
+          msg = res['result']
+          response = Cnvrg::Result.new(Cnvrg::CLI.is_response_success(res, true), msg)
+          unless response.is_success?
+            raise SignalException.new(1, res.msg)
+          end
         end
         log_message("Uploading files finished Successfully", Thor::Shell::Color::GREEN)
       rescue SignalException => e
@@ -1835,8 +1837,6 @@ module Cnvrg
           end
           clone_resp = Project.clone_dir(slug, owner, project_name,git)
           project_home = Dir.pwd + "/" + project_name
-
-
         end
 
         if clone_resp
@@ -1984,31 +1984,7 @@ module Cnvrg
         files_to_upload, upload_errors = invoke :upload_data_files,[commit, files_list: files_list],:new_branch=>new_branch, :verbose =>verbose, :force =>force, :sync =>sync, :chunk_size => chunk_size
 
         upload_size = files_to_upload + upload_errors.try(:size) rescue 0
-
-        if files_to_upload <= 0
-          check = Helpers.checkmark
-          log_message("#{check} Data sync finished", Thor::Shell::Color::GREEN)
-          if total_deleted > 0
-            log_message("#{total_deleted} files deleted successfully.", Thor::Shell::Color::GREEN)
-          end
-
-          if total_downloaded > 0
-            log_message("#{total_downloaded} files downloaded successfully.", Thor::Shell::Color::GREEN)
-          end
-
-          if upload_errors.try(:size) > 0
-            log_message("#{upload_errors.try(:size)}/#{upload_size} files didn't upload:", Thor::Shell::Color::RED)
-            upload_errors.each do |file_hash|
-              log_message("#{file_hash[:absolute_path]}", Thor::Shell::Color::RED)
-            end
-          end
-
-          if upload_errors.blank? and total_downloaded == 0 and total_deleted == 0
-            Cnvrg::CLI.log_message("You are up to date", Thor::Shell::Color::GREEN)
-          end
-          return
-        end
-        invoke :end_commit_data,[commit, success: true, uploaded_files: files_to_upload], :new_branch=>new_branch, :force =>force
+        invoke :end_commit_data,[commit, success: true, uploaded_files: files_to_upload, sync: sync], :new_branch=>new_branch, :force =>force
         if tags
           log_message('Uploading Tags', Thor::Shell::Color::BLUE)
           dataset_dir = is_cnvrg_dir(Dir.pwd)
@@ -2033,8 +2009,10 @@ module Cnvrg
         if total_downloaded > 0
           log_message("#{total_downloaded} files downloaded successfully.", Thor::Shell::Color::GREEN)
         end
+        if upload_size > 0
+          log_message("#{files_to_upload}/#{upload_size} files uploaded successfully.", Thor::Shell::Color::GREEN)
+        end
 
-        log_message("#{files_to_upload}/#{upload_size} files uploaded successfully.", Thor::Shell::Color::GREEN)
         if upload_errors.try(:size) > 0
           log_message("#{upload_errors.try(:size)}/#{upload_size} files didn't upload:", Thor::Shell::Color::RED)
           upload_errors.each do |file_hash|
@@ -2098,7 +2076,7 @@ module Cnvrg
     method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
     method_option :force, :type => :boolean, :aliases => ["-f","--force"], :default => false
 
-    def end_commit_data(commit, success: true, uploaded_files: 0)
+    def end_commit_data(commit, success: true, uploaded_files: 0, sync: false)
       begin
         verify_logged_in(true)
         log_start(__method__, args, options)
@@ -2113,7 +2091,12 @@ module Cnvrg
             @dataset.revert_next_commit #removes the next commit
             log_message("#{check} Dataset is up to date", Thor::Shell::Color::GREEN)
           else
-            log_message("#{check} Data sync finished", Thor::Shell::Color::GREEN)
+            if sync
+              message = "#{check} Data sync finished"
+            else
+              message = "#{check} Data upload finished"
+            end
+            log_message(message, Thor::Shell::Color::GREEN)
             @dataset.remove_next_commit #takes the next commit and put it as current commit
             @dataset.set_partial_commit(nil)
             @dataset.backup_idx
@@ -2909,7 +2892,7 @@ module Cnvrg
     method_option :job_type, :type => :string, :aliases => ["-jt", "--job_type"], :default => nil
     method_option :files, :type => :string, :aliases => ["--files"], :default => nil
     method_option :output_dir, :type => :string, :aliases => ["--output_dir"], :default => nil
-    def  sync(direct = true)
+    def sync(direct = true)
       verify_logged_in(true) if direct
       @project = Project.new(get_project_home)
       log_start(__method__, args, options)
@@ -2971,6 +2954,7 @@ module Cnvrg
     method_option :requirements, :type => :boolean, :aliases => ["-r", "--requirements"], :default => true
     method_option :notify_on_error, :type => :boolean, :aliases => ["-noe", "--notify_on_error"], :default => nil
     method_option :notify_on_success, :type => :boolean, :aliases => ["-nos", "--notify_on_success"], :default => nil
+    method_option :emails, :type => :string, :aliases => ["-es", "--emails"], :default => "", :desc => "additional emails to notify on success / or error, comma separated"
 
     def run(*cmd)
       verify_logged_in(true)
@@ -3004,6 +2988,7 @@ module Cnvrg
       requirements = options["requirements"]
       email_notification_error = options["notify_on_error"]
       email_notification_success = options["notify_on_success"]
+      emails = options["emails"]
 
       if !data.present? and data_query.present?
         log_message("Please provide data with data_query", Thor::Shell::Color::RED)
@@ -3018,7 +3003,6 @@ module Cnvrg
       restart_if_stuck = options["restart_if_stuck"]
 
       options_hash = Hash[options]
-
       if local
         if Cnvrg::Helpers.windows?
             say "Windows is currently not supported for running experiments locally"
@@ -3054,7 +3038,7 @@ module Cnvrg
                :periodic_sync => periodic_sync, :dataset_only_tree=> dataset_only_tree,
                :output_dir=>output_dir, :data_query=>data_query, :git_commit =>git_commit, :git_branch=> git_branch,
                :restart_if_stuck =>restart_if_stuck, :local_folders => local_folders, :datasets => datasets, :prerun => prerun, :requirements => requirements,
-               :email_notification_error => email_notification_error, :email_notification_success => email_notification_success
+               :email_notification_error => email_notification_error, :email_notification_success => email_notification_success, :emails => emails
         return
       end
 
@@ -3360,8 +3344,9 @@ module Cnvrg
     method_option :datasets, :type => :string, :aliases => ["--datasets"], :default => nil
     method_option :prerun, :type => :boolean, :aliases => ["-p", "--prerun"], :default => true
     method_option :requirements, :type => :boolean, :aliases => ["-r", "--requirements"], :default => true
-    method_option :email_notification_error, :type => :boolean, :aliases => ["--email_notification_error"], :default => true
-    method_option :email_notification_success, :type => :boolean, :aliases => ["--email_notification_success"], :default => true
+    method_option :email_notification_error, :type => :boolean, :aliases => ["-noe", "--email_notification_error"], :default => true
+    method_option :email_notification_success, :type => :boolean, :aliases => ["-nos", "--email_notification_success"], :default => true
+    method_option :emails, :type => :string, :aliases => ["-es", "--emails"], :default => "", :desc => "additional emails to notify on success / or error"
 
     def exec_remote(*cmd)
 
@@ -3383,6 +3368,7 @@ module Cnvrg
         requirements = options["requirements"]
         email_notification_error = options["email_notification_error"]
         email_notification_success = options["email_notification_success"]
+        emails = options["emails"]
         max_time = options["max_time"]
         if !max_time.nil? and !max_time.empty?
           max_time = max_time.to_i
@@ -3422,7 +3408,7 @@ module Cnvrg
         options_hash.except!("schedule", "recurring", "machine_type", "image", "upload_output", "grid", "data", "data_commit", "title",
                              "local", "small", "medium", "large", "gpu", "gpuxl", "gpuxxl","max_time","dataset_only_tree",
                              "data_query", "git_commit","git_branch", "restart_if_stuck","local_folders","output_dir", "commit", "datasets", "requirements", "prerun",
-                             "email_notification_error", "email_notification_success")
+                             "email_notification_error", "email_notification_success", "emails")
         exec_options = options_hash.map {|x| "--#{x[0]}=#{x[1]}"}.flatten.join(" ")
         command = "#{exec_options} #{remote}  #{upload_output_option} #{cmd.flatten.join(" ")}"
         commit_to_run = options["commit"] || nil
@@ -3490,7 +3476,7 @@ module Cnvrg
         res = exp.exec_remote(command, commit_to_run, instance_type, image, schedule, local_timestamp, grid, path_to_cmd, data, data_commit,
                               periodic_sync, sync_before_terminate, max_time, ds_sync_options,output_dir,
                               data_query, git_commit, git_branch, restart_if_stuck,local_folders_options, title, datasets, prerun: prerun, requirements: requirements, recurring: recurring,
-                              email_notification_error: email_notification_error, email_notification_success: email_notification_success)
+                              email_notification_error: email_notification_error, email_notification_success: email_notification_success, emails_to_notify: emails)
         if Cnvrg::CLI.is_response_success(res)
           check = Helpers.checkmark()
           str = "#{check} Experiment's is on: #{Cnvrg::Helpers.remote_url}/#{project.owner}/projects/#{project.slug}/experiments/#{res["result"]["exp_url"]}"

data/lib/cnvrg/data.rb

@@ -186,11 +186,14 @@ module Cnvrg
     desc 'data put DATASET_URL FILES_PREFIX', 'Upload selected files from local dataset directory to remote server'
     method_option :dir, :type => :string, :aliases => ["-d", "--dir"], :default => ''
     method_option :commit, :type => :string, :aliases => ["-c", "--commit"], :default => ''
+    method_option :message, :type => :string, :aliases => ["--message"], :desc => "create commit with message", :default => nil
     def put(dataset_url, *files)
       cli = Cnvrg::CLI.new()
       dir = options[:dir]
       commit = options[:commit]
-      cli.data_put(dataset_url, files: files, dir: dir, commit: commit)
+      message = options[:message]
+
+      cli.data_put(dataset_url, files: files, dir: dir, commit: commit, message: message)
     end
 
     desc 'data clone_query --query=QUERY_SLUG DATASET_URL', 'Clone dataset with specific query'

data/lib/cnvrg/datafiles.rb

@@ -20,6 +20,15 @@ module Cnvrg
       @dataset = dataset
       @base_resource = "users/#{owner}/datasets/#{dataset_slug}/"
       @downloader = @dataset.get_storage_client
+      @token_issue_time = Time.current
+    end
+
+    def refresh_storage_token
+      current_time = Time.current
+      if current_time - @token_issue_time > 3.hours
+        @downloader = @dataset.get_storage_client
+        @token_issue_time = Time.current
+      end
     end
 
     def check_file_sha1(filename, org_sha1, tag: 'conflict')
@@ -117,6 +126,7 @@ module Cnvrg
     def upload_multiple_files(commit_sha1, tree, threads: ParallelThreads, force: false, new_branch: false, prefix: '', partial_commit: nil, total: nil)
       begin
         Cnvrg::Logger.log_info("Sending Upload Files request")
+        refresh_storage_token
         error = nil
         upload_resp = nil
         10.times do
@@ -914,7 +924,7 @@ module Cnvrg
       begin
         response = Cnvrg::API.request("#{base_resource}/commit/end", 'POST', {commit_sha1: commit_sha1,force:force, success: success, uploaded_files: uploaded_files})
         Cnvrg::CLI.is_response_success(response, true)
-        #return response
+        return response
       rescue => e
         return false
       end
@@ -1007,6 +1017,7 @@ module Cnvrg
 
     def download_multiple_files_s3(files, project_home, conflict: false, progressbar: nil, read_only:false)
       begin
+        refresh_storage_token
         parallel_options = {
             in_threads: ParallelThreads,
             isolation: true

data/lib/cnvrg/experiment.rb

@@ -108,7 +108,7 @@ module Cnvrg
     def exec_remote(command, commit_to_run, instance_type, image_slug,schedule,local_timestamp, grid,path_to_cmd,data, data_commit,periodic_sync,
                     sync_before_terminate, max_time, ds_sync_options=0,output_dir=nil,data_query=nil,
                     git_commit=nil, git_branch=nil, restart_if_stuck=nil, local_folders=nil,title=nil, datasets=nil, prerun: true, requirements: true, recurring: nil,
-                    email_notification_error: false, email_notification_success: false)
+                    email_notification_error: false, email_notification_success: false, emails_to_notify: nil)
       response = Cnvrg::API.request("users/#{@owner}/projects/#{@project_slug}/experiment/remote", 'POST', {command: command, image_slug: image_slug,
                                                                                                             commit_sha1: commit_to_run,
                                                                                                             instance_type: instance_type,
@@ -124,7 +124,8 @@ module Cnvrg
                                                                                                             restart_if_stuck:restart_if_stuck, local_folders: local_folders, title:title,
                                                                                                             prerun: prerun, requirements: requirements, recurring: recurring,
                                                                                                             email_notification_error: email_notification_error,
-                                                                                                            email_notification_success: email_notification_success})
+                                                                                                            email_notification_success: email_notification_success,
+                                                                                                            emails_to_notify: emails_to_notify})
 
       return response
     end

data/lib/cnvrg/files.rb

@@ -7,7 +7,7 @@ require 'cnvrg/result'
 module Cnvrg
   class Files
     ParallelThreads = Cnvrg::Helpers.parallel_threads
-    VALID_FILE_NAME = /[\x00\\:\*\?\"<>\|]/
+    VALID_FILE_NAME = /[\x00\\\*\?\"<>\|]/
     LARGE_FILE=1024*1024*5
     MULTIPART_SPLIT=10000000
 
@@ -27,8 +27,16 @@ module Cnvrg
       @custom_progess = false
       @cli = cli
       @options = options
+      @token_issue_time = Time.current
     end
 
+    def refresh_storage_token
+      current_time = Time.current
+      if current_time - @token_issue_time > 3.hours
+        @client = @project.get_storage_client
+        @token_issue_time = Time.current
+      end
+    end
 
     def self.valid_file_name?(fullpath)
       VALID_FILE_NAME.match(fullpath).blank?
@@ -97,7 +105,13 @@ module Cnvrg
       files = res['files']
 
       #upload files
+      token_mutex = Mutex.new
       blob_ids = Parallel.map(files.keys, in_threads: ParallelThreads) do |file|
+
+        token_mutex.synchronize {
+          refresh_storage_token
+        }
+
         begin
           Cnvrg::Helpers.try_until_success{self.upload_single_file(files[file].merge(files_list[file]))}
         rescue => e
@@ -721,8 +735,15 @@ module Cnvrg
             in_threads: Cnvrg::Helpers.parallel_threads,
             isolation: true
         }
+
+        token_mutex = Mutex.new
+
         Parallel.map(files["keys"], parallel_options) do |f|
 
+          token_mutex.synchronize {
+            refresh_storage_token
+          }
+
           file_path = f["name"]
           if file_path.end_with? "/"
             # dir

data/lib/cnvrg/version.rb

@@ -1,3 +1,3 @@
 module Cnvrg
-  VERSION = '1.6.26'
+  VERSION = '1.6.35'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 1.6.26
+  version: 1.6.35
 platform: ruby
 authors:
 - Yochay Ettun
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-02-23 00:00:00.000000000 Z
+date: 2020-06-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -477,7 +477,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.4
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: A CLI tool for interacting with cnvrg.io.