cnvrg 1.6.26 → 1.6.35
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/cnvrg/cli.rb +31 -45
- data/lib/cnvrg/data.rb +4 -1
- data/lib/cnvrg/datafiles.rb +12 -1
- data/lib/cnvrg/experiment.rb +3 -2
- data/lib/cnvrg/files.rb +22 -1
- data/lib/cnvrg/version.rb +1 -1
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 5f90f59c9a970e96f5ac718fa295cc2ce4f81ff143d8d9f8848b039a4ecdd004
|
4
|
+
data.tar.gz: 66beb8643ccbe900863b674b20f6138d69380cd0719deba77b15ce836c14d3dd
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 771c2fc62849c1d33363d49aa555ffd7408ef3f821dc3b3106c5c94004ab1076f4d96574b12af92b80f5a7fe8ca38270875cd06801edd26145d6c7eaef249266
|
7
|
+
data.tar.gz: 53fd6f5f3653ed67777022e9c50f9b042f2a4b35739e23a28ff4777c5480c62e227b62633cd7b34ff44bf19a8d1ed2268555194a5650f2c103578960f2d0bf5f
|
data/lib/cnvrg/cli.rb
CHANGED
@@ -1184,7 +1184,7 @@ module Cnvrg
|
|
1184
1184
|
end
|
1185
1185
|
|
1186
1186
|
desc '', '', :hide => true
|
1187
|
-
def data_put(dataset_url, files: [], dir: '', commit: '', chunk_size: 1000)
|
1187
|
+
def data_put(dataset_url, files: [], dir: '', commit: '', chunk_size: 1000, message: nil)
|
1188
1188
|
begin
|
1189
1189
|
verify_logged_in(false)
|
1190
1190
|
log_start(__method__, args, options)
|
@@ -1204,7 +1204,7 @@ module Cnvrg
|
|
1204
1204
|
log_message("Uploading #{@files.size} files", Thor::Shell::Color::GREEN)
|
1205
1205
|
number_of_chunks = (@files.size.to_f / chunk_size).ceil
|
1206
1206
|
if commit.blank?
|
1207
|
-
response = @datafiles.start_commit(false, true, chunks: number_of_chunks)
|
1207
|
+
response = @datafiles.start_commit(false, true, chunks: number_of_chunks, message: message )
|
1208
1208
|
unless response #means we failed in the start commit.
|
1209
1209
|
raise SignalException.new(1, "Cant put files into dataset, check the dataset id")
|
1210
1210
|
end
|
@@ -1229,14 +1229,16 @@ module Cnvrg
|
|
1229
1229
|
end
|
1230
1230
|
if commit.blank?
|
1231
1231
|
res = @datafiles.put_commit(@commit)
|
1232
|
-
|
1233
|
-
|
1234
|
-
|
1232
|
+
unless res.is_success?
|
1233
|
+
raise SignalException.new(1, res.msg)
|
1234
|
+
end
|
1235
1235
|
else
|
1236
1236
|
res = @datafiles.end_commit(@commit,false, success: true )
|
1237
|
-
|
1238
|
-
|
1239
|
-
|
1237
|
+
msg = res['result']
|
1238
|
+
response = Cnvrg::Result.new(Cnvrg::CLI.is_response_success(res, true), msg)
|
1239
|
+
unless response.is_success?
|
1240
|
+
raise SignalException.new(1, res.msg)
|
1241
|
+
end
|
1240
1242
|
end
|
1241
1243
|
log_message("Uploading files finished Successfully", Thor::Shell::Color::GREEN)
|
1242
1244
|
rescue SignalException => e
|
@@ -1835,8 +1837,6 @@ module Cnvrg
|
|
1835
1837
|
end
|
1836
1838
|
clone_resp = Project.clone_dir(slug, owner, project_name,git)
|
1837
1839
|
project_home = Dir.pwd + "/" + project_name
|
1838
|
-
|
1839
|
-
|
1840
1840
|
end
|
1841
1841
|
|
1842
1842
|
if clone_resp
|
@@ -1984,31 +1984,7 @@ module Cnvrg
|
|
1984
1984
|
files_to_upload, upload_errors = invoke :upload_data_files,[commit, files_list: files_list],:new_branch=>new_branch, :verbose =>verbose, :force =>force, :sync =>sync, :chunk_size => chunk_size
|
1985
1985
|
|
1986
1986
|
upload_size = files_to_upload + upload_errors.try(:size) rescue 0
|
1987
|
-
|
1988
|
-
if files_to_upload <= 0
|
1989
|
-
check = Helpers.checkmark
|
1990
|
-
log_message("#{check} Data sync finished", Thor::Shell::Color::GREEN)
|
1991
|
-
if total_deleted > 0
|
1992
|
-
log_message("#{total_deleted} files deleted successfully.", Thor::Shell::Color::GREEN)
|
1993
|
-
end
|
1994
|
-
|
1995
|
-
if total_downloaded > 0
|
1996
|
-
log_message("#{total_downloaded} files downloaded successfully.", Thor::Shell::Color::GREEN)
|
1997
|
-
end
|
1998
|
-
|
1999
|
-
if upload_errors.try(:size) > 0
|
2000
|
-
log_message("#{upload_errors.try(:size)}/#{upload_size} files didn't upload:", Thor::Shell::Color::RED)
|
2001
|
-
upload_errors.each do |file_hash|
|
2002
|
-
log_message("#{file_hash[:absolute_path]}", Thor::Shell::Color::RED)
|
2003
|
-
end
|
2004
|
-
end
|
2005
|
-
|
2006
|
-
if upload_errors.blank? and total_downloaded == 0 and total_deleted == 0
|
2007
|
-
Cnvrg::CLI.log_message("You are up to date", Thor::Shell::Color::GREEN)
|
2008
|
-
end
|
2009
|
-
return
|
2010
|
-
end
|
2011
|
-
invoke :end_commit_data,[commit, success: true, uploaded_files: files_to_upload], :new_branch=>new_branch, :force =>force
|
1987
|
+
invoke :end_commit_data,[commit, success: true, uploaded_files: files_to_upload, sync: sync], :new_branch=>new_branch, :force =>force
|
2012
1988
|
if tags
|
2013
1989
|
log_message('Uploading Tags', Thor::Shell::Color::BLUE)
|
2014
1990
|
dataset_dir = is_cnvrg_dir(Dir.pwd)
|
@@ -2033,8 +2009,10 @@ module Cnvrg
|
|
2033
2009
|
if total_downloaded > 0
|
2034
2010
|
log_message("#{total_downloaded} files downloaded successfully.", Thor::Shell::Color::GREEN)
|
2035
2011
|
end
|
2012
|
+
if upload_size > 0
|
2013
|
+
log_message("#{files_to_upload}/#{upload_size} files uploaded successfully.", Thor::Shell::Color::GREEN)
|
2014
|
+
end
|
2036
2015
|
|
2037
|
-
log_message("#{files_to_upload}/#{upload_size} files uploaded successfully.", Thor::Shell::Color::GREEN)
|
2038
2016
|
if upload_errors.try(:size) > 0
|
2039
2017
|
log_message("#{upload_errors.try(:size)}/#{upload_size} files didn't upload:", Thor::Shell::Color::RED)
|
2040
2018
|
upload_errors.each do |file_hash|
|
@@ -2098,7 +2076,7 @@ module Cnvrg
|
|
2098
2076
|
method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
|
2099
2077
|
method_option :force, :type => :boolean, :aliases => ["-f","--force"], :default => false
|
2100
2078
|
|
2101
|
-
def end_commit_data(commit, success: true, uploaded_files: 0)
|
2079
|
+
def end_commit_data(commit, success: true, uploaded_files: 0, sync: false)
|
2102
2080
|
begin
|
2103
2081
|
verify_logged_in(true)
|
2104
2082
|
log_start(__method__, args, options)
|
@@ -2113,7 +2091,12 @@ module Cnvrg
|
|
2113
2091
|
@dataset.revert_next_commit #removes the next commit
|
2114
2092
|
log_message("#{check} Dataset is up to date", Thor::Shell::Color::GREEN)
|
2115
2093
|
else
|
2116
|
-
|
2094
|
+
if sync
|
2095
|
+
message = "#{check} Data sync finished"
|
2096
|
+
else
|
2097
|
+
message = "#{check} Data upload finished"
|
2098
|
+
end
|
2099
|
+
log_message(message, Thor::Shell::Color::GREEN)
|
2117
2100
|
@dataset.remove_next_commit #takes the next commit and put it as current commit
|
2118
2101
|
@dataset.set_partial_commit(nil)
|
2119
2102
|
@dataset.backup_idx
|
@@ -2909,7 +2892,7 @@ module Cnvrg
|
|
2909
2892
|
method_option :job_type, :type => :string, :aliases => ["-jt", "--job_type"], :default => nil
|
2910
2893
|
method_option :files, :type => :string, :aliases => ["--files"], :default => nil
|
2911
2894
|
method_option :output_dir, :type => :string, :aliases => ["--output_dir"], :default => nil
|
2912
|
-
def
|
2895
|
+
def sync(direct = true)
|
2913
2896
|
verify_logged_in(true) if direct
|
2914
2897
|
@project = Project.new(get_project_home)
|
2915
2898
|
log_start(__method__, args, options)
|
@@ -2971,6 +2954,7 @@ module Cnvrg
|
|
2971
2954
|
method_option :requirements, :type => :boolean, :aliases => ["-r", "--requirements"], :default => true
|
2972
2955
|
method_option :notify_on_error, :type => :boolean, :aliases => ["-noe", "--notify_on_error"], :default => nil
|
2973
2956
|
method_option :notify_on_success, :type => :boolean, :aliases => ["-nos", "--notify_on_success"], :default => nil
|
2957
|
+
method_option :emails, :type => :string, :aliases => ["-es", "--emails"], :default => "", :desc => "additional emails to notify on success / or error, comma separated"
|
2974
2958
|
|
2975
2959
|
def run(*cmd)
|
2976
2960
|
verify_logged_in(true)
|
@@ -3004,6 +2988,7 @@ module Cnvrg
|
|
3004
2988
|
requirements = options["requirements"]
|
3005
2989
|
email_notification_error = options["notify_on_error"]
|
3006
2990
|
email_notification_success = options["notify_on_success"]
|
2991
|
+
emails = options["emails"]
|
3007
2992
|
|
3008
2993
|
if !data.present? and data_query.present?
|
3009
2994
|
log_message("Please provide data with data_query", Thor::Shell::Color::RED)
|
@@ -3018,7 +3003,6 @@ module Cnvrg
|
|
3018
3003
|
restart_if_stuck = options["restart_if_stuck"]
|
3019
3004
|
|
3020
3005
|
options_hash = Hash[options]
|
3021
|
-
|
3022
3006
|
if local
|
3023
3007
|
if Cnvrg::Helpers.windows?
|
3024
3008
|
say "Windows is currently not supported for running experiments locally"
|
@@ -3054,7 +3038,7 @@ module Cnvrg
|
|
3054
3038
|
:periodic_sync => periodic_sync, :dataset_only_tree=> dataset_only_tree,
|
3055
3039
|
:output_dir=>output_dir, :data_query=>data_query, :git_commit =>git_commit, :git_branch=> git_branch,
|
3056
3040
|
:restart_if_stuck =>restart_if_stuck, :local_folders => local_folders, :datasets => datasets, :prerun => prerun, :requirements => requirements,
|
3057
|
-
:email_notification_error => email_notification_error, :email_notification_success => email_notification_success
|
3041
|
+
:email_notification_error => email_notification_error, :email_notification_success => email_notification_success, :emails => emails
|
3058
3042
|
return
|
3059
3043
|
end
|
3060
3044
|
|
@@ -3360,8 +3344,9 @@ module Cnvrg
|
|
3360
3344
|
method_option :datasets, :type => :string, :aliases => ["--datasets"], :default => nil
|
3361
3345
|
method_option :prerun, :type => :boolean, :aliases => ["-p", "--prerun"], :default => true
|
3362
3346
|
method_option :requirements, :type => :boolean, :aliases => ["-r", "--requirements"], :default => true
|
3363
|
-
method_option :email_notification_error, :type => :boolean, :aliases => ["--email_notification_error"], :default => true
|
3364
|
-
method_option :email_notification_success, :type => :boolean, :aliases => ["--email_notification_success"], :default => true
|
3347
|
+
method_option :email_notification_error, :type => :boolean, :aliases => ["-noe", "--email_notification_error"], :default => true
|
3348
|
+
method_option :email_notification_success, :type => :boolean, :aliases => ["-nos", "--email_notification_success"], :default => true
|
3349
|
+
method_option :emails, :type => :string, :aliases => ["-es", "--emails"], :default => "", :desc => "additional emails to notify on success / or error"
|
3365
3350
|
|
3366
3351
|
def exec_remote(*cmd)
|
3367
3352
|
|
@@ -3383,6 +3368,7 @@ module Cnvrg
|
|
3383
3368
|
requirements = options["requirements"]
|
3384
3369
|
email_notification_error = options["email_notification_error"]
|
3385
3370
|
email_notification_success = options["email_notification_success"]
|
3371
|
+
emails = options["emails"]
|
3386
3372
|
max_time = options["max_time"]
|
3387
3373
|
if !max_time.nil? and !max_time.empty?
|
3388
3374
|
max_time = max_time.to_i
|
@@ -3422,7 +3408,7 @@ module Cnvrg
|
|
3422
3408
|
options_hash.except!("schedule", "recurring", "machine_type", "image", "upload_output", "grid", "data", "data_commit", "title",
|
3423
3409
|
"local", "small", "medium", "large", "gpu", "gpuxl", "gpuxxl","max_time","dataset_only_tree",
|
3424
3410
|
"data_query", "git_commit","git_branch", "restart_if_stuck","local_folders","output_dir", "commit", "datasets", "requirements", "prerun",
|
3425
|
-
"email_notification_error", "email_notification_success")
|
3411
|
+
"email_notification_error", "email_notification_success", "emails")
|
3426
3412
|
exec_options = options_hash.map {|x| "--#{x[0]}=#{x[1]}"}.flatten.join(" ")
|
3427
3413
|
command = "#{exec_options} #{remote} #{upload_output_option} #{cmd.flatten.join(" ")}"
|
3428
3414
|
commit_to_run = options["commit"] || nil
|
@@ -3490,7 +3476,7 @@ module Cnvrg
|
|
3490
3476
|
res = exp.exec_remote(command, commit_to_run, instance_type, image, schedule, local_timestamp, grid, path_to_cmd, data, data_commit,
|
3491
3477
|
periodic_sync, sync_before_terminate, max_time, ds_sync_options,output_dir,
|
3492
3478
|
data_query, git_commit, git_branch, restart_if_stuck,local_folders_options, title, datasets, prerun: prerun, requirements: requirements, recurring: recurring,
|
3493
|
-
email_notification_error: email_notification_error, email_notification_success: email_notification_success)
|
3479
|
+
email_notification_error: email_notification_error, email_notification_success: email_notification_success, emails_to_notify: emails)
|
3494
3480
|
if Cnvrg::CLI.is_response_success(res)
|
3495
3481
|
check = Helpers.checkmark()
|
3496
3482
|
str = "#{check} Experiment's is on: #{Cnvrg::Helpers.remote_url}/#{project.owner}/projects/#{project.slug}/experiments/#{res["result"]["exp_url"]}"
|
data/lib/cnvrg/data.rb
CHANGED
@@ -186,11 +186,14 @@ module Cnvrg
|
|
186
186
|
desc 'data put DATASET_URL FILES_PREFIX', 'Upload selected files from local dataset directory to remote server'
|
187
187
|
method_option :dir, :type => :string, :aliases => ["-d", "--dir"], :default => ''
|
188
188
|
method_option :commit, :type => :string, :aliases => ["-c", "--commit"], :default => ''
|
189
|
+
method_option :message, :type => :string, :aliases => ["--message"], :desc => "create commit with message", :default => nil
|
189
190
|
def put(dataset_url, *files)
|
190
191
|
cli = Cnvrg::CLI.new()
|
191
192
|
dir = options[:dir]
|
192
193
|
commit = options[:commit]
|
193
|
-
|
194
|
+
message = options[:message]
|
195
|
+
|
196
|
+
cli.data_put(dataset_url, files: files, dir: dir, commit: commit, message: message)
|
194
197
|
end
|
195
198
|
|
196
199
|
desc 'data clone_query --query=QUERY_SLUG DATASET_URL', 'Clone dataset with specific query'
|
data/lib/cnvrg/datafiles.rb
CHANGED
@@ -20,6 +20,15 @@ module Cnvrg
|
|
20
20
|
@dataset = dataset
|
21
21
|
@base_resource = "users/#{owner}/datasets/#{dataset_slug}/"
|
22
22
|
@downloader = @dataset.get_storage_client
|
23
|
+
@token_issue_time = Time.current
|
24
|
+
end
|
25
|
+
|
26
|
+
def refresh_storage_token
|
27
|
+
current_time = Time.current
|
28
|
+
if current_time - @token_issue_time > 3.hours
|
29
|
+
@downloader = @dataset.get_storage_client
|
30
|
+
@token_issue_time = Time.current
|
31
|
+
end
|
23
32
|
end
|
24
33
|
|
25
34
|
def check_file_sha1(filename, org_sha1, tag: 'conflict')
|
@@ -117,6 +126,7 @@ module Cnvrg
|
|
117
126
|
def upload_multiple_files(commit_sha1, tree, threads: ParallelThreads, force: false, new_branch: false, prefix: '', partial_commit: nil, total: nil)
|
118
127
|
begin
|
119
128
|
Cnvrg::Logger.log_info("Sending Upload Files request")
|
129
|
+
refresh_storage_token
|
120
130
|
error = nil
|
121
131
|
upload_resp = nil
|
122
132
|
10.times do
|
@@ -914,7 +924,7 @@ module Cnvrg
|
|
914
924
|
begin
|
915
925
|
response = Cnvrg::API.request("#{base_resource}/commit/end", 'POST', {commit_sha1: commit_sha1,force:force, success: success, uploaded_files: uploaded_files})
|
916
926
|
Cnvrg::CLI.is_response_success(response, true)
|
917
|
-
|
927
|
+
return response
|
918
928
|
rescue => e
|
919
929
|
return false
|
920
930
|
end
|
@@ -1007,6 +1017,7 @@ module Cnvrg
|
|
1007
1017
|
|
1008
1018
|
def download_multiple_files_s3(files, project_home, conflict: false, progressbar: nil, read_only:false)
|
1009
1019
|
begin
|
1020
|
+
refresh_storage_token
|
1010
1021
|
parallel_options = {
|
1011
1022
|
in_threads: ParallelThreads,
|
1012
1023
|
isolation: true
|
data/lib/cnvrg/experiment.rb
CHANGED
@@ -108,7 +108,7 @@ module Cnvrg
|
|
108
108
|
def exec_remote(command, commit_to_run, instance_type, image_slug,schedule,local_timestamp, grid,path_to_cmd,data, data_commit,periodic_sync,
|
109
109
|
sync_before_terminate, max_time, ds_sync_options=0,output_dir=nil,data_query=nil,
|
110
110
|
git_commit=nil, git_branch=nil, restart_if_stuck=nil, local_folders=nil,title=nil, datasets=nil, prerun: true, requirements: true, recurring: nil,
|
111
|
-
email_notification_error: false, email_notification_success: false)
|
111
|
+
email_notification_error: false, email_notification_success: false, emails_to_notify: nil)
|
112
112
|
response = Cnvrg::API.request("users/#{@owner}/projects/#{@project_slug}/experiment/remote", 'POST', {command: command, image_slug: image_slug,
|
113
113
|
commit_sha1: commit_to_run,
|
114
114
|
instance_type: instance_type,
|
@@ -124,7 +124,8 @@ module Cnvrg
|
|
124
124
|
restart_if_stuck:restart_if_stuck, local_folders: local_folders, title:title,
|
125
125
|
prerun: prerun, requirements: requirements, recurring: recurring,
|
126
126
|
email_notification_error: email_notification_error,
|
127
|
-
email_notification_success: email_notification_success
|
127
|
+
email_notification_success: email_notification_success,
|
128
|
+
emails_to_notify: emails_to_notify})
|
128
129
|
|
129
130
|
return response
|
130
131
|
end
|
data/lib/cnvrg/files.rb
CHANGED
@@ -7,7 +7,7 @@ require 'cnvrg/result'
|
|
7
7
|
module Cnvrg
|
8
8
|
class Files
|
9
9
|
ParallelThreads = Cnvrg::Helpers.parallel_threads
|
10
|
-
VALID_FILE_NAME = /[\x00
|
10
|
+
VALID_FILE_NAME = /[\x00\\\*\?\"<>\|]/
|
11
11
|
LARGE_FILE=1024*1024*5
|
12
12
|
MULTIPART_SPLIT=10000000
|
13
13
|
|
@@ -27,8 +27,16 @@ module Cnvrg
|
|
27
27
|
@custom_progess = false
|
28
28
|
@cli = cli
|
29
29
|
@options = options
|
30
|
+
@token_issue_time = Time.current
|
30
31
|
end
|
31
32
|
|
33
|
+
def refresh_storage_token
|
34
|
+
current_time = Time.current
|
35
|
+
if current_time - @token_issue_time > 3.hours
|
36
|
+
@client = @project.get_storage_client
|
37
|
+
@token_issue_time = Time.current
|
38
|
+
end
|
39
|
+
end
|
32
40
|
|
33
41
|
def self.valid_file_name?(fullpath)
|
34
42
|
VALID_FILE_NAME.match(fullpath).blank?
|
@@ -97,7 +105,13 @@ module Cnvrg
|
|
97
105
|
files = res['files']
|
98
106
|
|
99
107
|
#upload files
|
108
|
+
token_mutex = Mutex.new
|
100
109
|
blob_ids = Parallel.map(files.keys, in_threads: ParallelThreads) do |file|
|
110
|
+
|
111
|
+
token_mutex.synchronize {
|
112
|
+
refresh_storage_token
|
113
|
+
}
|
114
|
+
|
101
115
|
begin
|
102
116
|
Cnvrg::Helpers.try_until_success{self.upload_single_file(files[file].merge(files_list[file]))}
|
103
117
|
rescue => e
|
@@ -721,8 +735,15 @@ module Cnvrg
|
|
721
735
|
in_threads: Cnvrg::Helpers.parallel_threads,
|
722
736
|
isolation: true
|
723
737
|
}
|
738
|
+
|
739
|
+
token_mutex = Mutex.new
|
740
|
+
|
724
741
|
Parallel.map(files["keys"], parallel_options) do |f|
|
725
742
|
|
743
|
+
token_mutex.synchronize {
|
744
|
+
refresh_storage_token
|
745
|
+
}
|
746
|
+
|
726
747
|
file_path = f["name"]
|
727
748
|
if file_path.end_with? "/"
|
728
749
|
# dir
|
data/lib/cnvrg/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: cnvrg
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.6.
|
4
|
+
version: 1.6.35
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Yochay Ettun
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2020-
|
13
|
+
date: 2020-06-03 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: bundler
|
@@ -477,7 +477,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
477
477
|
- !ruby/object:Gem::Version
|
478
478
|
version: '0'
|
479
479
|
requirements: []
|
480
|
-
rubygems_version: 3.
|
480
|
+
rubygems_version: 3.1.2
|
481
481
|
signing_key:
|
482
482
|
specification_version: 4
|
483
483
|
summary: A CLI tool for interacting with cnvrg.io.
|