cnvrg 1.6.21 → 1.6.22
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/cnvrg/cli.rb +147 -116
- data/lib/cnvrg/datafiles.rb +37 -24
- data/lib/cnvrg/dataset.rb +6 -10
- data/lib/cnvrg/downloader/client.rb +43 -0
- data/lib/cnvrg/downloader/clients/azure_client.rb +1 -17
- data/lib/cnvrg/downloader/clients/s3_client.rb +2 -1
- data/lib/cnvrg/files.rb +7 -8
- data/lib/cnvrg/version.rb +1 -1
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: edee87a5b2d2c1693d36ddcfeb992b4e87a88521988e6335844ab522401cee46
|
4
|
+
data.tar.gz: bd0286b12dc83235c57bcf64726d2af6860608e0a061c885888bf044129d1ab0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e1421a63d3a34350f181deaead9c9b5f1cebe91f75f675a81d2a0cb49de0990d690efc08ba9e8ef74315b44041889ed093e19fbf4a9a271ec0f1394b39c45748
|
7
|
+
data.tar.gz: 67872a7abe1dbac6d001c28c4a556c0b8c09b153dade45eb6a6e54ed318a804235e089504c5e2e409cef27dde5235ea1ab35d7a239bc7aefad976ce60969e111
|
data/lib/cnvrg/cli.rb
CHANGED
@@ -1212,16 +1212,10 @@ module Cnvrg
|
|
1212
1212
|
dir = dir[0..-2] if dir.end_with? '/'
|
1213
1213
|
dir = dir[1..-1] if dir.start_with? '/'
|
1214
1214
|
|
1215
|
-
progressbar = ProgressBar.create(:title => "Upload Progress",
|
1216
|
-
:progress_mark => '=',
|
1217
|
-
:format => "%b>>%i| %p%% %t",
|
1218
|
-
:starting_at => 0,
|
1219
|
-
:total => @files.size,
|
1220
|
-
:autofinish => true)
|
1221
1215
|
@files.each_slice(chunk_size).each do |list_files|
|
1222
1216
|
temp_tree = @dataset.generate_chunked_idx(list_files, prefix: dir)
|
1223
1217
|
#will throw a signal exception if something goes wrong.
|
1224
|
-
@datafiles.upload_multiple_files(@commit, temp_tree, force: true,
|
1218
|
+
@datafiles.upload_multiple_files(@commit, temp_tree, force: true, prefix: dir, total: @files.size)
|
1225
1219
|
end
|
1226
1220
|
res = @datafiles.put_commit(@commit)
|
1227
1221
|
unless res.is_success?
|
@@ -1951,12 +1945,11 @@ module Cnvrg
|
|
1951
1945
|
log_message('Syncing dataset', Thor::Shell::Color::BLUE, !options["verbose"])
|
1952
1946
|
if !force and !init
|
1953
1947
|
# w(verbose=false, new_branch=false,sync=false, commit=nil,all_files=true)
|
1954
|
-
invoke :download_data_new,[verbose, new_branch, true, commit, all_files], :new_branch=>new_branch, :direct=>false, :force =>force
|
1948
|
+
total_deleted, total_downloaded = invoke :download_data_new,[verbose, new_branch, true, commit, all_files], :new_branch=>new_branch, :direct=>false, :force =>force
|
1955
1949
|
end
|
1956
|
-
|
1957
1950
|
# w(new_branch, verbose,sync,force, tags, chunk_size)
|
1958
|
-
invoke :upload_data_new,[new_branch, verbose, true, force, tags, chunk_size, message:message
|
1959
|
-
:direct=>false, :force =>force, :sync =>true, :tags =>tags, :parallel => parallel, :message => message
|
1951
|
+
invoke :upload_data_new,[new_branch, verbose, true, force, tags, chunk_size, message:message, total_deleted: total_deleted, total_downloaded: total_downloaded],
|
1952
|
+
:new_branch=>new_branch, :direct=>false, :force =>force, :sync =>true, :tags =>tags, :parallel => parallel, :message => message
|
1960
1953
|
|
1961
1954
|
end
|
1962
1955
|
desc 'upload_data_new', 'upload_data_new', :hide => true
|
@@ -1968,31 +1961,69 @@ module Cnvrg
|
|
1968
1961
|
method_option :parallel, :type => :numeric, :aliases => ["-p", "--parallel"], :desc => "uparallel upload at the same time", :default => 15
|
1969
1962
|
method_option :message, :type => :string, :aliases => ["--message"], :desc => "create commit with message", :default => nil
|
1970
1963
|
|
1971
|
-
def upload_data_new(new_branch, verbose, sync, force, tags, chunk_size, message:nil)
|
1964
|
+
def upload_data_new(new_branch, verbose, sync, force, tags, chunk_size, message:nil, total_deleted: 0, total_downloaded: 0)
|
1972
1965
|
begin
|
1973
|
-
|
1974
|
-
|
1975
|
-
|
1976
|
-
|
1977
|
-
|
1978
|
-
|
1979
|
-
|
1980
|
-
|
1981
|
-
|
1982
|
-
|
1983
|
-
|
1984
|
-
|
1985
|
-
|
1986
|
-
|
1987
|
-
|
1966
|
+
commit, files_list = invoke :start_commit_data,[], :new_branch=> new_branch, :direct=>false, :force =>force, :chunk_size => chunk_size, :message => message
|
1967
|
+
files_to_upload, upload_errors = invoke :upload_data_files,[commit, files_list: files_list],:new_branch=>new_branch, :verbose =>verbose, :force =>force, :sync =>sync, :chunk_size => chunk_size
|
1968
|
+
|
1969
|
+
upload_size = files_to_upload + upload_errors.try(:size) rescue 0
|
1970
|
+
|
1971
|
+
if files_to_upload <= 0
|
1972
|
+
check = Helpers.checkmark
|
1973
|
+
log_message("#{check} Data sync finished", Thor::Shell::Color::GREEN)
|
1974
|
+
if total_deleted > 0
|
1975
|
+
log_message("#{total_deleted} files deleted successfully.", Thor::Shell::Color::GREEN)
|
1976
|
+
end
|
1977
|
+
|
1978
|
+
if total_downloaded > 0
|
1979
|
+
log_message("#{total_downloaded} files downloaded successfully.", Thor::Shell::Color::GREEN)
|
1980
|
+
end
|
1981
|
+
|
1982
|
+
if upload_errors.try(:size) > 0
|
1983
|
+
log_message("#{upload_errors.try(:size)}/#{upload_size} files didn't upload:", Thor::Shell::Color::RED)
|
1984
|
+
upload_errors.each do |file_hash|
|
1985
|
+
log_message("#{file_hash[:absolute_path]}", Thor::Shell::Color::RED)
|
1986
|
+
end
|
1987
|
+
end
|
1988
|
+
|
1989
|
+
if upload_errors.blank? and total_downloaded == 0 and total_deleted == 0
|
1990
|
+
Cnvrg::CLI.log_message("You are up to date", Thor::Shell::Color::GREEN)
|
1991
|
+
end
|
1988
1992
|
return
|
1989
1993
|
end
|
1990
|
-
|
1991
|
-
|
1992
|
-
|
1993
|
-
|
1994
|
+
invoke :end_commit_data,[commit, success: true, uploaded_files: files_to_upload], :new_branch=>new_branch, :force =>force
|
1995
|
+
if tags
|
1996
|
+
log_message('Uploading Tags', Thor::Shell::Color::BLUE)
|
1997
|
+
dataset_dir = is_cnvrg_dir(Dir.pwd)
|
1998
|
+
@dataset = Dataset.new(dataset_dir)
|
1999
|
+
begin
|
2000
|
+
tag_file = File.open(options[:tags], "r+")
|
2001
|
+
status = @dataset.upload_tags_via_yml(tag_file)
|
2002
|
+
rescue
|
2003
|
+
log_message('Tags file not found', Thor::Shell::Color::RED)
|
2004
|
+
return
|
2005
|
+
end
|
2006
|
+
if status
|
2007
|
+
log_message('Tags are successfully uploaded', Thor::Shell::Color::GREEN)
|
2008
|
+
else
|
2009
|
+
log_message('There was some error in uploading Tags', Thor::Shell::Color::RED)
|
2010
|
+
end
|
2011
|
+
end
|
2012
|
+
if total_deleted > 0
|
2013
|
+
log_message("#{total_deleted} files deleted successfully.", Thor::Shell::Color::GREEN)
|
2014
|
+
end
|
2015
|
+
|
2016
|
+
if total_downloaded > 0
|
2017
|
+
log_message("#{total_downloaded} files downloaded successfully.", Thor::Shell::Color::GREEN)
|
2018
|
+
end
|
2019
|
+
|
2020
|
+
log_message("#{files_to_upload}/#{upload_size} files uploaded successfully.", Thor::Shell::Color::GREEN)
|
2021
|
+
if upload_errors.try(:size) > 0
|
2022
|
+
log_message("#{upload_errors.try(:size)}/#{upload_size} files didn't upload:", Thor::Shell::Color::RED)
|
2023
|
+
upload_errors.each do |file_hash|
|
2024
|
+
log_message("#{file_hash[:absolute_path]}", Thor::Shell::Color::RED)
|
2025
|
+
end
|
1994
2026
|
end
|
1995
|
-
end
|
1996
2027
|
rescue => e
|
1997
2028
|
Cnvrg::CLI.log_message(e.message, 'red')
|
1998
2029
|
Cnvrg::Logger.log_error(e)
|
@@ -2022,17 +2053,16 @@ module Cnvrg
|
|
2022
2053
|
verify_logged_in(true)
|
2023
2054
|
log_start(__method__, args, options)
|
2024
2055
|
dataset_dir = is_cnvrg_dir(Dir.pwd)
|
2025
|
-
direct = options[:direct]
|
2026
2056
|
new_branch = options["new_branch"] || false
|
2027
2057
|
force = options["force"] || false
|
2028
2058
|
chunk_size = options["chunk_size"] || false
|
2029
2059
|
message = options["message"]
|
2030
|
-
commit_sha1 = nil
|
2031
2060
|
@dataset = Dataset.new(dataset_dir)
|
2032
2061
|
@dataset.backup_idx
|
2033
2062
|
@files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug, dataset: @dataset)
|
2034
2063
|
next_commit = @dataset.get_next_commit #if there was a partial commit..
|
2035
|
-
|
2064
|
+
files_list = @dataset.list_all_files
|
2065
|
+
chunks = (files_list.length.to_f / chunk_size).ceil
|
2036
2066
|
resp = @files.start_commit(new_branch, force, chunks: chunks, dataset: @dataset, message: message)
|
2037
2067
|
if !resp['result']['can_commit']
|
2038
2068
|
log_message("Cant upload files because a new version of this dataset exists, please download it or upload with --force", Thor::Shell::Color::RED)
|
@@ -2043,7 +2073,7 @@ module Cnvrg
|
|
2043
2073
|
@dataset.set_partial_commit(next_commit)
|
2044
2074
|
end
|
2045
2075
|
@dataset.set_next_commit(commit_sha1)
|
2046
|
-
return commit_sha1
|
2076
|
+
return commit_sha1, files_list
|
2047
2077
|
end
|
2048
2078
|
|
2049
2079
|
|
@@ -2066,7 +2096,7 @@ module Cnvrg
|
|
2066
2096
|
@dataset.revert_next_commit #removes the next commit
|
2067
2097
|
log_message("#{check} Dataset is up to date", Thor::Shell::Color::GREEN)
|
2068
2098
|
else
|
2069
|
-
log_message("#{check} Data
|
2099
|
+
log_message("#{check} Data sync finished", Thor::Shell::Color::GREEN)
|
2070
2100
|
@dataset.remove_next_commit #takes the next commit and put it as current commit
|
2071
2101
|
@dataset.set_partial_commit(nil)
|
2072
2102
|
@dataset.backup_idx
|
@@ -2119,55 +2149,53 @@ module Cnvrg
|
|
2119
2149
|
# method_option :tags_yml, :type => :boolean, :aliases => ["--file_tag_yml"], :default => false
|
2120
2150
|
method_option :parallel, :type => :numeric, :aliases => ["-p", "--parallel"], :desc => "uparallel upload at the same time", :default => 15
|
2121
2151
|
|
2122
|
-
def upload_data_files(new_commit,
|
2152
|
+
def upload_data_files(new_commit, files_list: [])
|
2123
2153
|
begin
|
2124
|
-
|
2125
|
-
|
2126
|
-
|
2127
|
-
|
2128
|
-
|
2129
|
-
|
2130
|
-
|
2131
|
-
|
2132
|
-
|
2133
|
-
|
2134
|
-
|
2135
|
-
|
2136
|
-
|
2137
|
-
|
2138
|
-
|
2139
|
-
|
2140
|
-
|
2141
|
-
|
2142
|
-
|
2143
|
-
|
2144
|
-
|
2145
|
-
|
2146
|
-
|
2147
|
-
|
2148
|
-
|
2149
|
-
|
2150
|
-
|
2151
|
-
|
2152
|
-
|
2153
|
-
|
2154
|
-
|
2155
|
-
|
2156
|
-
|
2157
|
-
|
2158
|
-
|
2159
|
-
|
2160
|
-
|
2161
|
-
|
2162
|
-
|
2163
|
-
|
2164
|
-
end
|
2165
|
-
@dataset.write_tree(new_tree) #we dont want to re-run it every time so just on finish.
|
2154
|
+
verify_logged_in(true)
|
2155
|
+
log_start(__method__, args, options)
|
2156
|
+
dataset_dir = is_cnvrg_dir(Dir.pwd)
|
2157
|
+
@dataset = Dataset.new(dataset_dir)
|
2158
|
+
@files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug, dataset: @dataset)
|
2159
|
+
new_commit ||= @dataset.get_next_commit
|
2160
|
+
partial_commit = @dataset.get_partial_commit
|
2161
|
+
if new_commit.blank?
|
2162
|
+
log_message("You must specify commit, run start_commit to create new commit", Thor::Shell::Color::RED)
|
2163
|
+
return false
|
2164
|
+
end
|
2165
|
+
chunk_size = options[:chunk_size]
|
2166
|
+
chunk_size = [chunk_size, 1].max
|
2167
|
+
new_branch = options["new_branch"] || false
|
2168
|
+
new_tree = {}
|
2169
|
+
force = options["force"] || false
|
2170
|
+
parallel_threads = options["parallel"] || ParallelThreads
|
2171
|
+
all_files = files_list
|
2172
|
+
all_files = @dataset.list_all_files if files_list.blank?
|
2173
|
+
files_uploaded = 0
|
2174
|
+
upload_errors = []
|
2175
|
+
|
2176
|
+
all_files.each_slice(chunk_size).each do |list_files|
|
2177
|
+
Cnvrg::Logger.log_info("Uploading files into #{@dataset.slug}, #{files_uploaded} files uploaded")
|
2178
|
+
temp_tree = @dataset.generate_chunked_idx(list_files, threads: parallel_threads)
|
2179
|
+
upload_resp, upload_error_files = @files.upload_multiple_files(new_commit, temp_tree,
|
2180
|
+
threads: parallel_threads,
|
2181
|
+
force: force,
|
2182
|
+
new_branch: new_branch,
|
2183
|
+
partial_commit: partial_commit,
|
2184
|
+
total: all_files.length)
|
2185
|
+
|
2186
|
+
files_uploaded += upload_resp
|
2187
|
+
upload_errors += upload_error_files if upload_error_files.present?
|
2188
|
+
temp_tree.each do |k, v|
|
2189
|
+
new_tree[k] = (v.present?) ? {sha1: v.try(:fetch, :sha1, nil), commit_time: nil} : nil
|
2190
|
+
end
|
2191
|
+
end
|
2192
|
+
|
2193
|
+
@dataset.write_tree(new_tree) #we dont want to re-run it every time so just on finish.
|
2166
2194
|
rescue => e
|
2167
2195
|
Cnvrg::Logger.log_error(e)
|
2168
2196
|
raise e
|
2169
2197
|
end
|
2170
|
-
return files_uploaded
|
2198
|
+
return files_uploaded, upload_errors.try(:flatten).try(:compact)
|
2171
2199
|
end
|
2172
2200
|
|
2173
2201
|
|
@@ -2435,52 +2463,51 @@ module Cnvrg
|
|
2435
2463
|
dataset_dir = is_cnvrg_dir(Dir.pwd)
|
2436
2464
|
@dataset = Dataset.new(dataset_dir)
|
2437
2465
|
@files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug, dataset: @dataset)
|
2438
|
-
all_files = all_files
|
2439
|
-
@dataset.generate_idx
|
2440
2466
|
res = @dataset.compare_idx_download(all_files: all_files, desired_commit: commit)
|
2441
2467
|
unless CLI.is_response_success(res, false)
|
2442
2468
|
log_message("Cant find the desired commit, please check it or try to download without it.", Thor::Shell::Color::RED)
|
2443
2469
|
exit(1)
|
2444
2470
|
end
|
2445
|
-
result = res["result"]
|
2446
|
-
|
2447
|
-
|
2448
|
-
|
2449
|
-
update_total = [result['added'], result["updated_on_server"], result["conflicts"], result["deleted"]].compact.flatten.size
|
2471
|
+
result = res["result"]
|
2472
|
+
tree = result["tree"]
|
2473
|
+
commit = result["commit"]
|
2474
|
+
update_total = [tree['added'], tree["updated_on_server"], tree["conflicts"], tree["deleted"]].compact.flatten.size
|
2450
2475
|
successful_changes = 0
|
2451
|
-
if update_total == 0
|
2452
|
-
log_message("Dataset is up to date", Thor::Shell::Color::GREEN
|
2453
|
-
return
|
2454
|
-
elsif options["verbose"]
|
2455
|
-
log_message("Downloading #{update_total} files", Thor::Shell::Color::BLUE)
|
2476
|
+
if update_total == 0
|
2477
|
+
log_message("Dataset is up to date", Thor::Shell::Color::GREEN, !sync)
|
2478
|
+
return 0, 0
|
2456
2479
|
else
|
2480
|
+
log_message("Downloading #{update_total} files", Thor::Shell::Color::BLUE, options["verbose"])
|
2457
2481
|
log_message("Syncing Dataset", Thor::Shell::Color::BLUE, !sync)
|
2458
2482
|
end
|
2459
|
-
Cnvrg::Logger.log_info("Current commit: #{@dataset.
|
2460
|
-
Cnvrg::Logger.log_info("Compare idx res: #{
|
2483
|
+
Cnvrg::Logger.log_info("Current commit: #{@dataset.last_local_commit}, destination commit: #{commit}")
|
2484
|
+
Cnvrg::Logger.log_info("Compare idx res: #{tree}")
|
2461
2485
|
progressbar = ProgressBar.create(:title => "Download Progress",
|
2462
|
-
|
2463
|
-
|
2464
|
-
|
2465
|
-
|
2466
|
-
|
2467
|
-
conflicts = @files.mark_conflicts(result)
|
2486
|
+
:progress_mark => '=',
|
2487
|
+
:format => "%b>>%i| %p%% %t",
|
2488
|
+
:starting_at => 0,
|
2489
|
+
:total => update_total,
|
2490
|
+
:autofinish => true)
|
2468
2491
|
|
2492
|
+
conflicts = @files.mark_conflicts(tree)
|
2469
2493
|
log_message("Found some conflicts, check .conflict files.", Thor::Shell::Color::BLUE) if conflicts > 0
|
2470
|
-
update_res = @files.download_files_in_chunks(
|
2471
|
-
added_res = @files.download_files_in_chunks(
|
2472
|
-
|
2473
|
-
deleted = result["deleted"].to_a
|
2494
|
+
update_res = @files.download_files_in_chunks(tree["updated_on_server"], progress: progressbar) if tree["updated_on_server"].present?
|
2495
|
+
added_res = @files.download_files_in_chunks(tree["added"], progress: progressbar) if tree["added"].present?
|
2496
|
+
deleted = tree["deleted"].to_a
|
2474
2497
|
delete_res = @files.delete_commit_files_local(deleted)
|
2475
|
-
|
2498
|
+
|
2476
2499
|
if !delete_res
|
2477
2500
|
log_message("Couldn't delete #{deleted.join(" ")}", Thor::Shell::Color::RED)
|
2478
2501
|
log_message("Couldn't download, Rolling Back all changes.", Thor::Shell::Color::RED)
|
2479
2502
|
exit(1)
|
2480
2503
|
end
|
2504
|
+
|
2505
|
+
progressbar.progress += deleted.size if progressbar.present? and deleted.size > 0
|
2506
|
+
|
2481
2507
|
success = (update_res.blank? or update_res.is_success?)
|
2482
2508
|
success &= (delete_res.blank? or delete_res.is_success?)
|
2483
2509
|
success &= (added_res.blank? or added_res.is_success?)
|
2510
|
+
|
2484
2511
|
if success
|
2485
2512
|
# update idx with latest commit
|
2486
2513
|
@dataset.update_idx_with_commit!(commit)
|
@@ -2490,17 +2517,24 @@ module Cnvrg
|
|
2490
2517
|
log_message(successful_changes.join("\n"), Thor::Shell::Color::GREEN)
|
2491
2518
|
log_message("Total of #{successful_changes.size} / #{update_total} files.", Thor::Shell::Color::GREEN)
|
2492
2519
|
else
|
2493
|
-
log_message("#{check} Downloaded changes successfully", Thor::Shell::Color::GREEN,
|
2520
|
+
log_message("#{check} Downloaded changes successfully", Thor::Shell::Color::GREEN, !sync)
|
2494
2521
|
end
|
2495
|
-
|
2522
|
+
|
2523
|
+
total_deleted = deleted.try(:size)
|
2524
|
+
total_downloaded = tree["added"].try(:size) || 0
|
2525
|
+
total_downloaded += tree["updated_on_server"].try(:size) if tree["updated_on_server"].present?
|
2526
|
+
|
2527
|
+
return total_deleted, total_downloaded
|
2528
|
+
else
|
2529
|
+
return []
|
2496
2530
|
end
|
2497
2531
|
rescue SignalException => e
|
2498
2532
|
Cnvrg::Logger.log_error(e)
|
2499
2533
|
say "\nAborting", Thor::Shell::Color::BLUE
|
2500
2534
|
exit(1)
|
2501
2535
|
rescue => e
|
2502
|
-
log_message("Error occurred, \nAborting", Thor::Shell::Color::BLUE)
|
2503
2536
|
Cnvrg::Logger.log_error(e)
|
2537
|
+
log_message("Error occurred, \nAborting", Thor::Shell::Color::RED)
|
2504
2538
|
exit(1)
|
2505
2539
|
end
|
2506
2540
|
end
|
@@ -3386,10 +3420,6 @@ module Cnvrg
|
|
3386
3420
|
output_dir = "output"
|
3387
3421
|
end
|
3388
3422
|
image = options["image"] || nil
|
3389
|
-
if image.blank?
|
3390
|
-
image = "cnvrg"
|
3391
|
-
end
|
3392
|
-
|
3393
3423
|
forced_commit = nil
|
3394
3424
|
if sync_before and !project.is_git
|
3395
3425
|
if force
|
@@ -5491,7 +5521,7 @@ module Cnvrg
|
|
5491
5521
|
end
|
5492
5522
|
end
|
5493
5523
|
|
5494
|
-
def self.log_message(message, type = Thor::Shell::Color::BLUE
|
5524
|
+
def self.log_message(message, type = Thor::Shell::Color::BLUE)
|
5495
5525
|
if $LOG.blank?
|
5496
5526
|
### handle case when $LOG is not initialized
|
5497
5527
|
CLI.new.log_handler
|
@@ -5513,11 +5543,12 @@ module Cnvrg
|
|
5513
5543
|
color = nil
|
5514
5544
|
$LOG.info message: message, type: "unknown"
|
5515
5545
|
end
|
5516
|
-
say "#{color}#{message}#{Thor::Shell::Color::CLEAR}"
|
5546
|
+
say "#{color}#{message}#{Thor::Shell::Color::CLEAR}"
|
5517
5547
|
end
|
5518
5548
|
|
5519
5549
|
def log_message(message, type=Thor::Shell::Color::GREEN, to_print = true)
|
5520
|
-
|
5550
|
+
return if not to_print
|
5551
|
+
CLI.log_message(message, type)
|
5521
5552
|
end
|
5522
5553
|
|
5523
5554
|
def log_error(e)
|
data/lib/cnvrg/datafiles.rb
CHANGED
@@ -114,33 +114,52 @@ module Cnvrg
|
|
114
114
|
end
|
115
115
|
|
116
116
|
|
117
|
-
def upload_multiple_files(commit_sha1, tree, threads: ParallelThreads, force: false, new_branch: false,
|
117
|
+
def upload_multiple_files(commit_sha1, tree, threads: ParallelThreads, force: false, new_branch: false, prefix: '', partial_commit: nil, total: nil)
|
118
118
|
begin
|
119
|
-
cli=CLI.new
|
120
119
|
Cnvrg::Logger.log_info("Sending Upload Files request")
|
121
|
-
|
122
|
-
|
120
|
+
error = nil
|
121
|
+
upload_resp = nil
|
122
|
+
10.times do
|
123
|
+
upload_resp = Cnvrg::API.request(@base_resource + "upload_files", 'POST_JSON', {commit_sha1: commit_sha1, tree: tree, force: force, is_branch: new_branch, partial_commit: partial_commit})
|
124
|
+
if Cnvrg::CLI.is_response_success(upload_resp, false)
|
125
|
+
error = nil
|
126
|
+
break
|
127
|
+
end
|
128
|
+
error = upload_resp
|
123
129
|
Cnvrg::Logger.log_method(bind: binding)
|
124
|
-
|
130
|
+
Cnvrg::CLI.log_info("Got an error message from server, #{upload_resp.try(:fetch, "message")}, trying again")
|
125
131
|
end
|
132
|
+
raise Exception.new("Can't upload data files: #{error["message"]}") if error.present?
|
133
|
+
|
126
134
|
Cnvrg::Logger.log_info("Uploading files")
|
127
135
|
results = upload_resp['result'].with_indifferent_access
|
136
|
+
|
128
137
|
if results['files'].blank?
|
129
|
-
|
130
|
-
|
138
|
+
return 0, []
|
139
|
+
end
|
140
|
+
|
141
|
+
if @temp_upload_progressbar.blank?
|
142
|
+
@temp_upload_progressbar = ProgressBar.create(:title => "Upload Progress",
|
143
|
+
:progress_mark => '=',
|
144
|
+
:format => "%b>>%i| %p%% %t",
|
145
|
+
:starting_at => 0,
|
146
|
+
:total => total,
|
147
|
+
:autofinish => true)
|
131
148
|
end
|
149
|
+
|
132
150
|
files = results['files']
|
133
|
-
|
151
|
+
upload_error_files = []
|
152
|
+
@temp_upload_progressbar.progress += tree.keys.length - files.length if @temp_upload_progressbar.present?
|
134
153
|
Parallel.map((files.keys), in_threads: threads) do |k|
|
135
154
|
o = tree[k].merge(files[k])
|
136
155
|
success = upload_single_file(o)
|
137
156
|
if not success
|
157
|
+
upload_error_files << {absolute_path: o[:absolute_path]}
|
138
158
|
files.except!(k)
|
139
159
|
tree.except!(k)
|
140
|
-
cli.log_message("Error while uploading file: #{o[:absolute_path]}", Thor::Shell::Color::RED)
|
141
160
|
Cnvrg::Logger.log_error_message("Error while upload single file #{o["path"]}")
|
142
161
|
end
|
143
|
-
|
162
|
+
@temp_upload_progressbar.progress += 1 if @temp_upload_progressbar.present?
|
144
163
|
end
|
145
164
|
blob_ids = files.values.map {|f| f['bv_id']}
|
146
165
|
if blob_ids.present?
|
@@ -153,25 +172,21 @@ module Cnvrg
|
|
153
172
|
end
|
154
173
|
end
|
155
174
|
Cnvrg::Logger.log_info("Upload Success")
|
156
|
-
return files.keys.length
|
175
|
+
return files.try(:keys).try(:length), upload_error_files
|
157
176
|
rescue => e
|
158
177
|
Cnvrg::Logger.log_method(bind: binding)
|
159
178
|
Cnvrg::Logger.log_error(e)
|
160
179
|
raise e
|
161
180
|
end
|
162
|
-
|
163
181
|
end
|
164
182
|
|
165
183
|
def upload_single_file(file)
|
166
184
|
begin
|
167
|
-
cli = CLI.new
|
168
185
|
file = file.as_json
|
169
186
|
Cnvrg::Logger.log_info("Uploading #{file["absolute_path"]}")
|
170
|
-
|
171
|
-
return false if not success
|
187
|
+
@downloader.safe_upload(file["path"], file["absolute_path"])
|
172
188
|
Cnvrg::Logger.log_info("#{file["absolute_path"]} uploaded.")
|
173
189
|
rescue => e
|
174
|
-
cli.log_message(e.message, Thor::Shell::Color::RED)
|
175
190
|
Cnvrg::Logger.log_error_message("Error while upload single file #{file["path"]}")
|
176
191
|
Cnvrg::Logger.log_error(e)
|
177
192
|
return false
|
@@ -815,6 +830,7 @@ module Cnvrg
|
|
815
830
|
end
|
816
831
|
return true
|
817
832
|
end
|
833
|
+
|
818
834
|
def delete_commit_files_local(deleted)
|
819
835
|
begin
|
820
836
|
FileUtils.rm_rf(deleted) unless (deleted.nil? or deleted.empty?)
|
@@ -822,9 +838,6 @@ module Cnvrg
|
|
822
838
|
rescue => e
|
823
839
|
return Cnvrg::Result.new(false, '')
|
824
840
|
end
|
825
|
-
|
826
|
-
return Cnvrg::Result.new(true, '')
|
827
|
-
|
828
841
|
end
|
829
842
|
|
830
843
|
def download_dir(dataset_home, absolute_path)
|
@@ -869,6 +882,7 @@ module Cnvrg
|
|
869
882
|
Cnvrg::CLI.is_response_success(response, true)
|
870
883
|
return response
|
871
884
|
end
|
885
|
+
|
872
886
|
def start_commit(new_branch,force=false,delete_commit=nil, chunks: 0, dataset: @dataset, message:nil)
|
873
887
|
begin
|
874
888
|
#if we are pushing with force or to branch we dont need to send current/next commit cause we want to
|
@@ -876,14 +890,13 @@ module Cnvrg
|
|
876
890
|
idx = (force || new_branch) ? {} : dataset.get_idx
|
877
891
|
commit = idx[:commit]
|
878
892
|
next_commit = idx[:next_commit]
|
879
|
-
|
880
|
-
|
881
|
-
|
882
|
-
|
893
|
+
response = Cnvrg::API.request("#{base_resource}/commit/start", 'POST', {dataset_slug: @dataset_slug, new_branch: new_branch,force:force,
|
894
|
+
username: @owner,current_commit: commit, next_commit: next_commit, total_chunks: chunks, message: message})
|
895
|
+
Cnvrg::CLI.is_response_success(response, true)
|
896
|
+
return response
|
883
897
|
rescue => e
|
884
898
|
return false
|
885
899
|
end
|
886
|
-
|
887
900
|
end
|
888
901
|
|
889
902
|
def end_commit(commit_sha1,force, success: true, uploaded_files: 0 )
|
data/lib/cnvrg/dataset.rb
CHANGED
@@ -103,8 +103,12 @@ module Cnvrg
|
|
103
103
|
|
104
104
|
def backup_idx
|
105
105
|
Cnvrg::Logger.log_info("Backup idx")
|
106
|
-
|
107
|
-
|
106
|
+
if File.exists? "#{self.local_path}/.cnvrg/idx.yml"
|
107
|
+
FileUtils.cp "#{self.local_path}/.cnvrg/idx.yml", "#{self.local_path}/.cnvrg/idx.yml.backup"
|
108
|
+
else
|
109
|
+
idx = {commit: nil, tree: {}}
|
110
|
+
File.open("#{self.local_path}/.cnvrg/idx.yml.backup", 'w') {|f| f.write idx.to_yaml}
|
111
|
+
end
|
108
112
|
end
|
109
113
|
|
110
114
|
def restore_idx
|
@@ -780,14 +784,6 @@ module Cnvrg
|
|
780
784
|
File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') {|f| f.write idx_hash.to_yaml}
|
781
785
|
end
|
782
786
|
|
783
|
-
def get_current_commit()
|
784
|
-
if !File.exist? "#{self.local_path}/.cnvrg/idx.yml"
|
785
|
-
return nil
|
786
|
-
end
|
787
|
-
idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
|
788
|
-
return idx_hash[:commit]
|
789
|
-
end
|
790
|
-
|
791
787
|
def compare_commit(commit)
|
792
788
|
if commit.nil? or commit.empty?
|
793
789
|
commit = last_local_commit
|
@@ -2,6 +2,8 @@
|
|
2
2
|
module Cnvrg
|
3
3
|
module Downloader
|
4
4
|
OLD_SERVER_VERSION_MESSAGE = "Your server version is not relevant for this cli version please contact support for further help."
|
5
|
+
MAXIMUM_BACKOFF = 64
|
6
|
+
RETRIES = ENV['UPLOAD_FILE_RETRIES'].try(:to_i) || 20
|
5
7
|
attr_accessor :bucket, :client
|
6
8
|
class Client
|
7
9
|
def initialize(params)
|
@@ -41,6 +43,35 @@ module Cnvrg
|
|
41
43
|
Cnvrg::Helpers.decrypt(@key, @iv, str)
|
42
44
|
end
|
43
45
|
|
46
|
+
def safe_upload(storage_path, local_path)
|
47
|
+
n = 1
|
48
|
+
error = nil
|
49
|
+
while n <= RETRIES
|
50
|
+
begin
|
51
|
+
self.upload(storage_path, local_path)
|
52
|
+
error = nil
|
53
|
+
break
|
54
|
+
rescue => e
|
55
|
+
backoff_time_seconds = backoff_time(n)
|
56
|
+
|
57
|
+
message = "Got error: #{e.class.name} with message: #{e.message} while uploading a single file: #{local_path}, retry: #{n} of: #{RETRIES}"
|
58
|
+
if n < RETRIES
|
59
|
+
message += ", next retry in: #{backoff_time_seconds} seconds"
|
60
|
+
else
|
61
|
+
message += ", done retry, continuing to the next file"
|
62
|
+
end
|
63
|
+
Cnvrg::Logger.log_error_message(message)
|
64
|
+
|
65
|
+
sleep backoff_time_seconds
|
66
|
+
|
67
|
+
n += 1
|
68
|
+
error = e
|
69
|
+
end
|
70
|
+
end
|
71
|
+
raise error if error.present?
|
72
|
+
true
|
73
|
+
end
|
74
|
+
|
44
75
|
def self.factory(params)
|
45
76
|
params = params.as_json
|
46
77
|
case params["storage"]
|
@@ -53,6 +84,18 @@ module Cnvrg
|
|
53
84
|
return Cnvrg::Downloader::Clients::GcpClient.new(project_id: params["project_id"], credentials: params["credentials"], bucket_name: params["bucket_name"], sts: params["sts"])
|
54
85
|
end
|
55
86
|
end
|
87
|
+
|
88
|
+
private
|
89
|
+
|
90
|
+
def random_number_milliseconds
|
91
|
+
rand(1000) / 1000.0
|
92
|
+
end
|
93
|
+
|
94
|
+
|
95
|
+
def backoff_time(n)
|
96
|
+
return [((2**n)+random_number_milliseconds), MAXIMUM_BACKOFF].min
|
97
|
+
end
|
98
|
+
|
56
99
|
end
|
57
100
|
end
|
58
101
|
end
|
@@ -21,28 +21,12 @@ module Cnvrg
|
|
21
21
|
|
22
22
|
def upload(storage_path, local_path)
|
23
23
|
begin
|
24
|
-
|
24
|
+
client.create_block_blob(@container, storage_path, File.open(local_path, "rb"))
|
25
25
|
rescue => e
|
26
26
|
raise e
|
27
27
|
end
|
28
28
|
end
|
29
29
|
|
30
|
-
def try_upload_file(storage_path, local_path)
|
31
|
-
c = 20
|
32
|
-
error = nil
|
33
|
-
while c > 0
|
34
|
-
begin
|
35
|
-
client.create_block_blob(@container, storage_path, File.open(local_path, "rb"))
|
36
|
-
break
|
37
|
-
rescue => e
|
38
|
-
c -= 1
|
39
|
-
error = e
|
40
|
-
end
|
41
|
-
end
|
42
|
-
raise error if error.present?
|
43
|
-
return true
|
44
|
-
end
|
45
|
-
|
46
30
|
def fetch_files(prefix: nil, marker: nil, limit: 10000)
|
47
31
|
blobs = client.list_blobs(@container, prefix: prefix, max_results: limit, marker: marker)
|
48
32
|
next_marker = blobs.continuation_token
|
@@ -53,7 +53,8 @@ module Cnvrg
|
|
53
53
|
### storage path is the path inside s3 (after the bucket)
|
54
54
|
# local path is fullpath for the file /home/ubuntu/user.../hazilim.py
|
55
55
|
o = aws_bucket.object(storage_path)
|
56
|
-
o.upload_file(local_path, @upload_options)
|
56
|
+
success = o.upload_file(local_path, @upload_options)
|
57
|
+
return success
|
57
58
|
rescue => e
|
58
59
|
raise e
|
59
60
|
end
|
data/lib/cnvrg/files.rb
CHANGED
@@ -880,6 +880,7 @@ module Cnvrg
|
|
880
880
|
end
|
881
881
|
|
882
882
|
end
|
883
|
+
|
883
884
|
def delete_commit_files_local(deleted)
|
884
885
|
begin
|
885
886
|
FileUtils.rm_rf(deleted) unless (deleted.nil? or deleted.empty?)
|
@@ -887,16 +888,14 @@ module Cnvrg
|
|
887
888
|
rescue => e
|
888
889
|
return false
|
889
890
|
end
|
891
|
+
end
|
890
892
|
|
891
|
-
|
892
|
-
|
893
|
+
def start_commit(new_branch,force:false, exp_start_commit:nil, job_slug: nil, job_type: nil, start_commit: nil, message: nil)
|
894
|
+
response = Cnvrg::API.request("#{base_resource}/commit/start", 'POST', {project_slug: @project_slug, new_branch: new_branch,force:force,
|
895
|
+
username: @owner, exp_start_commit:exp_start_commit, job_slug: job_slug, job_type: job_type, start_commit: start_commit, message: message})
|
896
|
+
Cnvrg::CLI.is_response_success(response,false)
|
897
|
+
return response
|
893
898
|
end
|
894
|
-
def start_commit(new_branch,force:false, exp_start_commit:nil, job_slug: nil, job_type: nil, start_commit: nil, message: nil)
|
895
|
-
response = Cnvrg::API.request("#{base_resource}/commit/start", 'POST', {project_slug: @project_slug, new_branch: new_branch,force:force,
|
896
|
-
username: @owner, exp_start_commit:exp_start_commit, job_slug: job_slug, job_type: job_type, start_commit: start_commit, message: message})
|
897
|
-
Cnvrg::CLI.is_response_success(response,false)
|
898
|
-
return response
|
899
|
-
end
|
900
899
|
|
901
900
|
def end_commit(commit_sha1,force:false,message:"")
|
902
901
|
response = Cnvrg::API.request("#{base_resource}/commit/end", 'POST', {commit_sha1: commit_sha1,force:force,message:message})
|
data/lib/cnvrg/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: cnvrg
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.6.
|
4
|
+
version: 1.6.22
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Yochay Ettun
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2020-02-
|
13
|
+
date: 2020-02-13 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: bundler
|
@@ -477,7 +477,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
477
477
|
- !ruby/object:Gem::Version
|
478
478
|
version: '0'
|
479
479
|
requirements: []
|
480
|
-
rubygems_version: 3.0.
|
480
|
+
rubygems_version: 3.0.3
|
481
481
|
signing_key:
|
482
482
|
specification_version: 4
|
483
483
|
summary: A CLI tool for interacting with cnvrg.io.
|