cnvrg 1.4.9.1 → 1.4.9.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/cnvrg/cli.rb +3 -5
- data/lib/cnvrg/datafiles.rb +2 -2
- data/lib/cnvrg/dataset.rb +17 -9
- data/lib/cnvrg/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 82d6237ddf965c30cbc18423f39929acf57df54047817c093c6db6d6263e1a31
|
4
|
+
data.tar.gz: 4738157c73f5f87fd4b4b3166eec8bdc1c0d1658749f209b51962813b75c50dd
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 800049d7ef5ee405b5350f5b649930ea7cd26616a66dd89b8e06ba8f7fd09bf223cbbfc05c9ec99e65c7df2096f31920bb1d268565f4537cc854160fb7fbd995
|
7
|
+
data.tar.gz: dc7a3afc3d2e39042327f085efb0c07efcd032d15d697913bf1e21648f3f3c2ba8c366385f4ba9e7b246025434e17ee4df9dcf52a6672254e24aebb76e26df4b
|
data/lib/cnvrg/cli.rb
CHANGED
@@ -868,7 +868,6 @@ module Cnvrg
|
|
868
868
|
return clone_data_query(dataset_url, query)
|
869
869
|
end
|
870
870
|
|
871
|
-
|
872
871
|
url_parts = dataset_url.split("/")
|
873
872
|
project_index = Cnvrg::Helpers.look_for_in_path(dataset_url, "datasets")
|
874
873
|
slug = url_parts[project_index + 1]
|
@@ -878,6 +877,7 @@ module Cnvrg
|
|
878
877
|
response["result"] = @dataset.get_dataset(commit: commit, query: query)
|
879
878
|
dataset_name = response["result"]["name"]
|
880
879
|
dataset_home = Dir.pwd+"/"+dataset_name
|
880
|
+
|
881
881
|
check = Helpers.checkmark
|
882
882
|
if @dataset.init_home(remote:remote)
|
883
883
|
log_message("Cloning #{dataset_name}", Thor::Shell::Color::BLUE)
|
@@ -902,9 +902,10 @@ module Cnvrg
|
|
902
902
|
:starting_at => 0,
|
903
903
|
:total => files_count,
|
904
904
|
:autofinish => true)
|
905
|
+
|
905
906
|
while files['keys'].length > 0
|
906
907
|
Cnvrg::Logger.log_info("download multiple files, #{downloaded_files.size} files downloaded")
|
907
|
-
@files.download_multiple_files_s3(files,
|
908
|
+
@files.download_multiple_files_s3(files, @dataset.local_path, progressbar: progressbar, read_only: read)
|
908
909
|
|
909
910
|
downloaded_files += files['keys'].length
|
910
911
|
files = @files.get_clone_chunk(commit: commit, latest_id: files['latest'])
|
@@ -912,7 +913,6 @@ module Cnvrg
|
|
912
913
|
progressbar.finish
|
913
914
|
if downloaded_files == files_count
|
914
915
|
Dataset.verify_cnvrgignore_exist(dataset_name, false)
|
915
|
-
@dataset = Dataset.new(dataset_home)
|
916
916
|
if !read
|
917
917
|
@dataset.write_idx(nil, commit) #nil means, generate idx
|
918
918
|
end
|
@@ -1603,8 +1603,6 @@ module Cnvrg
|
|
1603
1603
|
|
1604
1604
|
rescue => e
|
1605
1605
|
|
1606
|
-
puts e.message
|
1607
|
-
puts e.backtrace
|
1608
1606
|
log_message("Error occurred, \nAborting", Thor::Shell::Color::RED)
|
1609
1607
|
log_error(e)
|
1610
1608
|
@files.rollback_commit(commit_sha1)
|
data/lib/cnvrg/datafiles.rb
CHANGED
@@ -940,7 +940,7 @@ module Cnvrg
|
|
940
940
|
isolation: true
|
941
941
|
}
|
942
942
|
Parallel.map(files["keys"], parallel_options) do |f|
|
943
|
-
local_path = @dataset.
|
943
|
+
local_path = @dataset.local_path + '/' + f['name']
|
944
944
|
Cnvrg::Logger.log_info("Downloading #{local_path}")
|
945
945
|
progressbar.progress += 1 if progressbar.present?
|
946
946
|
if local_path.end_with? "/"
|
@@ -956,7 +956,7 @@ module Cnvrg
|
|
956
956
|
Cnvrg::Logger.log_info("Download #{local_path} success")
|
957
957
|
end
|
958
958
|
end
|
959
|
-
|
959
|
+
rescue => e
|
960
960
|
return Cnvrg::Result.new(false,"Could not download some files", e.message, e.backtrace)
|
961
961
|
end
|
962
962
|
end
|
data/lib/cnvrg/dataset.rb
CHANGED
@@ -29,7 +29,10 @@ module Cnvrg
|
|
29
29
|
end
|
30
30
|
|
31
31
|
def init_home(remote: false)
|
32
|
-
dataset_home = Dir.pwd
|
32
|
+
dataset_home = Dir.pwd
|
33
|
+
if not remote
|
34
|
+
dataset_home += "/"+@slug
|
35
|
+
end
|
33
36
|
if !remote and (Dir.exists? dataset_home)
|
34
37
|
Cnvrg::CLI.log_message("Error: Conflict with dir #{@slug}", Thor::Shell::Color::RED)
|
35
38
|
if Thor::Shell::Basic.new.no? "Sync to repository anyway? (current data might lost)", Thor::Shell::Color::YELLOW
|
@@ -257,6 +260,7 @@ module Cnvrg
|
|
257
260
|
if not File.exists?(self.local_path + "/.cnvrgignore")
|
258
261
|
self.generate_cnvrg_ignore
|
259
262
|
end
|
263
|
+
|
260
264
|
ignore_list = []
|
261
265
|
File.open(self.local_path + "/.cnvrgignore", "r").each_line do |line|
|
262
266
|
line = line.strip
|
@@ -380,25 +384,29 @@ module Cnvrg
|
|
380
384
|
end
|
381
385
|
|
382
386
|
def self.clone(owner, dataset_name, dataset_slug, remote = false)
|
383
|
-
|
384
387
|
begin
|
385
388
|
list_dirs = []
|
386
|
-
|
387
|
-
|
388
|
-
end
|
389
|
-
list_dirs << "#{dataset_name}/.cnvrg"
|
389
|
+
prefix = dataset_name + "/"
|
390
|
+
list_dirs << ".cnvrg"
|
390
391
|
list_files = [
|
391
|
-
"
|
392
|
+
".cnvrgignore",
|
393
|
+
".cnvrg/config.yml"
|
392
394
|
]
|
393
395
|
|
394
396
|
config = {dataset_name: dataset_name,
|
395
397
|
dataset_slug: dataset_slug,
|
396
398
|
owner: owner}
|
397
|
-
|
399
|
+
config_yml_path = ".cnvrg/config.yml"
|
400
|
+
if not remote
|
401
|
+
list_dirs = list_dirs.map {|x| prefix+x}
|
402
|
+
list_dirs << dataset_name
|
403
|
+
list_files = list_files.map {|x| prefix+x}
|
404
|
+
config_yml_path = prefix + config_yml_path
|
405
|
+
end
|
398
406
|
|
399
407
|
FileUtils.mkdir_p list_dirs
|
400
408
|
FileUtils.touch list_files
|
401
|
-
File.open(
|
409
|
+
File.open(config_yml_path, "w+") {|f| f.write config.to_yaml}
|
402
410
|
rescue => e
|
403
411
|
puts "Exception in clone request:#{e.message}"
|
404
412
|
return false
|
data/lib/cnvrg/version.rb
CHANGED