cnvrg 1.2.7 → 1.3.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/cnvrg.gemspec +3 -2
- data/lib/cnvrg/api.rb +2 -2
- data/lib/cnvrg/cli.rb +51 -36
- data/lib/cnvrg/data.rb +14 -3
- data/lib/cnvrg/datafiles.rb +20 -62
- data/lib/cnvrg/dataset.rb +198 -144
- data/lib/cnvrg/downloader/client.rb +53 -0
- data/lib/cnvrg/downloader/clients/azure_client.rb +22 -0
- data/lib/cnvrg/downloader/clients/gcp_client.rb +46 -0
- data/lib/cnvrg/downloader/clients/s3_client.rb +51 -0
- data/lib/cnvrg/files.rb +17 -44
- data/lib/cnvrg/helpers/executer.rb +171 -0
- data/lib/cnvrg/job_cli.rb +33 -0
- data/lib/cnvrg/project.rb +45 -6
- data/lib/cnvrg/storage.rb +128 -0
- data/lib/cnvrg/version.rb +1 -1
- metadata +28 -7
- data/lib/cnvrg/job.rb +0 -40
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c562ccc1d7fd69f3f2b0d28db538d27d57e4a55df5f1fef3eee6cab511bd30ca
|
4
|
+
data.tar.gz: 624e78308a885897c9fc31402aff21198a3f2db6551b676f14d82d9126624255
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 928e1bb402e0dac773bb9d8150c65815b2af8878c4969ca555eeb79ac7ad51a8f9a7f1d8c69cd58651788100c0cf6b2cd9a2e2b427d7efd667d0d4abf6f41629
|
7
|
+
data.tar.gz: 123068cd21d3900cdf3e9ae982feb2e9e52ee9303236877f41a6653cb8e4e6294f7afa8cb8c7c3a1e665fa0b331d8ee1e337b3e6a1b9e34a4f8293f25381d027
|
data/cnvrg.gemspec
CHANGED
@@ -6,7 +6,7 @@ require 'cnvrg/version'
|
|
6
6
|
Gem::Specification.new do |spec|
|
7
7
|
spec.name = 'cnvrg'
|
8
8
|
spec.version = Cnvrg::VERSION
|
9
|
-
spec.authors = ['Yochay Ettun', 'Leah Kolben']
|
9
|
+
spec.authors = ['Yochay Ettun', 'Leah Kolben', 'Omer Shacham']
|
10
10
|
spec.email = ['info@cnvrg.io']
|
11
11
|
spec.summary = %q{A CLI tool for interacting with cnvrg.io.}
|
12
12
|
spec.description = %q{A CLI tool for interacting with cnvrg.io.}
|
@@ -18,7 +18,7 @@ Gem::Specification.new do |spec|
|
|
18
18
|
spec.executables = ['cnvrg']
|
19
19
|
spec.require_paths = ['lib']
|
20
20
|
|
21
|
-
spec.add_development_dependency 'bundler'
|
21
|
+
spec.add_development_dependency 'bundler'
|
22
22
|
spec.add_development_dependency 'rake', '~> 10.0'
|
23
23
|
spec.add_development_dependency 'rspec', '~> 3.0'
|
24
24
|
spec.add_development_dependency 'vcr', '~> 3.0'
|
@@ -32,6 +32,7 @@ Gem::Specification.new do |spec|
|
|
32
32
|
spec.add_runtime_dependency 'highline', '~> 1.7', '>= 1.7.8'
|
33
33
|
spec.add_runtime_dependency 'thor', '~> 0.19.0','>=0.19.1'
|
34
34
|
spec.add_runtime_dependency 'aws-sdk', '~> 2'
|
35
|
+
spec.add_runtime_dependency 'google-cloud-storage', '~> 1.16'
|
35
36
|
spec.add_runtime_dependency 'sucker_punch', '~> 2.0'
|
36
37
|
spec.add_runtime_dependency 'urlcrypt', '~> 0.1.1'
|
37
38
|
spec.add_runtime_dependency 'parallel', '~> 1.12.0'
|
data/lib/cnvrg/api.rb
CHANGED
@@ -53,7 +53,7 @@ module Cnvrg
|
|
53
53
|
end
|
54
54
|
conn.headers['Auth-Token'] = @pass
|
55
55
|
conn.headers['User-Agent'] = "#{Cnvrg::API::USER_AGENT}"
|
56
|
-
|
56
|
+
conn.options.timeout = 420
|
57
57
|
conn.options.open_timeout=180
|
58
58
|
case method
|
59
59
|
when 'GET'
|
@@ -201,7 +201,7 @@ module Cnvrg
|
|
201
201
|
else
|
202
202
|
end
|
203
203
|
rescue => e
|
204
|
-
|
204
|
+
Cnvrg::Logger.log_error(e)
|
205
205
|
return nil
|
206
206
|
rescue SignalException
|
207
207
|
return false
|
data/lib/cnvrg/cli.rb
CHANGED
@@ -11,6 +11,18 @@ require 'yaml'
|
|
11
11
|
require 'digest' # sha1up
|
12
12
|
require "highline/import"
|
13
13
|
require 'socket'
|
14
|
+
require 'thor'
|
15
|
+
require 'docker'
|
16
|
+
require 'launchy'
|
17
|
+
require 'socket'
|
18
|
+
require 'timeout'
|
19
|
+
require 'fileutils'
|
20
|
+
require 'active_support/all'
|
21
|
+
require 'pathname'
|
22
|
+
require 'enumerator'
|
23
|
+
require 'ruby-progressbar'
|
24
|
+
require 'open3'
|
25
|
+
require 'logstash-logger'
|
14
26
|
require 'cnvrg/helpers'
|
15
27
|
require 'cnvrg/api'
|
16
28
|
require 'cnvrg/auth'
|
@@ -21,25 +33,15 @@ require 'cnvrg/Images'
|
|
21
33
|
require 'cnvrg/dataset'
|
22
34
|
require 'cnvrg/datafiles'
|
23
35
|
require 'cnvrg/data'
|
36
|
+
require 'cnvrg/storage'
|
24
37
|
require 'cnvrg/ssh'
|
25
38
|
require 'cnvrg/result'
|
26
39
|
require 'cnvrg/logger'
|
27
40
|
require 'cnvrg/org_helpers'
|
28
|
-
require 'cnvrg/
|
29
|
-
require
|
30
|
-
require
|
31
|
-
require
|
32
|
-
require 'docker'
|
33
|
-
require 'launchy'
|
34
|
-
require 'socket'
|
35
|
-
require 'timeout'
|
36
|
-
require 'fileutils'
|
37
|
-
require 'active_support/all'
|
38
|
-
require 'thor'
|
39
|
-
require 'pathname'
|
40
|
-
require 'enumerator'
|
41
|
-
require 'ruby-progressbar'
|
42
|
-
require 'open3'
|
41
|
+
require 'cnvrg/job_cli'
|
42
|
+
Gem.find_files("cnvrg/downloader/**/*.rb").reverse.each{|r| require r}
|
43
|
+
Gem.find_files("cnvrg/helpers/**/*.rb").reverse.each{|r| require r}
|
44
|
+
Gem.find_files("cnvrg/cli/**/*.rb").each{|r| require r}
|
43
45
|
|
44
46
|
|
45
47
|
class Thor
|
@@ -164,6 +166,9 @@ module Cnvrg
|
|
164
166
|
desc "data", "upload and manage datasets", :hide => false
|
165
167
|
subcommand "data", Data
|
166
168
|
|
169
|
+
desc "job", "manage running jobs", :hide => false
|
170
|
+
subcommand "job", JobCli
|
171
|
+
|
167
172
|
desc "flow", "mange project flows", :hide => true
|
168
173
|
subcommand "flow", Cnvrg::Commands::Flow
|
169
174
|
|
@@ -605,12 +610,14 @@ module Cnvrg
|
|
605
610
|
desc 'new', 'Create a new cnvrg project'
|
606
611
|
method_option :clean, :type => :boolean, :aliases => ["-c"], :default => false
|
607
612
|
method_option :docker_image, :type => :string, :aliases => ["-d"], :default => ""
|
613
|
+
method_option :bucket, :type => :string, :aliases => ["-b", "--bucket"], :default => ""
|
608
614
|
|
609
615
|
def new(project_name)
|
610
616
|
begin
|
611
617
|
verify_logged_in(false)
|
612
618
|
log_start(__method__, args, options)
|
613
619
|
clean = options["clean"]
|
620
|
+
bucket = options["bucket"]
|
614
621
|
docker_image = options["docker_image"]
|
615
622
|
working_dir = Dir.pwd + "/" + project_name
|
616
623
|
docker = false
|
@@ -662,6 +669,7 @@ module Cnvrg
|
|
662
669
|
method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
|
663
670
|
method_option :docker_image, :type => :string, :aliases => ["-d"], :default => ""
|
664
671
|
method_option :git, :type => :boolean, :aliases => ["-g","--git"], :default => false
|
672
|
+
method_option :bucket, :type => :string, :aliases => ["-b","--bucket"], :default => ''
|
665
673
|
method_option :title, :type => :string, :aliases => ["-t","--title"], :default => nil
|
666
674
|
|
667
675
|
def link
|
@@ -669,7 +677,8 @@ module Cnvrg
|
|
669
677
|
verify_logged_in(false)
|
670
678
|
log_start(__method__, args, options)
|
671
679
|
docker_image = options["docker_image"]
|
672
|
-
|
680
|
+
bucket = options["bucket"]
|
681
|
+
if docker_image.present?
|
673
682
|
docker = true
|
674
683
|
else
|
675
684
|
docker = false
|
@@ -687,7 +696,7 @@ module Cnvrg
|
|
687
696
|
end
|
688
697
|
working_dir = Dir.getwd
|
689
698
|
owner = CLI.get_owner
|
690
|
-
if Project.link(owner, project_name, docker,git)
|
699
|
+
if Project.link(owner, project_name, docker,git, bucket: bucket)
|
691
700
|
path = Dir.pwd
|
692
701
|
@project = Project.new(path)
|
693
702
|
if sync
|
@@ -718,7 +727,7 @@ module Cnvrg
|
|
718
727
|
desc 'data init', 'Init dataset directory'
|
719
728
|
method_option :public, :type => :boolean, :aliases => ["-p", "--public"], :default => false
|
720
729
|
|
721
|
-
def init_data(public)
|
730
|
+
def init_data(public, bucket: nil)
|
722
731
|
begin
|
723
732
|
verify_logged_in(false)
|
724
733
|
log_start(__method__, args, options)
|
@@ -733,7 +742,7 @@ module Cnvrg
|
|
733
742
|
|
734
743
|
working_dir = Dir.getwd
|
735
744
|
owner = CLI.get_owner
|
736
|
-
if Dataset.init(owner, dataset_name, options["public"])
|
745
|
+
if Dataset.init(owner, dataset_name, options["public"], bucket: bucket)
|
737
746
|
path = Dir.pwd
|
738
747
|
@dataset = Dataset.new(path)
|
739
748
|
|
@@ -837,7 +846,7 @@ module Cnvrg
|
|
837
846
|
check = Helpers.checkmark
|
838
847
|
if Dataset.clone(owner, dataset_name, slug, remote)
|
839
848
|
log_message("Cloning #{dataset_name}", Thor::Shell::Color::BLUE)
|
840
|
-
@files = Cnvrg::Datafiles.new(owner, slug)
|
849
|
+
@files = Cnvrg::Datafiles.new(owner, slug, dataset: Cnvrg::Dataset.new(dataset_home, dataset_url: dataset_url))
|
841
850
|
log_message("Downloading files", Thor::Shell::Color::BLUE)
|
842
851
|
if only_tree
|
843
852
|
success = Dataset.clone_tree(commit: commit, dataset_home: dataset_home)
|
@@ -853,7 +862,6 @@ module Cnvrg
|
|
853
862
|
:starting_at => 0,
|
854
863
|
:total => files_count,
|
855
864
|
:autofinish => true)
|
856
|
-
|
857
865
|
while files['keys'].length > 0
|
858
866
|
Cnvrg::Logger.log_info("download multiple files, #{downloaded_files.size} files downloaded")
|
859
867
|
@files.download_multiple_files_s3(files, dataset_home, progressbar: progressbar, read_only: read)
|
@@ -1781,7 +1789,7 @@ module Cnvrg
|
|
1781
1789
|
|
1782
1790
|
if clone_resp
|
1783
1791
|
@project = Project.new(project_home)
|
1784
|
-
@files = Cnvrg::Files.new(@project.owner, slug, project_home: project_home)
|
1792
|
+
@files = Cnvrg::Files.new(@project.owner, slug, project_home: project_home, project: @project)
|
1785
1793
|
response = @project.clone(remote, commit_to_clone)
|
1786
1794
|
Cnvrg::CLI.is_response_success response
|
1787
1795
|
commit_sha1 = response["result"]["commit"]
|
@@ -1974,7 +1982,7 @@ module Cnvrg
|
|
1974
1982
|
commit_sha1 = nil
|
1975
1983
|
@dataset = Dataset.new(dataset_dir)
|
1976
1984
|
@dataset.backup_idx
|
1977
|
-
@files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
|
1985
|
+
@files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug, dataset: @dataset)
|
1978
1986
|
next_commit = @dataset.get_next_commit #if there was a partial commit..
|
1979
1987
|
chunks = (@dataset.list_all_files.length.to_f / chunk_size).ceil
|
1980
1988
|
resp = @files.start_commit(new_branch, force, chunks: chunks, dataset: @dataset)
|
@@ -2001,7 +2009,7 @@ module Cnvrg
|
|
2001
2009
|
log_start(__method__, args, options)
|
2002
2010
|
dataset_dir = is_cnvrg_dir(Dir.pwd)
|
2003
2011
|
@dataset = Dataset.new(dataset_dir)
|
2004
|
-
@files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
|
2012
|
+
@files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug, dataset: @dataset)
|
2005
2013
|
force = options["force"] || false
|
2006
2014
|
resp = @files.end_commit(commit, force, success: success, uploaded_files: uploaded_files)
|
2007
2015
|
if (resp.present? and resp["result"])
|
@@ -2069,7 +2077,7 @@ module Cnvrg
|
|
2069
2077
|
log_start(__method__, args, options)
|
2070
2078
|
dataset_dir = is_cnvrg_dir(Dir.pwd)
|
2071
2079
|
@dataset = Dataset.new(dataset_dir)
|
2072
|
-
@files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
|
2080
|
+
@files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug, dataset: @dataset)
|
2073
2081
|
new_commit ||= @dataset.get_next_commit
|
2074
2082
|
partial_commit = @dataset.get_partial_commit
|
2075
2083
|
if new_commit.blank?
|
@@ -2374,7 +2382,7 @@ module Cnvrg
|
|
2374
2382
|
log_start(__method__, args, options)
|
2375
2383
|
dataset_dir = is_cnvrg_dir(Dir.pwd)
|
2376
2384
|
@dataset = Dataset.new(dataset_dir)
|
2377
|
-
@files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
|
2385
|
+
@files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug, dataset: @dataset)
|
2378
2386
|
all_files = all_files
|
2379
2387
|
res = @dataset.compare_idx_download(all_files: all_files, desired_commit: commit)
|
2380
2388
|
unless CLI.is_response_success(res, false)
|
@@ -2476,7 +2484,7 @@ module Cnvrg
|
|
2476
2484
|
log_start(__method__, args, options)
|
2477
2485
|
project_home = get_project_home
|
2478
2486
|
@project = Project.new(project_home)
|
2479
|
-
@files = Cnvrg::Files.new(@project.owner, @project.slug, project_home: project_home)
|
2487
|
+
@files = Cnvrg::Files.new(@project.owner, @project.slug, project_home: project_home, project: @project)
|
2480
2488
|
git = options["git"]
|
2481
2489
|
commit = options["commit"]
|
2482
2490
|
if git or @project.is_git
|
@@ -2607,7 +2615,7 @@ module Cnvrg
|
|
2607
2615
|
exit(0)
|
2608
2616
|
end
|
2609
2617
|
log_message("Jumping to commit #{commit_sha1}")
|
2610
|
-
@files = Cnvrg::Files.new(@project.owner, @project.slug, project_home: project_home)
|
2618
|
+
@files = Cnvrg::Files.new(@project.owner, @project.slug, project_home: project_home, project: @project)
|
2611
2619
|
resp = @project.jump_idx(destination: commit_sha1)
|
2612
2620
|
if resp.blank?
|
2613
2621
|
log_message("Cant find the given commit", Thor::Shell::Color::RED)
|
@@ -2651,7 +2659,7 @@ module Cnvrg
|
|
2651
2659
|
|
2652
2660
|
|
2653
2661
|
|
2654
|
-
@files = Cnvrg::Files.new(@project.owner, @project.slug)
|
2662
|
+
@files = Cnvrg::Files.new(@project.owner, @project.slug, project: @project)
|
2655
2663
|
begin
|
2656
2664
|
|
2657
2665
|
file = @files.show_file_s3(path, commit)
|
@@ -2788,7 +2796,6 @@ module Cnvrg
|
|
2788
2796
|
method_option :local_folders, :type => :string, :aliases => ["--local_folders"], :default => nil
|
2789
2797
|
|
2790
2798
|
def run(*cmd)
|
2791
|
-
|
2792
2799
|
verify_logged_in(true)
|
2793
2800
|
log_start(__method__, args, options)
|
2794
2801
|
datasets = options["datasets"]
|
@@ -4816,7 +4823,6 @@ module Cnvrg
|
|
4816
4823
|
user = options[:user]
|
4817
4824
|
is_gpu = options[:gpu]
|
4818
4825
|
res = @image.upload_docker_image(image_path, image_name, workdir, user, description, is_gpu)
|
4819
|
-
|
4820
4826
|
if res["status"] == 200
|
4821
4827
|
image_slug = res["id"]
|
4822
4828
|
owner = CLI.get_owner
|
@@ -5326,19 +5332,24 @@ module Cnvrg
|
|
5326
5332
|
end
|
5327
5333
|
|
5328
5334
|
def self.log_message(message, type = Thor::Shell::Color::BLUE, to_print: true)
|
5329
|
-
say "#{type}#{message}" if to_print
|
5330
5335
|
case type
|
5331
|
-
when Thor::Shell::Color::BLUE
|
5336
|
+
when Thor::Shell::Color::BLUE, 'blue', 'progress'
|
5337
|
+
color = Thor::Shell::Color::BLUE
|
5332
5338
|
$LOG.info message: message, type: "info"
|
5333
|
-
when Thor::Shell::Color::RED
|
5339
|
+
when Thor::Shell::Color::RED, 'red', 'error'
|
5340
|
+
color = Thor::Shell::Color::RED
|
5334
5341
|
$LOG.error message: message, type: "error"
|
5335
|
-
when Thor::Shell::Color::YELLOW
|
5342
|
+
when Thor::Shell::Color::YELLOW, 'yellow', 'warning'
|
5343
|
+
color = Thor::Shell::Color::YELLOW
|
5336
5344
|
$LOG.warn message: message, type: "warning"
|
5337
|
-
when Thor::Shell::Color::GREEN
|
5345
|
+
when Thor::Shell::Color::GREEN, 'green', 'info'
|
5346
|
+
color = Thor::Shell::Color::GREEN
|
5338
5347
|
$LOG.info message: message, type: "success"
|
5339
5348
|
else
|
5349
|
+
color = nil
|
5340
5350
|
$LOG.info message: message, type: "unknown"
|
5341
5351
|
end
|
5352
|
+
say "#{color}#{message}" if to_print
|
5342
5353
|
end
|
5343
5354
|
|
5344
5355
|
def log_message(message, type=Thor::Shell::Color::GREEN, to_print = true)
|
@@ -5450,6 +5461,10 @@ module Cnvrg
|
|
5450
5461
|
return Cnvrg::CLI.get_project_home
|
5451
5462
|
end
|
5452
5463
|
|
5464
|
+
def get_job
|
5465
|
+
|
5466
|
+
end
|
5467
|
+
|
5453
5468
|
def self.get_project_home
|
5454
5469
|
absolute_path = Dir.pwd
|
5455
5470
|
dirs = absolute_path.split("/")
|
data/lib/cnvrg/data.rb
CHANGED
@@ -1,5 +1,3 @@
|
|
1
|
-
require 'cnvrg/cli'
|
2
|
-
require 'thor'
|
3
1
|
class SubCommandBase < Thor
|
4
2
|
def self.banner(command, namespace = nil, subcommand = false)
|
5
3
|
"#{basename} #{command.usage}"
|
@@ -15,10 +13,12 @@ module Cnvrg
|
|
15
13
|
class_option :no_compression, :type => :boolean, :aliases => ["-nc", "--no_compression"], :default => false
|
16
14
|
desc "data init", "init data folder"
|
17
15
|
method_option :public, :type => :boolean, :aliases => ["-p", "--public"], :default => false
|
16
|
+
method_option :bucket, :type => :string, :aliases => ["-b", "--bucket"], :default => ""
|
18
17
|
def init
|
19
18
|
cli = Cnvrg::CLI.new()
|
20
19
|
public = options["public"]
|
21
|
-
|
20
|
+
bucket = options["bucket"]
|
21
|
+
cli.init_data(public, bucket: bucket)
|
22
22
|
end
|
23
23
|
desc "data upload", "upload data folder"
|
24
24
|
method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
|
@@ -187,5 +187,16 @@ module Cnvrg
|
|
187
187
|
cli.download_tags_yaml()
|
188
188
|
end
|
189
189
|
|
190
|
+
|
191
|
+
desc 'data testclient', 'test client'
|
192
|
+
def test
|
193
|
+
cli = Cnvrg::CLI.new
|
194
|
+
cli.verify_logged_in(true)
|
195
|
+
cli.log_start(__method__, args, options)
|
196
|
+
dataset_dir = cli.is_cnvrg_dir(Dir.pwd)
|
197
|
+
@dataset = Project.new(dataset_dir)
|
198
|
+
resp = @dataset.get_storage_client
|
199
|
+
end
|
200
|
+
|
190
201
|
end
|
191
202
|
end
|
data/lib/cnvrg/datafiles.rb
CHANGED
@@ -19,6 +19,7 @@ module Cnvrg
|
|
19
19
|
@owner = owner
|
20
20
|
@dataset = dataset
|
21
21
|
@base_resource = "users/#{owner}/datasets/#{dataset_slug}/"
|
22
|
+
@downloader = @dataset.get_storage_client
|
22
23
|
end
|
23
24
|
|
24
25
|
def check_file_sha1(filename, org_sha1, tag: 'conflict')
|
@@ -96,16 +97,11 @@ module Cnvrg
|
|
96
97
|
progressbar.progress += tree.keys.length if progressbar.present?
|
97
98
|
return 0
|
98
99
|
end
|
99
|
-
props = Cnvrg::Helpers.get_s3_props(results)
|
100
|
-
client = props[:client]
|
101
|
-
bucket = props[:bucket]
|
102
|
-
upload_options = props[:upload_options]
|
103
|
-
s3_bucket = Aws::S3::Resource.new(client: client).bucket(bucket)
|
104
100
|
files = results['files']
|
105
101
|
progressbar.progress += tree.keys.length - files.length if progressbar.present?
|
106
102
|
Parallel.map((files.keys), {in_threads: threads}) do |k|
|
107
103
|
o = tree[k].merge(files[k])
|
108
|
-
upload_single_file(o
|
104
|
+
upload_single_file(o)
|
109
105
|
progressbar.progress += 1 if progressbar.present?
|
110
106
|
end
|
111
107
|
blob_ids = files.values.map {|f| f['bv_id']}
|
@@ -126,16 +122,14 @@ module Cnvrg
|
|
126
122
|
|
127
123
|
end
|
128
124
|
|
129
|
-
def upload_single_file(file
|
125
|
+
def upload_single_file(file)
|
130
126
|
begin
|
131
|
-
file = file.
|
132
|
-
Cnvrg::Logger.log_info("Uploading #{file[
|
133
|
-
|
134
|
-
|
135
|
-
obj.upload_file(file[:absolute_path], upload_options)
|
136
|
-
Cnvrg::Logger.log_info("#{file[:absolute_path]} uploaded.")
|
127
|
+
file = file.as_json
|
128
|
+
Cnvrg::Logger.log_info("Uploading #{file["absolute_path"]}")
|
129
|
+
@downloader.upload(file["path"], file["absolute_path"])
|
130
|
+
Cnvrg::Logger.log_info("#{file["absolute_path"]} uploaded.")
|
137
131
|
rescue => e
|
138
|
-
Cnvrg::Logger.log_error_message("Error while upload single file #{file[
|
132
|
+
Cnvrg::Logger.log_error_message("Error while upload single file #{file["path"]}")
|
139
133
|
Cnvrg::Logger.log_error(e)
|
140
134
|
end
|
141
135
|
end
|
@@ -889,9 +883,8 @@ module Cnvrg
|
|
889
883
|
|
890
884
|
def download_files_in_chunks(files, chunk_size: 1000, conflict: false, commit: 'latest', progress: nil)
|
891
885
|
begin
|
892
|
-
ttl_files = 0
|
893
886
|
files.each_slice(chunk_size).each do |files|
|
894
|
-
|
887
|
+
download_files_chunk(files, conflict: conflict, progress: progress)
|
895
888
|
end
|
896
889
|
return Cnvrg::Result.new(true, "Download Completed")
|
897
890
|
rescue Exception => e
|
@@ -942,61 +935,26 @@ module Cnvrg
|
|
942
935
|
|
943
936
|
def download_multiple_files_s3(files, project_home, conflict: false, progressbar: nil, read_only:false)
|
944
937
|
begin
|
945
|
-
props = Cnvrg::Helpers.get_s3_props(files)
|
946
|
-
client = props[:client]
|
947
|
-
iv = props[:iv]
|
948
|
-
key = props[:key]
|
949
|
-
bucket = props[:bucket]
|
950
|
-
download_succ_count = 0
|
951
|
-
if read_only
|
952
|
-
in_threads = 1000
|
953
|
-
else
|
954
|
-
in_threads = ParallelThreads
|
955
|
-
end
|
956
938
|
parallel_options = {
|
957
|
-
in_threads:
|
939
|
+
in_threads: ParallelThreads,
|
958
940
|
isolation: true
|
959
941
|
}
|
960
942
|
Parallel.map(files["keys"], parallel_options) do |f|
|
961
|
-
|
962
|
-
|
943
|
+
local_path = @dataset.working_dir + '/' + f['name']
|
944
|
+
Cnvrg::Logger.log_info("Downloading #{local_path}")
|
963
945
|
progressbar.progress += 1 if progressbar.present?
|
964
|
-
if
|
965
|
-
|
966
|
-
|
967
|
-
|
968
|
-
else
|
969
|
-
return Cnvrg::Result.new(false,"Could not create directory: #{file_path}")
|
970
|
-
raise Parallel::Kill
|
971
|
-
end
|
972
|
-
else
|
946
|
+
if local_path.end_with? "/"
|
947
|
+
@downloader.mkdir(local_path, recursive: true)
|
948
|
+
next
|
949
|
+
end
|
973
950
|
# blob
|
974
951
|
begin
|
975
|
-
|
976
|
-
|
977
|
-
file_path = "#{file_path}.conflict" if conflict
|
978
|
-
dirname = File.dirname file_path
|
979
|
-
download_dir(project_home, dirname) unless dirname.eql? "."
|
980
|
-
Cnvrg::Logger.info("Download #{file_path}")
|
981
|
-
File.open(project_home+"/"+file_path, 'w+') do |file|
|
982
|
-
resp = client.get_object({bucket:bucket,
|
983
|
-
key:file_key}, target: file)
|
984
|
-
end
|
985
|
-
Cnvrg::Logger.log_info("Download #{file_path} success")
|
952
|
+
local_path = "#{local_path}.conflict" if conflict
|
953
|
+
storage_path = f["path"]
|
986
954
|
|
987
|
-
|
988
|
-
|
989
|
-
else
|
990
|
-
return Cnvrg::Result.new(false,"Could not create file: #{file_path}")
|
955
|
+
@downloader.download(storage_path, local_path)
|
956
|
+
Cnvrg::Logger.log_info("Download #{local_path} success")
|
991
957
|
end
|
992
|
-
rescue => e
|
993
|
-
return Cnvrg::Result.new(false,"Could not create file: #{file_path}", e.message, e.backtrace)
|
994
|
-
raise Parallel::Kill
|
995
|
-
end
|
996
|
-
end
|
997
|
-
end
|
998
|
-
if download_succ_count == files["keys"].size
|
999
|
-
return Cnvrg::Result.new(true,"Done.\nDownloaded #{download_succ_count} files")
|
1000
958
|
end
|
1001
959
|
rescue => e
|
1002
960
|
return Cnvrg::Result.new(false,"Could not download some files", e.message, e.backtrace)
|