cnvrg 0.7.3 → 0.7.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/cnvrg.gemspec +1 -1
- data/lib/cnvrg/cli.rb +17 -23
- data/lib/cnvrg/datafiles.rb +11 -4
- data/lib/cnvrg/experiment.rb +2 -2
- data/lib/cnvrg/helpers.rb +1 -1
- data/lib/cnvrg/version.rb +1 -1
- metadata +6 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 34ca8a94802b76adb5ab7b8d2c2d6a2c10ff3824
|
4
|
+
data.tar.gz: 4dc880d0b84ccb072ac878c8c815be43a0689446
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 14a5b0e7db3d5bddd7449a0da6583acf5fc069a08443e3889dd754cd9a27fe9b9de10608b1df85f1b8602ac853bcb29e71498f252c9384b515c104227edf4ec0
|
7
|
+
data.tar.gz: 2646237fecf5996079985e1138a493f1dee678423c2ce39455e4e03c3ff52723823ebd79ee9e94b64bafc260b6bb36822852b3cc0e969e7b86a4988822bd3a56
|
data/cnvrg.gemspec
CHANGED
@@ -31,7 +31,7 @@ Gem::Specification.new do |spec|
|
|
31
31
|
spec.add_runtime_dependency 'open4', '~> 1.3', '>= 1.3.4'
|
32
32
|
spec.add_runtime_dependency 'highline', '~> 1.7', '>= 1.7.8'
|
33
33
|
spec.add_runtime_dependency 'thor', '~> 0.19.0','>=0.19.1'
|
34
|
-
spec.add_runtime_dependency 'aws-sdk', '2
|
34
|
+
spec.add_runtime_dependency 'aws-sdk', '~> 2'
|
35
35
|
spec.add_runtime_dependency 'sucker_punch', '~> 2.0'
|
36
36
|
spec.add_runtime_dependency 'urlcrypt', '~> 0.1.1'
|
37
37
|
spec.add_runtime_dependency 'parallel', '~> 1.12.0'
|
data/lib/cnvrg/cli.rb
CHANGED
@@ -838,6 +838,7 @@ module Cnvrg
|
|
838
838
|
project_index = Cnvrg::Helpers.look_for_in_path(dataset_url, "datasets")
|
839
839
|
slug = url_parts[project_index + 1]
|
840
840
|
owner = url_parts[project_index - 1]
|
841
|
+
log_message("#{Time.now}: Fetching Dataset files list", Thor::Shell::Color::GREEN)
|
841
842
|
|
842
843
|
response = Cnvrg::API.request("users/#{owner}/datasets/#{slug}/clone", 'POST',{ commit: commit, query:query})
|
843
844
|
Cnvrg::CLI.is_response_success(response,true)
|
@@ -872,9 +873,11 @@ module Cnvrg
|
|
872
873
|
:starting_at => 0,
|
873
874
|
:total => files_count,
|
874
875
|
:autofinish => true)
|
875
|
-
|
876
|
+
|
876
877
|
while files['keys'].length > 0
|
877
|
-
|
878
|
+
|
879
|
+
@files.download_multiple_files_s3(files, dataset_home, progressbar: progressbar, read_only: read)
|
880
|
+
|
878
881
|
downloaded_files += files['keys'].length
|
879
882
|
files = @files.get_clone_chunk(commit: commit, latest_id: files['latest'])
|
880
883
|
end
|
@@ -884,6 +887,7 @@ module Cnvrg
|
|
884
887
|
if !read
|
885
888
|
@dataset.write_idx(nil, commit) #nil means, generate idx
|
886
889
|
end
|
890
|
+
log_message("#{Time.now} Clone finished successfully", Thor::Shell::Color::GREEN)
|
887
891
|
|
888
892
|
log_message("#{check} Clone finished successfully", Thor::Shell::Color::GREEN)
|
889
893
|
end
|
@@ -3356,7 +3360,7 @@ module Cnvrg
|
|
3356
3360
|
method_option :upload_output, :type => :string, :aliases => ["-uo", "--upload_output"], :default => ""
|
3357
3361
|
method_option :commit, :type => :string, :aliases => ["-c", "--commit"], :default => ""
|
3358
3362
|
method_option :schedule, :type => :string, :aliases => ["-s", "--schedule"], :default => ""
|
3359
|
-
method_option :image, :type => :string, :aliases => ["--image"], :default =>
|
3363
|
+
method_option :image, :type => :string, :aliases => ["--image"], :default => nil
|
3360
3364
|
method_option :grid, :type => :string, :aliases => ["-g", "--grid"], :default => ""
|
3361
3365
|
method_option :data, :type => :string, :aliases => ["-d", "--data"], :default => ""
|
3362
3366
|
method_option :data_commit, :type => :string, :aliases => ["--data_commit"], :default => ""
|
@@ -3385,7 +3389,7 @@ module Cnvrg
|
|
3385
3389
|
upload_output = options["upload_output"]
|
3386
3390
|
local = options["local"]
|
3387
3391
|
schedule = options["schedule"]
|
3388
|
-
image = options["image"]
|
3392
|
+
image = options["image"] || nil
|
3389
3393
|
grid = options["grid"]
|
3390
3394
|
data = options["data"]
|
3391
3395
|
data_commit = options["data_commit"]
|
@@ -3848,7 +3852,6 @@ module Cnvrg
|
|
3848
3852
|
output_dir = options["output_dir"] || nil
|
3849
3853
|
git_commit = options["git_commit"]
|
3850
3854
|
git_branch = options["git_branch"]
|
3851
|
-
|
3852
3855
|
options_hash = Hash[options]
|
3853
3856
|
local_folders_options = options["local_folders"]
|
3854
3857
|
options_hash.except!("schedule", "machine_type", "image", "upload_output", "grid", "data", "data_commit",
|
@@ -3867,22 +3870,11 @@ module Cnvrg
|
|
3867
3870
|
if project.is_git and output_dir.blank?
|
3868
3871
|
output_dir = "output"
|
3869
3872
|
end
|
3870
|
-
|
3871
|
-
|
3872
|
-
|
3873
|
-
invoke :set_image, [choose_image]
|
3873
|
+
image = options["image"] || nil
|
3874
|
+
if image.blank?
|
3875
|
+
image = "cnvrg"
|
3874
3876
|
end
|
3875
|
-
image = is_project_with_docker(working_dir)
|
3876
|
-
if !image or !image.is_docker
|
3877
|
-
# say "Couldn't find image related to project", Thor::Shell::Color::RED
|
3878
3877
|
|
3879
|
-
image_slug = "cnvrg"
|
3880
|
-
if instance_type.eql? "gpu" or instance_type.eql? "gpuxl"
|
3881
|
-
image_slug = "cnvrg_gpu"
|
3882
|
-
end
|
3883
|
-
else
|
3884
|
-
image_slug = image.image_slug
|
3885
|
-
end
|
3886
3878
|
forced_commit = nil
|
3887
3879
|
if sync_before and !project.is_git
|
3888
3880
|
if force
|
@@ -3923,7 +3915,7 @@ module Cnvrg
|
|
3923
3915
|
end
|
3924
3916
|
commit_to_run = commit_to_run.presence || project.last_local_commit
|
3925
3917
|
|
3926
|
-
res = exp.exec_remote(command, commit_to_run, instance_type,
|
3918
|
+
res = exp.exec_remote(command, commit_to_run, instance_type, image, schedule, local_timestamp, grid, path_to_cmd, data, data_commit,
|
3927
3919
|
periodic_sync, sync_before_terminate, max_time, ds_sync_options,output_dir,
|
3928
3920
|
data_query, git_commit, git_branch, restart_if_stuck,local_folders_options, title )
|
3929
3921
|
if Cnvrg::CLI.is_response_success(res)
|
@@ -4042,7 +4034,7 @@ module Cnvrg
|
|
4042
4034
|
method_option :gpu, :type => :boolean, :aliases => ["--gpu"], :default => false
|
4043
4035
|
method_option :gpuxl, :type => :boolean, :aliases => ["--gpuxl"], :default => false
|
4044
4036
|
method_option :gpuxxl, :type => :boolean, :aliases => ["--gpuxxl"], :default => false
|
4045
|
-
method_option :image, :type => :string, :aliases => ["-i", "--image"], :default =>
|
4037
|
+
method_option :image, :type => :string, :aliases => ["-i", "--image"], :default => nil
|
4046
4038
|
method_option :data, :type => :string, :aliases => ["-d", "--data"], :default => ""
|
4047
4039
|
method_option :data_commit, :type => :string, :aliases => ["--data_commit"], :default => ""
|
4048
4040
|
method_option :dataset_only_tree, :type => :boolean, :aliases => [ "--dataset_only_tree"], :default => false
|
@@ -4056,7 +4048,7 @@ module Cnvrg
|
|
4056
4048
|
local = options["local"]
|
4057
4049
|
notebook_dir = options["notebook_dir"]
|
4058
4050
|
kernel = options["kernel"]
|
4059
|
-
image = options["image"]
|
4051
|
+
image = options["image"] || nil
|
4060
4052
|
data = options["data"]
|
4061
4053
|
data_commit = options["data_commit"]
|
4062
4054
|
dataset_only_tree = options["dataset_only_tree"]
|
@@ -4198,6 +4190,7 @@ module Cnvrg
|
|
4198
4190
|
method_option :commit, :type => :string, :aliases => ["--commit"], :default => ""
|
4199
4191
|
method_option :dataset_only_tree, :type => :boolean, :aliases => [ "--dataset_only_tree"], :default => false
|
4200
4192
|
method_option :data_query, :type => :string, :aliases => ["-q","--data_query"], :default => ""
|
4193
|
+
method_option :image, :type => :string, :aliases => ["--image"], :default => nil
|
4201
4194
|
|
4202
4195
|
def remote_notebook()
|
4203
4196
|
verify_logged_in(true)
|
@@ -4210,6 +4203,7 @@ module Cnvrg
|
|
4210
4203
|
commit = options["commit"]
|
4211
4204
|
notebook_type = options["notebook_type"]
|
4212
4205
|
dataset_only_tree = options["dataset_only_tree"]
|
4206
|
+
image = options["image"]
|
4213
4207
|
ds_sync_options = 0
|
4214
4208
|
if dataset_only_tree
|
4215
4209
|
ds_sync_options = 1
|
@@ -4234,7 +4228,7 @@ module Cnvrg
|
|
4234
4228
|
end
|
4235
4229
|
invoke :sync, [false], []
|
4236
4230
|
slug = ""
|
4237
|
-
res = exp.remote_notebook(instance_type, commit, data, data_commit, notebook_type,ds_sync_options,data_query)
|
4231
|
+
res = exp.remote_notebook(instance_type, commit, data, data_commit, notebook_type,ds_sync_options,data_query, image)
|
4238
4232
|
if Cnvrg::CLI.is_response_success(res)
|
4239
4233
|
slug = res["result"]["notebook_url"]
|
4240
4234
|
log_message("#{Helpers.checkmark} Notebook is ready: #{Cnvrg::Helpers.remote_url}/#{project.owner}/projects/#{project.slug}/notebook_sessions/show/#{slug}", Thor::Shell::Color::GREEN)
|
data/lib/cnvrg/datafiles.rb
CHANGED
@@ -872,7 +872,7 @@ module Cnvrg
|
|
872
872
|
next unless Cnvrg::CLI.is_response_success(response, false) #trying to api request 5 times.
|
873
873
|
files_to_download = response['files']
|
874
874
|
data_home = "#{Dir.pwd}/#{response['name']}"
|
875
|
-
res = download_multiple_files_s3(files_to_download, data_home, conflict: conflict)
|
875
|
+
res = download_multiple_files_s3(files_to_download, data_home, conflict: conflict, read_only: false)
|
876
876
|
next unless res.is_success? #try again..
|
877
877
|
return files_to_download['keys'].length
|
878
878
|
end
|
@@ -890,7 +890,7 @@ module Cnvrg
|
|
890
890
|
data_home = "#{Dir.pwd}/#{response['name']}"
|
891
891
|
last_chunk_size = files['keys'].length
|
892
892
|
break if last_chunk_size == 0
|
893
|
-
res = download_multiple_files_s3(files, data_home)
|
893
|
+
res = download_multiple_files_s3(files, data_home, read_only: false)
|
894
894
|
overall += last_chunk_size
|
895
895
|
q[:latest] = files['latest']
|
896
896
|
else
|
@@ -907,7 +907,7 @@ module Cnvrg
|
|
907
907
|
|
908
908
|
end
|
909
909
|
|
910
|
-
def download_multiple_files_s3(files, project_home, conflict: false, progressbar: nil)
|
910
|
+
def download_multiple_files_s3(files, project_home, conflict: false, progressbar: nil, read_only:false)
|
911
911
|
begin
|
912
912
|
props = Cnvrg::Helpers.get_s3_props(files)
|
913
913
|
client = props[:client]
|
@@ -915,11 +915,17 @@ module Cnvrg
|
|
915
915
|
key = props[:key]
|
916
916
|
bucket = props[:bucket]
|
917
917
|
download_succ_count = 0
|
918
|
+
if read_only
|
919
|
+
in_threads = 1000
|
920
|
+
else
|
921
|
+
in_threads = ParallelThreads
|
922
|
+
end
|
918
923
|
parallel_options = {
|
919
|
-
in_threads:
|
924
|
+
in_threads: in_threads,
|
920
925
|
isolation: true
|
921
926
|
}
|
922
927
|
Parallel.map(files["keys"], parallel_options) do |f|
|
928
|
+
|
923
929
|
file_path = f["name"]
|
924
930
|
progressbar.progress += 1 if progressbar.present?
|
925
931
|
if file_path.end_with? "/"
|
@@ -943,6 +949,7 @@ module Cnvrg
|
|
943
949
|
resp = client.get_object({bucket:bucket,
|
944
950
|
key:file_key}, target: file)
|
945
951
|
end
|
952
|
+
|
946
953
|
if resp
|
947
954
|
download_succ_count +=1
|
948
955
|
else
|
data/lib/cnvrg/experiment.rb
CHANGED
@@ -106,9 +106,9 @@ module Cnvrg
|
|
106
106
|
|
107
107
|
return response
|
108
108
|
end
|
109
|
-
def remote_notebook(instance_type, commit, data, data_commit, notebook_type,ds_sync_options=0,data_query=nil)
|
109
|
+
def remote_notebook(instance_type, commit, data, data_commit, notebook_type,ds_sync_options=0,data_query=nil, image = nil)
|
110
110
|
response = Cnvrg::API.request("users/#{@owner}/projects/#{@project_slug}/notebook/remote", 'POST', {instance_type: instance_type,dataset_slug:data,
|
111
|
-
dataset_commit: data_commit,
|
111
|
+
dataset_commit: data_commit,image_slug:image,
|
112
112
|
commit:commit,notebook_type:notebook_type,dataset_sync_options:ds_sync_options,
|
113
113
|
dataset_query:data_query})
|
114
114
|
return response
|
data/lib/cnvrg/helpers.rb
CHANGED
data/lib/cnvrg/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: cnvrg
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.7.
|
4
|
+
version: 0.7.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Yochay Ettun
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2018-09-
|
12
|
+
date: 2018-09-25 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: bundler
|
@@ -207,16 +207,16 @@ dependencies:
|
|
207
207
|
name: aws-sdk
|
208
208
|
requirement: !ruby/object:Gem::Requirement
|
209
209
|
requirements:
|
210
|
-
- -
|
210
|
+
- - "~>"
|
211
211
|
- !ruby/object:Gem::Version
|
212
|
-
version: 2
|
212
|
+
version: '2'
|
213
213
|
type: :runtime
|
214
214
|
prerelease: false
|
215
215
|
version_requirements: !ruby/object:Gem::Requirement
|
216
216
|
requirements:
|
217
|
-
- -
|
217
|
+
- - "~>"
|
218
218
|
- !ruby/object:Gem::Version
|
219
|
-
version: 2
|
219
|
+
version: '2'
|
220
220
|
- !ruby/object:Gem::Dependency
|
221
221
|
name: sucker_punch
|
222
222
|
requirement: !ruby/object:Gem::Requirement
|