cnvrg 0.9.9.6 → 1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/cnvrg/api.rb +2 -2
- data/lib/cnvrg/cli.rb +56 -17
- data/lib/cnvrg/data.rb +12 -1
- data/lib/cnvrg/dataset.rb +37 -3
- data/lib/cnvrg/experiment.rb +4 -2
- data/lib/cnvrg/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 12febac69e1c9f8c035af8af4f971130b94de970
|
4
|
+
data.tar.gz: 30935d7322d87135629f944f3d5588bc4f3c9a80
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6b0447676f52c0dc2ff437a3d2ce67aee5326485a4ac5dcf81814706713d2ffffcc16ca1520372080d895b73a87ca8fe3a27277870ec7f0ea05a6db27ded7ac8
|
7
|
+
data.tar.gz: c687e4ec459d503675171b0711a56c2cc048675833dbf1db97821b7f456eb05d5eca0772ab0ac89d60534e9d1e2c4db951b0b87b23748f5ddce8e08d5cbb985b
|
data/lib/cnvrg/api.rb
CHANGED
@@ -54,7 +54,7 @@ module Cnvrg
|
|
54
54
|
conn.headers['Auth-Token'] = @pass
|
55
55
|
conn.headers['User-Agent'] = "#{Cnvrg::API::USER_AGENT}"
|
56
56
|
conn.options.timeout = 420
|
57
|
-
conn.options.open_timeout
|
57
|
+
conn.options.open_timeout=180
|
58
58
|
case method
|
59
59
|
when 'GET'
|
60
60
|
retries = 0
|
@@ -82,7 +82,7 @@ module Cnvrg
|
|
82
82
|
end
|
83
83
|
when 'POST', 'PUT'
|
84
84
|
conn.options.timeout = 4200
|
85
|
-
conn.options.open_timeout
|
85
|
+
conn.options.open_timeout=180
|
86
86
|
retries = 0
|
87
87
|
success = false
|
88
88
|
while !success and retries < 20
|
data/lib/cnvrg/cli.rb
CHANGED
@@ -771,21 +771,42 @@ module Cnvrg
|
|
771
771
|
end
|
772
772
|
end
|
773
773
|
|
774
|
+
desc 'data verify', 'Verify datasets'
|
775
|
+
method_option :timeout, :type => :numeric, :aliases => ["-t", "--timeout"], :desc => "Time to wait before returning final answer", :default => 15
|
776
|
+
|
777
|
+
def verify_datasets(dataset_titles, timeout=0)
|
778
|
+
begin
|
779
|
+
verify_logged_in(false)
|
780
|
+
log_start(__method__, args, options)
|
781
|
+
log_message("Verifying datasets #{dataset_titles}", Thor::Shell::Color::BLUE)
|
782
|
+
verified = Dataset.verify_datasets(dataset_titles, timeout)
|
783
|
+
log_message("All datasets are verified", Thor::Shell::Color::BLUE) if verified
|
784
|
+
log_message("Failed to verify datasets", Thor::Shell::Color::RED) if !verified
|
785
|
+
exit(1) if !verified
|
786
|
+
|
787
|
+
rescue SignalException
|
788
|
+
say "\nAborting", Thor::Shell::Color::RED
|
789
|
+
exit(1)
|
790
|
+
end
|
791
|
+
end
|
792
|
+
|
774
793
|
desc 'data clone', 'Clone dataset'
|
775
794
|
method_option :commit, :type => :string, :aliases => ["-c", "--commit"], :default => ""
|
776
795
|
method_option :only_tree, :type => :boolean, :aliases => ["-t", "--tree"], :default => false
|
777
796
|
method_option :query, :type => :string, :aliases => ["-q", "--query"], :default => nil
|
778
797
|
method_option :read, :type => :boolean, :aliases => ["-r", "--read"], :default => false
|
798
|
+
method_option :remote, :type => :boolean, :aliases => ["-h", "--remote"], :default => false
|
779
799
|
|
780
|
-
def clone_data(dataset_url,only_tree=false,commit=nil,query=nil,read=false)
|
800
|
+
def clone_data(dataset_url,only_tree=false,commit=nil,query=nil,read=false,remote=false)
|
781
801
|
begin
|
782
802
|
verify_logged_in(false)
|
783
803
|
log_start(__method__, args, options)
|
784
804
|
commit = options["commit"] || commit
|
785
805
|
only_tree = options["only_tree"] || only_tree
|
786
806
|
read = options["read"] || read || false
|
807
|
+
remote = options["remote"] || remote || false
|
787
808
|
query = options['query'].presence || query.presence
|
788
|
-
if query.present?
|
809
|
+
if query.present?
|
789
810
|
return clone_data_query(dataset_url, query)
|
790
811
|
end
|
791
812
|
|
@@ -798,7 +819,7 @@ module Cnvrg
|
|
798
819
|
Cnvrg::CLI.is_response_success(response,true)
|
799
820
|
dataset_name = response["result"]["name"]
|
800
821
|
dataset_home = Dir.pwd+"/"+dataset_name
|
801
|
-
if (Dir.exists? dataset_name)
|
822
|
+
if !remote and (Dir.exists? dataset_name)
|
802
823
|
log_message("Error: Conflict with dir #{dataset_name}", Thor::Shell::Color::RED)
|
803
824
|
if no? "Sync to repository anyway? (current data might lost)", Thor::Shell::Color::YELLOW
|
804
825
|
log_message("Remove dir in order to clone #{dataset_name}", Thor::Shell::Color::RED)
|
@@ -806,7 +827,7 @@ module Cnvrg
|
|
806
827
|
end
|
807
828
|
end
|
808
829
|
check = Helpers.checkmark
|
809
|
-
if Dataset.clone(owner, dataset_name, slug)
|
830
|
+
if Dataset.clone(owner, dataset_name, slug, remote)
|
810
831
|
log_message("Cloning #{dataset_name}", Thor::Shell::Color::BLUE)
|
811
832
|
@files = Cnvrg::Datafiles.new(owner, slug)
|
812
833
|
log_message("Downloading files", Thor::Shell::Color::BLUE)
|
@@ -844,6 +865,7 @@ module Cnvrg
|
|
844
865
|
end
|
845
866
|
|
846
867
|
log_message("#{check} Clone finished successfully", Thor::Shell::Color::GREEN)
|
868
|
+
@dataset.write_success
|
847
869
|
end
|
848
870
|
else
|
849
871
|
log_message("Error: Couldn't create directory: #{dataset_name}", Thor::Shell::Color::RED)
|
@@ -923,6 +945,7 @@ module Cnvrg
|
|
923
945
|
dataset.generate_idx()
|
924
946
|
check = Helpers.checkmark
|
925
947
|
log_message("#{check} Clone finished successfully", Thor::Shell::Color::GREEN)
|
948
|
+
dataset.write_success(in_folder=true)
|
926
949
|
rescue
|
927
950
|
exit(1)
|
928
951
|
end
|
@@ -1874,6 +1897,7 @@ module Cnvrg
|
|
1874
1897
|
# w(verbose=false, new_branch=false,sync=false, commit=nil,all_files=true)
|
1875
1898
|
invoke :download_data_new,[verbose, new_branch, true, commit, all_files], :new_branch=>new_branch, :direct=>false, :force =>force
|
1876
1899
|
end
|
1900
|
+
|
1877
1901
|
# w(new_branch, verbose,sync,force, tags, chunk_size)
|
1878
1902
|
invoke :upload_data_new,[new_branch, verbose,true,force, tags, chunk_size], :new_branch=>new_branch,
|
1879
1903
|
:direct=>false, :force =>force, :sync =>true, :tags =>tags, :parallel => parallel
|
@@ -2126,7 +2150,7 @@ module Cnvrg
|
|
2126
2150
|
if git_output_dir.ends_with? "/"
|
2127
2151
|
git_output_dir = git_output_dir[0..-2]
|
2128
2152
|
end
|
2129
|
-
list = @project.
|
2153
|
+
list = @project.generate_output_dir(git_output_dir)
|
2130
2154
|
spec_files_to_upload = list
|
2131
2155
|
if spec_files_to_upload.blank?
|
2132
2156
|
log_message("#{check} Project is up to date", Thor::Shell::Color::GREEN, (((options["sync"] or sync) and !direct) ? false : true))
|
@@ -2235,6 +2259,7 @@ module Cnvrg
|
|
2235
2259
|
log_message("Total of #{update_count} / #{update_total} files.", Thor::Shell::Color::GREEN)
|
2236
2260
|
else
|
2237
2261
|
if return_id
|
2262
|
+
puts "\n"
|
2238
2263
|
print_res = {
|
2239
2264
|
'success' => "true",
|
2240
2265
|
'commit_sha1' => res["result"]["commit_id"]
|
@@ -2263,6 +2288,7 @@ module Cnvrg
|
|
2263
2288
|
'success' => "false",
|
2264
2289
|
'message' => 'couldn\'t commit changes, Rolling Back all changes.'
|
2265
2290
|
}
|
2291
|
+
puts "\n"
|
2266
2292
|
puts JSON[print_res] if return_id
|
2267
2293
|
return false
|
2268
2294
|
end
|
@@ -2686,7 +2712,7 @@ module Cnvrg
|
|
2686
2712
|
end
|
2687
2713
|
end
|
2688
2714
|
|
2689
|
-
desc 'sync', 'Sync with
|
2715
|
+
desc 'sync', 'Sync with remote server'
|
2690
2716
|
method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
|
2691
2717
|
method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
|
2692
2718
|
method_option :ignore, :type => :string, :aliases => ["-i", "--ignore"], :default => ""
|
@@ -2741,6 +2767,7 @@ module Cnvrg
|
|
2741
2767
|
method_option :image, :type => :string, :aliases => ["--image"], :default => nil
|
2742
2768
|
method_option :grid, :type => :string, :aliases => ["-g", "--grid"], :default => ""
|
2743
2769
|
method_option :data, :type => :string, :aliases => ["-d", "--data"], :default => ""
|
2770
|
+
method_option :datasets, :type => :string, :aliases => ["--datasets"], :desc => "'[{\"id\": \"dataset id\", \"commit\": \"commit id\", \"query\": \"query name\", \"tree_only\": true]'", :default => ""
|
2744
2771
|
method_option :data_commit, :type => :string, :aliases => ["--data_commit"], :default => ""
|
2745
2772
|
method_option :ignore, :type => :string, :aliases => ["-i", "--ignore"], :desc => "ignore following files", :default => ""
|
2746
2773
|
method_option :force, :type => :boolean, :aliases => ["-f", "--force"], :default => false
|
@@ -2758,6 +2785,7 @@ module Cnvrg
|
|
2758
2785
|
def run(*cmd)
|
2759
2786
|
verify_logged_in(true)
|
2760
2787
|
log_start(__method__, args, options)
|
2788
|
+
datasets = options["datasets"]
|
2761
2789
|
sync_before = options["sync_before"]
|
2762
2790
|
sync_after = options["sync_after"]
|
2763
2791
|
log = options["log"]
|
@@ -2794,7 +2822,6 @@ module Cnvrg
|
|
2794
2822
|
git_branch = options["git_branch"]
|
2795
2823
|
restart_if_stuck = options["restart_if_stuck"]
|
2796
2824
|
|
2797
|
-
|
2798
2825
|
options_hash = Hash[options]
|
2799
2826
|
|
2800
2827
|
if local
|
@@ -2829,7 +2856,8 @@ module Cnvrg
|
|
2829
2856
|
:image => image, :grid => grid, :data => data, :data_commit => data_commit, :ignore => ignore, :force => force, :sync_before_terminate => sync_before_terminate,
|
2830
2857
|
:max_time => max_time,
|
2831
2858
|
:periodic_sync => periodic_sync, :dataset_only_tree=> dataset_only_tree,
|
2832
|
-
:output_dir=>output_dir, :data_query=>data_query, :git_commit =>git_commit, :git_branch=> git_branch,
|
2859
|
+
:output_dir=>output_dir, :data_query=>data_query, :git_commit =>git_commit, :git_branch=> git_branch,
|
2860
|
+
:restart_if_stuck =>restart_if_stuck, :local_folders => local_folders, :datasets => datasets
|
2833
2861
|
return
|
2834
2862
|
end
|
2835
2863
|
|
@@ -3166,6 +3194,7 @@ module Cnvrg
|
|
3166
3194
|
method_option :git_branch, :type => :string, :aliases => [ "--git_branch"], :default => nil
|
3167
3195
|
method_option :restart_if_stuck, :type => :boolean, :aliases => ["--restart"], :default => nil
|
3168
3196
|
method_option :local_folders, :type => :string, :aliases => ["--local_folders"], :default => nil
|
3197
|
+
method_option :datasets, :type => :string, :aliases => ["--datasets"], :default => nil
|
3169
3198
|
|
3170
3199
|
def exec_remote(*cmd)
|
3171
3200
|
|
@@ -3178,6 +3207,7 @@ module Cnvrg
|
|
3178
3207
|
title = options["title"] || nil
|
3179
3208
|
grid = options["grid"] || nil
|
3180
3209
|
data = options["data"] || nil
|
3210
|
+
datasets = options["datasets"] || nil
|
3181
3211
|
data_commit = options["data_commit"] || nil
|
3182
3212
|
data_query = options["data_query"] || nil
|
3183
3213
|
sync_before = options["sync_before"]
|
@@ -3237,7 +3267,7 @@ module Cnvrg
|
|
3237
3267
|
local_folders_options = options["local_folders"]
|
3238
3268
|
options_hash.except!("schedule", "machine_type", "image", "upload_output", "grid", "data", "data_commit", "title",
|
3239
3269
|
"local", "small", "medium", "large", "gpu", "gpuxl", "gpuxxl","max_time","dataset_only_tree",
|
3240
|
-
"data_query", "git_commit","git_branch", "restart_if_stuck","local_folders",
|
3270
|
+
"data_query", "git_commit","git_branch", "restart_if_stuck","local_folders","output_dir", "commit", "datasets" )
|
3241
3271
|
exec_options = options_hash.map {|x| "--#{x[0]}=#{x[1]}"}.flatten.join(" ")
|
3242
3272
|
command = "#{exec_options} #{remote} #{upload_output_option} #{cmd.flatten.join(" ")}"
|
3243
3273
|
commit_to_run = options["commit"] || nil
|
@@ -3308,7 +3338,7 @@ module Cnvrg
|
|
3308
3338
|
|
3309
3339
|
res = exp.exec_remote(command, commit_to_run, instance_type, image, schedule, local_timestamp, grid, path_to_cmd, data, data_commit,
|
3310
3340
|
periodic_sync, sync_before_terminate, max_time, ds_sync_options,output_dir,
|
3311
|
-
data_query, git_commit, git_branch, restart_if_stuck,local_folders_options, title )
|
3341
|
+
data_query, git_commit, git_branch, restart_if_stuck,local_folders_options, title, datasets)
|
3312
3342
|
if Cnvrg::CLI.is_response_success(res)
|
3313
3343
|
check = Helpers.checkmark()
|
3314
3344
|
str = "#{check} Experiment's is on: #{Cnvrg::Helpers.remote_url}/#{project.owner}/projects/#{project.slug}/experiments/#{res["result"]["exp_url"]}"
|
@@ -3368,11 +3398,8 @@ module Cnvrg
|
|
3368
3398
|
project = Project.new(working_dir)
|
3369
3399
|
commit_to_run = options["commit"] || nil
|
3370
3400
|
|
3371
|
-
workers = options["workers"]
|
3401
|
+
workers = options["workers"] || nil
|
3372
3402
|
begin
|
3373
|
-
if workers.blank?
|
3374
|
-
workers = 2
|
3375
|
-
end
|
3376
3403
|
num_workers = workers.to_i
|
3377
3404
|
rescue
|
3378
3405
|
log_message("Number of workers should be a number between 1 to 10", Thor::Shell::Color::RED)
|
@@ -3381,10 +3408,18 @@ module Cnvrg
|
|
3381
3408
|
file_as_input = options["file_as_input"] || false
|
3382
3409
|
|
3383
3410
|
|
3411
|
+
image = is_project_with_docker(working_dir)
|
3412
|
+
if !image or !image.is_docker
|
3413
|
+
image = Images.new(working_dir, "cnvrg")
|
3414
|
+
image_slug = image.image_slug
|
3415
|
+
else
|
3416
|
+
image_slug = image.image_slug
|
3417
|
+
end
|
3418
|
+
|
3384
3419
|
|
3385
3420
|
invoke :sync, [false], []
|
3386
3421
|
|
3387
|
-
res = project.deploy(file_to_run, function, nil, commit_to_run, instance_type,
|
3422
|
+
res = project.deploy(file_to_run, function, nil, commit_to_run, instance_type, image_slug, schedule, local_timestamp, num_workers, file_as_input, title)
|
3388
3423
|
|
3389
3424
|
if Cnvrg::CLI.is_response_success(res)
|
3390
3425
|
|
@@ -3416,6 +3451,7 @@ module Cnvrg
|
|
3416
3451
|
method_option :local, :type => :boolean, :aliases => ["-l"], :default => false
|
3417
3452
|
method_option :small, :type => :boolean, :aliases => ["-sm", "--small"], :default => false
|
3418
3453
|
method_option :medium, :type => :boolean, :aliases => ["-md", "--medium"], :default => false
|
3454
|
+
method_option :datasets, :type => :string, :aliases => ["--datasets"], :desc => "'[{\"id\": \"dataset id\", \"commit\": \"commit id\", \"query\": \"query name\", \"tree_only\": true]'", :default => ""
|
3419
3455
|
method_option :large, :type => :boolean, :aliases => ["-lg", "--large"], :default => false
|
3420
3456
|
method_option :gpu, :type => :boolean, :aliases => ["--gpu"], :default => false
|
3421
3457
|
method_option :gpuxl, :type => :boolean, :aliases => ["--gpuxl"], :default => false
|
@@ -3433,6 +3469,7 @@ module Cnvrg
|
|
3433
3469
|
log_start(__method__, args, options)
|
3434
3470
|
local = options["local"]
|
3435
3471
|
notebook_dir = options["notebook_dir"]
|
3472
|
+
datasets = options["datasets"]
|
3436
3473
|
kernel = options["kernel"]
|
3437
3474
|
image = options["image"] || nil
|
3438
3475
|
data = options["data"]
|
@@ -3457,7 +3494,7 @@ module Cnvrg
|
|
3457
3494
|
instance_type = get_instance_type(instances)
|
3458
3495
|
|
3459
3496
|
invoke :remote_notebook, [], :notebook_dir => notebook_dir, :kernel => kernel, :machine_type => instance_type, :image => image,
|
3460
|
-
:data => data, :data_commit => data_commit , :dataset_only_tree => dataset_only_tree, :data_query => data_query
|
3497
|
+
:data => data, :data_commit => data_commit , :dataset_only_tree => dataset_only_tree, :data_query => data_query, :datasets => datasets
|
3461
3498
|
return
|
3462
3499
|
|
3463
3500
|
end
|
@@ -3562,6 +3599,7 @@ module Cnvrg
|
|
3562
3599
|
method_option :dataset_only_tree, :type => :boolean, :aliases => [ "--dataset_only_tree"], :default => false
|
3563
3600
|
method_option :data_query, :type => :string, :aliases => ["-q","--data_query"], :default => ""
|
3564
3601
|
method_option :image, :type => :string, :aliases => ["--image"], :default => nil
|
3602
|
+
method_option :datasets, :type => :string, :aliases => ["--datasets"], :desc => "'[{\"id\": \"dataset id\", \"commit\": \"commit id\", \"query\": \"query name\", \"tree_only\": true]'", :default => ""
|
3565
3603
|
|
3566
3604
|
def remote_notebook()
|
3567
3605
|
verify_logged_in(true)
|
@@ -3569,6 +3607,7 @@ module Cnvrg
|
|
3569
3607
|
|
3570
3608
|
working_dir = is_cnvrg_dir()
|
3571
3609
|
instance_type = options["machine_type"] || nil
|
3610
|
+
datasets = options["datasets"]
|
3572
3611
|
data = options["data"]
|
3573
3612
|
data_commit = options["data_commit"]
|
3574
3613
|
commit = options["commit"]
|
@@ -3599,7 +3638,7 @@ module Cnvrg
|
|
3599
3638
|
end
|
3600
3639
|
invoke :sync, [false], []
|
3601
3640
|
slug = ""
|
3602
|
-
res = exp.remote_notebook(instance_type, commit, data, data_commit, notebook_type,ds_sync_options,data_query, image)
|
3641
|
+
res = exp.remote_notebook(instance_type, commit, data, data_commit, notebook_type,ds_sync_options,data_query, image, datasets)
|
3603
3642
|
if Cnvrg::CLI.is_response_success(res)
|
3604
3643
|
slug = res["result"]["notebook_url"]
|
3605
3644
|
log_message("#{Helpers.checkmark} Notebook is ready: #{Cnvrg::Helpers.remote_url}/#{project.owner}/projects/#{project.slug}/notebook_sessions/show/#{slug}", Thor::Shell::Color::GREEN)
|
data/lib/cnvrg/data.rb
CHANGED
@@ -82,6 +82,7 @@ module Cnvrg
|
|
82
82
|
method_option :commit, :type => :string, :aliases => ["-c", "--commit"], :default => nil
|
83
83
|
method_option :query, :type => :string, :aliases => ["-q", "--query"], :default => nil
|
84
84
|
method_option :read, :type => :boolean, :aliases => ["-r", "--read"], :default => false
|
85
|
+
method_option :remote, :type => :boolean, :aliases => ["-h", "--remote"], :default => false
|
85
86
|
|
86
87
|
def clone(dataset_url)
|
87
88
|
cli = Cnvrg::CLI.new()
|
@@ -89,7 +90,17 @@ module Cnvrg
|
|
89
90
|
commit =options[:commit]
|
90
91
|
query =options[:query]
|
91
92
|
read = options[:read]
|
92
|
-
|
93
|
+
remote = options[:remote]
|
94
|
+
cli.clone_data(dataset_url, only_tree=only_tree,commit=commit, query=query, read=read, remote=remote)
|
95
|
+
end
|
96
|
+
|
97
|
+
desc 'data verify', 'verify datasets', :hide => true
|
98
|
+
method_option :timeout, :type => :numeric, :aliases => ["-t", "--timeout"], :desc => "Time to wait before returning final answer", :default => 15
|
99
|
+
|
100
|
+
def verify(*dataset_titles)
|
101
|
+
cli = Cnvrg::CLI.new()
|
102
|
+
timeout =options[:timeout]
|
103
|
+
cli.verify_datasets(dataset_titles, timeout)
|
93
104
|
end
|
94
105
|
|
95
106
|
desc 'data set --url=dataset_url', 'Set dataset url to other url'
|
data/lib/cnvrg/dataset.rb
CHANGED
@@ -250,6 +250,28 @@ module Cnvrg
|
|
250
250
|
return true
|
251
251
|
end
|
252
252
|
|
253
|
+
def self.verify_datasets(dataset_titles, timeout=100)
|
254
|
+
start_time = Time.now.to_i
|
255
|
+
Cnvrg::Logger.log_info("Verifying datasets #{dataset_titles}")
|
256
|
+
Cnvrg::Logger.log_info("Timeout is #{timeout}")
|
257
|
+
while true
|
258
|
+
begin
|
259
|
+
current_time = Time.now.to_i
|
260
|
+
return false if timeout < current_time - start_time
|
261
|
+
all_are_ready = dataset_titles.all? do |dataset_title|
|
262
|
+
config = YAML.load_file("/data/#{dataset_title}/.cnvrg/config.yml")
|
263
|
+
config[:success] == true
|
264
|
+
end
|
265
|
+
return true if all_are_ready
|
266
|
+
Cnvrg::Logger.log_info("Sleeping..")
|
267
|
+
sleep 10
|
268
|
+
rescue => e
|
269
|
+
Cnvrg::Logger.log_info("Got error")
|
270
|
+
Cnvrg::Logger.log_error(e)
|
271
|
+
sleep 10
|
272
|
+
end
|
273
|
+
end
|
274
|
+
end
|
253
275
|
|
254
276
|
def clone( commit)
|
255
277
|
|
@@ -257,11 +279,14 @@ module Cnvrg
|
|
257
279
|
return response
|
258
280
|
end
|
259
281
|
|
260
|
-
def self.clone(owner, dataset_name, dataset_slug)
|
282
|
+
def self.clone(owner, dataset_name, dataset_slug, remote=false)
|
261
283
|
|
262
284
|
begin
|
263
|
-
list_dirs = [
|
264
|
-
|
285
|
+
list_dirs = []
|
286
|
+
if !remote
|
287
|
+
list_dirs << dataset_name
|
288
|
+
end
|
289
|
+
list_dirs << "#{dataset_name}/.cnvrg"
|
265
290
|
list_files = [
|
266
291
|
"#{dataset_name}/.cnvrg/config.yml"
|
267
292
|
]
|
@@ -281,6 +306,15 @@ module Cnvrg
|
|
281
306
|
return true
|
282
307
|
end
|
283
308
|
|
309
|
+
def write_success(in_folder = false)
|
310
|
+
file_path = ".cnvrg/config.yml"
|
311
|
+
file_path = "#{@slug}/" + file_path if !in_folder
|
312
|
+
if File.exist?(file_path)
|
313
|
+
File.open(file_path, "a") { |f| f.puts(":success: true") }
|
314
|
+
end
|
315
|
+
rescue
|
316
|
+
end
|
317
|
+
|
284
318
|
def self.init_container(owner, dataset_slug,dataset_name)
|
285
319
|
|
286
320
|
cnvrgignore = Helpers.cnvrgignore_content
|
data/lib/cnvrg/experiment.rb
CHANGED
@@ -90,12 +90,13 @@ module Cnvrg
|
|
90
90
|
|
91
91
|
def exec_remote(command, commit_to_run, instance_type, image_slug,scheduling_query,local_timestamp, grid,path_to_cmd,data, data_commit,periodic_sync,
|
92
92
|
sync_before_terminate, max_time, ds_sync_options=0,output_dir=nil,data_query=nil,
|
93
|
-
git_commit=nil, git_branch=nil, restart_if_stuck=nil, local_folders=nil,title=nil)
|
93
|
+
git_commit=nil, git_branch=nil, restart_if_stuck=nil, local_folders=nil,title=nil, datasets=nil)
|
94
94
|
response = Cnvrg::API.request("users/#{@owner}/projects/#{@project_slug}/experiment/remote", 'POST', {command: command, image_slug: image_slug,
|
95
95
|
commit_sha1: commit_to_run,
|
96
96
|
instance_type: instance_type,
|
97
97
|
scheduling_query:scheduling_query,
|
98
98
|
local_timestamp:local_timestamp,
|
99
|
+
datasets: datasets,
|
99
100
|
grid: grid,
|
100
101
|
path_to_cmd:path_to_cmd,dataset_slug:data,
|
101
102
|
dataset_commit: data_commit,max_time:max_time,
|
@@ -106,9 +107,10 @@ module Cnvrg
|
|
106
107
|
|
107
108
|
return response
|
108
109
|
end
|
109
|
-
def remote_notebook(instance_type, commit, data, data_commit, notebook_type,ds_sync_options=0,data_query=nil, image = nil)
|
110
|
+
def remote_notebook(instance_type, commit, data, data_commit, notebook_type,ds_sync_options=0,data_query=nil, image = nil, datasets = nil)
|
110
111
|
response = Cnvrg::API.request("users/#{@owner}/projects/#{@project_slug}/notebook/remote", 'POST', {instance_type: instance_type,dataset_slug:data,
|
111
112
|
dataset_commit: data_commit,image_slug:image,
|
113
|
+
datasets: datasets,
|
112
114
|
commit:commit,notebook_type:notebook_type,dataset_sync_options:ds_sync_options,
|
113
115
|
dataset_query:data_query})
|
114
116
|
return response
|
data/lib/cnvrg/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: cnvrg
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0
|
4
|
+
version: '1.0'
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Yochay Ettun
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2019-01-
|
12
|
+
date: 2019-01-13 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: bundler
|