cnvrg 0.0.1530000 → 0.0.1540000
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/cnvrg.gemspec +2 -0
- data/lib/cnvrg/api.rb +7 -4
- data/lib/cnvrg/cli.rb +796 -619
- data/lib/cnvrg/datafiles.rb +16 -2
- data/lib/cnvrg/files.rb +31 -6
- data/lib/cnvrg/helpers.rb +7 -0
- data/lib/cnvrg/project.rb +0 -1
- data/lib/cnvrg/version.rb +1 -3
- metadata +18 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 021cf92b7a10351f3e2b5dffb061acbba7e335e2
|
4
|
+
data.tar.gz: 087dac2c6aacf82f7ca15c9e51f4858e7cfd1035
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 34b286d7d0c4e9c3d93acc09c3f857674d0b75f575730bfa7bae48b8fa5396cf631714b53bb7c99b48d5e7603925574011cef44801e235e208b2992fcf9ef2bb
|
7
|
+
data.tar.gz: 44dacf68e883b199b0ceabc6921a5c1d853155942d8ad5dadb207effeb4e7096ae54e61edf819ca2385a5142b1b49a274648da7ab259faac5da721d4dd154e1c
|
data/cnvrg.gemspec
CHANGED
@@ -34,6 +34,8 @@ Gem::Specification.new do |spec|
|
|
34
34
|
spec.add_runtime_dependency 'aws-sdk'
|
35
35
|
spec.add_runtime_dependency 'sucker_punch', '~> 2.0'
|
36
36
|
spec.add_runtime_dependency 'urlcrypt', '~> 0.1.1'
|
37
|
+
spec.add_runtime_dependency 'parallel', '~> 1.12.0'
|
38
|
+
|
37
39
|
spec.add_runtime_dependency 'logstash-logger', '~> 0.22.1'
|
38
40
|
spec.add_runtime_dependency 'launchy', '~> 2.4'
|
39
41
|
spec.add_runtime_dependency 'docker-api', '~> 1.33'
|
data/lib/cnvrg/api.rb
CHANGED
@@ -81,14 +81,17 @@ module Cnvrg
|
|
81
81
|
|
82
82
|
# what if windows?
|
83
83
|
# data[:file] = Faraday::UploadIO.new(data[:absolute_path], content_type)
|
84
|
-
|
85
|
-
|
86
|
-
|
84
|
+
file_base = File.basename(data[:relative_path])
|
85
|
+
temp_path = File.expand_path('~')+"/.cnvrg/tmp_files/#{file_base}"
|
86
|
+
FileUtils.touch(temp_path)
|
87
|
+
|
88
|
+
|
87
89
|
data[:file] = Faraday::UploadIO.new("#{temp_path}", "plain/text")
|
88
90
|
|
89
91
|
response = conn.post "#{endpoint_uri}/#{resource}", data
|
90
92
|
|
91
|
-
FileUtils.
|
93
|
+
FileUtils.rm_rf(temp_path)
|
94
|
+
|
92
95
|
|
93
96
|
if parse_request == true
|
94
97
|
JSON.parse(response.body)
|
data/lib/cnvrg/cli.rb
CHANGED
@@ -1,5 +1,7 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
2
|
require "pty"
|
3
|
+
require 'etc'
|
4
|
+
require 'parallel'
|
3
5
|
require "open4"
|
4
6
|
require 'netrc'
|
5
7
|
require 'net/http'
|
@@ -22,7 +24,6 @@ require 'cnvrg/dataset'
|
|
22
24
|
require 'cnvrg/datafiles'
|
23
25
|
require 'cnvrg/data'
|
24
26
|
require 'cnvrg/ssh'
|
25
|
-
require 'etc'
|
26
27
|
require 'logstash-logger'
|
27
28
|
require 'cnvrg/job'
|
28
29
|
require 'docker'
|
@@ -35,6 +36,7 @@ require 'active_support/all'
|
|
35
36
|
require 'thor'
|
36
37
|
require 'pathname'
|
37
38
|
require 'enumerator'
|
39
|
+
require 'ruby-progressbar'
|
38
40
|
|
39
41
|
class Thor
|
40
42
|
module Base
|
@@ -143,6 +145,10 @@ module Cnvrg
|
|
143
145
|
INSTALLATION_URLS = {docker: "https://docs.docker.com/engine/installation/", jupyter: "http://jupyter.readthedocs.io/en/latest/install.html"}
|
144
146
|
IP="localhost"
|
145
147
|
PORT=7654
|
148
|
+
|
149
|
+
ParallelThreads ||= 10
|
150
|
+
ParallelProcesses ||= Parallel.processor_count
|
151
|
+
|
146
152
|
class << self
|
147
153
|
# Hackery.Take the run method away from Thor so that we can redefine it.
|
148
154
|
def is_thor_reserved_word?(word, type)
|
@@ -203,11 +209,11 @@ module Cnvrg
|
|
203
209
|
end
|
204
210
|
config = YAML.load_file(home_dir+"/.cnvrg/config.yml")
|
205
211
|
owner = config.to_h[:owner]
|
206
|
-
compression_path =
|
212
|
+
compression_path = "#{File.expand_path('~')}/.cnvrg/tmp"
|
207
213
|
|
208
214
|
say "Setting default api to be: #{url}", Thor::Shell::Color::BLUE
|
209
215
|
if config.empty?
|
210
|
-
config = {owner: "", username: "", version_last_check: get_start_day(), api: url,compression_path: compression_path
|
216
|
+
config = {owner: "", username: "", version_last_check: get_start_day(), api: url, compression_path: compression_path}
|
211
217
|
else
|
212
218
|
if !config.to_h[:compression_path].nil?
|
213
219
|
compression_path = config.to_h[:compression_path]
|
@@ -215,7 +221,7 @@ module Cnvrg
|
|
215
221
|
config = {owner: config.to_h[:owner], username: config.to_h[:username], version_last_check: config.to_h[:version_last_check], api: url, compression_path: compression_path}
|
216
222
|
end
|
217
223
|
res = Cnvrg::API.request("/users/#{owner}/custom_api", 'POST', {custom_api: url})
|
218
|
-
if Cnvrg::CLI.is_response_success(res,false)
|
224
|
+
if Cnvrg::CLI.is_response_success(res, false)
|
219
225
|
|
220
226
|
checks = Helpers.checkmark
|
221
227
|
|
@@ -249,8 +255,8 @@ module Cnvrg
|
|
249
255
|
end
|
250
256
|
config = YAML.load_file(home_dir+"/.cnvrg/config.yml")
|
251
257
|
|
252
|
-
compression_path =
|
253
|
-
config = {owner: owner, username: current_user, version_last_check: get_start_day(), api: url,compression_path:compression_path}
|
258
|
+
compression_path = "#{home_dir}/.cnvrg/tmp"
|
259
|
+
config = {owner: owner, username: current_user, version_last_check: get_start_day(), api: url, compression_path: compression_path}
|
254
260
|
File.open(home_dir+"/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
|
255
261
|
say "Done"
|
256
262
|
rescue
|
@@ -326,12 +332,12 @@ module Cnvrg
|
|
326
332
|
end
|
327
333
|
|
328
334
|
desc 'set_compression_path', 'set compression path'
|
329
|
-
method_option :reset, :type => :boolean, :aliases => ["-r","--reset"], :default => false
|
335
|
+
method_option :reset, :type => :boolean, :aliases => ["-r", "--reset"], :default => false
|
330
336
|
|
331
337
|
def set_compression_path(*compression_path)
|
332
338
|
begin
|
333
339
|
if (compression_path.nil? or compression_path.empty?) and options["reset"]
|
334
|
-
compression_path =
|
340
|
+
compression_path = ["#{File.expand_path('~')}/.cnvrg/tmp"]
|
335
341
|
end
|
336
342
|
compression_path = compression_path.join(" ")
|
337
343
|
if !Dir.exist? compression_path
|
@@ -348,7 +354,7 @@ module Cnvrg
|
|
348
354
|
end
|
349
355
|
config = YAML.load_file(path)
|
350
356
|
config_new = {owner: config.to_h[:owner], username: config.to_h[:username],
|
351
|
-
|
357
|
+
version_last_check: config.to_h[:version_last_check], api: config.to_h[:api], compression_path: compression_path}
|
352
358
|
File.open(home_dir+"/.cnvrg/config.yml", "w+") { |f| f.write config_new.to_yaml }
|
353
359
|
checks = Helpers.checkmark
|
354
360
|
say "#{checks} Done", Thor::Shell::Color::GREEN
|
@@ -360,11 +366,13 @@ module Cnvrg
|
|
360
366
|
end
|
361
367
|
|
362
368
|
|
363
|
-
|
364
369
|
desc 'login', 'Authenticate with cnvrg.io platform'
|
365
370
|
|
366
371
|
def login
|
367
372
|
begin
|
373
|
+
log_handler()
|
374
|
+
log_start(__method__, args, options)
|
375
|
+
|
368
376
|
cmd = HighLine.new
|
369
377
|
|
370
378
|
say 'Authenticating with cnvrg', Thor::Shell::Color::YELLOW
|
@@ -374,7 +382,7 @@ module Cnvrg
|
|
374
382
|
@email, token = netrc[Cnvrg::Helpers.netrc_domain]
|
375
383
|
|
376
384
|
if @email and token
|
377
|
-
|
385
|
+
log_message('Seems you\'re already logged in', Thor::Shell::Color::BLUE)
|
378
386
|
exit(0)
|
379
387
|
end
|
380
388
|
@email = ask("Enter your email:")
|
@@ -385,7 +393,8 @@ module Cnvrg
|
|
385
393
|
netrc[Cnvrg::Helpers.netrc_domain] = @email, result["token"]
|
386
394
|
netrc.save
|
387
395
|
|
388
|
-
|
396
|
+
log_message("Authenticated successfully as #{@email}", Thor::Shell::Color::GREEN)
|
397
|
+
|
389
398
|
owners = result["owners"]
|
390
399
|
urls = result["urls"]
|
391
400
|
choose_owner = result["username"]
|
@@ -431,22 +440,24 @@ module Cnvrg
|
|
431
440
|
# end
|
432
441
|
|
433
442
|
if set_owner(choose_owner, result["username"], urls[ow_index])
|
434
|
-
|
443
|
+
log_message("Setting default owner: #{choose_owner}", Thor::Shell::Color::GREEN)
|
435
444
|
|
436
445
|
else
|
437
|
-
|
446
|
+
log_message("Setting default owenr has failed, logging out", Thor::Shell::Color::RED)
|
438
447
|
|
439
448
|
return logout()
|
440
449
|
end
|
441
450
|
|
442
451
|
else
|
443
|
-
|
452
|
+
log_message("Failed to authenticate, wrong email/password", Thor::Shell::Color::RED)
|
444
453
|
|
445
454
|
exit(1)
|
446
455
|
end
|
447
|
-
rescue
|
456
|
+
rescue => e
|
457
|
+
|
458
|
+
log_message("Error Occurred, aborting", Thor::Shell::Color::RED)
|
459
|
+
log_error(e)
|
448
460
|
|
449
|
-
say "Error Occurred, aborting", Thor::Shell::Color::RED
|
450
461
|
logout()
|
451
462
|
exit(1)
|
452
463
|
rescue SignalException
|
@@ -462,10 +473,12 @@ module Cnvrg
|
|
462
473
|
|
463
474
|
def logout
|
464
475
|
begin
|
476
|
+
log_handler()
|
477
|
+
log_start(__method__, args, options)
|
465
478
|
netrc = Netrc.read
|
466
479
|
netrc.delete(Cnvrg::Helpers.netrc_domain)
|
467
480
|
netrc.save
|
468
|
-
|
481
|
+
log_message("Logged out successfully.\n", Thor::Shell::Color::GREEN)
|
469
482
|
rescue SignalException
|
470
483
|
say "\nAborting"
|
471
484
|
exit(1)
|
@@ -482,16 +495,13 @@ module Cnvrg
|
|
482
495
|
log_start(__method__, args, options)
|
483
496
|
auth = Cnvrg::Auth.new
|
484
497
|
if (email = auth.get_email)
|
485
|
-
|
498
|
+
log_message("Logged in as: #{email}", Thor::Shell::Color::GREEN)
|
486
499
|
else
|
487
|
-
|
500
|
+
log_message("You're not logged in.", Thor::Shell::Color::RED)
|
488
501
|
end
|
489
502
|
|
490
|
-
log_end(0)
|
491
503
|
|
492
504
|
rescue SignalException
|
493
|
-
log_end(-1)
|
494
|
-
|
495
505
|
say "\nAborting"
|
496
506
|
exit(1)
|
497
507
|
end
|
@@ -513,10 +523,9 @@ module Cnvrg
|
|
513
523
|
if !docker_image.nil? and !docker_image.empty?
|
514
524
|
docker = true
|
515
525
|
end
|
516
|
-
|
526
|
+
log_message("Creating #{project_name}", Thor::Shell::Color::BLUE)
|
517
527
|
if Dir.exists? project_name or File.exists? project_name
|
518
|
-
|
519
|
-
log_end(1, "conflict with dir/file #{project_name}")
|
528
|
+
log_message("Conflict with dir/file #{project_name}", Thor::Shell::Color::RED)
|
520
529
|
exit(1)
|
521
530
|
|
522
531
|
end
|
@@ -532,24 +541,24 @@ module Cnvrg
|
|
532
541
|
if yes? "Image wasn't found locally, pull image from cnvrg repository?", Thor::Shell::Color::YELLOW
|
533
542
|
image = pull(docker_image)
|
534
543
|
if image
|
535
|
-
|
544
|
+
log_message("downloaded image: #{docker_image}", Thor::Shell::Color::BLUE)
|
536
545
|
@image = Images.new(working_dir, docker_image)
|
537
546
|
else
|
538
|
-
|
547
|
+
log_message("Could not create a new project with docker, image was not found", Thor::Shell::Color::RED)
|
539
548
|
@project.revert(working_dir)
|
540
549
|
exit(1)
|
541
550
|
end
|
542
551
|
else
|
543
|
-
|
552
|
+
log_message("Could not create a new project with docker, image was not found", Thor::Shell::Color::RED)
|
544
553
|
@project.revert(working_dir)
|
545
554
|
exit(1)
|
546
555
|
|
547
556
|
end
|
548
557
|
elsif docker_image_local.size == 1
|
549
|
-
|
558
|
+
log_message("found image: #{docker_image_local[0]}, setting it up..", Thor::Shell::Color::BLUE)
|
550
559
|
@image = Images.new(working_dir, docker_image_local[0])
|
551
560
|
elsif docker_image_local.size >1
|
552
|
-
|
561
|
+
log_message("found #{docker_image_local.size} images, choose the image name you want to use", Thor::Shell::Color::BLUE)
|
553
562
|
image_name = ask "#{docker_image_local.join("\n")}\n", Thor::Shell::Color::BLUE
|
554
563
|
image_name = image_name.strip
|
555
564
|
@image = Images.new(working_dir, image_name)
|
@@ -558,18 +567,17 @@ module Cnvrg
|
|
558
567
|
|
559
568
|
end
|
560
569
|
else
|
561
|
-
|
570
|
+
log_message("Error creating project, please contact support.", Thor::Shell::Color::RED)
|
562
571
|
@project.revert(working_dir)
|
563
|
-
log_end(1, "can't create project #{project_name}")
|
564
572
|
|
565
573
|
exit(0)
|
566
574
|
end
|
567
575
|
|
568
|
-
|
569
|
-
|
576
|
+
log_message("created project successfully", Thor::Shell::Color::GREEN)
|
577
|
+
log_message("Linked directory to\t#{@project.url}", Thor::Shell::Color::GREEN)
|
570
578
|
rescue => e
|
571
|
-
|
572
|
-
|
579
|
+
log_message("Error occurred, aborting", Thor::Shell::Color::RED)
|
580
|
+
log_error(e)
|
573
581
|
if Dir.exist? working_dir
|
574
582
|
|
575
583
|
@project.revert(working_dir)
|
@@ -642,11 +650,10 @@ module Cnvrg
|
|
642
650
|
|
643
651
|
sync = options["sync"]
|
644
652
|
project_name =File.basename(Dir.getwd)
|
645
|
-
|
653
|
+
log_message("Linking #{project_name}", Thor::Shell::Color::BLUE)
|
646
654
|
if File.directory?(Dir.getwd+"/.cnvrg")
|
647
655
|
config = YAML.load_file("#{Dir.getwd}/.cnvrg/config.yml")
|
648
|
-
|
649
|
-
|
656
|
+
log_message("Directory is already linked to #{config[:project_slug]}", Thor::Shell::Color::RED)
|
650
657
|
exit(0)
|
651
658
|
end
|
652
659
|
working_dir = Dir.getwd
|
@@ -662,47 +669,46 @@ module Cnvrg
|
|
662
669
|
if yes? "Image wasn't found locally, pull image from cnvrg repository?", Thor::Shell::Color::YELLOW
|
663
670
|
image = pull(docker_image)
|
664
671
|
if image
|
665
|
-
|
672
|
+
log_message("downloaded image: #{docker_image}", Thor::Shell::Color::BLUE)
|
666
673
|
@image = Images.new(working_dir, docker_image)
|
667
674
|
else
|
668
|
-
|
675
|
+
log_message("Could not create a new project with docker, image was not found", Thor::Shell::Color::RED)
|
669
676
|
@project.revert(working_dir)
|
670
677
|
exit(1)
|
671
678
|
end
|
672
679
|
else
|
673
|
-
|
680
|
+
log_message("Could not create a new project with docker, image was not found", Thor::Shell::Color::RED)
|
674
681
|
@project.revert(working_dir)
|
675
682
|
exit(1)
|
676
683
|
|
677
684
|
end
|
678
685
|
elsif docker_image_local.size == 1
|
679
|
-
|
686
|
+
log_message("found image: #{docker_image_local[0]}, setting it up..", Thor::Shell::Color::BLUE)
|
680
687
|
@image = Images.new(working_dir, docker_image_local[0])
|
681
688
|
elsif docker_image_local.size >1
|
682
|
-
|
689
|
+
log_message("found #{docker_image_local.size} images, choose the image name you want to use", Thor::Shell::Color::BLUE)
|
683
690
|
image_name = ask "#{docker_image_local.join("\n")}\n", Thor::Shell::Color::BLUE
|
684
691
|
@image = Images.new(working_dir, image_name)
|
685
692
|
end
|
686
693
|
end
|
687
694
|
if sync
|
688
|
-
|
695
|
+
log_message("Syncing project", Thor::Shell::Color::BLUE)
|
689
696
|
upload(true)
|
690
697
|
end
|
691
698
|
|
692
699
|
url = @project.url
|
693
700
|
check = Helpers.checkmark
|
694
|
-
|
695
|
-
|
696
|
-
log_end(0)
|
701
|
+
log_message("#{check} Link finished successfully", Thor::Shell::Color::GREEN)
|
702
|
+
log_message("#{project_name}'s location is: #{url}\n", Thor::Shell::Color::GREEN)
|
697
703
|
|
698
704
|
else
|
699
|
-
log_end(1, "can't link project")
|
700
705
|
@project.revert(working_dir) unless @project.nil?
|
701
|
-
|
706
|
+
log_message("Error linking project, please contact support.", Thor::Shell::Color::RED)
|
702
707
|
exit(0)
|
703
708
|
end
|
709
|
+
rescue => e
|
710
|
+
log_error(e)
|
704
711
|
rescue SignalException
|
705
|
-
log_end(-1)
|
706
712
|
|
707
713
|
say "\nAborting"
|
708
714
|
exit(1)
|
@@ -719,11 +725,11 @@ module Cnvrg
|
|
719
725
|
dataset_name =File.basename(Dir.getwd)
|
720
726
|
if File.directory?(Dir.getwd+"/.cnvrg")
|
721
727
|
config = YAML.load_file("#{Dir.getwd}/.cnvrg/config.yml")
|
722
|
-
|
728
|
+
log_message("Directory is already linked to #{config[:dataset_slug]}", Thor::Shell::Color::RED)
|
723
729
|
|
724
730
|
exit(0)
|
725
731
|
end
|
726
|
-
|
732
|
+
log_message("Init dataset: #{dataset_name}", Thor::Shell::Color::BLUE)
|
727
733
|
|
728
734
|
working_dir = Dir.getwd
|
729
735
|
owner = CLI.get_owner
|
@@ -733,19 +739,17 @@ module Cnvrg
|
|
733
739
|
|
734
740
|
url = @dataset.url
|
735
741
|
check = Helpers.checkmark
|
736
|
-
|
737
|
-
|
738
|
-
log_end(0)
|
742
|
+
log_message("#{check} Link finished successfully", Thor::Shell::Color::GREEN)
|
743
|
+
log_message("#{dataset_name}'s location is: #{url}\n", Thor::Shell::Color::GREEN)
|
739
744
|
|
740
745
|
else
|
741
|
-
log_end(1, "can't create dataset")
|
742
746
|
@dataset.revert(working_dir) unless @dataset.nil?
|
743
|
-
|
747
|
+
log_message("Error creating dataset, please contact support.", Thor::Shell::Color::RED)
|
744
748
|
exit(0)
|
745
749
|
end
|
750
|
+
rescue => e
|
751
|
+
log_error(e)
|
746
752
|
rescue SignalException
|
747
|
-
log_end(-1)
|
748
|
-
|
749
753
|
say "\nAborting"
|
750
754
|
exit(1)
|
751
755
|
end
|
@@ -767,17 +771,15 @@ module Cnvrg
|
|
767
771
|
Cnvrg::CLI.is_response_success(response)
|
768
772
|
dataset_name = response["result"]["name"]
|
769
773
|
if (Dir.exists? dataset_name)
|
770
|
-
|
774
|
+
log_messgae("Error: Conflict with dir #{dataset_name}", Thor::Shell::Color::RED)
|
771
775
|
if no? "Sync to repository anyway? (current data might lost)", Thor::Shell::Color::YELLOW
|
772
|
-
|
773
|
-
log_end(1, "conflict with dir #{dataset_name}")
|
774
|
-
|
776
|
+
log_message("Remove dir in order to clone #{dataset_name}", Thor::Shell::Color::RED)
|
775
777
|
exit(1)
|
776
778
|
end
|
777
779
|
|
778
780
|
end
|
779
781
|
if Dataset.clone(owner, dataset_name, slug)
|
780
|
-
|
782
|
+
log_message("Cloning #{dataset_name}", Thor::Shell::Color::BLUE)
|
781
783
|
|
782
784
|
commit_to_clone = options["commit"] || nil
|
783
785
|
working_dir = "#{Dir.pwd}/#{dataset_name}"
|
@@ -785,21 +787,23 @@ module Cnvrg
|
|
785
787
|
@dataset.generate_idx()
|
786
788
|
|
787
789
|
|
788
|
-
download_data(false, false, path = working_dir, in_dir=false)
|
790
|
+
download_res = download_data(false, false, path = working_dir, in_dir=false)
|
791
|
+
if !download_res
|
792
|
+
exit(1)
|
793
|
+
end
|
789
794
|
|
790
795
|
|
791
796
|
check = Helpers.checkmark
|
792
|
-
|
793
|
-
log_end(0)
|
797
|
+
log_message("#{check} Clone finished successfully", Thor::Shell::Color::GREEN)
|
794
798
|
|
795
799
|
else
|
796
|
-
log_end(1, "can't create dataset")
|
797
800
|
@dataset.revert(working_dir) unless @dataset.nil?
|
798
|
-
|
801
|
+
log_message("Error creating dataset, please contact support.", Thor::Shell::Color::RED)
|
799
802
|
exit(0)
|
800
803
|
end
|
804
|
+
rescue => e
|
805
|
+
log_error(e)
|
801
806
|
rescue SignalException
|
802
|
-
log_end(-1)
|
803
807
|
|
804
808
|
say "\nAborting"
|
805
809
|
exit(1)
|
@@ -826,7 +830,6 @@ module Cnvrg
|
|
826
830
|
container.exec(command, tty: true)
|
827
831
|
|
828
832
|
rescue SignalException
|
829
|
-
log_end(-1)
|
830
833
|
|
831
834
|
say "\nAborting"
|
832
835
|
exit(1)
|
@@ -862,10 +865,8 @@ module Cnvrg
|
|
862
865
|
if Dataset.init_container(owner, dataset_slug, dataset_name)
|
863
866
|
|
864
867
|
say "init finished successfully", Thor::Shell::Color::GREEN
|
865
|
-
log_end(0)
|
866
868
|
|
867
869
|
else
|
868
|
-
log_end(1, "can't create dataset")
|
869
870
|
say "error creating dataset, please contact support.", Thor::Shell::Color::RED
|
870
871
|
exit(0)
|
871
872
|
end
|
@@ -894,33 +895,70 @@ module Cnvrg
|
|
894
895
|
|
895
896
|
commit = res["commit"]
|
896
897
|
if result["updated_on_server"].empty? and result["conflicts"].empty? and result["deleted"].empty?
|
897
|
-
|
898
|
-
log_end(0)
|
898
|
+
log_message("Project is up to date", Thor::Shell::Color::GREEN, ((options["sync"] or sync)) ? false : true)
|
899
899
|
return true
|
900
900
|
end
|
901
|
-
|
901
|
+
log_message("Downloading data", Thor::Shell::Color::BLUE)
|
902
902
|
|
903
903
|
result = @dataset.downlowd_updated_data(@dataset.last_local_commit)
|
904
904
|
|
905
905
|
delete = result["result"]["delete"]
|
906
906
|
commits = result["result"]["commits"]
|
907
907
|
updated_idx = result["result"]["idx"]
|
908
|
-
|
908
|
+
parallel_options = {
|
909
|
+
:progress => {
|
910
|
+
:title => "Download Progress",
|
911
|
+
:progress_mark => '=',
|
912
|
+
:format => "%b>>%i| %p%% %t",
|
913
|
+
:starting_at => 0,
|
914
|
+
:total => commits.size,
|
915
|
+
:autofinish => true
|
916
|
+
},
|
917
|
+
in_processes: ParallelProcesses,
|
918
|
+
in_thread: ParallelThreads
|
919
|
+
}
|
920
|
+
|
921
|
+
begin
|
922
|
+
tar_files = []
|
923
|
+
download_result = Parallel.map(commits, parallel_options) do |c|
|
924
|
+
|
909
925
|
file_name = @files.download_data_file(c, dataset_dir)
|
910
926
|
|
911
927
|
if file_name.eql? false or file_name.nil?
|
912
|
-
|
913
|
-
|
928
|
+
count = 0
|
929
|
+
success_download = false
|
930
|
+
while count < 3 and !success_download
|
931
|
+
log_message("Couldn't download data files, retrying.. ", Thor::Shell::Color::BLUE)
|
932
|
+
|
933
|
+
file_name = @files.download_data_file(c, dataset_dir)
|
934
|
+
success_download = (file_name.eql? false or file_name.nil?)
|
935
|
+
count +=1
|
936
|
+
|
937
|
+
end
|
938
|
+
if count > 3 or !success_download
|
939
|
+
log_message("Couldn't download data files,revoking", Thor::Shell::Color::RED)
|
940
|
+
|
941
|
+
raise Parallel::Kill
|
942
|
+
end
|
914
943
|
end
|
915
944
|
file_path = "#{dataset_dir}/#{file_name}"
|
945
|
+
tar_files << file_path
|
916
946
|
success = extarct_tar(file_path, dataset_dir)
|
917
947
|
if !success
|
918
|
-
|
948
|
+
log_message("Couldn't extract data files,revoking", Thor::Shell::Color::RED)
|
949
|
+
|
950
|
+
raise Parallel::Kill
|
919
951
|
end
|
920
952
|
|
921
953
|
FileUtils.rm_rf([file_path])
|
922
954
|
|
923
955
|
end
|
956
|
+
rescue Interrupt
|
957
|
+
@files.revoke_download(tar_files,updated_idx[:tree].keys )
|
958
|
+
return false
|
959
|
+
end
|
960
|
+
|
961
|
+
|
924
962
|
to_delete = []
|
925
963
|
delete.each do |d|
|
926
964
|
to_delete << "#{dataset_dir}/#{d}"
|
@@ -930,28 +968,19 @@ module Cnvrg
|
|
930
968
|
@dataset.update_idx(updated_idx)
|
931
969
|
|
932
970
|
|
933
|
-
# result["conflicts"].each do |f|
|
934
|
-
# relative_path = f.gsub(/^#{@dataset.local_path}/, "")
|
935
|
-
# if @files.download_file_s3(f, relative_path, dataset_dir, conflict=true)
|
936
|
-
# successful_changes << relative_path
|
937
|
-
# end
|
938
|
-
#
|
939
|
-
# end
|
940
|
-
# update idx with latest commit
|
941
|
-
# @dataset.update_idx_with_commit!(commit)
|
942
971
|
check = Helpers.checkmark()
|
943
|
-
|
972
|
+
log_message("#{check} Downloaded changes successfully", Thor::Shell::Color::GREEN)
|
973
|
+
return true
|
944
974
|
|
945
975
|
|
946
|
-
log_end(0)
|
947
976
|
end
|
948
|
-
rescue
|
949
|
-
|
977
|
+
rescue => e
|
978
|
+
log_message("Error occurd, \nAborting", Thor::Shell::Color::BLUE)
|
979
|
+
log_error(e)
|
980
|
+
@files.revoke_download(tar_files,updated_idx[:tree].keys )
|
950
981
|
|
951
|
-
say "Error occurd, \nAborting", Thor::Shell::Color::BLUE
|
952
982
|
exit(1)
|
953
983
|
rescue SignalException
|
954
|
-
log_end(-1)
|
955
984
|
say "\nAborting", Thor::Shell::Color::BLUE
|
956
985
|
exit(1)
|
957
986
|
end
|
@@ -974,19 +1003,18 @@ module Cnvrg
|
|
974
1003
|
@files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
|
975
1004
|
ignore = options[:ignore] || []
|
976
1005
|
if !@dataset.update_ignore_list(ignore)
|
977
|
-
|
1006
|
+
log_message("Couldn't append new ignore files to .cnvrgignore", Thor::Shell::Color::YELLOW)
|
978
1007
|
end
|
979
1008
|
result = @dataset.compare_idx(false)
|
980
1009
|
|
981
1010
|
commit = result["result"]["commit"]
|
982
1011
|
if commit != @dataset.last_local_commit and !@dataset.last_local_commit.nil? and !result["result"]["tree"]["updated_on_server"].empty?
|
983
|
-
log_end(0)
|
984
1012
|
|
985
|
-
|
1013
|
+
log_message("Remote server has an updated version, please run `cnvrg download` first, or alternatively: `cnvrg sync`", Thor::Shell::Color::BLUE)
|
986
1014
|
exit(1)
|
987
1015
|
end
|
988
1016
|
|
989
|
-
|
1017
|
+
log_message("Comparing local changes with remote version:", Thor::Shell::Color::BLUE, options["verbose"] ? true : false)
|
990
1018
|
result = result["result"]["tree"]
|
991
1019
|
# if result["added"].any? {|x| x.include? ".conflict"} or !result["conflicts"].empty?
|
992
1020
|
# all = result["added"].select {|x| x.include? ".conflict"} +result["conflicts"].flatten
|
@@ -1004,8 +1032,7 @@ module Cnvrg
|
|
1004
1032
|
check = Helpers.checkmark()
|
1005
1033
|
|
1006
1034
|
if result["added"].empty? and result["updated_on_local"].empty? and result["deleted"].empty?
|
1007
|
-
|
1008
|
-
say "#{check} Dataset is up to date", Thor::Shell::Color::GREEN unless ((options["sync"] or sync) and !direct)
|
1035
|
+
log_message("#{check} Dataset is up to date", Thor::Shell::Color::GREEN, (((options["sync"] or sync) and !direct) ? false : true))
|
1009
1036
|
return true
|
1010
1037
|
end
|
1011
1038
|
update_count = 0
|
@@ -1014,12 +1041,12 @@ module Cnvrg
|
|
1014
1041
|
successful_deletions = []
|
1015
1042
|
if options["verbose"]
|
1016
1043
|
if update_total == 1
|
1017
|
-
|
1044
|
+
log_message("Updating #{update_total} file", Thor::Shell::Color::BLUE)
|
1018
1045
|
else
|
1019
|
-
|
1046
|
+
log_message("Updating #{update_total} files", Thor::Shell::Color::BLUE)
|
1020
1047
|
end
|
1021
1048
|
else
|
1022
|
-
|
1049
|
+
log_message("Syncing files", Thor::Shell::Color::BLUE, ((options["sync"] or sync)) ? false : true)
|
1023
1050
|
|
1024
1051
|
end
|
1025
1052
|
|
@@ -1045,8 +1072,7 @@ module Cnvrg
|
|
1045
1072
|
successful_updates<< relative_path
|
1046
1073
|
else
|
1047
1074
|
@files.rollback_commit(commit_sha1)
|
1048
|
-
|
1049
|
-
say "Couldn't upload, Rolling Back all changes.", Thor::Shell::Color::RED
|
1075
|
+
log_message("Couldn't upload, Rolling Back all changes.", Thor::Shell::Color::RED)
|
1050
1076
|
exit(0)
|
1051
1077
|
end
|
1052
1078
|
end
|
@@ -1070,14 +1096,13 @@ module Cnvrg
|
|
1070
1096
|
end
|
1071
1097
|
|
1072
1098
|
rescue SignalException
|
1073
|
-
log_end(-1)
|
1074
1099
|
@files.rollback_commit(commit_sha1)
|
1075
1100
|
say "User aborted, Rolling Back all changes.", Thor::Shell::Color::RED
|
1076
1101
|
exit(0)
|
1077
1102
|
rescue => e
|
1078
|
-
|
1103
|
+
log_message("Exception while trying to upload, Rolling back", Thor::Shell::Color::RED)
|
1104
|
+
log_error(e)
|
1079
1105
|
@files.rollback_commit(commit_sha1)
|
1080
|
-
say "Exception while trying to upload, Rolling back", Thor::Shell::Color::RED
|
1081
1106
|
exit(0)
|
1082
1107
|
end
|
1083
1108
|
if !result["deleted"].nil? and !result["deleted"].empty?
|
@@ -1092,57 +1117,52 @@ module Cnvrg
|
|
1092
1117
|
|
1093
1118
|
@dataset.update_idx_with_commit!(commit_sha1)
|
1094
1119
|
rescue => e
|
1095
|
-
|
1120
|
+
log_message("Couldn't commit updates, Rolling Back all changes.", Thor::Shell::Color::RED)
|
1121
|
+
log_error(e)
|
1096
1122
|
@files.rollback_commit(commit_sha1)
|
1097
|
-
say "Couldn't commit updates, Rolling Back all changes.", Thor::Shell::Color::RED
|
1098
1123
|
exit(1)
|
1099
1124
|
|
1100
1125
|
end
|
1101
1126
|
if options["verbose"]
|
1102
|
-
|
1127
|
+
log_message("#{check} Done", Thor::Shell::Color::BLUE)
|
1103
1128
|
if successful_updates.size >0
|
1104
|
-
|
1129
|
+
log_message("Updated:", Thor::Shell::Color::GREEN)
|
1105
1130
|
suc = successful_updates.map { |x| x=Helpers.checkmark() +" "+x }
|
1106
|
-
|
1131
|
+
log_message(suc.join("\n"), Thor::Shell::Color::GREEN)
|
1107
1132
|
end
|
1108
1133
|
if successful_deletions.size >0
|
1109
|
-
|
1134
|
+
log_message("Deleted:", Thor::Shell::Color::GREEN)
|
1110
1135
|
del = successful_updates.map { |x| x=Helpers.checkmark() +" "+x }
|
1111
|
-
|
1136
|
+
log_message(del.join("\n"), Thor::Shell::Color::GREEN)
|
1112
1137
|
end
|
1113
|
-
|
1138
|
+
log_message("Total of #{update_count} / #{update_total} files.", Thor::Shell::Color::GREEN)
|
1114
1139
|
else
|
1115
1140
|
if (options["sync"] or sync) and direct
|
1116
|
-
|
1141
|
+
log_message("#{check} Syncing dataset completed successfully", Thor::Shell::Color::GREEN)
|
1117
1142
|
|
1118
1143
|
else
|
1119
|
-
|
1144
|
+
log_message("#{check} Changes were updated successfully", Thor::Shell::Color::GREEN)
|
1120
1145
|
|
1121
1146
|
end
|
1122
1147
|
|
1123
1148
|
end
|
1124
1149
|
|
1125
|
-
log_end(0)
|
1126
1150
|
else
|
1127
1151
|
@files.rollback_commit(commit_sha1)
|
1128
|
-
|
1129
|
-
say "Error. Rolling Back all changes.", Thor::Shell::Color::RED
|
1152
|
+
log_message("Error: Couldn't commit. \nRolling Back all changes.", Thor::Shell::Color::RED)
|
1130
1153
|
end
|
1131
1154
|
else
|
1132
|
-
|
1133
|
-
say "Error occurd, \nRolling back", Thor::Shell::Color::RED
|
1155
|
+
log_message("Error: Uploaded only #{update_count}/#{update_total} files, \nRolling back", Thor::Shell::Color::RED)
|
1134
1156
|
|
1135
1157
|
@files.rollback_commit(commit_sha1)
|
1136
1158
|
end
|
1137
1159
|
rescue => e
|
1160
|
+
log_message("Error occurd, \nAborting", Thor::Shell::Color::RED)
|
1161
|
+
log_error(e)
|
1138
1162
|
|
1139
|
-
log_end(-1)
|
1140
|
-
|
1141
|
-
say "Error occurd, \nAborting", Thor::Shell::Color::RED
|
1142
1163
|
@files.rollback_commit(commit_sha1)
|
1143
1164
|
exit(1)
|
1144
1165
|
rescue SignalException
|
1145
|
-
log_end(-1)
|
1146
1166
|
|
1147
1167
|
say "\nAborting", Thor::Shell::Color::BLUE
|
1148
1168
|
say "\nRolling back all changes", Thor::Shell::Color::BLUE
|
@@ -1159,7 +1179,7 @@ module Cnvrg
|
|
1159
1179
|
method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
|
1160
1180
|
method_option :no_compression, :type => :boolean, :aliases => ["-nc", "--no_compression"], :default => false
|
1161
1181
|
|
1162
|
-
def upload_data_tar(ignore, verbose, sync,no_compression)
|
1182
|
+
def upload_data_tar(ignore, verbose, sync, no_compression)
|
1163
1183
|
|
1164
1184
|
begin
|
1165
1185
|
verify_logged_in(true)
|
@@ -1170,29 +1190,27 @@ module Cnvrg
|
|
1170
1190
|
|
1171
1191
|
@files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
|
1172
1192
|
if !@dataset.update_ignore_list(ignore)
|
1173
|
-
|
1193
|
+
log_message("Couldn't append new ignore files to .cnvrgignore", Thor::Shell::Color::RED)
|
1174
1194
|
exit(1)
|
1175
1195
|
end
|
1176
|
-
|
1196
|
+
log_message("Checking dataset", Thor::Shell::Color::BLUE)
|
1177
1197
|
local_idx = @dataset.generate_idx
|
1178
1198
|
result = @dataset.compare_idx(false, commit=@dataset.last_local_commit, local_idx= local_idx)
|
1179
1199
|
|
1180
1200
|
|
1181
1201
|
commit = result["result"]["commit"]
|
1182
1202
|
if commit != @dataset.last_local_commit and !@dataset.last_local_commit.nil? and !result["result"]["tree"]["updated_on_server"].empty?
|
1183
|
-
log_end(0)
|
1184
1203
|
|
1185
|
-
|
1204
|
+
log_message("Remote server has an updated version, please run `cnvrg data download` first", Thor::Shell::Color::BLUE)
|
1186
1205
|
exit(1)
|
1187
1206
|
end
|
1188
1207
|
|
1189
|
-
|
1208
|
+
log_message("Comparing local changes with remote version:", Thor::Shell::Color::BLUE, verbose)
|
1190
1209
|
result = result["result"]["tree"]
|
1191
1210
|
check = Helpers.checkmark()
|
1192
1211
|
|
1193
1212
|
if result["added"].empty? and result["updated_on_local"].empty? and result["deleted"].empty?
|
1194
|
-
|
1195
|
-
say "#{check} Dataset is up to date", Thor::Shell::Color::GREEN unless (sync)
|
1213
|
+
log_message("#{check} Dataset is up to date", Thor::Shell::Color::GREEN, (sync ? false : true))
|
1196
1214
|
return true
|
1197
1215
|
end
|
1198
1216
|
update_count = 0
|
@@ -1221,7 +1239,7 @@ module Cnvrg
|
|
1221
1239
|
@dataset.update_idx_with_files_commits!((successful_deletions+successful_updates), commit_time)
|
1222
1240
|
|
1223
1241
|
@dataset.update_idx_with_commit!(commit_sha1)
|
1224
|
-
|
1242
|
+
log_message("Compressing data", Thor::Shell::Color::BLUE)
|
1225
1243
|
|
1226
1244
|
home_dir = File.expand_path('~')
|
1227
1245
|
compression_path = get_compression_path
|
@@ -1229,22 +1247,22 @@ module Cnvrg
|
|
1229
1247
|
tar_files_path = "#{home_dir}/.cnvrg/tmp/#{@dataset.slug}_#{commit_sha1}.txt"
|
1230
1248
|
tar_files = (result["added"] + result["updated_on_local"]).join("\n")
|
1231
1249
|
File.open(tar_files_path, 'w') { |f| f.write tar_files }
|
1232
|
-
is_tar = create_tar(dataset_dir, tar_path, tar_files_path,no_compression)
|
1250
|
+
is_tar = create_tar(dataset_dir, tar_path, tar_files_path, no_compression)
|
1233
1251
|
if !is_tar
|
1234
|
-
|
1252
|
+
log_message("ERROR: Couldn't compress data", Thor::Shell::Color::RED)
|
1235
1253
|
FileUtils.rm_rf([tar_path]) if File.exist? tar_path
|
1236
1254
|
FileUtils.rm_rf([tar_files_path]) if File.exist? tar_files_path
|
1237
1255
|
@files.rollback_commit(commit_sha1)
|
1238
|
-
|
1256
|
+
log_message("Rolling Back all changes.", Thor::Shell::Color::RED)
|
1239
1257
|
exit(1)
|
1240
1258
|
end
|
1241
|
-
|
1259
|
+
log_message("Uploading data", Thor::Shell::Color::BLUE)
|
1242
1260
|
log_file = "#{home_dir}/.cnvrg/tmp/upload_#{File.basename(tar_path)}.log"
|
1243
1261
|
res = false
|
1244
1262
|
res = @files.upload_tar_file(tar_path, tar_path, commit_sha1)
|
1245
1263
|
|
1246
1264
|
if res
|
1247
|
-
|
1265
|
+
log_message("Commiting data", Thor::Shell::Color::BLUE)
|
1248
1266
|
|
1249
1267
|
cur_idx = @dataset.get_idx.to_h
|
1250
1268
|
|
@@ -1255,13 +1273,13 @@ module Cnvrg
|
|
1255
1273
|
|
1256
1274
|
|
1257
1275
|
@files.rollback_commit(commit_sha1)
|
1258
|
-
|
1276
|
+
log_message("Can't commit, Rolling Back all changes.", Thor::Shell::Color::RED)
|
1259
1277
|
exit(1)
|
1260
1278
|
end
|
1261
1279
|
|
1262
1280
|
else
|
1263
1281
|
if File.exist? log_file
|
1264
|
-
|
1282
|
+
@files.upload_data_log_file(log_file, log_file, commit_sha1)
|
1265
1283
|
end
|
1266
1284
|
|
1267
1285
|
|
@@ -1270,8 +1288,8 @@ module Cnvrg
|
|
1270
1288
|
|
1271
1289
|
|
1272
1290
|
@files.rollback_commit(commit_sha1)
|
1273
|
-
|
1274
|
-
|
1291
|
+
log_message("Can't upload, Rolling Back all changes.", Thor::Shell::Color::RED)
|
1292
|
+
log_message("Upload error log: #{log_file}", Thor::Shell::Color::RED)
|
1275
1293
|
|
1276
1294
|
exit(1)
|
1277
1295
|
end
|
@@ -1281,11 +1299,10 @@ module Cnvrg
|
|
1281
1299
|
FileUtils.rm_rf([tar_path, tar_files_path])
|
1282
1300
|
|
1283
1301
|
rescue SignalException
|
1284
|
-
log_end(-1)
|
1285
1302
|
FileUtils.rm_rf([tar_files_path]) if File.exist? tar_files_path
|
1286
1303
|
FileUtils.rm_rf([tar_path]) if File.exist? tar_path
|
1287
1304
|
if File.exist? log_file
|
1288
|
-
@files.upload_data_log_file(log_file,log_file,commit_sha1)
|
1305
|
+
@files.upload_data_log_file(log_file, log_file, commit_sha1)
|
1289
1306
|
end
|
1290
1307
|
|
1291
1308
|
|
@@ -1293,36 +1310,36 @@ module Cnvrg
|
|
1293
1310
|
say "User aborted, Rolling Back all changes.", Thor::Shell::Color::RED
|
1294
1311
|
exit(0)
|
1295
1312
|
rescue => e
|
1296
|
-
|
1313
|
+
log_error(e)
|
1297
1314
|
if !Cnvrg::Helpers.internet_connection?
|
1298
1315
|
say "Seems there is no internet connection", Thor::Shell::Color::RED
|
1299
1316
|
|
1300
1317
|
end
|
1301
1318
|
if File.exist? log_file
|
1302
|
-
@files.upload_data_log_file(log_file,log_file,commit_sha1)
|
1319
|
+
@files.upload_data_log_file(log_file, log_file, commit_sha1)
|
1303
1320
|
end
|
1304
1321
|
FileUtils.rm_rf([tar_files_path]) if File.exist? tar_files_path
|
1305
1322
|
FileUtils.rm_rf([tar_path]) if File.exist? tar_path
|
1306
1323
|
|
1307
|
-
log_end(-1, e.message)
|
1308
1324
|
@files.rollback_commit(commit_sha1)
|
1309
|
-
|
1310
|
-
|
1325
|
+
log_message("Exception while trying to upload, \nRolling back,\n look at the log for more details", Thor::Shell::Color::RED)
|
1326
|
+
log_message("Error log: #{log_file}", Thor::Shell::Color::RED)
|
1311
1327
|
|
1312
1328
|
|
1313
1329
|
exit(0)
|
1314
1330
|
end
|
1315
|
-
|
1316
|
-
|
1331
|
+
log_message("#{check} Changes were updated successfully", Thor::Shell::Color::GREEN)
|
1317
1332
|
|
1318
1333
|
|
1319
1334
|
rescue => e
|
1320
|
-
|
1321
|
-
|
1335
|
+
|
1336
|
+
puts e.message
|
1337
|
+
puts e.backtrace
|
1338
|
+
log_message("Error occurred, \nAborting", Thor::Shell::Color::RED)
|
1339
|
+
log_error(e)
|
1322
1340
|
@files.rollback_commit(commit_sha1)
|
1323
1341
|
exit(1)
|
1324
1342
|
rescue SignalException
|
1325
|
-
log_end(-1)
|
1326
1343
|
|
1327
1344
|
say "\nAborting", Thor::Shell::Color::BLUE
|
1328
1345
|
say "\nRolling back all changes", Thor::Shell::Color::BLUE
|
@@ -1330,7 +1347,6 @@ module Cnvrg
|
|
1330
1347
|
exit(1)
|
1331
1348
|
end
|
1332
1349
|
|
1333
|
-
log_end(0)
|
1334
1350
|
|
1335
1351
|
end
|
1336
1352
|
|
@@ -1424,18 +1440,16 @@ module Cnvrg
|
|
1424
1440
|
project_name = response["title"]
|
1425
1441
|
commit_to_clone = options["commit"] || nil
|
1426
1442
|
|
1427
|
-
|
1443
|
+
log_message("Cloning #{project_name}", Thor::Shell::Color::BLUE)
|
1428
1444
|
clone_resp = false
|
1429
1445
|
if remote
|
1430
1446
|
clone_resp = Project.clone_dir_remote(slug, owner, project_name)
|
1431
1447
|
project_home = Dir.pwd
|
1432
1448
|
else
|
1433
1449
|
if (Dir.exists? project_name)
|
1434
|
-
|
1450
|
+
log_message("Error: Conflict with dir #{project_name}", Thor::Shell::Color::RED)
|
1435
1451
|
if no? "Sync to repository anyway? (current data might lost)", Thor::Shell::Color::YELLOW
|
1436
|
-
|
1437
|
-
log_end(1, "conflict with dir #{project_name}")
|
1438
|
-
|
1452
|
+
log_message("Remove dir in order to clone #{project_name}", Thor::Shell::Color::RED)
|
1439
1453
|
exit(1)
|
1440
1454
|
end
|
1441
1455
|
|
@@ -1463,20 +1477,20 @@ module Cnvrg
|
|
1463
1477
|
if yes? "Image wasn't found locally, pull image from cnvrg repository?", Thor::Shell::Color::YELLOW
|
1464
1478
|
image = pull(docker_image)
|
1465
1479
|
if image
|
1466
|
-
|
1480
|
+
log_message("downloaded image: #{docker_image}", Thor::Shell::Color::BLUE)
|
1467
1481
|
@image = Images.new(working_dir, docker_image)
|
1468
1482
|
else
|
1469
|
-
|
1483
|
+
log_message("Could not create a new project with docker, image was not found", Thor::Shell::Color::RED)
|
1470
1484
|
@project.revert(working_dir)
|
1471
1485
|
exit(1)
|
1472
1486
|
end
|
1473
1487
|
end
|
1474
1488
|
|
1475
1489
|
elsif docker_image_local.size == 1
|
1476
|
-
|
1490
|
+
log_message("found image: #{docker_image_local[0]}, setting it up..", Thor::Shell::Color::BLUE)
|
1477
1491
|
@image = Images.new(working_dir, docker_image_local[0])
|
1478
1492
|
elsif docker_image_local.size >1
|
1479
|
-
|
1493
|
+
log_message("found #{docker_image_local.size} images, choose the image name you want to use", Thor::Shell::Color::BLUE)
|
1480
1494
|
image_name = ask "#{docker_image_local.join("\n")}\n", Thor::Shell::Color::BLUE
|
1481
1495
|
@image = Images.new(working_dir, image_name)
|
1482
1496
|
end
|
@@ -1484,35 +1498,65 @@ module Cnvrg
|
|
1484
1498
|
end
|
1485
1499
|
|
1486
1500
|
successful_changes = []
|
1487
|
-
|
1501
|
+
log_message("Downloading files", Thor::Shell::Color::BLUE)
|
1488
1502
|
if !response["result"]["tree"].nil?
|
1489
|
-
|
1490
|
-
|
1491
|
-
|
1492
|
-
|
1493
|
-
|
1494
|
-
|
1495
|
-
|
1496
|
-
|
1497
|
-
|
1503
|
+
parallel_options = {
|
1504
|
+
:progress => {
|
1505
|
+
:title => "Download Progress",
|
1506
|
+
:progress_mark => '=',
|
1507
|
+
:format => "%b>>%i| %p%% %t",
|
1508
|
+
:starting_at => 0,
|
1509
|
+
:total => response["result"]["tree"].size,
|
1510
|
+
:autofinish => true
|
1511
|
+
},
|
1512
|
+
in_processes: ParallelProcesses,
|
1513
|
+
in_thread: ParallelThreads
|
1514
|
+
}
|
1515
|
+
begin
|
1516
|
+
clone_result = Parallel.map((response["result"]["tree"]), parallel_options) do |f|
|
1498
1517
|
|
1499
|
-
|
1500
|
-
|
1518
|
+
relative_path = f[0].gsub(/^#{@project.local_path}/, "")
|
1519
|
+
if f[0].end_with? "/"
|
1520
|
+
# dir
|
1521
|
+
if @files.download_dir(f[0], relative_path, project_home)
|
1522
|
+
f
|
1523
|
+
else
|
1524
|
+
log_message("Could not create directory: #{f[0]}", Thor::Shell::Color::RED)
|
1525
|
+
raise Parallel::Kill
|
1526
|
+
end
|
1527
|
+
else
|
1528
|
+
# blob
|
1529
|
+
|
1530
|
+
if @files.download_file_s3(f[0], relative_path, project_home, commit_sha1=current_commit)
|
1531
|
+
f
|
1532
|
+
successful_changes << relative_path
|
1533
|
+
else
|
1534
|
+
log_message("Could not download file: #{f[0]}", Thor::Shell::Color::RED)
|
1535
|
+
raise Parallel::Kill
|
1536
|
+
|
1537
|
+
end
|
1501
1538
|
end
|
1502
1539
|
end
|
1540
|
+
rescue Interrupt
|
1541
|
+
|
1542
|
+
log_message("Couldn't download, Rolling Back all changes.", Thor::Shell::Color::RED)
|
1543
|
+
|
1544
|
+
@files.revoke_download([], response["result"]["tree"])
|
1545
|
+
exit(1)
|
1503
1546
|
end
|
1547
|
+
|
1548
|
+
|
1504
1549
|
end
|
1505
1550
|
|
1506
|
-
|
1507
|
-
|
1551
|
+
successful_changes = clone_result
|
1552
|
+
|
1553
|
+
log_message("Done.\nDownloaded total of #{successful_changes.size} files", Thor::Shell::Color::GREEN)
|
1508
1554
|
else
|
1509
|
-
log_end(1, "can't create directory")
|
1510
1555
|
|
1511
|
-
|
1556
|
+
log_message("Error: Couldn't create directory: #{project_name}", Thor::Shell::Color::RED)
|
1512
1557
|
exit(1)
|
1513
1558
|
end
|
1514
1559
|
rescue SignalException
|
1515
|
-
log_end(-1)
|
1516
1560
|
say "\nAborting"
|
1517
1561
|
exit(1)
|
1518
1562
|
end
|
@@ -1534,49 +1578,46 @@ module Cnvrg
|
|
1534
1578
|
result = @project.compare_idx(new_branch)["result"]
|
1535
1579
|
commit = result["commit"]
|
1536
1580
|
result = result["tree"]
|
1537
|
-
|
1581
|
+
log_message("Comparing local changes with remote version:", Thor::Shell::Color::BLUE)
|
1538
1582
|
|
1539
1583
|
if result["added"].empty? and result["updated_on_local"].empty? and result["updated_on_server"].empty? and result["deleted"].empty? and result["conflicts"].empty?
|
1540
|
-
|
1541
|
-
log_end(0)
|
1584
|
+
log_message("Project is up to date", Thor::Shell::Color::GREEN)
|
1542
1585
|
return true
|
1543
1586
|
end
|
1544
1587
|
if result["added"].size > 0
|
1545
|
-
|
1588
|
+
log_message("Added files:\n", Thor::Shell::Color::BLUE)
|
1546
1589
|
result["added"].each do |a|
|
1547
|
-
|
1590
|
+
log_message("\t\tA:\t#{a}", Thor::Shell::Color::GREEN)
|
1548
1591
|
end
|
1549
1592
|
end
|
1550
1593
|
|
1551
1594
|
if result["deleted"].size > 0
|
1552
|
-
|
1595
|
+
log_message("Deleted files:\n", Thor::Shell::Color::BLUE)
|
1553
1596
|
result["deleted"].each do |a|
|
1554
|
-
|
1597
|
+
log_message("\t\tD:\t#{a}", Thor::Shell::Color::GREEN)
|
1555
1598
|
end
|
1556
1599
|
end
|
1557
1600
|
if result["updated_on_local"].size > 0
|
1558
|
-
|
1601
|
+
log_message("Local changes:\n", Thor::Shell::Color::BLUE)
|
1559
1602
|
result["updated_on_local"].each do |a|
|
1560
|
-
|
1603
|
+
log_message("\t\tM:\t#{a}", Thor::Shell::Color::GREEN)
|
1561
1604
|
end
|
1562
1605
|
end
|
1563
1606
|
|
1564
1607
|
if result["updated_on_server"].size > 0
|
1565
|
-
|
1608
|
+
log_message("Remote changes:\n", Thor::Shell::Color::BLUE)
|
1566
1609
|
result["updated_on_server"].each do |a|
|
1567
|
-
|
1610
|
+
log_message("\t\tM:\t#{a}", Thor::Shell::Color::GREEN)
|
1568
1611
|
end
|
1569
1612
|
end
|
1570
1613
|
|
1571
1614
|
if result["conflicts"].size > 0
|
1572
|
-
|
1615
|
+
log_message("Conflicted changes:\n", Thor::Shell::Color::BLUE)
|
1573
1616
|
result["conflicts"].each do |a|
|
1574
|
-
|
1617
|
+
log_message("\t\tC:\t#{a}", Thor::Shell::Color::RED)
|
1575
1618
|
end
|
1576
1619
|
end
|
1577
|
-
log_end(0)
|
1578
1620
|
rescue SignalException
|
1579
|
-
log_end(-1)
|
1580
1621
|
say "\nAborting"
|
1581
1622
|
exit(1)
|
1582
1623
|
end
|
@@ -1634,7 +1675,7 @@ module Cnvrg
|
|
1634
1675
|
end
|
1635
1676
|
end
|
1636
1677
|
if !@project.update_ignore_list(ignore)
|
1637
|
-
|
1678
|
+
log_message("Couldn't append new ignore files to .cnvrgignore", Thor::Shell::Color::YELLOW)
|
1638
1679
|
end
|
1639
1680
|
new_branch = options["new_branch"] || false
|
1640
1681
|
|
@@ -1648,13 +1689,12 @@ module Cnvrg
|
|
1648
1689
|
commit = result["result"]["commit"]
|
1649
1690
|
if !link
|
1650
1691
|
if commit != @project.last_local_commit and !@project.last_local_commit.nil? and !result["result"]["tree"]["updated_on_server"].empty?
|
1651
|
-
log_end(0)
|
1652
1692
|
|
1653
|
-
|
1693
|
+
log_message("Remote server has an updated version, please run `cnvrg download` first, or alternatively: `cnvrg sync`", Thor::Shell::Color::BLUE)
|
1654
1694
|
exit(1)
|
1655
1695
|
end
|
1656
1696
|
|
1657
|
-
|
1697
|
+
log_message("Comparing local changes with remote version:", Thor::Shell::Color::BLUE, (options["verbose"]))
|
1658
1698
|
end
|
1659
1699
|
result = result["result"]["tree"]
|
1660
1700
|
# if result["added"].any? {|x| x.include? ".conflict"} or !result["conflicts"].empty?
|
@@ -1672,8 +1712,7 @@ module Cnvrg
|
|
1672
1712
|
# end
|
1673
1713
|
check = Helpers.checkmark()
|
1674
1714
|
if result["added"].empty? and result["updated_on_local"].empty? and result["deleted"].empty?
|
1675
|
-
|
1676
|
-
say "#{check} Project is up to date", Thor::Shell::Color::GREEN unless ((options["sync"] or sync) and !direct)
|
1715
|
+
log_message("#{check} Project is up to date", Thor::Shell::Color::GREEN, (((options["sync"] or sync) and !direct) ? false : true))
|
1677
1716
|
return true
|
1678
1717
|
end
|
1679
1718
|
update_count = 0
|
@@ -1682,77 +1721,133 @@ module Cnvrg
|
|
1682
1721
|
successful_deletions = []
|
1683
1722
|
if options["verbose"]
|
1684
1723
|
if update_total == 1
|
1685
|
-
|
1724
|
+
log_message("Updating #{update_total} file", Thor::Shell::Color::BLUE)
|
1686
1725
|
else
|
1687
|
-
|
1726
|
+
log_message("Updating #{update_total} files", Thor::Shell::Color::BLUE)
|
1688
1727
|
end
|
1689
1728
|
else
|
1690
|
-
|
1729
|
+
log_message("Syncing files", Thor::Shell::Color::BLUE, ((options["sync"] or sync)) ? false : true)
|
1691
1730
|
|
1692
1731
|
end
|
1693
|
-
|
1694
1732
|
# Start commit
|
1695
|
-
|
1696
1733
|
commit_sha1 = @files.start_commit(new_branch)["result"]["commit_sha1"]
|
1697
1734
|
|
1698
1735
|
# upload / update
|
1699
1736
|
begin
|
1700
|
-
(result["added"] + result["updated_on_local"]).each do |f|
|
1701
|
-
absolute_path = "#{@project.local_path}/#{f}"
|
1702
|
-
relative_path = f.gsub(/^#{@project.local_path + "/"}/, "")
|
1703
|
-
if File.directory?(absolute_path)
|
1704
|
-
resDir = @files.create_dir(absolute_path, relative_path, commit_sha1)
|
1705
|
-
if resDir
|
1706
|
-
update_count += 1
|
1707
|
-
successful_updates<< relative_path
|
1708
|
-
end
|
1709
|
-
else
|
1710
|
-
res = @files.upload_file(absolute_path, relative_path, commit_sha1)
|
1711
|
-
if res
|
1712
|
-
update_count += 1
|
1713
|
-
successful_updates<< relative_path
|
1714
|
-
else
|
1715
|
-
say "Failed to upload: #{ File.basename(absolute_path) }", Thor::Shell::Color::RED
|
1716
1737
|
|
1717
|
-
|
1718
|
-
|
1719
|
-
|
1720
|
-
|
1738
|
+
parallel_options = {
|
1739
|
+
:progress => {
|
1740
|
+
:title => "Upload Progress",
|
1741
|
+
:progress_mark => '=',
|
1742
|
+
:format => "%b>>%i| %p%% %t",
|
1743
|
+
:starting_at => 0,
|
1744
|
+
:total => update_total,
|
1745
|
+
:autofinish => true
|
1746
|
+
},
|
1747
|
+
in_processes: ParallelProcesses,
|
1748
|
+
in_thread: ParallelThreads,
|
1749
|
+
isolation: true
|
1750
|
+
}
|
1751
|
+
if (result["added"] + result["updated_on_local"]).size > 0
|
1752
|
+
|
1753
|
+
begin
|
1754
|
+
upload_result = Parallel.map((result["added"] + result["updated_on_local"]), parallel_options) do |f|
|
1755
|
+
absolute_path = "#{@project.local_path}/#{f}"
|
1756
|
+
relative_path = f.gsub(/^#{@project.local_path + "/"}/, "")
|
1757
|
+
if File.directory?(absolute_path)
|
1758
|
+
resDir = @files.create_dir(absolute_path, relative_path, commit_sha1)
|
1759
|
+
if resDir
|
1760
|
+
f
|
1761
|
+
# progressbar.increment
|
1762
|
+
update_count += 1
|
1763
|
+
successful_updates<< relative_path
|
1764
|
+
else
|
1765
|
+
log_message("Failed to upload directory: #{ relative_path }", Thor::Shell::Color::RED)
|
1766
|
+
|
1767
|
+
raise Parallel::Kill
|
1768
|
+
end
|
1769
|
+
|
1770
|
+
else
|
1771
|
+
|
1772
|
+
res = @files.upload_file(absolute_path, relative_path, commit_sha1)
|
1773
|
+
if res
|
1774
|
+
f
|
1775
|
+
update_count += 1
|
1776
|
+
|
1777
|
+
successful_updates<< relative_path
|
1778
|
+
else
|
1779
|
+
log_message("Failed to upload: #{ File.basename(absolute_path) }", Thor::Shell::Color::RED)
|
1780
|
+
|
1781
|
+
raise Parallel::Kill
|
1782
|
+
|
1783
|
+
end
|
1784
|
+
end
|
1721
1785
|
end
|
1786
|
+
rescue Interrupt
|
1787
|
+
log_message("Couldn't upload, Rolling Back all changes.", Thor::Shell::Color::RED)
|
1788
|
+
@files.rollback_commit(commit_sha1)
|
1789
|
+
|
1790
|
+
exit(1)
|
1722
1791
|
end
|
1723
1792
|
end
|
1724
1793
|
|
1794
|
+
|
1795
|
+
successful_updates = upload_result.to_a
|
1796
|
+
|
1725
1797
|
# delete
|
1798
|
+
|
1726
1799
|
deleted = update_deleted(result["deleted"])
|
1727
|
-
|
1728
|
-
|
1729
|
-
|
1730
|
-
|
1731
|
-
|
1732
|
-
|
1733
|
-
|
1734
|
-
|
1735
|
-
|
1736
|
-
|
1737
|
-
|
1800
|
+
begin
|
1801
|
+
|
1802
|
+
deleted_result = Parallel.map(deleted, in_processes: ParallelProcesses, in_thread: ParallelThreads) do |f|
|
1803
|
+
|
1804
|
+
relative_path = f.gsub(/^#{@project.local_path + "/"}/, "")
|
1805
|
+
if relative_path.end_with?("/")
|
1806
|
+
if @files.delete_dir(f, relative_path, commit_sha1)
|
1807
|
+
f
|
1808
|
+
else
|
1809
|
+
log_message("Failed to delete directory: #{ f }", Thor::Shell::Color::RED)
|
1810
|
+
|
1811
|
+
end
|
1812
|
+
else
|
1813
|
+
if @files.delete_file(f, relative_path, commit_sha1)
|
1814
|
+
f
|
1815
|
+
else
|
1816
|
+
log_message("Failed to delete file: #{ f }", Thor::Shell::Color::RED)
|
1817
|
+
|
1818
|
+
end
|
1738
1819
|
end
|
1739
1820
|
end
|
1821
|
+
rescue Interrupt
|
1822
|
+
log_message("Couldn't upload, Rolling Back all changes.", Thor::Shell::Color::RED)
|
1823
|
+
@files.rollback_commit(commit_sha1)
|
1824
|
+
|
1825
|
+
exit(1)
|
1826
|
+
|
1740
1827
|
end
|
1741
1828
|
|
1829
|
+
|
1830
|
+
successful_deletions += successful_deletions.select { |x| not x.nil? }
|
1831
|
+
|
1832
|
+
|
1833
|
+
successful_updates = successful_updates.select { |x| not x.nil? }
|
1834
|
+
|
1835
|
+
update_count = successful_updates.size
|
1836
|
+
|
1742
1837
|
rescue SignalException
|
1743
|
-
log_end(-1)
|
1744
1838
|
@files.rollback_commit(commit_sha1)
|
1745
1839
|
say "User aborted, Rolling Back all changes.", Thor::Shell::Color::RED
|
1746
1840
|
exit(0)
|
1747
1841
|
rescue => e
|
1748
|
-
log_end(-1, e.message)
|
1749
1842
|
@files.rollback_commit(commit_sha1)
|
1750
|
-
|
1843
|
+
log_message("Exception while trying to upload, Rolling back", Thor::Shell::Color::RED)
|
1844
|
+
log_error(e)
|
1751
1845
|
exit(0)
|
1752
1846
|
end
|
1753
1847
|
if !result["deleted"].nil? and !result["deleted"].empty?
|
1754
1848
|
update_count += result["deleted"].size
|
1755
1849
|
end
|
1850
|
+
|
1756
1851
|
if update_count == update_total
|
1757
1852
|
res = @files.end_commit(commit_sha1)
|
1758
1853
|
if (Cnvrg::CLI.is_response_success(res, false))
|
@@ -1762,9 +1857,9 @@ module Cnvrg
|
|
1762
1857
|
|
1763
1858
|
@project.update_idx_with_commit!(commit_sha1)
|
1764
1859
|
rescue => e
|
1765
|
-
log_end(-1, e.message)
|
1766
1860
|
@files.rollback_commit(commit_sha1)
|
1767
|
-
|
1861
|
+
log_message("Couldn't commit updates, Rolling Back all changes.", Thor::Shell::Color::RED)
|
1862
|
+
log_error(e)
|
1768
1863
|
exit(1)
|
1769
1864
|
|
1770
1865
|
end
|
@@ -1774,49 +1869,47 @@ module Cnvrg
|
|
1774
1869
|
end
|
1775
1870
|
|
1776
1871
|
if options["verbose"]
|
1777
|
-
|
1872
|
+
log_message("#{check} Done", Thor::Shell::Color::BLUE)
|
1778
1873
|
if successful_updates.size >0
|
1779
|
-
|
1874
|
+
log_message("Updated:", Thor::Shell::Color::GREEN)
|
1780
1875
|
suc = successful_updates.map { |x| x=Helpers.checkmark() +" "+x }
|
1781
|
-
|
1876
|
+
log_message(suc.join("\n"), Thor::Shell::Color::GREEN)
|
1782
1877
|
end
|
1783
1878
|
if successful_deletions.size >0
|
1784
|
-
|
1879
|
+
log_message("Deleted:", Thor::Shell::Color::GREEN)
|
1785
1880
|
del = successful_updates.map { |x| x=Helpers.checkmark() +" "+x }
|
1786
|
-
|
1881
|
+
log_message(del.join("\n"), Thor::Shell::Color::GREEN)
|
1787
1882
|
end
|
1788
|
-
|
1883
|
+
log_message("Total of #{update_count} / #{update_total} files.", Thor::Shell::Color::GREEN)
|
1789
1884
|
else
|
1790
1885
|
if (options["sync"] or sync) and direct
|
1791
|
-
|
1886
|
+
log_message("#{check} Syncing project completed successfully", Thor::Shell::Color::GREEN)
|
1792
1887
|
|
1793
1888
|
else
|
1794
|
-
|
1889
|
+
log_message("#{check} Changes were updated successfully", Thor::Shell::Color::GREEN)
|
1795
1890
|
|
1796
1891
|
end
|
1797
1892
|
|
1798
1893
|
end
|
1799
1894
|
|
1800
|
-
log_end(0)
|
1801
1895
|
else
|
1802
1896
|
@files.rollback_commit(commit_sha1)
|
1803
|
-
|
1804
|
-
say "Error. Rolling Back all changes.", Thor::Shell::Color::RED
|
1897
|
+
log_message("Error: couldn't commit changes, Rolling Back all changes.", Thor::Shell::Color::RED)
|
1805
1898
|
end
|
1806
1899
|
else
|
1807
|
-
|
1808
|
-
say "Error occurd, \nRolling back", Thor::Shell::Color::RED
|
1900
|
+
log_message("Error: uploaded only: #{update_count} / #{update_total}, \n Rolling back", Thor::Shell::Color::RED)
|
1809
1901
|
|
1810
1902
|
@files.rollback_commit(commit_sha1)
|
1811
1903
|
end
|
1812
1904
|
rescue => e
|
1813
|
-
log_end(-1,e.message)
|
1814
1905
|
|
1815
|
-
|
1906
|
+
log_message("Error occurred, \nAborting", Thor::Shell::Color::RED)
|
1907
|
+
log_error(e)
|
1816
1908
|
@files.rollback_commit(commit_sha1)
|
1909
|
+
puts e.message
|
1910
|
+
|
1817
1911
|
exit(1)
|
1818
1912
|
rescue SignalException
|
1819
|
-
log_end(-1)
|
1820
1913
|
|
1821
1914
|
say "\nAborting", Thor::Shell::Color::BLUE
|
1822
1915
|
say "\nRolling back all changes", Thor::Shell::Color::BLUE
|
@@ -1852,7 +1945,7 @@ module Cnvrg
|
|
1852
1945
|
end
|
1853
1946
|
end
|
1854
1947
|
if !@project.update_ignore_list(ignore)
|
1855
|
-
|
1948
|
+
log_message("Couldn't append new ignore files to .cnvrgignore", Thor::Shell::Color::YELLOW)
|
1856
1949
|
end
|
1857
1950
|
new_branch = options["new_branch"] || false
|
1858
1951
|
|
@@ -1862,8 +1955,7 @@ module Cnvrg
|
|
1862
1955
|
|
1863
1956
|
commit = res["commit"]
|
1864
1957
|
if result["updated_on_server"].empty? and result["conflicts"].empty? and result["deleted"].empty?
|
1865
|
-
|
1866
|
-
log_end(0)
|
1958
|
+
log_message("Project is up to date", Thor::Shell::Color::GREEN, ((options["sync"] or sync) ? false : true))
|
1867
1959
|
return true
|
1868
1960
|
end
|
1869
1961
|
# if result["added"].any? {|x| x.include? ".conflict"} or !result["conflicts"].empty?
|
@@ -1879,106 +1971,149 @@ module Cnvrg
|
|
1879
1971
|
# exit(1)
|
1880
1972
|
# end
|
1881
1973
|
update_count = 0
|
1882
|
-
update_total = result["updated_on_server"].size + result["conflicts"].size
|
1974
|
+
update_total = result["updated_on_server"].size + result["conflicts"].size + result["deleted"].size
|
1883
1975
|
|
1884
1976
|
|
1885
1977
|
successful_changes = []
|
1886
1978
|
if update_total ==1
|
1887
|
-
|
1979
|
+
log_message("Downloading #{update_total} file", Thor::Shell::Color::BLUE, !options["sync"])
|
1888
1980
|
elsif update_total == 0
|
1889
|
-
|
1890
|
-
log_end(0)
|
1981
|
+
log_message("Project is up to date", Thor::Shell::Color::GREEN, !options["sync"])
|
1891
1982
|
return true
|
1892
1983
|
elsif options["verbose"]
|
1893
|
-
|
1984
|
+
log_message("Downloading #{update_total} files", Thor::Shell::Color::BLUE)
|
1894
1985
|
else
|
1895
|
-
|
1896
|
-
|
1986
|
+
log_message("Syncing files", Thor::Shell::Color::BLUE, !options["sync"])
|
1897
1987
|
|
1898
1988
|
end
|
1989
|
+
parallel_options = {
|
1990
|
+
:progress => {
|
1991
|
+
:title => "Download Progress",
|
1992
|
+
:progress_mark => '=',
|
1993
|
+
:format => "%b>>%i| %p%% %t",
|
1994
|
+
:starting_at => 0,
|
1995
|
+
:total => result["updated_on_server"].size,
|
1996
|
+
:autofinish => true
|
1997
|
+
},
|
1998
|
+
in_processes: ParallelProcesses,
|
1999
|
+
in_thread: ParallelThreads
|
2000
|
+
}
|
2001
|
+
if !result["conflicts"].empty?
|
2002
|
+
begin
|
1899
2003
|
|
1900
|
-
result["conflicts"].each do |f|
|
1901
|
-
relative_path = f.gsub(/^#{@project.local_path}/, "")
|
1902
|
-
if @files.download_file_s3(f, relative_path, project_home, commit_sha1=nil, conflict=true)
|
1903
|
-
successful_changes << relative_path
|
1904
|
-
end
|
1905
2004
|
|
1906
|
-
|
1907
|
-
result["updated_on_server"].each do |f|
|
1908
|
-
relative_path = f.gsub(/^#{@project.local_path}/, "")
|
1909
|
-
if f.end_with? "/"
|
1910
|
-
# dir
|
1911
|
-
if @files.download_dir(f, relative_path, project_home)
|
1912
|
-
successful_changes << relative_path
|
2005
|
+
conflicts_result = Parallel.map(result["conflicts"], in_processes: ParallelProcesses, in_thread: ParallelThreads) do |f|
|
1913
2006
|
|
2007
|
+
relative_path = f.gsub(/^#{@project.local_path}/, "")
|
2008
|
+
if @files.download_file_s3(f, relative_path, project_home, commit_sha1=nil, conflict=true)
|
2009
|
+
f
|
2010
|
+
else
|
2011
|
+
log_message("Couldn't download: #{f}", Thor::Shell::Color::RED)
|
2012
|
+
raise Parallel::Kill
|
2013
|
+
|
2014
|
+
end
|
1914
2015
|
end
|
1915
|
-
|
1916
|
-
|
1917
|
-
|
1918
|
-
|
2016
|
+
rescue Interrupt
|
2017
|
+
|
2018
|
+
log_message("Couldn't download, Rolling Back all changes.", Thor::Shell::Color::RED)
|
2019
|
+
|
2020
|
+
@files.revoke_download(result["conflicts"], [])
|
2021
|
+
exit(1)
|
2022
|
+
end
|
2023
|
+
end
|
2024
|
+
|
2025
|
+
|
2026
|
+
successful_changes += conflicts_result.to_a
|
2027
|
+
if !result["updated_on_server"].empty?
|
2028
|
+
begin
|
2029
|
+
updated_on_server_result = Parallel.map(result["updated_on_server"], parallel_options) do |f|
|
2030
|
+
|
2031
|
+
relative_path = f.gsub(/^#{@project.local_path}/, "")
|
2032
|
+
if f.end_with? "/"
|
2033
|
+
# dir
|
2034
|
+
if @files.download_dir(f, relative_path, project_home)
|
2035
|
+
f
|
2036
|
+
else
|
2037
|
+
log_message("Couldn't dcreate directory: #{f}", Thor::Shell::Color::RED)
|
2038
|
+
raise Parallel::Kill
|
2039
|
+
|
2040
|
+
|
2041
|
+
end
|
2042
|
+
|
2043
|
+
else
|
2044
|
+
# blob
|
2045
|
+
if @files.download_file_s3(f, relative_path, project_home)
|
2046
|
+
f
|
2047
|
+
else
|
2048
|
+
|
2049
|
+
|
2050
|
+
log_message("Couldn't download: #{f}", Thor::Shell::Color::RED)
|
2051
|
+
raise Parallel::Kill
|
2052
|
+
|
2053
|
+
|
2054
|
+
end
|
2055
|
+
end
|
2056
|
+
|
2057
|
+
|
1919
2058
|
end
|
2059
|
+
successful_changes += updated_on_server_result.to_a
|
2060
|
+
rescue Interrupt
|
2061
|
+
log_message("Couldn't download, Rolling Back all changes.", Thor::Shell::Color::RED)
|
2062
|
+
|
2063
|
+
@files.revoke_download(result["conflicts"], result["updated_on_server"])
|
2064
|
+
exit(1)
|
2065
|
+
|
1920
2066
|
end
|
2067
|
+
end
|
2068
|
+
|
2069
|
+
deleted = result["deleted"].to_a
|
2070
|
+
delete_res = @files.delete_commit_files_local(deleted)
|
2071
|
+
if !delete_res
|
2072
|
+
log_message("Couldn't delete #{deleted.join(" ")}", Thor::Shell::Color::RED)
|
2073
|
+
log_message("Couldn't download, Rolling Back all changes.", Thor::Shell::Color::RED)
|
2074
|
+
|
2075
|
+
@files.revoke_download(result["conflicts"], result["updated_on_server"])
|
2076
|
+
exit(1)
|
1921
2077
|
|
1922
2078
|
end
|
2079
|
+
successful_changes += deleted
|
2080
|
+
|
2081
|
+
|
2082
|
+
successful_changes = successful_changes.select { |x| not x.nil? }
|
2083
|
+
|
1923
2084
|
if update_total == successful_changes.size
|
1924
2085
|
# update idx with latest commit
|
1925
2086
|
@project.update_idx_with_commit!(commit)
|
1926
2087
|
check = Helpers.checkmark()
|
1927
2088
|
|
1928
2089
|
if options["verbose"]
|
1929
|
-
|
1930
|
-
|
1931
|
-
|
2090
|
+
log_message("#{check} Done, Downloaded:", Thor::Shell::Color::GREEN)
|
2091
|
+
log_message(successful_changes.join("\n"), Thor::Shell::Color::GREEN)
|
2092
|
+
log_message("Total of #{successful_changes.size} / #{update_total} files.", Thor::Shell::Color::GREEN)
|
1932
2093
|
else
|
1933
|
-
|
2094
|
+
log_message("#{check} Downloaded changes successfully", Thor::Shell::Color::GREEN, ((sync or options["sync"]) ? false : true))
|
1934
2095
|
end
|
1935
2096
|
|
1936
2097
|
|
1937
|
-
log_end(0)
|
1938
2098
|
end
|
1939
2099
|
rescue => e
|
1940
|
-
log_end(-1)
|
1941
2100
|
|
1942
|
-
|
2101
|
+
log_message("Error occurred, \nAborting", Thor::Shell::Color::BLUE)
|
2102
|
+
log_error(e)
|
1943
2103
|
if successful_changes.nil?
|
1944
2104
|
exit(1)
|
1945
2105
|
end
|
1946
|
-
|
1947
|
-
|
1948
|
-
abs_path = "#{@project.local_path}/#{f}"
|
1949
|
-
filename = File.basename abs_path
|
1950
|
-
say "revoking #{filename}"
|
1951
|
-
if result["conflicts"].include? f
|
1952
|
-
@files.revoke_download_file(abs_path, f, filename, true)
|
1953
|
-
elsif result["updated_on_server"].include? f
|
1954
|
-
if File.directory? abs_path
|
1955
|
-
@files.revoke_download_dir(abs_path, f, project_home)
|
1956
|
-
else
|
1957
|
-
@files.revoke_download_file(project_home, abs_path, filename)
|
1958
|
-
end
|
1959
|
-
end
|
2106
|
+
begin
|
2107
|
+
@files.revoke_download(result["conflicts"], result["updated_on_server"])
|
1960
2108
|
end
|
1961
2109
|
exit(1)
|
1962
2110
|
rescue SignalException
|
1963
|
-
log_end(-1)
|
1964
2111
|
say "\nAborting", Thor::Shell::Color::BLUE
|
1965
2112
|
if successful_changes.nil?
|
1966
2113
|
exit(1)
|
1967
2114
|
end
|
1968
|
-
|
1969
|
-
|
1970
|
-
abs_path = "#{@project.local_path}/#{f}"
|
1971
|
-
filename = File.basename abs_path
|
1972
|
-
say "revoking #{filename}"
|
1973
|
-
if result["conflicts"].include? f
|
1974
|
-
@files.revoke_download_file(abs_path, f, filename, true)
|
1975
|
-
elsif result["updated_on_server"].include? f
|
1976
|
-
if File.directory? abs_path
|
1977
|
-
@files.revoke_download_dir(abs_path, f, project_home)
|
1978
|
-
else
|
1979
|
-
@files.revoke_download_file(project_home, abs_path, filename)
|
1980
|
-
end
|
1981
|
-
end
|
2115
|
+
begin
|
2116
|
+
@files.revoke_download(result["conflicts"], result["updated_on_server"])
|
1982
2117
|
end
|
1983
2118
|
exit(1)
|
1984
2119
|
end
|
@@ -2014,30 +2149,36 @@ module Cnvrg
|
|
2014
2149
|
File.open(project_home + "/.cnvrg/idx.yml", "w+") { |f| f.write idx.to_yaml }
|
2015
2150
|
current_tree = Dir.entries(".").reject { |file| file.start_with? '.' or file.eql? "__init__.py" or file.eql? "uwsgi.ini" }
|
2016
2151
|
|
2017
|
-
response["result"]["tree"]
|
2152
|
+
jump_result = Parallel.map(response["result"]["tree"], in_processes: ParallelProcesses, in_thread: ParallelThreads) do |f|
|
2153
|
+
|
2018
2154
|
relative_path = f[0].gsub(/^#{@project.local_path}/, "")
|
2019
2155
|
if f[0].end_with? "/"
|
2020
2156
|
# dir
|
2021
2157
|
if @files.download_dir(f[0], relative_path, project_home)
|
2022
2158
|
current_tree.delete(relative_path[0, relative_path.size-1])
|
2023
|
-
|
2159
|
+
f[0]
|
2024
2160
|
end
|
2025
2161
|
else
|
2026
2162
|
# blob
|
2027
2163
|
if @files.download_file_s3(f[0], relative_path, project_home, commit_sha1=commit_sha1)
|
2028
2164
|
current_tree.delete(relative_path)
|
2029
|
-
|
2030
|
-
successful_changes << relative_path
|
2165
|
+
f[0]
|
2031
2166
|
end
|
2032
2167
|
end
|
2168
|
+
|
2033
2169
|
end
|
2170
|
+
successful_changes = jump_result.select { |x| not x.nil? }
|
2034
2171
|
end
|
2172
|
+
|
2173
|
+
|
2035
2174
|
FileUtils.rm_rf(current_tree)
|
2036
|
-
|
2037
|
-
|
2175
|
+
log_message("Done. Jumped to #{commit_sha1} completed successfully", Thor::Shell::Color::GREEN)
|
2176
|
+
rescue => e
|
2177
|
+
log_message("Error occurred, Aborting", Thor::Shell::Color::RED)
|
2178
|
+
log_error(e)
|
2038
2179
|
rescue SignalException
|
2039
|
-
|
2040
|
-
|
2180
|
+
|
2181
|
+
exit(1)
|
2041
2182
|
end
|
2042
2183
|
end
|
2043
2184
|
|
@@ -2083,13 +2224,14 @@ module Cnvrg
|
|
2083
2224
|
end
|
2084
2225
|
|
2085
2226
|
|
2086
|
-
|
2087
|
-
log_end(0)
|
2227
|
+
log_message("Done. Jumped to #{commit_sha1} completed successfully", Thor::Shell::Color::GREEN)
|
2088
2228
|
end
|
2229
|
+
rescue => e
|
2230
|
+
log_message("Error occurred, Aborting", Thor::Shell::Color::RED)
|
2231
|
+
log_error(e)
|
2089
2232
|
|
2090
2233
|
rescue SignalException
|
2091
|
-
|
2092
|
-
exi(1)
|
2234
|
+
exit(1)
|
2093
2235
|
end
|
2094
2236
|
end
|
2095
2237
|
|
@@ -2100,11 +2242,9 @@ module Cnvrg
|
|
2100
2242
|
|
2101
2243
|
def sync(direct=true)
|
2102
2244
|
verify_logged_in(true) if direct
|
2103
|
-
|
2104
|
-
|
2105
|
-
|
2106
|
-
say 'Syncing project', Thor::Shell::Color::BLUE
|
2107
|
-
end
|
2245
|
+
log_start(__method__, args, options)
|
2246
|
+
log_message('Checking for new updates from remote version', Thor::Shell::Color::BLUE, options["verbose"])
|
2247
|
+
log_message('Syncing project', Thor::Shell::Color::BLUE, !options["verbose"])
|
2108
2248
|
|
2109
2249
|
|
2110
2250
|
invoke :download, [], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync => true
|
@@ -2206,6 +2346,7 @@ module Cnvrg
|
|
2206
2346
|
method_option :ignore, :type => :string, :aliases => ["-i", "--ignore"], :desc => "ignore following files", :default => ""
|
2207
2347
|
method_option :remote, :type => :boolean, :aliases => ["--remote"], :default => false
|
2208
2348
|
method_option :gpu, :type => :boolean, :aliases => ["--gpu"], :default => false
|
2349
|
+
|
2209
2350
|
def exec(*cmd)
|
2210
2351
|
|
2211
2352
|
log = []
|
@@ -2256,7 +2397,7 @@ module Cnvrg
|
|
2256
2397
|
if image_proj and image_proj.is_docker
|
2257
2398
|
container= image_proj.get_container
|
2258
2399
|
if !container
|
2259
|
-
|
2400
|
+
log_message("Couldn't create container with image #{image_proj.image_name}:#{image_proj.image_tag}", Thor::Shell::Color::RED)
|
2260
2401
|
exit(1)
|
2261
2402
|
end
|
2262
2403
|
|
@@ -2288,8 +2429,8 @@ module Cnvrg
|
|
2288
2429
|
begin
|
2289
2430
|
machine_activity = @exp.get_machine_activity(working_dir)
|
2290
2431
|
@exp.start(cmd, platform, machine_name, start_commit, title, email_notification, machine_activity, script_path)
|
2291
|
-
|
2292
|
-
|
2432
|
+
log_message("Experiment's live results: #{Cnvrg::Helpers.remote_url}/#{@project.owner}/projects/#{@project.slug}/experiments/#{@exp.slug}", Thor::Shell::Color::GREEN)
|
2433
|
+
log_message("Running: #{cmd}\n", Thor::Shell::Color::BLUE)
|
2293
2434
|
unless @exp.slug.nil?
|
2294
2435
|
real = Time.now
|
2295
2436
|
exp_success = true
|
@@ -2303,7 +2444,7 @@ module Cnvrg
|
|
2303
2444
|
cur_time = Time.now
|
2304
2445
|
real_time= Time.now-real
|
2305
2446
|
|
2306
|
-
cur_log = {
|
2447
|
+
cur_log = {time: cur_time,
|
2307
2448
|
message: line,
|
2308
2449
|
type: "stdout",
|
2309
2450
|
real: real_time
|
@@ -2311,7 +2452,7 @@ module Cnvrg
|
|
2311
2452
|
}
|
2312
2453
|
|
2313
2454
|
|
2314
|
-
|
2455
|
+
if print_log
|
2315
2456
|
puts cur_log
|
2316
2457
|
end
|
2317
2458
|
log << cur_log
|
@@ -2323,7 +2464,7 @@ module Cnvrg
|
|
2323
2464
|
stats = usage_metrics_in_docker(docker_id)
|
2324
2465
|
cpu = stats[0]
|
2325
2466
|
memory = stats[1]
|
2326
|
-
if
|
2467
|
+
if is_on_gpu
|
2327
2468
|
gpu_stats = gpu_util
|
2328
2469
|
gpu_utilization = gpu_stats[0]
|
2329
2470
|
gpu_memory_util = gpu_stats[1]
|
@@ -2334,9 +2475,9 @@ module Cnvrg
|
|
2334
2475
|
end
|
2335
2476
|
log.each do |l|
|
2336
2477
|
if remote and is_on_gpu
|
2337
|
-
l.merge!(cpu:cpu,memory:memory, gpu_util:gpu_utilization,gpu_memory_util:gpu_memory_util)
|
2478
|
+
l.merge!(cpu: cpu, memory: memory, gpu_util: gpu_utilization, gpu_memory_util: gpu_memory_util)
|
2338
2479
|
else
|
2339
|
-
l.merge!(cpu:cpu,memory:memory)
|
2480
|
+
l.merge!(cpu: cpu, memory: memory)
|
2340
2481
|
end
|
2341
2482
|
end
|
2342
2483
|
|
@@ -2351,8 +2492,10 @@ module Cnvrg
|
|
2351
2492
|
end
|
2352
2493
|
|
2353
2494
|
end
|
2354
|
-
rescue
|
2355
|
-
|
2495
|
+
rescue => e
|
2496
|
+
|
2497
|
+
log_message("Failed to upload ongoing results, continuing with experiment", Thor::Shell::Color::YELLOW)
|
2498
|
+
log_error(e)
|
2356
2499
|
end
|
2357
2500
|
|
2358
2501
|
end
|
@@ -2367,22 +2510,23 @@ module Cnvrg
|
|
2367
2510
|
end
|
2368
2511
|
|
2369
2512
|
rescue Errno::EIO => e
|
2370
|
-
|
2371
|
-
|
2372
|
-
|
2513
|
+
log_error(e)
|
2514
|
+
|
2515
|
+
# break
|
2516
|
+
rescue Errno::ENOENT => e
|
2373
2517
|
|
2374
2518
|
exp_success = false
|
2375
2519
|
|
2376
|
-
|
2520
|
+
log_message("command \"#{cmd}\" couldn't be executed, verify command is valid", Thor::Shell::Color::RED)
|
2521
|
+
log_error(e)
|
2377
2522
|
rescue PTY::ChildExited
|
2378
|
-
log_end(1, "proccess exited")
|
2379
2523
|
exp_success = false
|
2380
|
-
|
2524
|
+
log_message("The process exited!", Thor::Shell::Color::RED)
|
2381
2525
|
rescue => e
|
2382
|
-
log_end(-1, e.message)
|
2383
2526
|
res = @exp.end(log, 1, start_commit, cpu_average, memory_average)
|
2384
2527
|
|
2385
|
-
|
2528
|
+
log_message("Error occurred,aborting", Thor::Shell::Color::RED)
|
2529
|
+
log_error(e)
|
2386
2530
|
exit(0)
|
2387
2531
|
end
|
2388
2532
|
::Process.wait pid
|
@@ -2396,21 +2540,19 @@ module Cnvrg
|
|
2396
2540
|
if !Cnvrg::Helpers.internet_connection?
|
2397
2541
|
wait_offline = agree "Seems like you're offline, wait until you're back online?", Thor::Shell::Color::YELLOW
|
2398
2542
|
if wait_offline
|
2399
|
-
|
2543
|
+
log_message("Waiting until your'e online..", Thor::Shell::Color::BLUE)
|
2400
2544
|
while !Cnvrg::Helpers.internet_connection?
|
2401
2545
|
end
|
2402
2546
|
exit_status = 0
|
2403
2547
|
else
|
2404
|
-
|
2405
|
-
log_end(1, "experiment has failed,computer is offline")
|
2548
|
+
log_message("Experiment has failed, your'e computer is offline", Thor::Shell::Color::RED)
|
2406
2549
|
exit(0)
|
2407
2550
|
end
|
2408
2551
|
else
|
2409
2552
|
|
2410
2553
|
end_commit = @project.last_local_commit
|
2411
2554
|
res = @exp.end(log, exit_status, end_commit, cpu_average, memory_average)
|
2412
|
-
|
2413
|
-
log_end(1, "experiment has failed")
|
2555
|
+
log_message("Experiment has failed, look at the log for more details or run cnvrg exec --log", Thor::Shell::Color::RED)
|
2414
2556
|
exit(0)
|
2415
2557
|
end
|
2416
2558
|
|
@@ -2427,15 +2569,15 @@ module Cnvrg
|
|
2427
2569
|
|
2428
2570
|
res = @exp.end(log, exit_status, end_commit, cpu_average, memory_average)
|
2429
2571
|
check = Helpers.checkmark()
|
2430
|
-
|
2431
|
-
log_end(0)
|
2572
|
+
log_message("#{check} Done. Experiment's results were updated!", Thor::Shell::Color::GREEN)
|
2432
2573
|
end
|
2433
2574
|
rescue => e
|
2434
|
-
log_end(-1, e.message)
|
2435
2575
|
if container
|
2436
2576
|
container.stop()
|
2437
2577
|
end
|
2438
|
-
|
2578
|
+
log_message("Couldn't run #{cmd}, check your input parameters", Thor::Shell::Color::RED)
|
2579
|
+
log_error(e)
|
2580
|
+
|
2439
2581
|
exit(1)
|
2440
2582
|
end
|
2441
2583
|
|
@@ -2445,7 +2587,6 @@ module Cnvrg
|
|
2445
2587
|
end
|
2446
2588
|
rescue SignalException
|
2447
2589
|
exit_status = -1
|
2448
|
-
log_end(-1)
|
2449
2590
|
end_commit = @project.last_local_commit
|
2450
2591
|
|
2451
2592
|
res = @exp.end(log, exit_status, end_commit, cpu_average, memory_average)
|
@@ -2488,7 +2629,7 @@ module Cnvrg
|
|
2488
2629
|
start_commit = @project.last_local_commit
|
2489
2630
|
cmd = cmd.join("\s")
|
2490
2631
|
|
2491
|
-
|
2632
|
+
log_message("Running: #{cmd}\n", Thor::Shell::Color::BLUE)
|
2492
2633
|
|
2493
2634
|
@exp = Experiment.new(@project.owner, @project.slug)
|
2494
2635
|
machine_activity = @exp.get_machine_activity(project_home)
|
@@ -2540,7 +2681,7 @@ module Cnvrg
|
|
2540
2681
|
|
2541
2682
|
end
|
2542
2683
|
rescue
|
2543
|
-
|
2684
|
+
log_message("Failed to upload ongoing results, continuing with experiment", Thor::Shell::Color::YELLOW)
|
2544
2685
|
end
|
2545
2686
|
|
2546
2687
|
end
|
@@ -2557,18 +2698,16 @@ module Cnvrg
|
|
2557
2698
|
rescue Errno::EIO => e
|
2558
2699
|
break
|
2559
2700
|
rescue Errno::ENOENT
|
2560
|
-
log_end(1, "command #{cmd} isn't valid")
|
2561
2701
|
|
2562
2702
|
exp_success = false
|
2563
2703
|
|
2564
|
-
|
2704
|
+
log_message("command \"#{cmd}\" couldn't be executed, verify command is valid", Thor::Shell::Color::RED)
|
2565
2705
|
rescue PTY::ChildExited
|
2566
|
-
log_end(1, "proccess exited")
|
2567
2706
|
exp_success = false
|
2568
|
-
|
2707
|
+
log_message("The process exited!", Thor::Shell::Color::RED)
|
2569
2708
|
rescue => e
|
2570
|
-
|
2571
|
-
|
2709
|
+
log_message("Error occurred, aborting", Thor::Shell::Color::RED)
|
2710
|
+
log_error(e)
|
2572
2711
|
exit(0)
|
2573
2712
|
end
|
2574
2713
|
::Process.wait pid
|
@@ -2582,13 +2721,12 @@ module Cnvrg
|
|
2582
2721
|
if !Cnvrg::Helpers.internet_connection?
|
2583
2722
|
wait_offline = agree "Seems like you're offline, wait until your'e back online?", Thor::Shell::Color::YELLOW
|
2584
2723
|
if wait_offline
|
2585
|
-
|
2724
|
+
log_message("Waiting until your'e online..", Thor::Shell::Color::BLUE)
|
2586
2725
|
while !Cnvrg::Helpers.internet_connection?
|
2587
2726
|
end
|
2588
2727
|
exit_status = 0
|
2589
2728
|
else
|
2590
|
-
|
2591
|
-
log_end(1, "experiment has failed,computer is offline")
|
2729
|
+
log_message("Experiment has failed, your'e computer is offline", Thor::Shell::Color::RED)
|
2592
2730
|
exit(0)
|
2593
2731
|
end
|
2594
2732
|
else
|
@@ -2596,18 +2734,17 @@ module Cnvrg
|
|
2596
2734
|
end_commit = @project.last_local_commit
|
2597
2735
|
res = @exp.end(log, exit_status, end_commit, cpu_average, memory_average)
|
2598
2736
|
@image.update_image_activity(@project.last_local_commit, @exp.slug)
|
2599
|
-
|
2600
|
-
log_end(1, "experiment has failed")
|
2737
|
+
log_message("Experiment has failed, look at the log for more details or run cnvrg exec --log", Thor::Shell::Color::RED)
|
2601
2738
|
exit(0)
|
2602
2739
|
end
|
2603
2740
|
|
2604
2741
|
end
|
2605
2742
|
if sync_after
|
2606
|
-
|
2743
|
+
log_message("Syncing project after running", Thor::Shell::Color::BLUE)
|
2607
2744
|
# Sync after run
|
2608
2745
|
download()
|
2609
2746
|
upload()
|
2610
|
-
|
2747
|
+
log_message("Done Syncing", Thor::Shell::Color::BLUE)
|
2611
2748
|
end
|
2612
2749
|
end_commit = @project.last_local_commit
|
2613
2750
|
|
@@ -2615,12 +2752,11 @@ module Cnvrg
|
|
2615
2752
|
@image.update_image_activity(@project.last_local_commit, @exp.slug)
|
2616
2753
|
|
2617
2754
|
check = Helpers.checkmark()
|
2618
|
-
|
2619
|
-
log_end(0)
|
2755
|
+
log_message("#{check} Done. Experiment's result: #{Cnvrg::Helpers.remote_url}/#{@project.owner}/projects/#{@project.slug}/experiments/#{@exp.slug}", Thor::Shell::Color::GREEN)
|
2620
2756
|
end
|
2621
2757
|
rescue => e
|
2622
|
-
|
2623
|
-
|
2758
|
+
log_message("Couldn't run #{cmd}, check your input parameters", Thor::Shell::Color::RED)
|
2759
|
+
log_error(e)
|
2624
2760
|
exit(1)
|
2625
2761
|
end
|
2626
2762
|
|
@@ -2628,9 +2764,12 @@ module Cnvrg
|
|
2628
2764
|
end
|
2629
2765
|
|
2630
2766
|
end
|
2767
|
+
rescue => e
|
2768
|
+
log_message("Error occurred, Aborting", Thor::Shell::Color::RED)
|
2769
|
+
log_error(e)
|
2770
|
+
|
2631
2771
|
rescue SignalException
|
2632
2772
|
exit_status = -1
|
2633
|
-
log_end(-1)
|
2634
2773
|
end_commit = @project.last_local_commit
|
2635
2774
|
if !@exp.nil?
|
2636
2775
|
|
@@ -2684,7 +2823,7 @@ module Cnvrg
|
|
2684
2823
|
elsif time[2].downcase().start_with? "days"
|
2685
2824
|
new = local + (time[1].to_i * 3600 * 24)
|
2686
2825
|
else
|
2687
|
-
|
2826
|
+
log_message("Could not undersatnd when to schedule experiment", Thor::Shell::Color::RED)
|
2688
2827
|
exit(1)
|
2689
2828
|
end
|
2690
2829
|
new_time = new.to_s
|
@@ -2738,13 +2877,13 @@ module Cnvrg
|
|
2738
2877
|
image_slug = image.image_slug
|
2739
2878
|
end
|
2740
2879
|
|
2741
|
-
invoke :sync, [false], []
|
2880
|
+
invoke :sync, [false], [] if sync_before
|
2742
2881
|
|
2743
2882
|
|
2744
2883
|
if command.include? "'"
|
2745
2884
|
oc = command.to_enum(:scan, /'/).map { Regexp.last_match }
|
2746
2885
|
pairs = oc.enum_for(:each_slice, 2).to_a
|
2747
|
-
pairs.each_with_index do |p,i|
|
2886
|
+
pairs.each_with_index do |p, i|
|
2748
2887
|
add=0
|
2749
2888
|
if i!=0
|
2750
2889
|
add=2*i
|
@@ -2755,7 +2894,7 @@ module Cnvrg
|
|
2755
2894
|
|
2756
2895
|
|
2757
2896
|
end
|
2758
|
-
|
2897
|
+
log_message("Running remote experiment", Thor::Shell::Color::BLUE)
|
2759
2898
|
exp = Experiment.new(project.owner, project.slug)
|
2760
2899
|
res = exp.exec_remote(command, commit_to_run, instance_type, image_slug, schedule, local_timestamp, grid, path_to_cmd, data, data_commit)
|
2761
2900
|
if Cnvrg::CLI.is_response_success(res)
|
@@ -2796,16 +2935,18 @@ module Cnvrg
|
|
2796
2935
|
str = "Running grid search. #{Cnvrg::Helpers.remote_url}/#{project.owner}/projects/#{project.slug}/experiments?grid=#{res["result"]["exp_url"]} "
|
2797
2936
|
end
|
2798
2937
|
|
2799
|
-
|
2938
|
+
log_message(str, Thor::Shell::Color::GREEN)
|
2800
2939
|
|
2801
2940
|
exit(0)
|
2802
2941
|
# end
|
2803
2942
|
end
|
2804
2943
|
|
2944
|
+
rescue => e
|
2945
|
+
log_message("Error occurred, Aborting", Thor::Shell::Color::RED)
|
2946
|
+
log_error(e)
|
2805
2947
|
|
2806
2948
|
rescue SignalException
|
2807
2949
|
exit_status = -1
|
2808
|
-
log_end(-1)
|
2809
2950
|
end_commit = project.last_local_commit
|
2810
2951
|
|
2811
2952
|
res = @exp.end(log, exit_status, end_commit, "", "")
|
@@ -2852,7 +2993,7 @@ module Cnvrg
|
|
2852
2993
|
begin
|
2853
2994
|
num_workers = workers.to_i
|
2854
2995
|
rescue
|
2855
|
-
|
2996
|
+
log_message("Number of workers should be a number between 1 to 10", Thor::Shell::Color::RED)
|
2856
2997
|
exit(1)
|
2857
2998
|
end
|
2858
2999
|
file_as_input = options["file_as_input"] || false
|
@@ -2910,16 +3051,18 @@ module Cnvrg
|
|
2910
3051
|
# end
|
2911
3052
|
# else
|
2912
3053
|
check = Helpers.checkmark()
|
2913
|
-
|
3054
|
+
log_message("#{check} Deployment process is on: #{Cnvrg::Helpers.remote_url}/#{project.owner}/projects/#{project.slug}/deploys/show/#{res["result"]["deploy_slug"]}", Thor::Shell::Color::GREEN)
|
2914
3055
|
|
2915
3056
|
exit(0)
|
2916
3057
|
# end
|
2917
3058
|
end
|
3059
|
+
rescue => e
|
3060
|
+
log_message("Error occurred, Aborting", Thor::Shell::Color::RED)
|
3061
|
+
log_error(e)
|
2918
3062
|
|
2919
3063
|
|
2920
3064
|
rescue SignalException
|
2921
3065
|
exit_status = -1
|
2922
|
-
log_end(-1)
|
2923
3066
|
end_commit = project.last_local_commit
|
2924
3067
|
|
2925
3068
|
res = @exp.end(log, exit_status, end_commit, "", "")
|
@@ -3017,7 +3160,7 @@ module Cnvrg
|
|
3017
3160
|
res = @image.remote_notebook(notebook_dir, instance_type, kernel, data, data_commit)
|
3018
3161
|
if Cnvrg::CLI.is_response_success(res)
|
3019
3162
|
if res["result"]["machine"] == -1
|
3020
|
-
|
3163
|
+
log_message("There are no available machines", Thor::Shell::Color::BLUE)
|
3021
3164
|
create = yes? "create new machine?", Thor::Shell::Color::YELLOW
|
3022
3165
|
if create
|
3023
3166
|
res = Cnvrg::API.request("users/#{@image.owner}/machines/list", 'GET')
|
@@ -3030,22 +3173,22 @@ module Cnvrg
|
|
3030
3173
|
if !url.nil? and !url.empty?
|
3031
3174
|
check = Helpers.checkmark()
|
3032
3175
|
|
3033
|
-
|
3176
|
+
log_message("#{check} Notebook server started successfully", Thor::Shell::Color::GREEN)
|
3034
3177
|
Launchy.open(url)
|
3035
3178
|
else
|
3036
|
-
|
3179
|
+
log_message("Couldn't run notebook server", Thor::Shell::Color::RED)
|
3037
3180
|
end
|
3038
3181
|
exit(0)
|
3039
3182
|
end
|
3040
3183
|
end
|
3041
3184
|
else
|
3042
|
-
|
3185
|
+
log_message("No machines are avilable", Thor::Shell::Color::RED)
|
3043
3186
|
exit(0)
|
3044
3187
|
end
|
3045
3188
|
|
3046
3189
|
|
3047
3190
|
else
|
3048
|
-
|
3191
|
+
log_message("Can't execute command on remote machine with local image", Thor::Shell::Color::RED)
|
3049
3192
|
exit(1)
|
3050
3193
|
|
3051
3194
|
end
|
@@ -3053,17 +3196,18 @@ module Cnvrg
|
|
3053
3196
|
note_url = res["result"]["notebook_url"]
|
3054
3197
|
@image.set_note_url(note_url)
|
3055
3198
|
check = Helpers.checkmark()
|
3056
|
-
|
3199
|
+
log_message("#{check} Notebook is on: #{Cnvrg::Helpers.remote_url}/#{@image.owner}/projects/#{@image.project_slug}/notebook_sessions/show/#{note_url}", Thor::Shell::Color::GREEN)
|
3057
3200
|
# Launchy.open(url)
|
3058
3201
|
|
3059
3202
|
exit(0)
|
3060
3203
|
end
|
3061
3204
|
end
|
3062
|
-
|
3205
|
+
rescue => e
|
3206
|
+
log_message("Error occurred, Aborting", Thor::Shell::Color::RED)
|
3207
|
+
log_error(e)
|
3063
3208
|
|
3064
3209
|
rescue SignalException
|
3065
3210
|
exit_status = -1
|
3066
|
-
log_end(-1)
|
3067
3211
|
end_commit = @project.last_local_commit
|
3068
3212
|
|
3069
3213
|
res = @exp.end(log, exit_status, end_commit, cpu_average, memory_average)
|
@@ -3073,7 +3217,7 @@ module Cnvrg
|
|
3073
3217
|
end
|
3074
3218
|
end
|
3075
3219
|
|
3076
|
-
desc 'search_libraries', 'search if libraries installed'
|
3220
|
+
desc 'search_libraries', 'search if libraries installed', :hide => true
|
3077
3221
|
|
3078
3222
|
def search_libraries(library)
|
3079
3223
|
begin
|
@@ -3135,7 +3279,7 @@ module Cnvrg
|
|
3135
3279
|
end
|
3136
3280
|
end
|
3137
3281
|
|
3138
|
-
desc 'show_libraries', 'show system libraries installed'
|
3282
|
+
desc 'show_libraries', 'show system libraries installed', :hide => true
|
3139
3283
|
method_option :system, :type => :boolean, :aliases => ["-s", "--s"], :default => false, :desc => "show also system libraries installed"
|
3140
3284
|
|
3141
3285
|
def show_libraries
|
@@ -3194,7 +3338,7 @@ module Cnvrg
|
|
3194
3338
|
end
|
3195
3339
|
|
3196
3340
|
|
3197
|
-
desc 'run_notebook', 'Starts a new notebook environment'
|
3341
|
+
desc 'run_notebook', 'Starts a new notebook environment', :hide => true
|
3198
3342
|
method_option :notebook_dir, :type => :string, :aliases => ["-n", "--n"], :default => "", :desc => "relative path to notebook dir from current directory"
|
3199
3343
|
method_option :remote, :type => :boolean, :aliases => ["-r", "--r"], :default => false, :desc => "run on remote machine"
|
3200
3344
|
method_option :kernel, :type => :string, :aliases => ["-k", "--k"], :default => "", :desc => "default kernel"
|
@@ -3343,7 +3487,7 @@ module Cnvrg
|
|
3343
3487
|
|
3344
3488
|
rescue => e
|
3345
3489
|
log_end(-1, e.message)
|
3346
|
-
say "Error
|
3490
|
+
say "Error occurred, aborting", Thor::Shell::Color::RED
|
3347
3491
|
if container
|
3348
3492
|
container.stop()
|
3349
3493
|
end
|
@@ -3380,10 +3524,10 @@ module Cnvrg
|
|
3380
3524
|
project_dir = is_cnvrg_dir()
|
3381
3525
|
|
3382
3526
|
|
3383
|
-
|
3527
|
+
log_message('Checking for new updates from remote version', Thor::Shell::Color::BLUE, options["verbose"])
|
3384
3528
|
invoke :sync, [false], :verbose => options["verbose"]
|
3385
3529
|
|
3386
|
-
|
3530
|
+
log_message("Done Syncing", Thor::Shell::Color::BLUE, options["verbose"])
|
3387
3531
|
|
3388
3532
|
|
3389
3533
|
@project = Project.new(project_dir)
|
@@ -3402,37 +3546,34 @@ module Cnvrg
|
|
3402
3546
|
if !container
|
3403
3547
|
check = Helpers.checkmark()
|
3404
3548
|
|
3405
|
-
|
3549
|
+
log_message("#{check} Notebook server stopped successfully", Thor::Shell::Color::GREEN)
|
3406
3550
|
exit(0)
|
3407
3551
|
end
|
3408
3552
|
|
3409
3553
|
|
3410
|
-
|
3554
|
+
log_message("Stopping notebook server...", Thor::Shell::Color::BLUE)
|
3411
3555
|
|
3412
3556
|
check = Helpers.checkmark()
|
3413
3557
|
image.remove_note_slug()
|
3414
3558
|
container.stop()
|
3415
3559
|
|
3416
|
-
|
3417
|
-
log_end(0)
|
3560
|
+
log_message("#{check} Notebook server stopped successfully", Thor::Shell::Color::GREEN)
|
3418
3561
|
|
3419
3562
|
exit(0)
|
3420
3563
|
elsif remote
|
3421
|
-
|
3564
|
+
log_message("Stopping notebook server...", Thor::Shell::Color::BLUE)
|
3422
3565
|
check = Helpers.checkmark()
|
3423
3566
|
|
3424
|
-
|
3425
|
-
log_end(0)
|
3567
|
+
log_message("#{check} Notebook server stopped successfully", Thor::Shell::Color::GREEN)
|
3426
3568
|
exit(0)
|
3427
3569
|
end
|
3428
3570
|
rescue => e
|
3429
|
-
|
3430
|
-
|
3571
|
+
log_message("Error occurd, aborting", Thor::Shell::Color::RED)
|
3572
|
+
log_error(e)
|
3431
3573
|
if container
|
3432
3574
|
container.stop()
|
3433
3575
|
end
|
3434
3576
|
rescue SignalException
|
3435
|
-
log_end(-1)
|
3436
3577
|
say "Aborting"
|
3437
3578
|
exit(1)
|
3438
3579
|
end
|
@@ -3441,7 +3582,7 @@ module Cnvrg
|
|
3441
3582
|
end
|
3442
3583
|
|
3443
3584
|
|
3444
|
-
desc 'install_system_libraries', 'Install libraries'
|
3585
|
+
desc 'install_system_libraries', 'Install libraries', :hide => true
|
3445
3586
|
|
3446
3587
|
def install_system_libraries(*command_to_run)
|
3447
3588
|
begin
|
@@ -3476,7 +3617,7 @@ module Cnvrg
|
|
3476
3617
|
log_end(0)
|
3477
3618
|
rescue => e
|
3478
3619
|
log_end(-1, e.message)
|
3479
|
-
say "Error
|
3620
|
+
say "Error occurred, aborting"
|
3480
3621
|
if container
|
3481
3622
|
container.stop()
|
3482
3623
|
end
|
@@ -3491,7 +3632,7 @@ module Cnvrg
|
|
3491
3632
|
|
3492
3633
|
end
|
3493
3634
|
|
3494
|
-
desc 'install_libraries', 'Install libraries'
|
3635
|
+
desc 'install_libraries', 'Install libraries', :hide => true
|
3495
3636
|
method_option :requirement, :type => :string, :aliases => ["-r", "--r"], :default => "", :desc => "Install from the given requirements file"
|
3496
3637
|
|
3497
3638
|
def install_python_libraries(*lib)
|
@@ -3538,7 +3679,7 @@ module Cnvrg
|
|
3538
3679
|
log_end(0)
|
3539
3680
|
rescue => e
|
3540
3681
|
log_end(-1, e.message)
|
3541
|
-
say "Error
|
3682
|
+
say "Error occurred, aborting"
|
3542
3683
|
if container
|
3543
3684
|
container.stop()
|
3544
3685
|
end
|
@@ -3558,121 +3699,117 @@ module Cnvrg
|
|
3558
3699
|
method_option :gpu, :type => :boolean, :aliases => ["--gpu"], :default => false
|
3559
3700
|
method_option :gpuxl, :type => :boolean, :aliases => ["--gpuxl"], :default => false
|
3560
3701
|
method_option :gpuxxl, :type => :boolean, :aliases => ["--gpuxxl"], :default => false
|
3561
|
-
method_option :image, :type => :string, :aliases => ["-i","--image"], :default => ""
|
3562
|
-
method_option :public, :type => :boolean, :aliases => ["-p","--public"], :default => false
|
3563
|
-
method_option :base, :type => :boolean, :aliases => ["-b","--base"], :default => false
|
3702
|
+
method_option :image, :type => :string, :aliases => ["-i", "--image"], :default => ""
|
3703
|
+
method_option :public, :type => :boolean, :aliases => ["-p", "--public"], :default => false
|
3704
|
+
method_option :base, :type => :boolean, :aliases => ["-b", "--base"], :default => false
|
3564
3705
|
method_option :python3, :type => :boolean, :aliases => ["--python3"], :default => false
|
3565
3706
|
method_option :docker_path, :type => :string, :aliases => ["--docker_path"], :default => ""
|
3566
3707
|
|
3567
3708
|
|
3568
|
-
desc 'create_custom_image', 'run commands inside containers', :hide=>true
|
3709
|
+
desc 'create_custom_image', 'run commands inside containers', :hide => true
|
3569
3710
|
|
3570
3711
|
def build_image(image_name)
|
3571
3712
|
begin
|
3572
|
-
|
3573
|
-
|
3574
|
-
|
3575
|
-
|
3576
|
-
|
3577
|
-
|
3578
|
-
|
3579
|
-
|
3580
|
-
|
3581
|
-
|
3582
|
-
|
3583
|
-
|
3584
|
-
|
3585
|
-
|
3586
|
-
|
3587
|
-
|
3588
|
-
|
3589
|
-
|
3590
|
-
|
3591
|
-
|
3592
|
-
|
3593
|
-
|
3594
|
-
|
3595
|
-
|
3596
|
-
|
3597
|
-
|
3598
|
-
|
3599
|
-
|
3600
|
-
|
3601
|
-
|
3602
|
-
|
3603
|
-
end
|
3713
|
+
verify_logged_in(false)
|
3714
|
+
log_start(__method__, args, options)
|
3715
|
+
instances = {"small" => options["small"], "medium" => options["medium"], "large" => options["large"],
|
3716
|
+
"gpu" => options["gpu"], "gpuxl" => options["gpuxl"], "gpuxxl" => options["gpuxxl"]}
|
3717
|
+
instance_type = get_instance_type(instances)
|
3718
|
+
image_extend = options["image"]
|
3719
|
+
public = options["public"]
|
3720
|
+
base = options["base"]
|
3721
|
+
python3 = options["python3"]
|
3722
|
+
docker_path = options["docker_path"]
|
3723
|
+
owner = CLI.get_owner
|
3724
|
+
checks = Helpers.checkmark()
|
3725
|
+
tar_path = nil
|
3726
|
+
if !docker_path.nil? and !docker_path.empty?
|
3727
|
+
docker_path = File.absolute_path(docker_path)
|
3728
|
+
#create tar of the docker path: it could be a docker file, and it could be a docker folder
|
3729
|
+
tar_path = File.expand_path('~')+"/.cnvrg/tmp/docker_#{File.basename docker_path}.tar.gz"
|
3730
|
+
resp = create_docker_tar(docker_path, tar_path)
|
3731
|
+
if !resp
|
3732
|
+
log_message("Couldn't create tar from docker path", Thor::Shell::Color::RED)
|
3733
|
+
FileUtils.rm_rf tar_path
|
3734
|
+
exit(1)
|
3735
|
+
end
|
3736
|
+
files = Cnvrg::Files.new(owner, "")
|
3737
|
+
resp = Images.create_new_custom_image_with_docker(instance_type, owner, image_name, public, base, image_extend, python3, tar_path, files)
|
3738
|
+
if resp
|
3739
|
+
end
|
3740
|
+
else
|
3741
|
+
log_message("Creating machine for your custom image, this may take a few moments...", Thor::Shell::Color::BLUE)
|
3742
|
+
resp = Images.create_new_custom_image(instance_type, owner, image_name, public, base, image_extend, python3, nil)
|
3604
3743
|
|
3605
|
-
if Cnvrg::CLI.is_response_success(resp,false)
|
3606
|
-
image_slug = resp["result"]["slug"]
|
3607
|
-
container = resp["result"]["machine_c"]
|
3608
|
-
say "#{checks} Created image and machine successfully", Thor::Shell::Color::GREEN
|
3609
|
-
say "Connecting to machine", Thor::Shell::Color::BLUE
|
3610
|
-
ssh = Ssh.new(resp)
|
3611
|
-
if !ssh.is_ssh
|
3612
|
-
say "Couldn't connect to machine,aborting", Thor::Shell::Color::RED
|
3613
|
-
Images.revoke_custom_new_image(owner, image_slug)
|
3614
|
-
log_end(-1,"Couldn't connect to machine,aborting")
|
3615
3744
|
end
|
3616
|
-
say "run command until ctrl + c or quit is initiated", Thor::Shell::Color::BLUE
|
3617
|
-
begin
|
3618
|
-
logs = []
|
3619
3745
|
|
3620
|
-
|
3621
|
-
|
3622
|
-
|
3623
|
-
|
3624
|
-
|
3625
|
-
|
3626
|
-
|
3627
|
-
|
3628
|
-
|
3629
|
-
|
3630
|
-
|
3631
|
-
|
3632
|
-
|
3633
|
-
res = res_parsed.join(",")
|
3634
|
-
end
|
3746
|
+
if Cnvrg::CLI.is_response_success(resp, false)
|
3747
|
+
image_slug = resp["result"]["slug"]
|
3748
|
+
container = resp["result"]["machine_c"]
|
3749
|
+
log_message("#{checks} Created image and machine successfully", Thor::Shell::Color::GREEN)
|
3750
|
+
log_message("Connecting to machine", Thor::Shell::Color::BLUE)
|
3751
|
+
ssh = Ssh.new(resp)
|
3752
|
+
if !ssh.is_ssh
|
3753
|
+
log_message("Couldn't connect to machine,aborting", Thor::Shell::Color::RED)
|
3754
|
+
Images.revoke_custom_new_image(owner, image_slug)
|
3755
|
+
end
|
3756
|
+
log_message("run command until ctrl + c or quit is initiated", Thor::Shell::Color::BLUE)
|
3757
|
+
begin
|
3758
|
+
logs = []
|
3635
3759
|
|
3636
|
-
|
3637
|
-
|
3760
|
+
while true
|
3761
|
+
command = ask("$>")
|
3762
|
+
logs << {time: Time.now,
|
3763
|
+
message: command,
|
3764
|
+
type: "stdout"
|
3765
|
+
}
|
3766
|
+
if command.eql? "quit"
|
3767
|
+
log_message("Commiting Image..", Thor::Shell::Color::BLUE)
|
3768
|
+
break
|
3769
|
+
end
|
3770
|
+
res = ssh.exec_command(command)
|
3771
|
+
begin
|
3772
|
+
res_parsed = JSON.parse(res)
|
3773
|
+
res = res_parsed.join(",")
|
3774
|
+
end
|
3775
|
+
|
3776
|
+
puts res
|
3777
|
+
logs << {time: Time.now,
|
3638
3778
|
message: res,
|
3639
3779
|
type: "stdout"
|
3640
|
-
|
3641
|
-
|
3780
|
+
}
|
3781
|
+
logs.flatten!
|
3642
3782
|
|
3643
|
-
|
3783
|
+
end
|
3644
3784
|
|
3645
|
-
|
3646
|
-
|
3785
|
+
rescue SignalException
|
3786
|
+
log_message("Commiting Image..", Thor::Shell::Color::BLUE)
|
3647
3787
|
|
3648
|
-
|
3649
|
-
|
3650
|
-
|
3651
|
-
|
3652
|
-
|
3653
|
-
|
3654
|
-
|
3788
|
+
end
|
3789
|
+
resp = Images.commit_custom_image(owner, image_slug, logs)
|
3790
|
+
if Cnvrg::CLI.is_response_success(resp, false)
|
3791
|
+
log_message("#{checks} Image commited successfuly, email will be sent when image is ready", Thor::Shell::Color::GREEN)
|
3792
|
+
else
|
3793
|
+
if image_slug
|
3794
|
+
Images.revoke_custom_new_image(owner, image_slug)
|
3795
|
+
end
|
3796
|
+
if ssh
|
3797
|
+
ssh.close_ssh()
|
3798
|
+
end
|
3799
|
+
log_message("Image couldn't be commited, rolling back changes", Thor::Shell::Color::RED)
|
3800
|
+
|
3801
|
+
exit(1)
|
3655
3802
|
end
|
3656
3803
|
if ssh
|
3657
3804
|
ssh.close_ssh()
|
3658
3805
|
end
|
3659
|
-
say "Image couldn't be commited, rolling back changes", Thor::Shell::Color::RED
|
3660
|
-
|
3661
|
-
exit(1)
|
3662
|
-
end
|
3663
|
-
if ssh
|
3664
|
-
ssh.close_ssh()
|
3665
|
-
end
|
3666
|
-
|
3667
|
-
|
3668
|
-
|
3669
|
-
|
3670
3806
|
|
3671
3807
|
|
3808
|
+
end
|
3809
|
+
rescue => e
|
3810
|
+
log_message("Error occurd, aborting", Thor::Shell::Color::RED)
|
3672
3811
|
|
3673
|
-
|
3674
|
-
rescue =>e
|
3675
|
-
puts e
|
3812
|
+
log_error(e)
|
3676
3813
|
if image_slug
|
3677
3814
|
Images.revoke_custom_new_image(owner, image_slug)
|
3678
3815
|
end
|
@@ -3695,7 +3832,7 @@ module Cnvrg
|
|
3695
3832
|
end
|
3696
3833
|
|
3697
3834
|
|
3698
|
-
desc 'build', 'run commands inside containers'
|
3835
|
+
desc 'build', 'run commands inside containers', :hide => true
|
3699
3836
|
method_option :install, :type => :string, :aliases => ["--i"], :default => nil, :desc => "Install from the given instructions file"
|
3700
3837
|
|
3701
3838
|
def build(*cmd)
|
@@ -3756,7 +3893,7 @@ module Cnvrg
|
|
3756
3893
|
log_end(0)
|
3757
3894
|
rescue => e
|
3758
3895
|
log_end(-1, e.message)
|
3759
|
-
say "Error
|
3896
|
+
say "Error occurred, aborting", Thor::Shell::Color::RED
|
3760
3897
|
if container
|
3761
3898
|
container.stop()
|
3762
3899
|
end
|
@@ -3771,7 +3908,7 @@ module Cnvrg
|
|
3771
3908
|
|
3772
3909
|
end
|
3773
3910
|
|
3774
|
-
desc 'commit_notebook', 'commit notebook changes to create a new notebook image'
|
3911
|
+
desc 'commit_notebook', 'commit notebook changes to create a new notebook image', :hide => true
|
3775
3912
|
|
3776
3913
|
def commit_image
|
3777
3914
|
verify_logged_in(true)
|
@@ -3804,7 +3941,7 @@ module Cnvrg
|
|
3804
3941
|
return new_image.id
|
3805
3942
|
rescue => e
|
3806
3943
|
log_end(-1, e.message)
|
3807
|
-
say "\nError occurred,aborting"
|
3944
|
+
say "\nError occurred, aborting"
|
3808
3945
|
exit(1)
|
3809
3946
|
rescue SignalException
|
3810
3947
|
log_end(-1)
|
@@ -3813,7 +3950,7 @@ module Cnvrg
|
|
3813
3950
|
end
|
3814
3951
|
end
|
3815
3952
|
|
3816
|
-
desc 'sync_image', 'sync current container into image, and push it to cnvrg repository'
|
3953
|
+
desc 'sync_image', 'sync current container into image, and push it to cnvrg repository', :hide => true
|
3817
3954
|
method_option :is_public, :type => :boolean, :aliases => ["-p", "--p"], :default => false, :desc => "is public"
|
3818
3955
|
method_option :is_base, :type => :boolean, :aliases => ["-b", "--b"], :default => false, :desc => "is base for others images"
|
3819
3956
|
method_option :message, :type => :string, :aliases => ["-m", "--m"], :default => "", :desc => "commit message for this image"
|
@@ -3854,13 +3991,13 @@ module Cnvrg
|
|
3854
3991
|
log_start(__method__, args, options)
|
3855
3992
|
working_dir = is_cnvrg_dir()
|
3856
3993
|
if !name.empty? and name == "cnvrg"
|
3857
|
-
|
3994
|
+
log_message("can't create image with the name cnvrg", Thor::Shell::Color::RED)
|
3858
3995
|
exit(1)
|
3859
3996
|
end
|
3860
3997
|
begin
|
3861
3998
|
image = is_project_with_docker(working_dir)
|
3862
3999
|
if !image or !image.is_docker
|
3863
|
-
|
4000
|
+
log_message("Couldn't find image related to project", Thor::Shell::Color::RED)
|
3864
4001
|
exit(0)
|
3865
4002
|
end
|
3866
4003
|
if !name.nil? and !name.empty?
|
@@ -3870,18 +4007,18 @@ module Cnvrg
|
|
3870
4007
|
end
|
3871
4008
|
stored_commands = File.open(working_dir+"/.cnvrg/custom_image.txt").read.chop.gsub("\n", ",")
|
3872
4009
|
if stored_commands.nil? or stored_commands.empty?
|
3873
|
-
|
4010
|
+
log_message("Nothing to push", Thor::Shell::Color::BLUE)
|
3874
4011
|
exit(0)
|
3875
4012
|
end
|
3876
4013
|
|
3877
|
-
|
4014
|
+
log_message("Pushing new image", Thor::Shell::Color::BLUE)
|
3878
4015
|
if image.create_custom_image(name, working_dir, stored_commands)
|
3879
4016
|
|
3880
|
-
|
4017
|
+
log_message("Image was updated successfully", Thor::Shell::Color::GREEN)
|
3881
4018
|
end
|
3882
4019
|
rescue => e
|
3883
|
-
|
3884
|
-
|
4020
|
+
log_message("error occurred, aborting", Thor::Shell::Color::RED)
|
4021
|
+
log_error(e)
|
3885
4022
|
|
3886
4023
|
end
|
3887
4024
|
end
|
@@ -3903,16 +4040,15 @@ module Cnvrg
|
|
3903
4040
|
message = message.join(" ")
|
3904
4041
|
end
|
3905
4042
|
|
3906
|
-
|
4043
|
+
log_message("Saving image's current state", Thor::Shell::Color::BLUE)
|
3907
4044
|
image.save(path)
|
3908
4045
|
|
3909
4046
|
begin
|
3910
|
-
|
4047
|
+
log_message("Compressing image file to upload", Thor::Shell::Color::BLUE)
|
3911
4048
|
gzipRes = system("gzip -f #{path}")
|
3912
4049
|
if !gzipRes
|
3913
|
-
log_End(1, "can't create gz file from image")
|
3914
4050
|
|
3915
|
-
|
4051
|
+
log_message("Couldn't create tar file from image", Thor::Shell::Color::RED)
|
3916
4052
|
exit(1)
|
3917
4053
|
end
|
3918
4054
|
path = path+".gz"
|
@@ -3920,7 +4056,7 @@ module Cnvrg
|
|
3920
4056
|
|
3921
4057
|
exit_status = $?.exitstatus
|
3922
4058
|
if exit_status == 0
|
3923
|
-
|
4059
|
+
log_message("Uploading image file", Thor::Shell::Color::BLUE)
|
3924
4060
|
|
3925
4061
|
diff =container_changes(Dir.pwd)
|
3926
4062
|
res = @files.upload_image(path, image_name, owner, is_public, is_base, diff[1], diff[0], diff[2], message, image.commit_id)
|
@@ -3930,23 +4066,19 @@ module Cnvrg
|
|
3930
4066
|
image_loc.update_slug(res["result"]["id"])
|
3931
4067
|
|
3932
4068
|
checks = Helpers.checkmark()
|
3933
|
-
|
3934
|
-
log_end(0)
|
4069
|
+
log_message("#{checks} Done", Thor::Shell::Color::GREEN)
|
3935
4070
|
else
|
3936
|
-
|
3937
|
-
log_end(1, "can't create upload imag")
|
4071
|
+
log_message("Couldn't upload image", Thor::Shell::Color::RED)
|
3938
4072
|
|
3939
4073
|
end
|
3940
4074
|
else
|
3941
|
-
|
3942
|
-
log_end(1, "can't create upload imag")
|
4075
|
+
log_message("Couldn't create image file for: #{image_name}", Thor::Shell::Color::RED)
|
3943
4076
|
exit(1)
|
3944
4077
|
end
|
3945
4078
|
rescue => e
|
3946
|
-
|
3947
|
-
|
4079
|
+
log_message("Couldn't upload image file for: #{image_name}", Thor::Shell::Color::RED)
|
4080
|
+
log_error(e)
|
3948
4081
|
rescue SignalException
|
3949
|
-
log_end(-1)
|
3950
4082
|
|
3951
4083
|
say "Couldn't upload image file for: #{image_name}", Thor::Shell::Color::RED
|
3952
4084
|
exit(1)
|
@@ -4026,7 +4158,7 @@ module Cnvrg
|
|
4026
4158
|
'8888/tcp' => {},
|
4027
4159
|
},
|
4028
4160
|
'HostConfig' => {
|
4029
|
-
'Binds'=> ["/var/run/docker.sock:/var/run/docker.sock","/usr/bin/docker:/usr/bin/docker"
|
4161
|
+
'Binds' => ["/var/run/docker.sock:/var/run/docker.sock", "/usr/bin/docker:/usr/bin/docker"],
|
4030
4162
|
'PortBindings' => {
|
4031
4163
|
'8888/tcp' => [
|
4032
4164
|
{'HostPort' => "#{port}", 'HostIp' => 'localhost'}
|
@@ -4264,14 +4396,13 @@ module Cnvrg
|
|
4264
4396
|
path = File.expand_path('~')+"/.cnvrg/tmp/#{image_name}.tar.gz"
|
4265
4397
|
@files = Cnvrg::Files.new(owner, "")
|
4266
4398
|
|
4267
|
-
|
4399
|
+
log_message("Downloading image file", Thor::Shell::Color::BLUE)
|
4268
4400
|
begin
|
4269
4401
|
if @files.download_image(path, image_slug, owner)
|
4270
4402
|
gzipRes = system("gunzip -f #{path}")
|
4271
4403
|
if !gzipRes
|
4272
|
-
log_End(1, "can't unzip gz file from image")
|
4273
4404
|
|
4274
|
-
|
4405
|
+
log_message("Couldn't create tar file from image", Thor::Shell::Color::RED)
|
4275
4406
|
exit(1)
|
4276
4407
|
else
|
4277
4408
|
path = path.gsub(".gz", "")
|
@@ -4279,17 +4410,14 @@ module Cnvrg
|
|
4279
4410
|
end
|
4280
4411
|
|
4281
4412
|
else
|
4282
|
-
|
4283
|
-
log_end(1, "can't download image")
|
4413
|
+
log_message("Couldn't download image #{image_name}", Thor::Shell::Color::RED)
|
4284
4414
|
return false
|
4285
4415
|
end
|
4286
4416
|
rescue Interrupt
|
4287
|
-
log_end(-1)
|
4288
4417
|
say "The user has exited to process, aborting", Thor::Shell::Color::BLUE
|
4289
4418
|
exit(1)
|
4290
4419
|
end
|
4291
4420
|
rescue SignalException
|
4292
|
-
log_end(-1)
|
4293
4421
|
say "\nAborting"
|
4294
4422
|
exit(1)
|
4295
4423
|
end
|
@@ -4363,10 +4491,8 @@ module Cnvrg
|
|
4363
4491
|
printf "%-20s %-20s %-30s %-20s %-20s\n", u["name"], u["project"], created_by, u["is_public"], update_at
|
4364
4492
|
end
|
4365
4493
|
end
|
4366
|
-
log_end(0)
|
4367
4494
|
return res["result"]["images"]
|
4368
4495
|
rescue SignalException
|
4369
|
-
log_end(-1)
|
4370
4496
|
say "\nAborting"
|
4371
4497
|
exit(1)
|
4372
4498
|
end
|
@@ -4390,10 +4516,8 @@ module Cnvrg
|
|
4390
4516
|
printf "%-20s %-20s %-20s\n", u["name"], u["created_by"], update_at
|
4391
4517
|
end
|
4392
4518
|
end
|
4393
|
-
log_end(0)
|
4394
4519
|
return res["result"]["images"]
|
4395
4520
|
rescue SignalException
|
4396
|
-
log_end(-1)
|
4397
4521
|
say "\nAborting"
|
4398
4522
|
exit(1)
|
4399
4523
|
end
|
@@ -4401,7 +4525,7 @@ module Cnvrg
|
|
4401
4525
|
|
4402
4526
|
end
|
4403
4527
|
|
4404
|
-
desc 'get_machine', 'create new aws machine'
|
4528
|
+
desc 'get_machine', 'create new aws machine', :hide => true
|
4405
4529
|
|
4406
4530
|
def get_machine()
|
4407
4531
|
begin
|
@@ -4449,12 +4573,12 @@ module Cnvrg
|
|
4449
4573
|
owner = Cnvrg::CLI.get_owner()
|
4450
4574
|
image = Cnvrg::Images.image_exist(owner, image_name)
|
4451
4575
|
if !image
|
4452
|
-
|
4576
|
+
log_message("Couldn't find image in cnvrg repository", Thor::Shell::Color::RED)
|
4453
4577
|
exit(1)
|
4454
4578
|
end
|
4455
4579
|
path = download_image(image_name, image["slug"])
|
4456
4580
|
if path
|
4457
|
-
|
4581
|
+
log_message("Building image", Thor::Shell::Color::BLUE)
|
4458
4582
|
Docker.options[:read_timeout]=216000
|
4459
4583
|
image = Docker::Image.build_from_dir(path, {'dockerfile' => 'Dockerfile.cpu', 't' => "#{image_name}:latest"}) do |v|
|
4460
4584
|
begin
|
@@ -4470,20 +4594,16 @@ module Cnvrg
|
|
4470
4594
|
if not image.nil?
|
4471
4595
|
FileUtils.rm_rf(path)
|
4472
4596
|
checks = Helpers.checkmark()
|
4473
|
-
|
4474
|
-
log_end(0)
|
4597
|
+
log_message("#{checks} Image built successfully", Thor::Shell::Color::GREEN)
|
4475
4598
|
return image
|
4476
|
-
log_end(0)
|
4477
4599
|
else
|
4478
4600
|
|
4479
|
-
|
4480
|
-
log_end(1, "Could build image")
|
4601
|
+
log_message("Could not build image", Thor::Shell::Color::RED)
|
4481
4602
|
return false
|
4482
4603
|
end
|
4483
4604
|
else
|
4484
4605
|
|
4485
|
-
|
4486
|
-
log_end(1, "Could build image")
|
4606
|
+
log_message("Could not download image", Thor::Shell::Color::RED)
|
4487
4607
|
return false
|
4488
4608
|
|
4489
4609
|
|
@@ -4511,12 +4631,11 @@ module Cnvrg
|
|
4511
4631
|
# end
|
4512
4632
|
rescue => e
|
4513
4633
|
|
4514
|
-
|
4515
|
-
|
4634
|
+
log_message "Error: couldn't build image", Thor::Shell::Color::RED
|
4635
|
+
log_error(e)
|
4516
4636
|
|
4517
4637
|
rescue SignalException
|
4518
4638
|
say "\nAborting"
|
4519
|
-
log_end(-1)
|
4520
4639
|
exit(1)
|
4521
4640
|
ensure
|
4522
4641
|
if path
|
@@ -4543,22 +4662,22 @@ module Cnvrg
|
|
4543
4662
|
if yes? "Image wasn't found locally, pull image from cnvrg repository?", Thor::Shell::Color::YELLOW
|
4544
4663
|
image = pull(docker_image)
|
4545
4664
|
if image
|
4546
|
-
|
4665
|
+
log_message("downloaded image: #{docker_image}", Thor::Shell::Color::BLUE)
|
4547
4666
|
@image = Images.new(working_dir, docker_image)
|
4548
4667
|
else
|
4549
|
-
|
4668
|
+
log_message("Could not create a new project with docker, image was not found", Thor::Shell::Color::RED)
|
4550
4669
|
exit(1)
|
4551
4670
|
end
|
4552
4671
|
else
|
4553
|
-
|
4672
|
+
log_message("Could not create a new project with docker, image was not found", Thor::Shell::Color::RED)
|
4554
4673
|
exit(1)
|
4555
4674
|
|
4556
4675
|
end
|
4557
4676
|
elsif docker_image_local.size == 1
|
4558
|
-
|
4677
|
+
log_message("found image: #{docker_image_local[0]}, setting it up..", Thor::Shell::Color::BLUE)
|
4559
4678
|
@image = Images.new(working_dir, docker_image_local[0])
|
4560
4679
|
elsif docker_image_local.size >1
|
4561
|
-
|
4680
|
+
log_message("found #{docker_image_local.size} images, choose the image name you want to use", Thor::Shell::Color::BLUE)
|
4562
4681
|
image_name = ask "#{docker_image_local.join("\n")}\n", Thor::Shell::Color::BLUE
|
4563
4682
|
image_name = image_name.strip
|
4564
4683
|
@image = Images.new(working_dir, image_name)
|
@@ -4619,11 +4738,11 @@ module Cnvrg
|
|
4619
4738
|
url = "https://cnvrg.io/api"
|
4620
4739
|
end
|
4621
4740
|
config = YAML.load_file(home_dir+"/.cnvrg/config.yml")
|
4622
|
-
compression_path =
|
4741
|
+
compression_path = "#{home_dir}/.cnvrg/tmp"
|
4623
4742
|
if config and !config.nil? and !config.empty? and !config.to_h[:compression_path].nil?
|
4624
4743
|
compression_path = config.to_h[:compression_path]
|
4625
4744
|
end
|
4626
|
-
config = {owner: owner, username: username, version_last_check: get_start_day(), api: url,compression_path:compression_path}
|
4745
|
+
config = {owner: owner, username: username, version_last_check: get_start_day(), api: url, compression_path: compression_path}
|
4627
4746
|
|
4628
4747
|
File.open(home_dir+"/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
|
4629
4748
|
return true
|
@@ -4668,13 +4787,48 @@ module Cnvrg
|
|
4668
4787
|
|
4669
4788
|
def log_start(command, args="", options={})
|
4670
4789
|
begin
|
4671
|
-
$LOG.info "----"
|
4672
4790
|
$LOG.info ruby_version: RUBY_VERSION, os: Cnvrg::Helpers.os(), cli_version: Cnvrg::VERSION
|
4673
4791
|
$LOG.info command: command, args: args, options: options
|
4674
4792
|
rescue
|
4675
4793
|
end
|
4676
4794
|
end
|
4677
4795
|
|
4796
|
+
def log_message(message, type, to_print=true)
|
4797
|
+
if to_print
|
4798
|
+
say message, type
|
4799
|
+
end
|
4800
|
+
begin
|
4801
|
+
if type == Thor::Shell::Color::BLUE
|
4802
|
+
$LOG.info message: message, type: "info"
|
4803
|
+
elsif type == Thor::Shell::Color::RED
|
4804
|
+
$LOG.error message: message, type: "error"
|
4805
|
+
elsif type == Thor::Shell::Color::YELLOW
|
4806
|
+
$LOG.warn message: message, type: "warning"
|
4807
|
+
elsif type == Thor::Shell::Color::GREEN
|
4808
|
+
$LOG.info message: message, type: "success"
|
4809
|
+
else
|
4810
|
+
$LOG.info message: message, type: "unknown"
|
4811
|
+
end
|
4812
|
+
|
4813
|
+
rescue => e
|
4814
|
+
puts e
|
4815
|
+
end
|
4816
|
+
end
|
4817
|
+
|
4818
|
+
def log_error(e)
|
4819
|
+
begin
|
4820
|
+
$LOG.error message: e.message, type: "error"
|
4821
|
+
size = e.backtrace.size
|
4822
|
+
min = 4
|
4823
|
+
if min>= size
|
4824
|
+
min = size
|
4825
|
+
end
|
4826
|
+
$LOG.error message: "Error: #{e.message}, Backtrace: #{e.backtrace[0..min].join(",")}", type: "error"
|
4827
|
+
rescue
|
4828
|
+
end
|
4829
|
+
end
|
4830
|
+
|
4831
|
+
|
4678
4832
|
def log_end(exit_status=0, error=nil)
|
4679
4833
|
begin
|
4680
4834
|
if exit_status != 0
|
@@ -4690,6 +4844,8 @@ module Cnvrg
|
|
4690
4844
|
if response.nil?
|
4691
4845
|
if !Cnvrg::Helpers.internet_connection?
|
4692
4846
|
say("<%= color('Error:You seems to be offline', RED) %>")
|
4847
|
+
$LOG.error message: "offline connection", type: "error"
|
4848
|
+
|
4693
4849
|
end
|
4694
4850
|
if should_exit
|
4695
4851
|
exit(1)
|
@@ -4702,6 +4858,7 @@ module Cnvrg
|
|
4702
4858
|
if response["status"] == 500
|
4703
4859
|
say("<%= color('Server Error', RED) %>")
|
4704
4860
|
else
|
4861
|
+
$LOG.error message: error, type: "error"
|
4705
4862
|
say("<%= color('Error:#{error}', RED) %>")
|
4706
4863
|
end
|
4707
4864
|
|
@@ -4734,7 +4891,8 @@ module Cnvrg
|
|
4734
4891
|
api = config.to_h[:api]
|
4735
4892
|
return api.gsub!("/api", "")
|
4736
4893
|
end
|
4737
|
-
|
4894
|
+
|
4895
|
+
def get_compression_path
|
4738
4896
|
home_dir = File.expand_path('~')
|
4739
4897
|
|
4740
4898
|
config = YAML.load_file(home_dir+ "/.cnvrg/config.yml")
|
@@ -4777,6 +4935,14 @@ module Cnvrg
|
|
4777
4935
|
def log_handler
|
4778
4936
|
begin
|
4779
4937
|
date = DateTime.now.strftime("%m_%d_%Y")
|
4938
|
+
home_dir = File.expand_path('~')
|
4939
|
+
|
4940
|
+
if !File.directory? home_dir+"/.cnvrg"
|
4941
|
+
FileUtils.mkdir_p([home_dir+"/.cnvrg", home_dir+"/.cnvrg/tmp"])
|
4942
|
+
end
|
4943
|
+
if !File.exist?(home_dir+"/.cnvrg/config.yml")
|
4944
|
+
FileUtils.touch [home_dir+"/.cnvrg/config.yml"]
|
4945
|
+
end
|
4780
4946
|
logfile = File.expand_path('~') +"/.cnvrg/log_#{date}.log"
|
4781
4947
|
if !File.exist? logfile
|
4782
4948
|
FileUtils.touch([logfile])
|
@@ -4798,27 +4964,34 @@ module Cnvrg
|
|
4798
4964
|
end
|
4799
4965
|
|
4800
4966
|
end
|
4801
|
-
|
4967
|
+
config = LogStashLogger.configure do |config|
|
4968
|
+
config.customize_event do |event|
|
4969
|
+
event.remove('@version')
|
4970
|
+
event.remove('severity')
|
4971
|
+
end
|
4972
|
+
end
|
4973
|
+
$LOG = LogStashLogger.new(type: :file, path: logfile, sync: true, config: config)
|
4802
4974
|
remove_old_log_files()
|
4975
|
+
rescue
|
4976
|
+
|
4803
4977
|
end
|
4804
4978
|
end
|
4805
4979
|
|
4806
4980
|
def remove_old_log_files()
|
4807
4981
|
begin
|
4808
|
-
|
4809
|
-
|
4810
|
-
|
4811
|
-
|
4812
|
-
|
4813
|
-
|
4982
|
+
last_week = (Time.now - (7*24*60*60)).strftime("%Y-%m-%d")
|
4983
|
+
home = File.expand_path('~')
|
4984
|
+
log_files = Dir["#{home}/.cnvrg/tmp/*.log"]
|
4985
|
+
log_files.each do |l|
|
4986
|
+
if File.mtime(l).strftime("%Y-%m-%d") < last_week
|
4987
|
+
FileUtils.rm_rf(l)
|
4988
|
+
end
|
4814
4989
|
end
|
4815
4990
|
end
|
4816
|
-
end
|
4817
4991
|
|
4818
4992
|
end
|
4819
4993
|
|
4820
4994
|
|
4821
|
-
|
4822
4995
|
def verify_logged_in(in_dir=true)
|
4823
4996
|
log_handler()
|
4824
4997
|
auth = Cnvrg::Auth.new
|
@@ -5109,15 +5282,16 @@ module Cnvrg
|
|
5109
5282
|
return relative
|
5110
5283
|
end
|
5111
5284
|
end
|
5112
|
-
|
5285
|
+
|
5286
|
+
def create_docker_tar(docker_path, tar_path)
|
5113
5287
|
begin
|
5114
5288
|
|
5115
|
-
|
5116
|
-
|
5117
|
-
|
5118
|
-
|
5119
|
-
|
5120
|
-
|
5289
|
+
if File.directory? (docker_path)
|
5290
|
+
`cd #{docker_path} && tar -czf #{tar_path} . `
|
5291
|
+
else
|
5292
|
+
dir_name = File.dirname docker_path
|
5293
|
+
`cd #{dir_name} && tar -czf #{tar_path} #{File.basename(docker_path)}`
|
5294
|
+
end
|
5121
5295
|
rescue => e
|
5122
5296
|
puts "Exception while compressing docker path: #{e.message}"
|
5123
5297
|
end
|
@@ -5125,7 +5299,7 @@ module Cnvrg
|
|
5125
5299
|
return $?.success?
|
5126
5300
|
end
|
5127
5301
|
|
5128
|
-
def create_tar(path_in, path_out, tar_files,no_compression=false)
|
5302
|
+
def create_tar(path_in, path_out, tar_files, no_compression=false)
|
5129
5303
|
#The cd is meant for cases when running cnvrg data uplaod not in the main folder
|
5130
5304
|
begin
|
5131
5305
|
if no_compression
|
@@ -5144,27 +5318,29 @@ module Cnvrg
|
|
5144
5318
|
`tar -xvf #{file_path} -C #{dir_path} > /dev/null 2>&1`
|
5145
5319
|
return $?.success?
|
5146
5320
|
end
|
5321
|
+
|
5147
5322
|
def cpu_usage
|
5148
5323
|
cpu_usage = 0.0
|
5149
5324
|
begin
|
5150
5325
|
cpu = `top b -n 2 -d 2 |grep %Cpu |tail -1 |awk '{print $2+$3}'`
|
5151
|
-
|
5152
|
-
|
5153
|
-
|
5326
|
+
if !cpu.nil?
|
5327
|
+
cpu_usage = cpu.to_f
|
5328
|
+
end
|
5154
5329
|
rescue
|
5155
5330
|
cpu_usage = 0.0
|
5156
5331
|
end
|
5157
5332
|
|
5158
5333
|
return cpu_usage
|
5159
5334
|
end
|
5335
|
+
|
5160
5336
|
def memory_usage
|
5161
5337
|
prec = 0.0
|
5162
5338
|
begin
|
5163
|
-
|
5164
|
-
|
5339
|
+
used = `free -mt |grep Mem: |awk '{print $3}'`
|
5340
|
+
total = `free -mt |grep Mem: |awk '{print $2}'`
|
5165
5341
|
|
5166
|
-
|
5167
|
-
|
5342
|
+
used_f = used.to_f if !used.nil?
|
5343
|
+
total_f = total.to_f if !total.nil?
|
5168
5344
|
prec = (used_f / total_f)*100
|
5169
5345
|
prec = prec.round(2)
|
5170
5346
|
rescue
|
@@ -5174,18 +5350,19 @@ module Cnvrg
|
|
5174
5350
|
|
5175
5351
|
|
5176
5352
|
end
|
5353
|
+
|
5177
5354
|
def gpu_util
|
5178
5355
|
gpu = [0.0, 0.0]
|
5179
5356
|
begin
|
5180
|
-
|
5357
|
+
gpu_stats = `nvidia-smi --query-gpu=utilization.gpu,utilization.memory --format=csv |tail -1`
|
5181
5358
|
|
5182
|
-
|
5183
|
-
|
5184
|
-
|
5185
|
-
|
5186
|
-
|
5187
|
-
|
5188
|
-
|
5359
|
+
if !gpu_stats.nil?
|
5360
|
+
gpu = gpu_stats.strip
|
5361
|
+
gpu = gpu_stats.gsub!("%", "").split(",")
|
5362
|
+
gpu[0] = gpu[0].to_f
|
5363
|
+
gpu[1] = gpu[1].to_f
|
5364
|
+
return gpu
|
5365
|
+
end
|
5189
5366
|
|
5190
5367
|
rescue
|
5191
5368
|
return gpu
|
@@ -5193,18 +5370,19 @@ module Cnvrg
|
|
5193
5370
|
|
5194
5371
|
|
5195
5372
|
end
|
5373
|
+
|
5196
5374
|
def usage_metrics_in_docker(docker_id)
|
5197
|
-
res = [0.0,0.0]
|
5375
|
+
res = [0.0, 0.0]
|
5198
5376
|
begin
|
5199
5377
|
if docker_id.nil?
|
5200
5378
|
docker_id = `cat /etc/hostname`
|
5201
5379
|
end
|
5202
|
-
|
5380
|
+
stats = `sudo docker stats #{docker_id} --no-stream --format "{{.CPUPerc}},{{.MemPerc}},{{.BlockIO}}"`
|
5203
5381
|
if !stats.nil?
|
5204
5382
|
conv = stats.split(",")
|
5205
|
-
cpu = conv[0].gsub!("%","").to_f
|
5206
|
-
memory = conv[1].gsub!("%","").to_f
|
5207
|
-
res = [cpu,memory]
|
5383
|
+
cpu = conv[0].gsub!("%", "").to_f
|
5384
|
+
memory = conv[1].gsub!("%", "").to_f
|
5385
|
+
res = [cpu, memory]
|
5208
5386
|
return res
|
5209
5387
|
end
|
5210
5388
|
rescue
|
@@ -5212,7 +5390,6 @@ module Cnvrg
|
|
5212
5390
|
end
|
5213
5391
|
|
5214
5392
|
|
5215
|
-
|
5216
5393
|
end
|
5217
5394
|
|
5218
5395
|
def with_progress_bar
|